Develop Reference

Why is my code not waiting for the completion of the function?

I am trying to read some data from a file and store it in a database.
This is part of a larger transaction and I need the returned ids for further steps.
async parseHeaders(mysqlCon, ghID, csv) {
var self = this;
var hIDs = [];
var skip = true;
var idx = 0;
console.log("Parsing headers");
return new Promise(async function(resolve, reject) {
try {
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(csv)
});
await lineReader.on('close', async function () {
console.log("done: ", JSON.stringify(hIDs));
resolve(hIDs);
});
await lineReader.on('line', async function (line) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx++] = await self.getParamID(mysqlCon, ghID, v, c, data);//, function(hID,sidx) { //add data in case the parameter is not in DB, yet.
}
});
} catch(e) {
console.log(JSON.stringify(e));
reject("some error occured: " + e);
}
});
}
async getParamID(mysqlCon,ghID,variable,category,data) {
return new Promise(function(resolve, reject) {
var sql = "SELECT ID FROM Parameter WHERE GreenHouseID="+ghID+" AND Variable = '" + variable + "' AND Category='" + category + "'";
mysqlCon.query(sql, function (err, result, fields) {
if(result.length === 0 || err) { //apparently not in DB, yet ... add it (Acronym and Machine need to be set manually).
sql = "INSERT INTO Parameter (GreenHouseID,Variable,Category,Control) VALUES ("+ghID+",'"+variable+"','"+category+"','"+data[3]+"')";
mysqlCon.query(sql, function (err, result) {
if(err) {
console.log(result,err,this.sql);
reject(err);
} else {
console.log("Inserting ",variable," into DB: ",JSON.stringify(result));
resolve(result.insertId); //added, return generated ID.
}
});
} else {
resolve(result[0].ID); //found in DB .. return ID.
}
});
});
}
The functions above are in the base class and called by the following code:
let headerIDs = await self.parseHeaders(mysqlCon, ghID, filePath);
console.log("headers:",JSON.stringify(headerIDs));
The sequence of events is that everything in parseHeaders completes except for the call to self.getParamID and control returns to the calling function which prints an empty array for headerIDs.
The console.log statements in self.getParamID are then printed afterward.
What am I missing?
Thank you

As you want to execute an asynchronous action for every line we could define a handler to do right that:
const once = (target, evt) => new Promise(res => target.on(evt, res));
function mapLines(reader, action) {
const results = [];
let index = 0;
reader.on("line", line => results.push(action(line, index++)));
return once(reader, "close").then(() => Promise.all(results));
}
So now you can solve that easily:
let skip = false;
const hIDs = [];
await mapLines(lineReader, async function (line, idx) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx] = await self.getParamID(mysqlCon, ghID, v, c, data);
}
});

why isn't this promise resolving?

I have a promise that is meant to collect metadata on a file and then resolve it with the metadata that was collected. Here is how I am trying to get its result:
getMetadata: function(text, url){
return this.getpageno(url).then(function(pagecount){
return new Promise(function(accept, reject){
var result = []
var dataMatch = rx.exec(text)
var produktDaten = dataMatch[1].split("&&&").filter(Boolean);
var dokuArr = dataMatch[2].split("&&&").filter(Boolean);
console.log('the produktdaten are ' + produktDaten)
for (var i=0; i<produktDaten.length; i+=4){
var entry = {}
for(var j=0; j<dokuArr.length; j+=3){
var seite = dokuArr[j+2];
// make sure seite is a digit
if (!(/^\d+$/.test(seite))){
console.log(seite + ' was not a valid page number')
throw err
}
if (/(A|a)lle?/i.test(nummer)){
entry.kks.pages[beschreibung] = seite;
// })
}
else if (nummer === kksNummer) {
entry.kks.pages[beschreibung] = seite;
}
}
entry.hersteller = produktDaten[i+1]
entry.typ = produktDaten[i+2]
entry.artikelNummer = produktDaten[i+3]
result.push(entry)
}
if (result.length>0){
return accept(result)
}
return reject()
})
})
},
getpageno: function(url){
var self=this
var pdf = pdfjs.getDocument(url);
return pdf.then(function(pdf){
var maxPages = pdf.pdfInfo.numPages;
var countPromises = []; // collecting all page promises
for (var j = 1; j <= maxPages; j++) {
try {
var page = pdf.getPage(j);
var txt = "";
countPromises.push(page.then(function(page) { // add page promise
var textContent = page.getTextContent();
return textContent.then(function(text){ // return content promise
return text.items.map(function (s) { return s.str; }).join('&&&'); // value page text
});
}));
}
catch(e){
console.log(e)
}
}
// Wait for all pages and join text
return Promise.all(countPromises).then(function (texts) {
// since doumentation pages do not add the final '&&&', must add one manually (only after rx has been found)
var fulltext = texts.reduce(function(full, text){
if (rx.test(full)){
var next = '&&&'+text
return full+=next
}
return full+=text
}, '')
return [fulltext, texts]
});
}).then(function(textarr){
var fulltext = textarr[0]
self.fulltext = fulltext;
var texts = textarr[1]
try {
var partialmatch = rx.exec(fulltext)[0]
var count = texts.reduce(function(pageno, text){
var tomatch = text.replace(/.*Typ&&&/, '')
if (tomatch.length>0 && partialmatch.indexOf(tomatch) > -1){
pageno++
}
return pageno;
}, 0)
}
catch(e){
console.log(e)
}
return count;
})
}
Edited the entry to show that I'm now returning the value of the gepageno function.
The data that I am expecting is logged but not available as a result of accept(). Can anyone tell what could be going wrong?

I have tried to simulate and give a solution of your promise. I have posted pseudo code.
Promise flow
var testFunction = ((result) => {
return new Promise(function (resolve, reject) {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("2. testFunction resolved");
resolve("Final response");
}
});
});
var getpageno = ((url) => {
return new Promise((resolve, reject) => {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("1. getpageno resolved");
resolve(45);
}
});
});
var getMetadata = ((text, url) => {
console.log("0. getmetadata resolved");
var self = this;
getpageno(url)
.then((pageCount) => {
console.log("> pageCount :", pageCount);
return testFunction(pageCount);
})
.then((data) => {
console.log(">", data);
});
});
getMetadata("hell", "https://www.google.com");
output
0. getmetadata resolved
1. getpageno resolved
> pageCount : 452. testFunction resolved
> Final response

you need to return the promise on the third line
return self.getpageno(url).then
to access its eventual fulfillment value.

How to wait for forEach to complete when each iteration calls an asynchronous options?

Alright, here's what the plan is. Go through each file, add the file into the array. Once all files are added, then combine them using the JSZipUtility and Docxtemplater:
'click .merge-icon': (e) => {
var programId = Router.current().url.split('/').pop();
var programObj = Programs.findOne(programId);
var insertedDocuments = [];
var i = 0;
var count = programObj.activityIds.count;
var fileDownloadPromise = new Promise((resolve, reject) => {
programObj.activityIds.forEach(function(activityId) {
var activityObj = Activities.findOne(activityId);
var documentObj = ActivityFiles.findOne(activityObj.documents.pop()._id);
JSZipUtils.getBinaryContent(documentObj.url(), callback);
function callback(error, content) {
var zip = new JSZip(content);
var doc = new Docxtemplater().loadZip(zip);
var xml = zip.files[doc.fileTypeConfig.textPath].asText();
xml = xml.substring(xml.indexOf("<w:body>") + 8);
xml = xml.substring(0, xml.indexOf("</w:body>"));
xml = xml.substring(0, xml.indexOf("<w:sectPr"));
insertedDocuments.push(xml);
i++;
if (i == count - 1) {
resolve();
}
}
});
});
fileDownloadPromise.then(() => {
JSZipUtils.getBinaryContent('/assets/template.docx', callback);
function callback(error, content) {
console.log(content);
var zip = new JSZip(content);
var doc = new Docxtemplater().loadZip(zip);
setData(doc);
}
function setData(doc) {
doc.setData({
body: insertedDocuments.join('<w:br/><w:br/>')
});
doc.render();
useResult(doc);
}
function useResult(doc) {
var out = doc.getZip().generate({
type: 'blob',
mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
});
saveAs(out, programObj.name + '.docx');
}
});
}
Turns out nothing's happening. What's wrong with the execution of this Promise here ?
I'm only calling resolve when every file has been loaded in the array.

how to make async html parser in nodejs with promises?

having fun with promises in JS and trying to craft simple xpath website parser, but I am struggling with logic on finishing overall parsing process, my code is:
var request = require('request');
var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var olxMain = 'https://www.some.site/';
var xpathRoot = '//a[contains(#href, "https://www.some.site/mask/")]';
var linksXpath = '//a';
var allGlobalLinks = [];
var getLink = function (node) {
for (key in node['attributes']) {
if (node['attributes'][key]['name'] === 'href') {
return node['attributes'][key]['value'];
}
}
}
var getData = function (url, xpathPattern) {
return new Promise(function (resolve, reject) {
console.log("Opening " + url);
var processResponse = function (error, response, body) {
var doc = new dom().parseFromString(body);
var childNodes = xpath.select(xpathPattern, doc);
var links = childNodes.map(function (n) {
return getLink(n);
});
resolve(links);
};
request({url: url}, processResponse);
}
);
}
var arrayUnique = function (x, i, a) {
return a.indexOf(x) == i;
};
var main = function () {
getData(olxMain, xpathRoot).then(function (links) {
links = links.filter(arrayUnique);
var maxThreads = 10, n = 0;
var chunks = [];
for (k in links) {
var url = links[k];
n++;
if (n <= maxThreads)
chunks.push(url);
else {
n = 0;
// console.log(chunks);
Promise.all(chunks.map(function (url) {
return getData(url, linksXpath);
})).then(function (links) {
// add these links to global scope list here
});
console.log("Finished mappings iteration");
});
chunks = [];
}
}
;
});
}
main();
So what I want is basically some kind of threadPool with promises, how to I manage these 10 promises, when they all are finished, I should spawn another 10 more, until list is finished and all Promises have finished ?

Prepending to a file in javascript using OS.File

Any idea why onSuccess is never called?
filesystem ..
rob#work:~$ cat test.txt
hello
hello2
rob#work:~$ pwd
/home/rob
addon code ..
var filePath = '/home/rob/test.txt',
combinedString = 'new file content';
const {TextDecoder, TextEncoder, OS} = Cu.import("resource://gre/modules/osfile.jsm", {});
OS.File.read(filePath).then(function(data) { // check if specified file exists
console.log(data)
function onSuccess(array) { // prepend to file
let text = new TextDecoder().decode(array);
let encodedArray = new TextEncoder().encode(combinedString + text);
console.log('exists: ' + combinedString + text);
var promise = OS.File.writeAtomic(filePath, encodedArray);
promise.then(
function() {
console.log('success');
},
function(ex) {
console.log('fail');
}
);
}
}, function(ex) { // file doesn't exist, create a new one
if (ex.becauseNoSuchFile) {
let encodedArray = new TextEncoder().encode(combinedString);
var promise = OS.File.writeAtomic(filePath, encodedArray);
console.log('new: ' + combinedString + text);
promise.then(
function() {
console.log('success');
},
function(ex) {
console.log('fail');
}
);
}
});
this is the only output in console ..
console.log: savetexttofile: Uint8Array {"0":104,"1":101,"2":108,"3":108,"4":111,"5":10,"6":104,"7":101,"8":108,"9":108,"10":111,"11":50,"12":10}
Total time: 5.596181 seconds
Program terminated successfully.

let promise = OS.File.read(filePath);
promise = promise.then(function onSuccess(contents) {
let text = new TextDecoder().decode(contents);
combinedString = combinedString + '\n\n' + text;
if (System.getPlatform().indexOf('win') >= 0)
combinedString = combinedString.replace(/[\n]/g, '\r\n');
let encodedArray = new TextEncoder().encode(combinedString);
var promise = OS.File.writeAtomic(filePath, encodedArray);
promise.then(
function() {
console.log('success');
},
function(ex) {
console.log('error');
}
);
return true;
},
function onError(reason) {
if (System.getPlatform().indexOf('win') >= 0)
combinedString = combinedString.replace(/[\n]/g, '\r\n');
let encodedArray = new TextEncoder().encode(combinedString);
if (reason.becauseNoSuchFile()) {
var promise = OS.File.writeAtomic(filePath, encodedArray);
promise.then(
function() {
console.log('success');
},
function(ex) {
console.log('error');
}
);
return false;
}
});