Develop Reference

using promises and async for protobufjs loadcall

I am trying to figure out the best way to re-write the following code:
var api = function(id, contract) {
var callback = function (error, root) {
if (error)
throw error;
var by = Buffer.from(contract,'base64')
var es = root.lookupType("Contract")
var esMsg = es.decode(by)
var esBytes = es.encode(esMsg).finish()
signature = id.sign(esBytes).toString('base64')
}
return new Promise((resolve, reject) => {
protobuf.load("contract.proto", callback)
})
}
var signContract = async(privateKey, contract) => {
let signature
var id = await crypto.keys.unmarshalPrivateKey(Buffer.from(privateKey, 'base64'))
result = await api(id,contract,signature)
}
function getSessionSignature (hash, time) {
return config.id + ":" + hash + ":" + time
}
module.exports = configure(({ ky }) => {
return async function * signbatch (input, options) {
var contracts = input.Contracts
for (var i = 0 ; i < contracts.length ; i++) {
contracts[i].contract = await signContract(config.PrivKey, contracts[i].contract)
}
//add signed contracts to the searchParams
searchParams.append("arg", JSON.stringify(contracts))
let res
res = await ky.post('storage/upload/signbatch', {
searchParams
}).json()
yield JSON.stringify({})
} else {
yield JSON.stringify({error:"Private key not found"})
}
}
})
My issue is how do I write the sign async code to pass in privateKey and contract variables to api var function and return the signature back to the result variable to be assigned to contracts[i].contract ? Please note that the id.sign(..) function is Promise inside the callback function.

You need to resolve the promise in the api function, the docs suggest you could use the single argument variant here, e.g.
var root = await protobuf.load("contract.proto");
... // (The code you currently by have in 'callback'
return signature;
As the generator is async, yield will emit a Promise which you can (obviously) handle with either .then or await

Why is my code not waiting for the completion of the function?

I am trying to read some data from a file and store it in a database.
This is part of a larger transaction and I need the returned ids for further steps.
async parseHeaders(mysqlCon, ghID, csv) {
var self = this;
var hIDs = [];
var skip = true;
var idx = 0;
console.log("Parsing headers");
return new Promise(async function(resolve, reject) {
try {
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(csv)
});
await lineReader.on('close', async function () {
console.log("done: ", JSON.stringify(hIDs));
resolve(hIDs);
});
await lineReader.on('line', async function (line) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx++] = await self.getParamID(mysqlCon, ghID, v, c, data);//, function(hID,sidx) { //add data in case the parameter is not in DB, yet.
}
});
} catch(e) {
console.log(JSON.stringify(e));
reject("some error occured: " + e);
}
});
}
async getParamID(mysqlCon,ghID,variable,category,data) {
return new Promise(function(resolve, reject) {
var sql = "SELECT ID FROM Parameter WHERE GreenHouseID="+ghID+" AND Variable = '" + variable + "' AND Category='" + category + "'";
mysqlCon.query(sql, function (err, result, fields) {
if(result.length === 0 || err) { //apparently not in DB, yet ... add it (Acronym and Machine need to be set manually).
sql = "INSERT INTO Parameter (GreenHouseID,Variable,Category,Control) VALUES ("+ghID+",'"+variable+"','"+category+"','"+data[3]+"')";
mysqlCon.query(sql, function (err, result) {
if(err) {
console.log(result,err,this.sql);
reject(err);
} else {
console.log("Inserting ",variable," into DB: ",JSON.stringify(result));
resolve(result.insertId); //added, return generated ID.
}
});
} else {
resolve(result[0].ID); //found in DB .. return ID.
}
});
});
}
The functions above are in the base class and called by the following code:
let headerIDs = await self.parseHeaders(mysqlCon, ghID, filePath);
console.log("headers:",JSON.stringify(headerIDs));
The sequence of events is that everything in parseHeaders completes except for the call to self.getParamID and control returns to the calling function which prints an empty array for headerIDs.
The console.log statements in self.getParamID are then printed afterward.
What am I missing?
Thank you

As you want to execute an asynchronous action for every line we could define a handler to do right that:
const once = (target, evt) => new Promise(res => target.on(evt, res));
function mapLines(reader, action) {
const results = [];
let index = 0;
reader.on("line", line => results.push(action(line, index++)));
return once(reader, "close").then(() => Promise.all(results));
}
So now you can solve that easily:
let skip = false;
const hIDs = [];
await mapLines(lineReader, async function (line, idx) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx] = await self.getParamID(mysqlCon, ghID, v, c, data);
}
});

why isn't this promise resolving?

I have a promise that is meant to collect metadata on a file and then resolve it with the metadata that was collected. Here is how I am trying to get its result:
getMetadata: function(text, url){
return this.getpageno(url).then(function(pagecount){
return new Promise(function(accept, reject){
var result = []
var dataMatch = rx.exec(text)
var produktDaten = dataMatch[1].split("&&&").filter(Boolean);
var dokuArr = dataMatch[2].split("&&&").filter(Boolean);
console.log('the produktdaten are ' + produktDaten)
for (var i=0; i<produktDaten.length; i+=4){
var entry = {}
for(var j=0; j<dokuArr.length; j+=3){
var seite = dokuArr[j+2];
// make sure seite is a digit
if (!(/^\d+$/.test(seite))){
console.log(seite + ' was not a valid page number')
throw err
}
if (/(A|a)lle?/i.test(nummer)){
entry.kks.pages[beschreibung] = seite;
// })
}
else if (nummer === kksNummer) {
entry.kks.pages[beschreibung] = seite;
}
}
entry.hersteller = produktDaten[i+1]
entry.typ = produktDaten[i+2]
entry.artikelNummer = produktDaten[i+3]
result.push(entry)
}
if (result.length>0){
return accept(result)
}
return reject()
})
})
},
getpageno: function(url){
var self=this
var pdf = pdfjs.getDocument(url);
return pdf.then(function(pdf){
var maxPages = pdf.pdfInfo.numPages;
var countPromises = []; // collecting all page promises
for (var j = 1; j <= maxPages; j++) {
try {
var page = pdf.getPage(j);
var txt = "";
countPromises.push(page.then(function(page) { // add page promise
var textContent = page.getTextContent();
return textContent.then(function(text){ // return content promise
return text.items.map(function (s) { return s.str; }).join('&&&'); // value page text
});
}));
}
catch(e){
console.log(e)
}
}
// Wait for all pages and join text
return Promise.all(countPromises).then(function (texts) {
// since doumentation pages do not add the final '&&&', must add one manually (only after rx has been found)
var fulltext = texts.reduce(function(full, text){
if (rx.test(full)){
var next = '&&&'+text
return full+=next
}
return full+=text
}, '')
return [fulltext, texts]
});
}).then(function(textarr){
var fulltext = textarr[0]
self.fulltext = fulltext;
var texts = textarr[1]
try {
var partialmatch = rx.exec(fulltext)[0]
var count = texts.reduce(function(pageno, text){
var tomatch = text.replace(/.*Typ&&&/, '')
if (tomatch.length>0 && partialmatch.indexOf(tomatch) > -1){
pageno++
}
return pageno;
}, 0)
}
catch(e){
console.log(e)
}
return count;
})
}
Edited the entry to show that I'm now returning the value of the gepageno function.
The data that I am expecting is logged but not available as a result of accept(). Can anyone tell what could be going wrong?

I have tried to simulate and give a solution of your promise. I have posted pseudo code.
Promise flow
var testFunction = ((result) => {
return new Promise(function (resolve, reject) {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("2. testFunction resolved");
resolve("Final response");
}
});
});
var getpageno = ((url) => {
return new Promise((resolve, reject) => {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("1. getpageno resolved");
resolve(45);
}
});
});
var getMetadata = ((text, url) => {
console.log("0. getmetadata resolved");
var self = this;
getpageno(url)
.then((pageCount) => {
console.log("> pageCount :", pageCount);
return testFunction(pageCount);
})
.then((data) => {
console.log(">", data);
});
});
getMetadata("hell", "https://www.google.com");
output
0. getmetadata resolved
1. getpageno resolved
> pageCount : 452. testFunction resolved
> Final response

you need to return the promise on the third line
return self.getpageno(url).then
to access its eventual fulfillment value.

JS Asynchronous execution is causing pain

I am developing a backend server code with NodeJS. What the code does is, periodically connect to a REST API, request updates and then write to my database.
I have no way of getting delta of the data from the API, so I drop my collection from MongoDB and then just insert the newly pulled data.
I implemented promises to make sure that the dependent methods are executed only after the previous methods resolve. This however doesn't seem to work as I anticipated.
So, I drop the collection and insert, this works. But the following method seems to execute before the new data is populated. It sometime works, when I have some new console.log statements which seems to induce a slight delay ever so slightly to make it all work.
setTimeout function didn't seem to help. Any suggestions?
Here is a sanitized version of the code: https://jsfiddle.net/ppbfrozg/
var request = require("request");
var q = require('q');
function authenticate() {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function getData(token) {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData);
console.log("Records Inserted");
} else {
db.collection('Details').drop();
db.collection('Details').insert(JSON.parse(a[0]));
console.log("Records Inserted");
}
deferred.resolve(token);
});
return deferred.promise;
}
function getMedia(dataContext) {
var deferred = q.defer();
var cursor = dataContext[0];
var token = dataContext[1];
if (cursor !== null) {
console.log("Inside cursor not null");
cursor.forEach(function(data) {
insertDataInMongo(data);
})
}
return deferred.promise;
}
function check(array, attr, value) {
for (var i = 0; i < array.length; i += 1) {
if (array[i][attr] === value) {
return false;
}
}
return true;
}
function get_value(array, attr) {
for (var i = 0; i < array.length; i += 1) {
if (array[i].hasOwnProperty(attr)) {
return array[i][attr];
}
}
}
function getNames(token) {
var deferred = q.defer();
var MongoClient2 = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
console.log("going to get Data");
MongoClient2.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err));
console.log("connected for select");
var data = db.collection('Details').find();
var dataContext = [data, token, 0, 0, 0, 0, 0, 0, 0, null];
deferred.resolve(dataContext);
});
return deferred.promise;
}
function convertDate(date) {
var yyyy = date.getFullYear().toString();
var mm = (date.getMonth() + 1).toString();
var dd = (date.getDate() - 3).toString();
var mmChars = mm.split('');
var ddChars = dd.split('');
return yyyy + '-' + (mmChars[1] ? mm : "0" + mmChars[0]) + '-' + (ddChars[1] ? dd : "0" + ddChars[0]);
}
authenticate()
.then(getData)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);

This should work. Let me know if any issue.
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData, function(){
console.log("Records Inserted");
return deferred.resolve(token);
});
} else {
db.collection('Details').drop(function(error, result){//Callback function that executes after drop operation has completed.
if(error){
return deferred.reject(error);//Reject the promise if there was an error
}
db.collection('Details').insert(JSON.parse(a[0]), function(err, res){//Callback function that executes after insert operation has completed.
if(err){
return deferred.reject(err);//Reject the promise if there was an error
}
console.log("Records Inserted");
return deferred.resolve(token);
});
});
}
});
return deferred.promise;
}

From what I see from nodeJS driver API for MongoDB :
https://mongodb.github.io/node-mongodb-native/api-generated/collection.html
hint : db.collection.drop is asynchronous, so you have to use a callback with it
db.collection('Details').drop(function(err, result){
// Do anything AFTER you dropped your collection
}
but since you are using promises, you should use something like that:
authenticate()
.then(getData)
.then(db.collection('Details').drop)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
or if you want to really keep the same code format :
function dropCollectionInMongo() {
db.collection('Details').drop()
}
authenticate()
.then(getData)
.then(dropCollectionInMongo)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);

Async PUT Requests

I'm building a script to handle some data modifications via the Shopify API, through JS.
My strategy is pull ALL the orders from one date, and to run a function against each order, then PUT the new data up.
I've gotten pretty close, I have an array containing all the orders from a specific date. I am returning this data through promises.
I need help with the async nature of this function to work as expected. Currently, my PUT request is being ignored completely, and I'm not sure why. Here's my code:
(function main() {
var yesterday = moment().add('-1', 'days').format('YYYY-MM-DD');
var today = moment().format('YYYY-MM-DD');
var scopeStart = 'created_at_min=' + yesterday + "%2000:00";
var scopeEnd = '&created_at_max=' + today + '%2000:00';
var promise = new Promise(function(resolve, reject) {
var apiEndpoint = '/admin/orders/count.json?limit=250&status=any&' + scopeStart + scopeEnd;
Shopify.get(apiEndpoint, function(err, data) {
if (err) reject(err);
else resolve(data);
});
});
promise.then(function(data) {
console.log('promise data', data.count);
if (data.count <= 250) {
var secondPromise = new Promise(function(resolve, reject) {
var apiEndpoint = '/admin/orders.json?limit=250&status=any&' + scopeStart + scopeEnd;
Shopify.get(apiEndpoint, function(err, data) {
if (err) reject(err);
else resolve(data);
});
});
secondPromise.then(function(data) {
var array = data.orders;
async.each(array, function(order, callback) {
console.log(order.name, order.id);
var thirdPromise = new Promise(function(resolve, reject) {
var fields = generateFields(order.id);
console.log('fields', fields);
console.log('----------------');
Shopify.put('/admin/orders/' + order.id, fields, function(err, data) {
if (err) reject(err);
else resolve(data);
})
});
thirdPromise.then(function(data) {
console.log('Updated', data);
});
});
});
}
});
}());
My third promise never returns a console log. The put request does not seem to go through. Any suggestions?