Develop Reference

javascript node.js async-await electron

I can't seem to find a relevant solution to this issue.
I am trying to use a foreach loop to call out for data. I have tried multiple variations of packaging this in promises but cannot seem to halt this function from proceeding before finishing the loop. Hoping someone may have some insight.
async function getData(){
temparray [] = { data to iterate over };
await getAgents();
console.log('done');
function getAgents(){
return new Promise(resolve => {
temparray.forEach((deal) => {
pipedrive.Persons.get(deal.agent, function(err, person) {
if (err) throw err;
console.log("trying to get agent " + deal.agent)
console.log(person.name);
});
});
}); // end of promise
};
};
So what I'm trying to do here is create a promise that resolves once all of the async calls are completed, but the process marches on before this has completed. Hence 'done' logs to the console before the getAgents() function has completed and logged the names. I suspect it has to do with the callbacks, but I've tried converting it to a promise, I've tried util.promiseify, and I've looked at building an array of promises and then using promise.all but haven't tested it.

Here's how you should do with another example
exec = async () => {
let tmp = [1,2,3,4,5,6];
let value = await Promise.all(tmp.map(id => getIdAfterTimeout(id)));
console.log(value);
}
getIdAfterTimeout = (id) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
return resolve(id);
}, 2000);
})
}
exec();

You could covert a function with callback style to a promise.
My example for you case:
async function getData() {
temparray[] = {data to iterate over};
let persons = await getAgents(temparray);
console.log(persons);
console.log('done');
};
async function getAgents(temparray) {
await Promise.all(
temparray.map((deal) => {
return new Promise(resolve => {
pipedrive.Persons.get(deal.agent, function(err, person) {
if (err) throw err;
console.log("trying to get agent " + deal.agent)
console.log(person.name);
resolve(person);
});
})
})
);
};
You can use Promise.all or old school for loop (fori), I think you need read more about Promise async/await.

async function getData(){
// this or you can do await
Promise.all(temparray.map(temp => getAgents(temp)))
.then(data => {
console.log('done');
})
.catch(e => {
console.log('error' );
})
// if you do await .. do like this
const {error, data } = await Promise.all(promises);
console.log(data);
}
function getAgents(id){
return new Promise((resolve, reject) => {
pipedrive.Persons.get(deal.agent, function(err, person){
if (err)
return resolve({error: err, data: null});
return resolve({error : null, data: person});
});
}); // end of promise
}

I even was stuck in similar problem yesterday. I applied following logic to solve this
Create a counter that tracks how many async requests are completed
Increment the counter in the callback function of async request
Add check if counter is equal to length of array
Resolve the promise once the counter is equal to the length of input array
const fs = require('fs');
async function getData() {
const array = [1, 2, 3, 4, 5, 6];
// Create a counter that tracks how many async requests are completed
let counter = 0;
await getAgents();
console.log('done');
async function getAgents() {
return new Promise(resolve => {
array.forEach((item, i) => {
//do async stuff here
fs.writeFile(`${i}.file.txt`, `line in file ${i}`, err => {
if (err) throw err;
// Increment the counter in the callback function of async request
// Add check if counter is equal to length of array
if (++counter == array.length) {
// Resolve the promise once the counter is equal to the length of input array
resolve(counter);
}
});
});
});
}
}
getData();

CosmosDB stored procedures - promise instead of callback

Is it possible to use javascript promises instead of regular callbacks within CosmosDB (DocumentDB) stored procedure API calls? An usage would be implementing pagination.
For example
token = getToken();
doSomething(token);
//....
function getToken(....) {
//...
collection.queryDocuments(link, query, queryOptions, function(error, documents, responseOptions) {
return responseOptions.continuation;
});
}
would not work because the token is returned within a callback, and the execution continues. Could you please give an example of how you would implement this?

The version of ECMAScript referenced in Cosmos DB docs supports async/await and Promises. I am able to use both of those in my stored procedures.
Here's a function that returns a promise that makes a parameterized document query:
function queryDocumentsAsync(sql, parameters, options) {
const querySpec = {
query: sql,
parameters: parameters
};
return new Promise((resolve, reject)=>{
let isAccepted = __.queryDocuments(__.getSelfLink(), querySpec, options || {}, (err, feed, options) => {
if(err) reject(err);
resolve({
feed: feed,
options: options
});
});
if(!isAccepted) throw "Query was not accepted.";
});
}
I am seeing some limitations around forcing a rollback with this approach, though. If you throw an Error, it gets swallowed by the promise chain and never gets out.

Here's an example on how to use async await for query and replace scenario.
function async_sample() {
const ERROR_CODE = {
NotAccepted: 429
};
const asyncHelper = {
queryDocuments(sqlQuery, options) {
return new Promise((resolve, reject) => {
const isAccepted = __.queryDocuments(__.getSelfLink(), sqlQuery, options, (err, feed, options) => {
if (err) reject(err);
resolve({ feed, options });
});
if (!isAccepted) reject(new Error(ERROR_CODE.NotAccepted, "replaceDocument was not accepted."));
});
},
replaceDocument(doc) {
return new Promise((resolve, reject) => {
const isAccepted = __.replaceDocument(doc._self, doc, (err, result, options) => {
if (err) reject(err);
resolve({ result, options });
});
if (!isAccepted) reject(new Error(ERROR_CODE.NotAccepted, "replaceDocument was not accepted."));
});
}
};
async function main() {
let continuation;
do {
let { feed, options } = await asyncHelper.queryDocuments("SELECT * from c", { continuation });
for (let doc of feed) {
doc.newProp = 1;
await asyncHelper.replaceDocument(doc);
}
continuation = options.continuation;
} while (continuation);
}
main().catch(err => getContext().abort(err));
}

With some cleverness you can use webpack to inline node dependencies, including promisify, which lets you do this:
https://github.com/Oblarg/cosmosdb-storedprocs-ts/blob/master/BuildStoredProcs.js

How do I send multiple queries from one endpoint with Express?

I am trying to query my database several times and construct an object which stores every response from my database in a field. Here is my code:
router.post('/search', (req, res) => {
var collection = db.get().collection('styles')
var data = [];
collection.distinct('make.name', (err, docs) => {
data.push({'make': docs });
});
collection.distinct('model', (function (err, docs) {
data.push({'model': docs });
}))
res.send(data);
});
Since NodeJS/Express is asynchronous, this isn't working as I would like. How can I reconstruct this endpoint to make several database calls (from the same collection) and return an object containing it?

There's more than one way to do it:
Nested callbacks
Without promises you could nest the callbacks:
router.post('/search', (req, res) => {
var collection = db.get().collection('styles')
var data = [];
collection.distinct('make.name', (err, docs) => {
if (err) {
// ALWAYS HANDLE ERRORS!
}
data.push({'make': docs });
collection.distinct('model', (function (err, docs) {
if (err) {
// ALWAYS HANDLE ERRORS!
}
data.push({'model': docs });
res.send(data);
}))
});
});
This would be the easiest way, but note that it is not efficient if those two requests could be done in parallel.
The async module
You can use the async module:
router.post('/search', (req, res) => {
var collection = db.get().collection('styles')
var data = [];
async.parallel({
make: cb => collection.distinct('make.name', cb),
model: cb => collection.distinct('model', cb),
}, (err, responses) => {
if (err) {
// ALWAYS HANDLE ERRORS!
}
data.push({'make': responses.make });
data.push({'model': responses.model });
res.send(data);
});
});
See: https://caolan.github.io/async/docs.html#parallel
But this may still not be the most convenient method.
ES2017 async/await
The most flexible way of doing that if you have 30 calls to make would be to:
Use functions that return promises instead of functions that take callbacks
Use async/await if you can or at least generator based coroutines
Await on promises (or yield promises) when the logic needs to run in sequence
Use Promise.all() for anything that can be done in parallel
With async/await your code could look like this:
// in sequence:
var make = await collection.distinct('make.name');
var model = await collection.distinct('model');
// use 'make' and 'model'
Or:
// in parallel:
var array = await Promise.all([
collection.distinct('make.name'),
collection.distinct('model'),
]);
// use array[0] and array[1]
A big advantage of async/await is the error handling:
try {
var x = await asyncFunc1();
var array = await Promise.all([asyncFunc2(x), asyncFunc3(x)]);
var y = asyncFunc4(array);
console.log(await asyncFunc5(y));
} catch (err) {
// handle any error here
}
You can only use it inside of a function created with the async keyword. For more info, see:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await
For support in browsers, see:
http://caniuse.com/async-functions
For support in Node, see:
http://node.green/#ES2017-features-async-functions
In places where you don't have native support for async and await you can use Babel:
https://babeljs.io/docs/plugins/transform-async-to-generator/
or with a slightly different syntax a generator based approach like in co or Bluebird coroutines:
https://www.npmjs.com/package/co
http://bluebirdjs.com/docs/api/promise.coroutine.html
See those answers for more info:
try/catch blocks with async/await
node.js ~ constructing chained sequence of Promise resolves
How to run Generator Functions in Parallel?
node.js ~ constructing chained sequence of Promise resolves
Using async/await + Bluebird to promisifyAll
jQuery: Return data after ajax call success

You can do it with Promises
router.post('/search', (req, res) => {
var collection = db.get().collection('styles');
// Create promise for "make.name" query
let firstQuery = new Promise((resolve, reject) => {
collection.distinct('make.name', (err, docs) => {
if (!err) {
resolve(docs);
} else {
reject(err);
}
});
});
// Create promise for "model" query
let secondQuery = new Promise((resolve, reject) => {
collection.distinct('model', (function (err, docs) {
if (!err) {
resolve(docs);
} else {
reject(err);
}
}))
})
// Run both queries at the same time and handle both resolve results or first reject
Promise.all([firstQuery, secondQuery])
.then((results) => {
res.send({ "make.name": results[0], "model": results[1] });
})
.catch((err) => {
// Catch error
res.send({});
});
});
Also you can use destructuring in callback functions like that:
Promise.all([firstQuery, secondQuery])
.then(([makeName, model]) => res.send({ "make.name": makeName, model }))
UPD: If you have a bunch of collection to request you can create an array of collections name, map it to promise requests and handle with Promise.all, for example
let collections = ["firstCollection", "secondCollection", "nCollection"];
let promises = collections.map((collectionName) => {
return new Promise((resolve, reject) => {
collection.distinct(collectionName, (err, docs) => {
if (!err) {
resolve(docs)
} else {
reject(err);
}
});
})
});
Promise.all(promises)
.then(results => {
// Do what you want to do
})
.catch(error => {
// or catch
});

Promise.all is never triggered due Promise rejection

I have two promises. One that reads a sample.txt file and another that reads all the files from a /books/ folder. The second promise uses a function called readFiles, which takes the dirnames and uses them to look though each file. When all the promises are ready the code inside then should run:
const p1 = new Promise((resolve, reject) => {
fs.readdir(__dirname + '/books/', (err, archives) => {
// archives = [ 'archive1.txt', 'archive2.txt']
readFiles(archives, result => {
if (archives.length === result.length) resolve(result)
else reject(result)
})
})
})
const p2 = new Promise((resolve, reject) => {
fs.readFile('sample.txt', 'utf-8', (err, sample) => {
resolve(sample)
})
})
Promise.all([p1, p2]).then(values => {
console.log('v:', values)
}).catch(reason => {
console.log('reason:', reason)
})
function readFiles (archives, callback) {
const result = []
archives.forEach(archive => {
fs.readFile(__dirname + '/books/' + archive, 'utf-8', (err, data) => {
result.push(data)
callback(result)
})
})
}
However, Promise.all always get rejected:
reason: [ 'archive 1\n' ]
What am I doing wrong?

Promises are one-shot devices. Once they've been rejected or resolved, their state can never change. With that in mind, readFiles() calls its callback for every file that it reads and you reject or resolve every time that callback is called, but the way you are using it, you check:
if (archives.length === result.length)
which will never be true on the first one and then you reject. Once that promise is rejected, its state cannot change. Subsequent calls to the callback will also call reject() and then the last one will call resolve(), but the state is long since set so only the first call to reject() or resolve() actually does anything. The others are simply ignored. So, p1 will always reject, thus Promise.all() that uses p1 will always reject.
You need to change readFiles() to either only call its callback once when it is done with all the files or change it to return a single promise that resolves when all the files are read or change how you're using the callback so you don't reject the first time it is called.
In general, if you're going to use promises, then you want to promisify at the lowest level and use the advantages of promises (particular for error propagation) everywhere rather than mix callbacks and promises. To that end, I'd suggest:
fs.readFileP = function(fname, encoding) {
return new Promise(function(resolve, reject) {
fs.readFile(fname, encoding, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
function readFiles(archives, encoding, callback) {
return Promise.all(archives.map(function(file) {
return fs.readFileP(file, encoding);
}));
}
Or, going a level deeper and promisifying fs.readdir() also, you'd get this:
// helper functions
fs.readdirP = function(dir) {
return new Promise(function(resolve, reject) {
fs.readdir(dir, function(err, files) {
if (err) return reject(err);
resolve(files);
});
});
}
fs.readFileP = function(fname, encoding) {
return new Promise(function(resolve, reject) {
fs.readFile(fname, encoding, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
function readFiles(archives, encoding) {
encoding = encoding || 'utf8';
return Promise.all(archives.map(function(file) {
return fs.readFileP(file, encoding);
}));
}
// actual logic for your operation
const p1 = fs.readdirP(__dirname + '/books/').then(readFiles);
const p2 = fs.readFileP('sample.txt', 'utf-8');
Promise.all([p1, p2]).then(values => {
console.log('v:', values);
}).catch(reason => {
console.log('reason:', reason);
});
If you use the Bluebird promise library which makes it easy to promisify whole modules at once and has some extra functions for managing Promise flow control, then the above code simplifies to this:
const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs'));
const p1 = fs.readdirAsync(__dirname + '/books/').then(files => {
return Promise.map(archives, file => {
return fs.readFileAsync(file, 'utf8');
});
});
const p2 = fs.readFileAsync('sample.txt', 'utf-8');
Promise.all([p1, p2]).then(values => {
console.log('v:', values);
}).catch(reason => {
console.log('reason:', reason);
});
In this block of code, the Promise.promisifyAll() line of code creates promisified versions of every method on the fs module with the Async suffix on them. Here, we use fs.readFileAsync() and fs.readdirAsync() so we can use promises for everything.

Promise will not resolve

The second part of the Promise below (inside the then) is never run. When I run the database query without using the Promise(in a node script that I run node myscript.js it returns the data but the console never returns the prompt--the console just hangs and I have to send an interrupt manually. Therefore, when I put it inside a Promise, I think the Promise doesn't know that the database query is complete even though it seems to have returned all the data, therefore the second part of the Promise isn't running ( I think). If that's the problem, how do I write the database query so that it doesn't hang and the Promise can run to completion?
const sqlite = require('/usr/local/lib/node_modules/sqlite3');
const express = require('/usr/local/lib/node_modules/express')
const promise = require('/usr/local/lib/node_modules/promise')
app.get('/', (request, res) => {
var res = [];
function getData() {
return new Promise(function(resolve, reject) {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, rows) => {
var d = {
a: rows['column_a'],
b: rows['column_b']
}
res.push(d)
});
});
}
getData().then(function(data) {
console.log("never run....", res, data) //never run
});
})

You need to resolve a promise by calling one of the functions it provides in the callback through its constructor.
const promise = new Promise((resolve, reject) => {
// you must call resolve() or reject() here
// otherwise the promise never resolves
});
Otherwise it will always stay in Pending state and never call the callbacks(s) you pass into then.
promise.then(() => {
// this never gets called if we don't resolve() or reject()
});
Additionally, promises allow you to resolve with values so there's usually no need to maintain global variables, you can just pass results through.
Finally, the callback in db.each will be called once for each row, so you would need to handle that by resolving the promise after all rows have been obtained
Here's how you could write your code:
function getData() {
const data = [];
return new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
// error reading a row, reject the Promise immediately
// optionally you could accumulate errors here in a similar manner to rows
reject(e);
return;
}
// success reading a row, store the row result
data.push({
a: row['column_a'],
b: row['column_b']
});
}, (e, rowCount) => { // the complete handler called when the operation is done, see docs: https://github.com/mapbox/node-sqlite3/wiki/API#databaseeachsql-param--callback-complete
if (e) {
// operation finished, there was an error
reject(e);
return;
}
// operation succeeded, resolve with rows
resolve(data);
});
});
}
app.get('/', (request, res) => {
getData().then((data) => {
// here `data` is an array of row objects
}, (e) => {
console.error(`Database error: ${e}`);
});
});
Side Note
Not sure why you are redeclaring the parameter res as an [], but there's no need for doing var res = []. Since you already have res, you can just say res = [] to point res to a new array. Of course that will overwrite the response object so I assume that you're doing it just for the purposes of this example. If not, you should probably create a new variable.

You've declared a Promise which means you're responsible for calling one of resolve or reject once and once only.
Here's a cleaned up example:
app.get('/', (request, res) => {
var res = [ ];
new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
reject(e);
return;
}
res.push({
a: row['column_a'],
b: row['column_b']
});
}, (err) => {
if (err) {
return reject(err);
}
resolve(res);
});
}).then((data) => {
console.log("Running ", res, data)//never run
}
});
If your database layer supports promises that usually makes this sort of code a lot less messy since you can simply chain that in there.
Edit: Since the Sqlite3 API is bizarrely non-standard and the each function has two callbacks you need to handle each row with the first, then the completion handler with the second.
If you design an API like this you're doing it wrong. Don't.

Several points :
resolve/reject must be called, otherwise a new Promise() will remain forever "pending".
always promisify at the lowest level possible, ie promisify db.each() not getData(). This gives you a testable, reusable utility and more comprehensible application code.
db.each() is a challenge to promisify because it has two possible sources of error; one in its iteration callback and one in its complete callback.
the sqlite3 documentation does not state what happens if an iteration error occurs but presumably the iteration continues, otherwise the error would simply appear as a completion error?
Here's a couple of ways to promisify :
1. First iteration error or completion error causes promise rejection - iteration errors are not exposed to your application code.
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, (iterationError, row) => {
if(iterationError) {
reject(iterationError);
} else {
iterationCallback(row);
}
}, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (row) => {
res.push({
a: row['column_a'],
b: row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
2. Only a completion error causes promise rejection - iteration errors are exposed to your application code
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, iterationCallback, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
// You can choose what to do on iterationError.
// Here, nulls are injected in place of values from the db,
// but you might choose not to push anything.
res.push({
a: iterationError ? null : row['column_a'],
b: iterationError ? null : row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
(2) is the better approach because exposing iteration errors affords you more flexibility. For example, you could choose to promisify with (2), and emulate (1) in your application :
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
var e = null;
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
if(iterationError && !e) {
// remember the first iteration error
e = iterationError;
} else {
// push only on success
res.push({
a: row['column_a'],
b: row['column_b']
});
}
}).then(n => {
if(e) {
throw e;
} else {
return res;
}
});
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
With (1), by rejecting on first iteration error rather than exposing iteration errors, the same flexibility is not available. (1) could not fully emulate (2).
Fortunately, the preferred approach (2) is the same as would be obtained with Bluebird's .promisify() method :
Promise.promisify(db.each);