I need some help with the asynchronous nature of node.js. I have a for loop, which collects data from the database. "result" is an array, which should then be returned to the main function.
user_collection.findOne({
_id : uid
}, function(error, user) {
if(error)
callback(error)
else {
for(var j = 0; j < user.contacts.length; j++) {
if(user.contacts[j].accepted == 'true') {
user_collection.findOne({
_id : user.contacts[j].contactId
}, function(error, user) {
result.push(user);
})
}
}
callback(null, result); // This callback executes before the for-loop ends, ofc
}
});
How can I ensure that the callback executes after the loop finished?
You might want to consider using helper library like async
https://github.com/caolan/async
It helps keep code more consistent..
In your case, you can look at the forEach() method
forEach(arr, iterator, callback)
The iterator is called with an item from the list and a callback for when it has finished.
Checkout the unit tests for examples
https://github.com/caolan/async/blob/master/mocha_test/each.js
Using ES6 Promises
(a promise library can be used for older browsers):
Process all requests guaranteeing synchronous execution (e.g. 1 then 2 then 3)
function asyncFunction (item, cb) {
setTimeout(() => {
console.log('done with', item);
cb();
}, 100);
}
let requests = [1, 2, 3].reduce((promiseChain, item) => {
return promiseChain.then(new Promise((resolve) => {
asyncFunction(item, resolve);
}));
}, Promise.resolve());
requests.then(() => console.log('done'))
Process all async requests without "synchronous" execution (2 may finish faster than 1)
let requests = [1,2,3].map((item) => {
return new Promise((resolve) => {
asyncFunction(item, resolve);
});
})
Promise.all(requests).then(() => console.log('done'));
I did it on that way
Promise.all(body.schedules.map(function(scheduleId) {
return new Promise(function(resolve, reject) {
return scheduleSchema.findOneAndRemove({
_id: scheduleId
})
.then(function() {
logSchema.insert({
userId: req.user.id,
actId: constants.LOG_SCHEDULE_DELETE.id,
extData: scheduleId
});
resolve();
})
.catch(function(err) {
reject(err);
});
});
})).then(function() {
return res.json({
code: constants.SUCCESS_CODE
});
}).catch(function(err) {
return res.json(constants.DATABASE_ERROR);
});
The last example
function callback (result) { console.log('all done'); }
[1, 2, 3].forEach((item, index, array) => {
asyncFunction(item, () => {
if (index === array.length - 1) {
callback();
}
});
});
This does not guarantee that callback will execute after all items are processed. It only guarantees that callback will execute after the very last item is processed.
More information
Michael.
With v1.5.2 of Async.js, It is each.
each(arr, iterator, [callback])
arr - An array to iterate over.
iterator(item, callback) - A function to apply to each item in arr.
callback(err) - Optional. A callback which is called when all iterator functions have finished, or an error occurs.
Related
I have a primary thread in my node application such as this:
function main_thread() {
console.log("Starting");
values = get_values(1);
console.log(values);
console.log("I expect to be after the values");
}
The get_values function calls the hgetall function using the node_redis package. This function provides a call back, but can be promisified:
function get_values(customer_id) {
// Uses a callback for result
new Promise(function(resolve, reject) {
redis_client.hgetall(customer_id, function (err, result) {
if (err) console.log(err)
console.log("About to resolve");
resolve(result);
});
})
.then(({result}) => {
console.log(result);
});
}
This works great for promise chaining within the function, however not so well in my main thread, as I can't wait and return the value.
Here's how I'd do it in ruby, the main language I use:
def get_values(customer_id)
return #redis_client.hgetall(customer_id)
end
How can I create a promise within a reusable function and make the main thread wait until the function returns the response from the promise?
EDIT:
It's been suggested the promise can be returned with a then chained in the main thread. However this still means any code in the main thread after after the function call executes before the then block.
EDIT 2:
After lengthy discussion with some IRL JS developer friends, it looks like trying to create a synchronous script is against the ethos of modern JS. I'm going to go back to my application design and work on making it async.
Here is a working example with async/await. I've replaced the redis with a timeout and an array for the data.
async function main_thread() {
console.log("Starting");
values = await get_values(1);
console.log(`After await: ${values}`);
console.log("I expect to be after the values");
}
async function get_values(customer_id) {
return new Promise((resolve, reject) => {
setTimeout(() => {
const result = [1, 2, 3];
console.log(`Resolving: ${result}`);
resolve(result);
}, 300);
});
}
main_thread();
Further reading:
Using Promises
Promise Constructor
Return the promise in get_values
function get_values(customer_id) {
// Uses a callback for result
return new Promise(function(resolve, reject) {
redis_client.hgetall(customer_id, function (err, result) {
if (err) console.log(err)
console.log("About to resolve");
resolve(result);
});
})
.then(({result}) => {
reject(result);
});
}
Now in your main thread, you could wait for it like:
function main_thread() {
console.log("Starting");
get_values(1).then(function(values) {
console.log(values);
}).catch(function(error) {
console.error(error);
});
}
Simple as returning the promise (chain) from your function
function get_values(customer_id) {
// Uses a callback for result
return new Promise(function(resolve, reject) {
redis_client.hgetall(customer_id, function (err, result) {
if (err) console.log(err)
console.log("About to resolve");
resolve(result);
});
})
.then(({result}) => {
console.log(result);
});
}
And then in your main async function or function
let result = await get_values(); or get_values.then(function(result){})
function main_thread() {
console.log("Starting");
values = get_values(1).then(function(values){
console.log(values);
console.log("I expect to be after the values");
});
}
async function main_thread() {
console.log("Starting");
let values = await get_values(1);
console.log(values);
console.log("I expect to be after the values");
}
I can't seem to find a relevant solution to this issue.
I am trying to use a foreach loop to call out for data. I have tried multiple variations of packaging this in promises but cannot seem to halt this function from proceeding before finishing the loop. Hoping someone may have some insight.
async function getData(){
temparray [] = { data to iterate over };
await getAgents();
console.log('done');
function getAgents(){
return new Promise(resolve => {
temparray.forEach((deal) => {
pipedrive.Persons.get(deal.agent, function(err, person) {
if (err) throw err;
console.log("trying to get agent " + deal.agent)
console.log(person.name);
});
});
}); // end of promise
};
};
So what I'm trying to do here is create a promise that resolves once all of the async calls are completed, but the process marches on before this has completed. Hence 'done' logs to the console before the getAgents() function has completed and logged the names. I suspect it has to do with the callbacks, but I've tried converting it to a promise, I've tried util.promiseify, and I've looked at building an array of promises and then using promise.all but haven't tested it.
Here's how you should do with another example
exec = async () => {
let tmp = [1,2,3,4,5,6];
let value = await Promise.all(tmp.map(id => getIdAfterTimeout(id)));
console.log(value);
}
getIdAfterTimeout = (id) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
return resolve(id);
}, 2000);
})
}
exec();
You could covert a function with callback style to a promise.
My example for you case:
async function getData() {
temparray[] = {data to iterate over};
let persons = await getAgents(temparray);
console.log(persons);
console.log('done');
};
async function getAgents(temparray) {
await Promise.all(
temparray.map((deal) => {
return new Promise(resolve => {
pipedrive.Persons.get(deal.agent, function(err, person) {
if (err) throw err;
console.log("trying to get agent " + deal.agent)
console.log(person.name);
resolve(person);
});
})
})
);
};
You can use Promise.all or old school for loop (fori), I think you need read more about Promise async/await.
async function getData(){
// this or you can do await
Promise.all(temparray.map(temp => getAgents(temp)))
.then(data => {
console.log('done');
})
.catch(e => {
console.log('error' );
})
// if you do await .. do like this
const {error, data } = await Promise.all(promises);
console.log(data);
}
function getAgents(id){
return new Promise((resolve, reject) => {
pipedrive.Persons.get(deal.agent, function(err, person){
if (err)
return resolve({error: err, data: null});
return resolve({error : null, data: person});
});
}); // end of promise
}
I even was stuck in similar problem yesterday. I applied following logic to solve this
Create a counter that tracks how many async requests are completed
Increment the counter in the callback function of async request
Add check if counter is equal to length of array
Resolve the promise once the counter is equal to the length of input array
const fs = require('fs');
async function getData() {
const array = [1, 2, 3, 4, 5, 6];
// Create a counter that tracks how many async requests are completed
let counter = 0;
await getAgents();
console.log('done');
async function getAgents() {
return new Promise(resolve => {
array.forEach((item, i) => {
//do async stuff here
fs.writeFile(`${i}.file.txt`, `line in file ${i}`, err => {
if (err) throw err;
// Increment the counter in the callback function of async request
// Add check if counter is equal to length of array
if (++counter == array.length) {
// Resolve the promise once the counter is equal to the length of input array
resolve(counter);
}
});
});
});
}
}
getData();
In my project I have two functions that retrieve respectively the number of completed tasks, and the number of uncompleted tasks. (I use NeDB but it does not matter).
After that, I want to add together these two numbers.
Here is my code :
var completed = 0
, uncompleted = 0;
// Get completed tasks
db.find({ completed: true }, function (err, tasks) {
completed = tasks.length;
console.log(completed); // output 7
});
// Get uncompleted tasks
db.find({ completed: false }, function (err, tasks) {
uncompleted = tasks.length;
console.log(uncompleted); // output 3
});
// This line is executed before the lines above
console.log(completed+uncompleted); // output 0 (I would like to output 10)
As you see the last line output 0 because it is executed before the two functions have finished.
So my question is how to make the last line output 10 ?
I think I have to use promises, but I read a lot of topics on SO and there is a lot of different approaches (with promise, then, done, deffered, array of promises, functions in variables ...) so I'm a little bit lost.
I wanted to know what is the correct way to do that :)
I tried this code, but I think I am completely mistaken ...
var completed = 0
, uncompleted = 0;
// Get completed tasks
var getCompleted = function(){
return db.find({ completed: true }, function (err, tasks) {
completed = tasks.length;
console.log(completed); // output 7
});
}
// Get uncompleted tasks
var getUncompleted = function(){
return db.find({ completed: false }, function (err, tasks) {
uncompleted = tasks.length;
console.log(uncompleted); // output 3
});
}
var promises = [];
promises.push(getCompleted());
promises.push(getUncompleted());
$.when.apply($, promises).done(function () {
console.log(completed+uncompleted); // output 0
});
If you want to use Promises, first you'll need to promisify any asynchronous tasks that don't return a promise
In your case
A
function dbFindPromise(db, params) {
return new Promise((resolve, reject) =>
db.find(params, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
})
)
};
Then, using this promisified db find, your code can be written as
B
Promise.all([
dbFindPromise(db, { completed: true }).then(tasks => {
console.log(tasks.length);
return tasks.length;
}),
dbFindPromise(db, { completed: false }).then(tasks => {
console.log(tasks.length);
return tasks.length;
})
]).then(([completed, uncompleted]) => {
console.log(completed + uncompleted);
});
Alternatively, using jquery Deferred and when - first the "promisified" function
C
function dbFindPromise(db, params) {
var deferred = $.Deferred();
db.find(params, (err, result) => {
if(err) {
deferred.reject(err);
} else {
deferred.resolve(result);
}
});
return deferred.promise();
};
Then to use it:
D
$.when(dbFindPromise(db, { completed: true }).then(tasks => {
console.log(tasks.length);
return tasks.length;
}), dbFindPromise(db, { completed: false }).then(tasks => {
console.log(tasks.length);
return tasks.length;
})
).done((completed, uncompleted) => {
console.log(completed + uncompleted);
});
So, native promises, code A and B, jquery Promises is code C and D
But, you can use code A with code D or code C with code B - because Promises should behave the same regardless of where they came from (in this case, native vs jquery)
The NeDB library you are using does not return promises - you can perform your work in nested callbacks like this:
db.find({completed: true}, function(err, completeTasks) {
db.find({completed: false}, function(error, incompleteTasks) {
console.log(completeTasks.length + incompleteTasks.length);
});
})
I'm trying to transfer 2 results from 2 callback to 1 lodash function (_.union) using recursive function.
I dont understand what am I doing wrong! I keep getting "undefined".
Here is my code:
EDITED:
My new code with the "promise" technique
the first function that check things in the remote DB-
function findFiles(kw, callback){
if (_.isArray(kw)) {return callback(kw)};
return new Promise((resolve, reject) => {
console.log(kw);
word.aggregate([
{ $match: { keyWord: kw } },
{ $project: { filesFound: '$filesFound.fileName' , '_id':0} },
{ $sort: { fileName: 1 } }
],function(err, obj){
console.log('checked' + kw);
console.log(obj);
if (err) return reject(err);
else
return resolve(obj[0].filesFound);//obj[0].filesFound
})
})
}
The main function:
function searchOperation(query, callback){
var stack=[];
if (!(_.includes(query, 'OR')) && !(_.includes(query, 'AND')) && !(_.includes(query, 'NOT'))){
findFiles(query)
.then((item) => {
console.log(item+'********');
callback(item)
})
.catch((err) => {
console.log(err)
})
}
else{
wordsArr = _.split(query, " ");
console.log("+++++:" + wordsArr);
wordsArr.forEach(function(w){
console.log('first check:'+w);
if(_.isEmpty(stack)){
if(_.includes(w, 'OR')){
console.log('found OR');
var statement = [];
console.log('query is:'+query);
statement = _.split(query, w, 2);
console.log(statement[0]+' , '+statement[1]);
return new Promise((resolve, reject)=>{
resolve(_.union(searchOperation(statement[0]),searchOperation(statement[1])))
})
//ANOTHER OPTION:
// searchOperation(statement[0],function(arr1){
// console.log('arr1');
// console.log('done part 1!');
// searchOperation(statement[1],function(arr2){
// console.log('who called arr2?');
// return(_.union(arr1,arr2));
// })
// });
}
}
})
}
}
now, inside the function findFile() the console.log return what i need. but then I need to use both returned values in another function (union) and it returns undefined.
in the main thread:
searchOperation('Expression1 OR Expression2', function(result){
res.json(result);
})
now i'm sure: something goes WRONG in the recursive function and the async of node...
I need it to work recursively and could get complicate expressions like:
'((A NOT B) AND (C OR D))'
does some know what is the right way to write it either with Promise or async.waterfall ??
thanks in advance!!!
Your code doesn't work because you're trying to get an Async response in a Synchronous fashion.
Look into Promises.
The second part of the Promise below (inside the then) is never run. When I run the database query without using the Promise(in a node script that I run node myscript.js it returns the data but the console never returns the prompt--the console just hangs and I have to send an interrupt manually. Therefore, when I put it inside a Promise, I think the Promise doesn't know that the database query is complete even though it seems to have returned all the data, therefore the second part of the Promise isn't running ( I think). If that's the problem, how do I write the database query so that it doesn't hang and the Promise can run to completion?
const sqlite = require('/usr/local/lib/node_modules/sqlite3');
const express = require('/usr/local/lib/node_modules/express')
const promise = require('/usr/local/lib/node_modules/promise')
app.get('/', (request, res) => {
var res = [];
function getData() {
return new Promise(function(resolve, reject) {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, rows) => {
var d = {
a: rows['column_a'],
b: rows['column_b']
}
res.push(d)
});
});
}
getData().then(function(data) {
console.log("never run....", res, data) //never run
});
})
You need to resolve a promise by calling one of the functions it provides in the callback through its constructor.
const promise = new Promise((resolve, reject) => {
// you must call resolve() or reject() here
// otherwise the promise never resolves
});
Otherwise it will always stay in Pending state and never call the callbacks(s) you pass into then.
promise.then(() => {
// this never gets called if we don't resolve() or reject()
});
Additionally, promises allow you to resolve with values so there's usually no need to maintain global variables, you can just pass results through.
Finally, the callback in db.each will be called once for each row, so you would need to handle that by resolving the promise after all rows have been obtained
Here's how you could write your code:
function getData() {
const data = [];
return new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
// error reading a row, reject the Promise immediately
// optionally you could accumulate errors here in a similar manner to rows
reject(e);
return;
}
// success reading a row, store the row result
data.push({
a: row['column_a'],
b: row['column_b']
});
}, (e, rowCount) => { // the complete handler called when the operation is done, see docs: https://github.com/mapbox/node-sqlite3/wiki/API#databaseeachsql-param--callback-complete
if (e) {
// operation finished, there was an error
reject(e);
return;
}
// operation succeeded, resolve with rows
resolve(data);
});
});
}
app.get('/', (request, res) => {
getData().then((data) => {
// here `data` is an array of row objects
}, (e) => {
console.error(`Database error: ${e}`);
});
});
Side Note
Not sure why you are redeclaring the parameter res as an [], but there's no need for doing var res = []. Since you already have res, you can just say res = [] to point res to a new array. Of course that will overwrite the response object so I assume that you're doing it just for the purposes of this example. If not, you should probably create a new variable.
You've declared a Promise which means you're responsible for calling one of resolve or reject once and once only.
Here's a cleaned up example:
app.get('/', (request, res) => {
var res = [ ];
new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
reject(e);
return;
}
res.push({
a: row['column_a'],
b: row['column_b']
});
}, (err) => {
if (err) {
return reject(err);
}
resolve(res);
});
}).then((data) => {
console.log("Running ", res, data)//never run
}
});
If your database layer supports promises that usually makes this sort of code a lot less messy since you can simply chain that in there.
Edit: Since the Sqlite3 API is bizarrely non-standard and the each function has two callbacks you need to handle each row with the first, then the completion handler with the second.
If you design an API like this you're doing it wrong. Don't.
Several points :
resolve/reject must be called, otherwise a new Promise() will remain forever "pending".
always promisify at the lowest level possible, ie promisify db.each() not getData(). This gives you a testable, reusable utility and more comprehensible application code.
db.each() is a challenge to promisify because it has two possible sources of error; one in its iteration callback and one in its complete callback.
the sqlite3 documentation does not state what happens if an iteration error occurs but presumably the iteration continues, otherwise the error would simply appear as a completion error?
Here's a couple of ways to promisify :
1. First iteration error or completion error causes promise rejection - iteration errors are not exposed to your application code.
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, (iterationError, row) => {
if(iterationError) {
reject(iterationError);
} else {
iterationCallback(row);
}
}, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (row) => {
res.push({
a: row['column_a'],
b: row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
2. Only a completion error causes promise rejection - iteration errors are exposed to your application code
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, iterationCallback, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
// You can choose what to do on iterationError.
// Here, nulls are injected in place of values from the db,
// but you might choose not to push anything.
res.push({
a: iterationError ? null : row['column_a'],
b: iterationError ? null : row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
(2) is the better approach because exposing iteration errors affords you more flexibility. For example, you could choose to promisify with (2), and emulate (1) in your application :
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
var e = null;
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
if(iterationError && !e) {
// remember the first iteration error
e = iterationError;
} else {
// push only on success
res.push({
a: row['column_a'],
b: row['column_b']
});
}
}).then(n => {
if(e) {
throw e;
} else {
return res;
}
});
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
With (1), by rejecting on first iteration error rather than exposing iteration errors, the same flexibility is not available. (1) could not fully emulate (2).
Fortunately, the preferred approach (2) is the same as would be obtained with Bluebird's .promisify() method :
Promise.promisify(db.each);