In my project I have two functions that retrieve respectively the number of completed tasks, and the number of uncompleted tasks. (I use NeDB but it does not matter).
After that, I want to add together these two numbers.
Here is my code :
var completed = 0
, uncompleted = 0;
// Get completed tasks
db.find({ completed: true }, function (err, tasks) {
completed = tasks.length;
console.log(completed); // output 7
});
// Get uncompleted tasks
db.find({ completed: false }, function (err, tasks) {
uncompleted = tasks.length;
console.log(uncompleted); // output 3
});
// This line is executed before the lines above
console.log(completed+uncompleted); // output 0 (I would like to output 10)
As you see the last line output 0 because it is executed before the two functions have finished.
So my question is how to make the last line output 10 ?
I think I have to use promises, but I read a lot of topics on SO and there is a lot of different approaches (with promise, then, done, deffered, array of promises, functions in variables ...) so I'm a little bit lost.
I wanted to know what is the correct way to do that :)
I tried this code, but I think I am completely mistaken ...
var completed = 0
, uncompleted = 0;
// Get completed tasks
var getCompleted = function(){
return db.find({ completed: true }, function (err, tasks) {
completed = tasks.length;
console.log(completed); // output 7
});
}
// Get uncompleted tasks
var getUncompleted = function(){
return db.find({ completed: false }, function (err, tasks) {
uncompleted = tasks.length;
console.log(uncompleted); // output 3
});
}
var promises = [];
promises.push(getCompleted());
promises.push(getUncompleted());
$.when.apply($, promises).done(function () {
console.log(completed+uncompleted); // output 0
});
If you want to use Promises, first you'll need to promisify any asynchronous tasks that don't return a promise
In your case
A
function dbFindPromise(db, params) {
return new Promise((resolve, reject) =>
db.find(params, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
})
)
};
Then, using this promisified db find, your code can be written as
B
Promise.all([
dbFindPromise(db, { completed: true }).then(tasks => {
console.log(tasks.length);
return tasks.length;
}),
dbFindPromise(db, { completed: false }).then(tasks => {
console.log(tasks.length);
return tasks.length;
})
]).then(([completed, uncompleted]) => {
console.log(completed + uncompleted);
});
Alternatively, using jquery Deferred and when - first the "promisified" function
C
function dbFindPromise(db, params) {
var deferred = $.Deferred();
db.find(params, (err, result) => {
if(err) {
deferred.reject(err);
} else {
deferred.resolve(result);
}
});
return deferred.promise();
};
Then to use it:
D
$.when(dbFindPromise(db, { completed: true }).then(tasks => {
console.log(tasks.length);
return tasks.length;
}), dbFindPromise(db, { completed: false }).then(tasks => {
console.log(tasks.length);
return tasks.length;
})
).done((completed, uncompleted) => {
console.log(completed + uncompleted);
});
So, native promises, code A and B, jquery Promises is code C and D
But, you can use code A with code D or code C with code B - because Promises should behave the same regardless of where they came from (in this case, native vs jquery)
The NeDB library you are using does not return promises - you can perform your work in nested callbacks like this:
db.find({completed: true}, function(err, completeTasks) {
db.find({completed: false}, function(error, incompleteTasks) {
console.log(completeTasks.length + incompleteTasks.length);
});
})
Related
I am deleting somefiles using fs.unlink and then I want to run some code. Due to the async nature of JS what is happening is that my code after unlinking is called before the callback of unlink. How can i Syncronise this? Is promises the only way ?
fs.unlink("FileName",function(err){
console.log("RUN");
})
for(let i = 0; i<10;i++) {
console.log(i);
}
RESULT :
1
2
3
4
5
6
7
8
9
RUN
The problem with using promises is that : If i have many files to delete, then i will have maintain a count of the promises and then check how many have been resolved. This i want to avoid
In this situation, you can use fs.unlinkSync, the synchronous version of fs.unlink:
try {
fs.unlinkSync("FileName");
console.log('Removing file successful!');
} catch(e) {
// TODO: handle errors here
};
console.log("RUN");
for(let i = 0; i<10;i++) {
console.log(i);
}
As #Keith rightfully mentions in the comments: synchronous operations like this should be use sparingly. If you have large numbers of files to delete, it may be better to use the asynchronous fs.unlink() because you can "start" more of those concurrently (tradeoff: start too many and the performance may suffer because of I/O saturation).
Using Promises is a good solution, you don't have to track the promises yourself, bluebird will do it for you:
const Promise = require('bluebird');
function unlinkFile(fileName) {
return new Promise((resolve, reject) => {
fs.unlink(fileName, function (err) {
if (err) {
return reject(err);
}
resolve();
});
});
}
Promise.all([unlinkFile('01.txt'), unlinkFile('02.txt'), unlinkFile('03.txt')])
.then(() => {
console.log('ALL FILES UNLINKED');
});
// OR YOU CAN USE promise.map
const filesToUnlink = ['01.txt', '02.txt', '3.txt'];
Promise.map(filesToUnlink, unlinkFile)
.then(() => {
console.log('ALL FILES UNLINKED');
});
Promise is not the only way. You can wrap your for loop in a function, and call that function from the unlink callback.
fs.unlink("FileName",function(err){
console.log("RUN");
loop();
})
function loop() {
for(let i = 0; i<10;i++) {
console.log(i);
}
}
On the other hand, with a Promise you would do this way :
new Promise((resolve, reject) => {
fs.unlink("FileName",function(err){
console.log("RUN");
return resolve();
});
})
.then(() => {
for(let i = 0; i<10;i++) {
console.log(i);
}
});
But, as #robertklep says, if your code does not need async, just call the synchronous function.
You should use Promise to get proper result.
let fs = require('fs');
new Promise((resolve,reject) => {
fs.unlink("demo1.txt", function (err) {
console.log("RUN");
if(err)
reject(err);
resolve();
})
})
.then(() => {
for (let i = 0; i < 10; i++) {
console.log(i);
}
},(error)=>{
console.log(error);
})
I'm trying to transfer 2 results from 2 callback to 1 lodash function (_.union) using recursive function.
I dont understand what am I doing wrong! I keep getting "undefined".
Here is my code:
EDITED:
My new code with the "promise" technique
the first function that check things in the remote DB-
function findFiles(kw, callback){
if (_.isArray(kw)) {return callback(kw)};
return new Promise((resolve, reject) => {
console.log(kw);
word.aggregate([
{ $match: { keyWord: kw } },
{ $project: { filesFound: '$filesFound.fileName' , '_id':0} },
{ $sort: { fileName: 1 } }
],function(err, obj){
console.log('checked' + kw);
console.log(obj);
if (err) return reject(err);
else
return resolve(obj[0].filesFound);//obj[0].filesFound
})
})
}
The main function:
function searchOperation(query, callback){
var stack=[];
if (!(_.includes(query, 'OR')) && !(_.includes(query, 'AND')) && !(_.includes(query, 'NOT'))){
findFiles(query)
.then((item) => {
console.log(item+'********');
callback(item)
})
.catch((err) => {
console.log(err)
})
}
else{
wordsArr = _.split(query, " ");
console.log("+++++:" + wordsArr);
wordsArr.forEach(function(w){
console.log('first check:'+w);
if(_.isEmpty(stack)){
if(_.includes(w, 'OR')){
console.log('found OR');
var statement = [];
console.log('query is:'+query);
statement = _.split(query, w, 2);
console.log(statement[0]+' , '+statement[1]);
return new Promise((resolve, reject)=>{
resolve(_.union(searchOperation(statement[0]),searchOperation(statement[1])))
})
//ANOTHER OPTION:
// searchOperation(statement[0],function(arr1){
// console.log('arr1');
// console.log('done part 1!');
// searchOperation(statement[1],function(arr2){
// console.log('who called arr2?');
// return(_.union(arr1,arr2));
// })
// });
}
}
})
}
}
now, inside the function findFile() the console.log return what i need. but then I need to use both returned values in another function (union) and it returns undefined.
in the main thread:
searchOperation('Expression1 OR Expression2', function(result){
res.json(result);
})
now i'm sure: something goes WRONG in the recursive function and the async of node...
I need it to work recursively and could get complicate expressions like:
'((A NOT B) AND (C OR D))'
does some know what is the right way to write it either with Promise or async.waterfall ??
thanks in advance!!!
Your code doesn't work because you're trying to get an Async response in a Synchronous fashion.
Look into Promises.
I have 3 total forEach loops and I want the first two to wait on a promise's then block to execute before they start over and iterate again. I've demonstrated this with console.log statements labeled 1, 2, and 3.
I want the console logs to go in order, like "1, 2, 3, 1, 2, 3 etc"
Instead what I'm getting is "1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 3, 3, 3, 3, 3"
How do I make all of this sequential?
fetchCandidatesByCity is the mongoose find method, which returns a promise.
axios.get(FEED_URL).then(data => {
let candidatesWithMessageSent = [];
data.data.jobs.forEach(job => {
console.log("1");
// console.log(candidatesWithMessageSent);
job.city.forEach(cityAndState => {
console.log("2");
let jobState = extractState(cityAndState);
let jobLocation = addSpaceAfterComma(cityAndState.toLowerCase());
let jobCity = extractCity(jobLocation);
fetchCandidatesByCity(jobCity)
.then(candidates => {
candidates.forEach((candidate, index) => {
console.log("3");
const candidateId = candidate._id.toString();
if (index <= MAX_MESSAGE_LIMIT &&
!containsUberLyft(job.title) &&
priceIsHigh(job.price) &&
!candidateHasMessage(candidatesWithMessageSent, candidateId)) {
const jobURL = `http://www.jobs2careers.com/click.php?id=${job.id}.${PUBLISHER_ID}`;
// candidate has received a job notification, so add candidate to candidatesWithMessageSent
candidatesWithMessageSent = [ ...candidatesWithMessageSent, candidateId ];
}
return;
});
})
.catch((error) => {
console.log(error);
});
});
});
});
A common design pattern to iterate an array serially using async operations is using .reduce() where you accumulate a promise. In outline, it works like this:
array.reduce(function(p, item) {
return p.then(function() {
return someAsyncPromise(item);
});
}, Promise.resolve()).then(function(finalValue) {
// all done here
}).catch(function(err) {
// error here
});
Promise.resolve() is passed in as the initial value of the accumulator. Each iteration of the loop, then does return p.then(someOperation).
This essentially chains a whole bunch of operations together like this:
Promise.resolve().then(...).then(...).then(...).then(...).then(...)
Where each time your custom function is called by a .then() handler, it is passed the next iteration value of the array and returns a promise that will be waited on before the next time f is called.
Since you have two nested array iteration loops, you can nest one inside the other like this:
myData.reduce(function(p, nestedArray) {
return p.then(function() {
return nestedArray.reduce(function(p2, item) {
return p2.then(function() {
return someAsyncPromise(item);
});
}, Promise.resolve());
});
}, Promise.resolve()).then(function(finalVal) {
// all done here
}).catch(function(err) {
// error here
});
Or, if you aren't using the promise resolved result of the prior iteration from the outer reduce(), you can simplify that a bit like this:
myData.reduce(function(p, nestedArray) {
return nestedArray.reduce(function(p2, item) {
return p2.then(function() {
return someAsyncPromise(item);
});
}, p);
}, Promise.resolve()).then(function(finalVal) {
// all done here
}).catch(function(err) {
// error here
});
This is a place where the Bluebird promise library makes things easier because you can use Promise.mapSeries() to iterate an array in series and accumulate an array of results:
Promise.mapSeries(myData, function(nestedArray) {
return Promise.mapSeries(nestedArray, function(item) {
return someAsyncPromise(item);
});
}).then(function(results) {
// all done here
}).catch(function(err) {
// error here
});
In all these scenarios, you can either accumulate the final result in one object that becomes the resolved value of the final promise or you can use it for side effects where you have some other object defined in a higher scope that you are modifying and you then use that object when you're all done.
let seq = Promise.resolve()
let myData = [[1.1,1.2],[2.1,2.2]];
function findUser () {
return new Promise(resolve => {
setTimeout(() => {
resolve(['a','b']);
}, 500);
});
}
myData.forEach(data => {
seq = seq.then(() => { // chain myData.forEach
console.log(data);
let seq = Promise.resolve();
data.forEach(data => {
seq = seq.then(() => { // chain data.forEach
console.log(data);
return findUser({}).then(users => {
users.forEach(user => {
console.log(data, user);
})
})
})
})
return seq;
});
});
The second part of the Promise below (inside the then) is never run. When I run the database query without using the Promise(in a node script that I run node myscript.js it returns the data but the console never returns the prompt--the console just hangs and I have to send an interrupt manually. Therefore, when I put it inside a Promise, I think the Promise doesn't know that the database query is complete even though it seems to have returned all the data, therefore the second part of the Promise isn't running ( I think). If that's the problem, how do I write the database query so that it doesn't hang and the Promise can run to completion?
const sqlite = require('/usr/local/lib/node_modules/sqlite3');
const express = require('/usr/local/lib/node_modules/express')
const promise = require('/usr/local/lib/node_modules/promise')
app.get('/', (request, res) => {
var res = [];
function getData() {
return new Promise(function(resolve, reject) {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, rows) => {
var d = {
a: rows['column_a'],
b: rows['column_b']
}
res.push(d)
});
});
}
getData().then(function(data) {
console.log("never run....", res, data) //never run
});
})
You need to resolve a promise by calling one of the functions it provides in the callback through its constructor.
const promise = new Promise((resolve, reject) => {
// you must call resolve() or reject() here
// otherwise the promise never resolves
});
Otherwise it will always stay in Pending state and never call the callbacks(s) you pass into then.
promise.then(() => {
// this never gets called if we don't resolve() or reject()
});
Additionally, promises allow you to resolve with values so there's usually no need to maintain global variables, you can just pass results through.
Finally, the callback in db.each will be called once for each row, so you would need to handle that by resolving the promise after all rows have been obtained
Here's how you could write your code:
function getData() {
const data = [];
return new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
// error reading a row, reject the Promise immediately
// optionally you could accumulate errors here in a similar manner to rows
reject(e);
return;
}
// success reading a row, store the row result
data.push({
a: row['column_a'],
b: row['column_b']
});
}, (e, rowCount) => { // the complete handler called when the operation is done, see docs: https://github.com/mapbox/node-sqlite3/wiki/API#databaseeachsql-param--callback-complete
if (e) {
// operation finished, there was an error
reject(e);
return;
}
// operation succeeded, resolve with rows
resolve(data);
});
});
}
app.get('/', (request, res) => {
getData().then((data) => {
// here `data` is an array of row objects
}, (e) => {
console.error(`Database error: ${e}`);
});
});
Side Note
Not sure why you are redeclaring the parameter res as an [], but there's no need for doing var res = []. Since you already have res, you can just say res = [] to point res to a new array. Of course that will overwrite the response object so I assume that you're doing it just for the purposes of this example. If not, you should probably create a new variable.
You've declared a Promise which means you're responsible for calling one of resolve or reject once and once only.
Here's a cleaned up example:
app.get('/', (request, res) => {
var res = [ ];
new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
reject(e);
return;
}
res.push({
a: row['column_a'],
b: row['column_b']
});
}, (err) => {
if (err) {
return reject(err);
}
resolve(res);
});
}).then((data) => {
console.log("Running ", res, data)//never run
}
});
If your database layer supports promises that usually makes this sort of code a lot less messy since you can simply chain that in there.
Edit: Since the Sqlite3 API is bizarrely non-standard and the each function has two callbacks you need to handle each row with the first, then the completion handler with the second.
If you design an API like this you're doing it wrong. Don't.
Several points :
resolve/reject must be called, otherwise a new Promise() will remain forever "pending".
always promisify at the lowest level possible, ie promisify db.each() not getData(). This gives you a testable, reusable utility and more comprehensible application code.
db.each() is a challenge to promisify because it has two possible sources of error; one in its iteration callback and one in its complete callback.
the sqlite3 documentation does not state what happens if an iteration error occurs but presumably the iteration continues, otherwise the error would simply appear as a completion error?
Here's a couple of ways to promisify :
1. First iteration error or completion error causes promise rejection - iteration errors are not exposed to your application code.
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, (iterationError, row) => {
if(iterationError) {
reject(iterationError);
} else {
iterationCallback(row);
}
}, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (row) => {
res.push({
a: row['column_a'],
b: row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
2. Only a completion error causes promise rejection - iteration errors are exposed to your application code
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, iterationCallback, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
// You can choose what to do on iterationError.
// Here, nulls are injected in place of values from the db,
// but you might choose not to push anything.
res.push({
a: iterationError ? null : row['column_a'],
b: iterationError ? null : row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
(2) is the better approach because exposing iteration errors affords you more flexibility. For example, you could choose to promisify with (2), and emulate (1) in your application :
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
var e = null;
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
if(iterationError && !e) {
// remember the first iteration error
e = iterationError;
} else {
// push only on success
res.push({
a: row['column_a'],
b: row['column_b']
});
}
}).then(n => {
if(e) {
throw e;
} else {
return res;
}
});
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
With (1), by rejecting on first iteration error rather than exposing iteration errors, the same flexibility is not available. (1) could not fully emulate (2).
Fortunately, the preferred approach (2) is the same as would be obtained with Bluebird's .promisify() method :
Promise.promisify(db.each);
I need some help with the asynchronous nature of node.js. I have a for loop, which collects data from the database. "result" is an array, which should then be returned to the main function.
user_collection.findOne({
_id : uid
}, function(error, user) {
if(error)
callback(error)
else {
for(var j = 0; j < user.contacts.length; j++) {
if(user.contacts[j].accepted == 'true') {
user_collection.findOne({
_id : user.contacts[j].contactId
}, function(error, user) {
result.push(user);
})
}
}
callback(null, result); // This callback executes before the for-loop ends, ofc
}
});
How can I ensure that the callback executes after the loop finished?
You might want to consider using helper library like async
https://github.com/caolan/async
It helps keep code more consistent..
In your case, you can look at the forEach() method
forEach(arr, iterator, callback)
The iterator is called with an item from the list and a callback for when it has finished.
Checkout the unit tests for examples
https://github.com/caolan/async/blob/master/mocha_test/each.js
Using ES6 Promises
(a promise library can be used for older browsers):
Process all requests guaranteeing synchronous execution (e.g. 1 then 2 then 3)
function asyncFunction (item, cb) {
setTimeout(() => {
console.log('done with', item);
cb();
}, 100);
}
let requests = [1, 2, 3].reduce((promiseChain, item) => {
return promiseChain.then(new Promise((resolve) => {
asyncFunction(item, resolve);
}));
}, Promise.resolve());
requests.then(() => console.log('done'))
Process all async requests without "synchronous" execution (2 may finish faster than 1)
let requests = [1,2,3].map((item) => {
return new Promise((resolve) => {
asyncFunction(item, resolve);
});
})
Promise.all(requests).then(() => console.log('done'));
I did it on that way
Promise.all(body.schedules.map(function(scheduleId) {
return new Promise(function(resolve, reject) {
return scheduleSchema.findOneAndRemove({
_id: scheduleId
})
.then(function() {
logSchema.insert({
userId: req.user.id,
actId: constants.LOG_SCHEDULE_DELETE.id,
extData: scheduleId
});
resolve();
})
.catch(function(err) {
reject(err);
});
});
})).then(function() {
return res.json({
code: constants.SUCCESS_CODE
});
}).catch(function(err) {
return res.json(constants.DATABASE_ERROR);
});
The last example
function callback (result) { console.log('all done'); }
[1, 2, 3].forEach((item, index, array) => {
asyncFunction(item, () => {
if (index === array.length - 1) {
callback();
}
});
});
This does not guarantee that callback will execute after all items are processed. It only guarantees that callback will execute after the very last item is processed.
More information
Michael.
With v1.5.2 of Async.js, It is each.
each(arr, iterator, [callback])
arr - An array to iterate over.
iterator(item, callback) - A function to apply to each item in arr.
callback(err) - Optional. A callback which is called when all iterator functions have finished, or an error occurs.