Develop Reference

Who can explain nodejs promises in a loop to a PHP programmer

I have searched high and low but can't get my head around promises. What I do understand is how to define one promise and use its result by using .then.
What I do not understand is how I can create a loop to query the database for different blocks of records. This is needed due to a limit set on the number of records to query.
The predefined promise api call is used like this:
let getRecords = (name) => {
return new Promise((resolve,reject) => {
xyz.api.getRecords(name, 1000, 1000, function(err, result){
// result gets processed here.
resolve(//somevariables here);
});
)};
going with what I am used to, I tried:
for (let i=1; i<90000; i+=500) {
xyz.api.getRecords('james', i, 500, function(err, result){
// result gets processed here.
});
}
But then I can't access the information (could be my wrong doing)
Also tried something like this:
function getRecords(name,i){
xyz.api.getRecords(name, i, 500, function(err, result){
// result gets processed here.
});
};
for (let i=1; i<90000; i+=500) {
var someThing = getRecords('james', i);
}
All tutorials only seem to use one query, and process the data.
How do I call the api function multiple times with different arguments, collect the data and process it once everything is retrieved?
Only thing I can think of is, to write info to a file, terrible thought.

Using async/await
(async () => {
function getRecords(name,i){
// create new Promise so you can await for it later
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, function(err, result){
if(err) {
return reject(err);
}
resolve(result);
});
});
}
for (let i = 1; i < 90000; i += 500) {
// wait for the result in every loop iteration
const someThing = await getRecords('james', i);
}
})();
To handle errors you need to use try/catch block
try {
const someThing = await getRecords('james', i);
} catch(e) {
// handle somehow
}
Using only Promises
function getRecords(name, i) {
// create Promise so you can use Promise.all
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, function (err, result) {
if (err) {
return reject(err);
}
resolve(result);
});
});
}
const results = [];
for (let i = 1; i < 90000; i += 500) {
// push Promise's to array without waiting for results
results.push(getRecords("james", i));
}
// wait for all pending Promise's
Promise.all(results).then((results) => {
console.log(results);
});
let count = 0;
function getRecords(name, i) {
return new Promise((resolve, reject) => {
setTimeout(() => {
// example results
resolve((new Array(10)).fill(0).map(() => ++count));
}, 100);
});
}
const results = [];
for (let i = 1; i < 9000; i += 500) {
results.push(getRecords("james", i));
}
Promise.all(results).then((results) => {
console.log("Results:", results);
console.log("Combined results:",[].concat(...results));
});
To handle errors you need to use .catch() block
Promise.all(results).then((results) => { ... }).catch((error) => {
// handle somehow
});

By returning a promise and calling your asynchronous function inside, you can resolve the result and then use it this way:
function getRecords (name, i) {
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
for (let i = 1; i < 90000; i * 500) {
getRecords('james', i)
.then(result => {
// do stuff with result
})
}
Or, using async / await syntax:
async function getRecords (name, i) {
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
// this needs to happen inside a function, node does not allow top level `await`
for (let i = 1; i < 90000; i *= 500) {
const result = await getRecords('james', i);
// do stuff
}
Get all of your records at once
const requests = [];
for (let i = 1; i < 90000; i * 500) {
requests.push(getRecords('james', i));
}
const results = await Promise.all(requests);

Collect asynchronous and synchronous data in loop

var data = [10,21,33,40,50,69];
var i = 0;
var dataSeq = [];
while(i<data.length){
if(data[i]%2 == 0){
store.findOne({'visibility': true},function(err, data){
dataSeq.push(i)
i++;
});
}
else{
dataSeq.push(i)
i++;
}
}
if(i==data.length-1){
console.log(dataSeq) // Should Print [1,2,3,4,5]
return res.status(200).send({ message: 'Task Completed'})
}
I want to collect data as per loop excecutes.
I am aware about how to handle async calls in nodejs. But I want the callbacks in sequence.
e.g. Though there is a async call in if condition i want to hault the loop, so that I can push value of i in dataSeq and it will result in [1,2,3,4,5] array. I want that sequence because my post operations are dependent on that sequence.

I think asyncjs#eachSeries has what you need.
Your code would become something like this:
async.each(data, (item, callback) => {
if(item%2 == 0){
store.findOne({'visibility': true},function(err, data){
dataSeq.push(i)
i++;
});
}
else{
dataSeq.push(i)
i++;
}
}, (err) => {
// if any of the callbacks produced an error, err would equal that error
});

You can use something like async#eachOf
var async = require('async');
var data = [10,21,33,40,50,69];
var dataSeq = [];
async.eachOf(data, function(value, key, cb) {
if (value % 2 == 0) {
store.findOne({ 'visibility': true })
.then(function(doc) {
dataSeq.push(key);
})
.catch(function(err) {
return cb(err);
});
} else {
cb();
}
}, function(err) {
if (err) {
console.error(err)
return res.status(500).send(); # handle the error as you want
}
return res.status(200).send({ message: 'Task Completed'})
})

Nested promise in a for loop not behaving as expected

I am having trouble extending a Promise inside a .then(). I am trying to perform DB updates in a for-loop and then close the database after all records are processed. However the application exits with process.exit() right away which means that process.exit() was executed even before all db updates were finished. I am pretty sure I am doing something wrong with the nested promise.
var myDB;
function doSomething() {
return MongoClient.connect(DB_CONNECTION).then(function(db) {
myDB = db;
var collection = db.collection(COLLETION_NAME);
for (var i = 0; i < 10; i++) {
promise.then(function{
collection.update({
symbol: items[i].symbol
}, {
$set: {
value: 123
}
}, {
upsert: true
});
});
}
})
}
var promise = doSomething();
promise.then(function(){
console.log("DONE");
myDB.close();
process.exit();
});

It looks like you are getting a promise back from the MongoClient.connect method so why not use that to chain together. I've put a quick sample together below based on your code:
function doSomething(db) {
return new Promise(function(resolve, reject){
var collection = db.collection(COLLETION_NAME);
for (var i = 0; i < 10; i++) {
collection.update({
symbol: items[i].symbol
}, {
$set: {
value: 123
}
}, {
upsert: true
});
}
resolve(db);
})
}
function connectToDB() {
return MongoClient.connect(DB_CONNECTION);
}
function closeDB(db) {
return new Promise(function(resolve, reject){
db.close();
resolve();
});
}
connectToDB().then(function(db){
return doSomething(db);
}).then(function(db){
return closeDB(db);
}).then(function(){
console.log("DONE");
process.exit();
}).catch(function(error){
console.log('Something went wrong: ' + error);
});

Updated code as per #RayonDabre 's suggestion
function doSomething() {
return MongoClient.connect(DB_CONNECTION).then(function(db) {
myDB = db;
var collection = db.collection(COLLECTION_NAME);
var promises = [];
for (var i = 0; i < 10; i++) {
var innerPromise = collection.update({
symbol: items[i].symbol
}, {
$set: {
value: 123
}
}, {
upsert: true
});
promises.push(innerPromise);
}
return Promise.all(promises);
});
}
var promise = doSomething();
promise.then(function(){
console.log("DONE");
myDB.close();
process.exit();
});

Cant send headers after already sent?

What does that even mean in terms of the following code, the res.send works fine however in my console i get the following message:
http.js:689
throw new Error('Can\'t set headers after they are sent.');
app.get('/summoner/:summonerName', function(req, res) {
lolapi.Summoner.getByName(req.params.summonerName, function(err, obj) {
var options = {
beginIndex: 0,
endIndex: 1
};
lolapi.MatchList.getBySummonerId(obj['savisaar2'].id, options, function(err, matches) {
var gameMatches = matches.matches;
for(var i = 0; i < gameMatches.length; i++) {
lolapi.Match.get(gameMatches[i].matchId, function(err, games) {
res.send({profile : obj, matchHistory : games});
});
}
});
});
});

As I explained in my comments, you are calling res.send() inside a for loop which means you are calling it more than once. You can only call it once per request. That is why you are seeing the error message in the console.
It is not clear exactly what your code really wants to do, but if the desire is to collect all the results into an array and send them all as the response, then you can do that like this:
app.get('/summoner/:summonerName', function (req, res) {
lolapi.Summoner.getByName(req.params.summonerName, function (err, obj) {
if (err) {
return res.status(500).end();
}
var options = {beginIndex: 0, endIndex: 1};
lolapi.MatchList.getBySummonerId(obj['savisaar2'].id, options, function (err, matches) {
var gameMatches = matches.matches;
var results = [];
for (var i = 0; i < gameMatches.length; i++) {
lolapi.Match.get(gameMatches[i].matchId, function (err, games) {
if (err) {
return res.status(500).end();
}
results.push({profile: obj, matchHistory: games});
// if all results are done, then send response
if (results.length === gameMatches.length) {
res.json(results);
}
});
}
});
});
});
Note: I've also added rudimentary error handling.
If you want the results in the particular order that you requested then, then you can add a little more code to do that like this:
app.get('/summoner/:summonerName', function (req, res) {
lolapi.Summoner.getByName(req.params.summonerName, function (err, obj) {
if (err) {
return res.status(500).end();
}
var options = {beginIndex: 0, endIndex: 1};
lolapi.MatchList.getBySummonerId(obj['savisaar2'].id, options, function (err, matches) {
var gameMatches = matches.matches;
var results = new Array(gameMatches.length);
var cntr = 0;
for (var i = 0; i < gameMatches.length; i++) {
(function(index) {
lolapi.Match.get(gameMatches[i].matchId, function (err, games) {
if (err) {
return res.status(500).end();
}
++cntr;
results[index] = {profile: obj, matchHistory: games};
// if all results are done, then send response
if (cntr === gameMatches.length) {
res.json(results);
}
});
})(i);
}
});
});
});
Since Promises are now standard in 2016, here's an idea what this could look like using the Bluebird promise library:
const Promise = require('bluebird');
Promise.promisifyAll(lolapi.Summoner);
Promise.promisifyAll(lolapi.MatchList);
Promise.promisifyAll(lolapi.Match);
app.get('/summoner/:summonerName', function (req, res) {
var main;
lolapi.Summoner.getByNameAsync(req.params.summonerName).then(function(obj) {
main = obj;
var options = {beginIndex: 0, endIndex: 1};
return lolapi.MatchList.getBySummonerIdAsync(obj['savisaar2'].id, options);
}).then(function(matches) {
var gameMatches = matches.matches;
return Promise.map(gameMatches, function(item){
return lolapi.Match.getAsync(item.matchId).then(function(games) {
return {profile: main, matchHistory: games};
});
});
}).then(function(results) {
res.json(results);
}).catch(function(err) {
res.status(500).end();
});
}

Q.js: How can I rewrite an async series flow in Q.js?

In an attempt to grasp Q.js, I'd like to convert the following code using async.series in Q.js. Basically I create a folder if it doesn't exist (using mkdirp), move a file into a backup folder and save a file into a main folder.
var async = require('async');
var fs = require('fs');
var path = require('path');
var sessiondId = new Date().getTime() % 2 == 0 ? new Date().getTime().toString() : '_1234';
var backupFolder = path.join(__dirname,sessiondId);
var backupFullPath = path.join(backupFolder,'a.txt');
var fullPath = path.join(__dirname,'main','a.txt');
var mkdirp = require('mkdirp');
async.series({
createOrSkip: function(callback) {
mkdirp(backupFolder, function (err, dir) {
if(err) {
callback(err, null);
} else {
callback(null, {created: !!dir, folderAt: backupFolder});
}
});
},
move: function(callback) {
fs.rename(fullPath, backupFullPath, function(err) {
if(err) {
callback(err, null);
} else {
callback(null, {backupAt: backupFullPath});
}
});
},
write: function(callback) {
fs.writeFile(fullPath, 'abc', function(err) {
if (err) {
callback(err, null);
} else {
callback(null, {saveAt: fullPath});
}
});
}
}, function(err, result) {
console.log(result);
});
Actually I don't know where to start. Thanks for your help.
R.

The key is to convert the node.js functions to return promises using Q.denodeify before you start, this means the header of your file should look like:
var Q = require('q')
var fs = require('fs');
var path = require('path');
var sessiondId = new Date().getTime() % 2 == 0 ? new Date().getTime().toString() : '_1234';
var backupFolder = path.join(__dirname,sessiondId);
var backupFullPath = path.join(backupFolder,'a.txt');
var fullPath = path.join(__dirname,'main','a.txt');
var mkdirp = Q.denodeify(require('mkdirp'));
var rename = Q.denodeify(fs.rename);
var writeFile = Q.denodeify(fs.writeFile);
That change wouldn't be needed if node.js natively supported promises.
Option 1
// createOrSkip
mkdirp(backupFolder)
.then(function (dir) {
// move
return rename(fullPath, backupFullPath);
})
.then(function () {
// write
return writeFile(fullPath, 'abc');
})
.done(function () {
console.log('operation complete')
});
I don't think it gets much simpler than that. Like #Bergi said though, it's more similar to "waterfall". If you want the exact behavior of series (but with promises) you'll have to use something like Option 2 or Option 3.
Option 2
You could write out the code manually to save the results. I usually find that, although this requires a little extra writing, it's by far the easiest to read:
var result = {}
mkdirp(backupFolder)
.then(function (dir) {
result.createOrSkip = {created: !!dir, folderAt: backupFolder};
return rename(fullPath, backupFullPath);
})
.then(function () {
result.move = {backupAt: backupFullPath};
return writeFile(fullPath, 'abc');
})
.then(function () {
result.write = {saveAt: fullPath};
return result;
})
.done(function (result) {
console.log(result);
});
Option 3
If you find yourself using this sort of code all the time, you could write a very simple series helper (I've never found the need to do this personally):
function promiseSeries(series) {
var ready = Q(null);
var result = {};
Object.keys(series)
.forEach(function (key) {
ready = ready.then(function () {
return series[key]();
}).then(function (res) {
result[key] = res;
});
});
return ready.then(function () {
return result;
});
}
promiseSeries({
createOrSkip: function () {
return mkdirp(backupFolder).then(function (dir) {
return {created: !!dir, folderAt: backupFolder};
});
},
move: function () {
return rename(fullPath, backupFullPath)
.thenResolve({backupAt: backupFullPath});
},
write: function () {
return writeFile(fullPath, 'abc')
.thenResolve({saveAt: fullPath});
}
}).done(function (result) {
console.log(result);
});
I'd say once you've written the helper, the code is a lot clearer for promises than with all the error handling cruft required to work with callbacks. I'd say it's clearer still when you either write it by hand or don't keep track of all those intermediate results.
Summing Up
You may or may not think these examples are clearer than the async.series version. Consider how well you might know that function though. It's actually doing something pretty complex in a very opaque manner. I initially assumed that only the last result would be returned (ala waterfall) and had to look it up in the documentation of Async. I almost never have to look something up int the documentation of a Promise library.

Make each of your functions return a promise. Construct them with a Deferred:
function createOrSkip(folder) {
var deferred = Q.defer();
mkdirp(folder, function (err, dir) {
if(err) {
deferred.reject(err);
} else {
deferred.resolve({created: !!dir, folderAt: backupFolder});
}
});
return deferred.promise;
}
However, there are helper functions for node-style callbacks so that you don't need to check for the err yourself everytime. With Q.nfcall it becomes
function createOrSkip(folder) {
return Q.nfcall(mkdirp, folder).then(function transform(dir) {
return {created: !!dir, folderAt: backupFolder};
});
}
The transform function will map the result (dir) to the object you expect.
If you have done this for all your functions, you can chain them with then:
createOrSkip(backupfolder).then(function(createResult) {
return move(fullPath, backupFullPath);
}).then(function(moveResult) {
return write(fullPath, 'abc');
}).then(function(writeResult) {
console.log("I'm done");
}, function(err) {
console.error("Something has failed:", err);
});
Notice that this works like async's waterfall, not series, i.e. the intermediate results will be lost. To achieve that, you would need to nest them:
createOrSkip(backupfolder).then(function(createResult) {
return move(fullPath, backupFullPath).then(function(moveResult) {
return write(fullPath, 'abc');.then(function(writeResult) {
return {
createOrSkip: createResult,
move: moveResult,
write: writeResult
};
});
});
}).then(function(res){
console.log(res);
}, function(err) {
console.error("Something has failed:", err);
});