My node project currently contains a sideway christmas tree of nested callbacks in order to fetch data and process them in the right order. Now I'm trying refactor that using Promises, but I'm unsure how to do it properly.
Let's say I'm fetching a list of offices, then for each office all their employees and then each employees' salary. In the end all entities (offices, employees and salaries) should be linked together and stored in a database.
Some pseudo-code illustrating my current code (error handling omitted):
fetch(officesEndpoint, function (data, response) {
parse(data, function (err, offices) {
offices.forEach(function (office) {
save(office);
fetch(employeesEndPoint, function (data, response) {
parse(data, function (err, employees) {
// link each employee to office
save(office);
save(employee);
employees.forEach(function () {
fetch(salaryEndpoint, function (data, response) {
parse(data, function (err, salaries) {
// link salary to employee
save(employee);
});
});
});
});
});
});
});
});
I tried solving this with promises, but I have a couple of problems:
kind of verbose?
each office needs to be linked to their respective employees, but in the saveEmployees function I only have access to the employees, not the office from further up in the chain:
var restClient = require('node-rest-client');
var client = new restClient.Client();
var xml2js = require('xml2js');
// some imaginary endpoints
var officesEndpoint = 'http://api/offices';
var employeesEndpoint = 'http://api/offices/employees';
var salaryEndpoint = 'http://api/employees/:id/salary';
function fetch (url) {
return new Promise(function (resolve, reject) {
client.get(url, function (data, response) {
if (response.statusCode !== 200) {
reject(statusCode);
}
resolve(data);
});
});
}
function parse (data) {
return new Promise(function (resolve, reject) {
xml2js.parseString(data, function (err, result) {
if (err) {
reject(err);
}
resolve(result);
});
});
}
function saveOffices (offices) {
var saveOffice = function (office) {
return new Promise(function (resolve, reject) {
setTimeout(function () { // simulating async save()
console.log('saved office in mongodb');
resolve(office);
}, 500);
})
}
return Promise.all(offices.map(saveOffice));
}
function saveEmployees (employees) {
var saveEmployee = function (employee) {
return new Promise(function (resolve, reject) {
setTimeout(function () { // simulating async save()
console.log('saved employee in mongodb');
resolve(office);
}, 500);
})
}
return Promise.all(offices.map(saveEmployee));
}
fetch(officesEndpoint)
.then(parse)
.then(saveOffices)
.then(function (savedOffices) {
console.log('all offices saved!', savedOffices);
return savedOffices;
})
.then(function (savedOffices) {
fetch(employeesEndPoint)
.then(parse)
.then(saveEmployees)
.then(function (savedEmployees) {
// repeat the chain for fetching salaries?
})
})
.catch(function (error) {
console.log('something went wrong:', error);
});
You don't necesseraly have to nest, this would work too:
fetch(officesEndpoint)
.then(parse)
.then(saveOffices)
.then(function(savedOffices) {
console.log('all offices saved!', savedOffices);
return savedOffices;
})
.then(function(savedOffices) {
// return a promise
return fetch(employeesEndPoint); // the returned promise can be more complex, like a Promise.all of fetchEmployeesOfThisOffice(officeId)
})
// so you can chain at this level
.then(parse)
.then(saveEmployees)
.then(function(savedEmployees) {
return fetch(salariesEndPoint);
})
.catch(function(error) {
console.log('something went wrong:', error);
});
Your promisified functions fetch, parse, saveOffices and saveEmployees are fine. With those, you can refactor your current code to use promises, chain instead of nest where applicable, and leave out a bunch of error handling boilerplate:
fetch(officesEndpoint)
.then(parse)
.then(function(offices) {
return Promise.all(offices.map(function(office) {
return save(office)
.then(function(){ return fetch(employeesEndPoint); })
.then(parse)
.then(function(employees) {
// link each employee to office
// throw in a Promise.all([save(office), save(employee)]) if needed here
return Promise.all(employees.map(function(employee) {
return fetch(salaryEndpoint)
.then(parse)
.then(function(salaries) {
return Promise.all(salaries.map(function(salary) {
// link salary to employee
return save(employee);
}));
});
}));
});
}));
});
In the innermost loop callback, you've got all of office, employee and salary available to interlink them to your liking. You cannot really avoid this kind of nesting.
You'll get back a promise for a huge array of arrays of arrays of save results, or for any error in the whole process.
It is good approach to change this
if (response.statusCode !== 200) {
reject(statusCode);
}
resolve(data);
to this
if (response.statusCode !== 200) {
return reject(statusCode);
}
resolve(data);
In your example, the result will be same, but if you are making more things (like doing something in database) the unexpected result may occure, because without return the whole method will be executed.
This example
var prom = new Promise((resolve,reject) => {
reject(new Error('error'));
console.log('What? It did not end');
resolve('Ok, promise will not be called twice');
});
prom.then(val => {
console.log(val);
}).catch(err => {
console.log(err.message);
});
is having this output
What? It did not end
error
To the question - if you need access to more than one returned value (i.e. offices AND employies), you have basically two options :
Nested promises - this is not generally bad, if it "makes sense". Altought promises are great to avoid huge callback nesting, it is ok to nest the promises, if the logic needs it.
Having "global" variables - you can define variable in the scope of the promise itself and save results to it, therefore the promises are using these variables as "global" (in their scope).
Related
Is it possible to use javascript promises instead of regular callbacks within CosmosDB (DocumentDB) stored procedure API calls? An usage would be implementing pagination.
For example
token = getToken();
doSomething(token);
//....
function getToken(....) {
//...
collection.queryDocuments(link, query, queryOptions, function(error, documents, responseOptions) {
return responseOptions.continuation;
});
}
would not work because the token is returned within a callback, and the execution continues. Could you please give an example of how you would implement this?
The version of ECMAScript referenced in Cosmos DB docs supports async/await and Promises. I am able to use both of those in my stored procedures.
Here's a function that returns a promise that makes a parameterized document query:
function queryDocumentsAsync(sql, parameters, options) {
const querySpec = {
query: sql,
parameters: parameters
};
return new Promise((resolve, reject)=>{
let isAccepted = __.queryDocuments(__.getSelfLink(), querySpec, options || {}, (err, feed, options) => {
if(err) reject(err);
resolve({
feed: feed,
options: options
});
});
if(!isAccepted) throw "Query was not accepted.";
});
}
I am seeing some limitations around forcing a rollback with this approach, though. If you throw an Error, it gets swallowed by the promise chain and never gets out.
Here's an example on how to use async await for query and replace scenario.
function async_sample() {
const ERROR_CODE = {
NotAccepted: 429
};
const asyncHelper = {
queryDocuments(sqlQuery, options) {
return new Promise((resolve, reject) => {
const isAccepted = __.queryDocuments(__.getSelfLink(), sqlQuery, options, (err, feed, options) => {
if (err) reject(err);
resolve({ feed, options });
});
if (!isAccepted) reject(new Error(ERROR_CODE.NotAccepted, "replaceDocument was not accepted."));
});
},
replaceDocument(doc) {
return new Promise((resolve, reject) => {
const isAccepted = __.replaceDocument(doc._self, doc, (err, result, options) => {
if (err) reject(err);
resolve({ result, options });
});
if (!isAccepted) reject(new Error(ERROR_CODE.NotAccepted, "replaceDocument was not accepted."));
});
}
};
async function main() {
let continuation;
do {
let { feed, options } = await asyncHelper.queryDocuments("SELECT * from c", { continuation });
for (let doc of feed) {
doc.newProp = 1;
await asyncHelper.replaceDocument(doc);
}
continuation = options.continuation;
} while (continuation);
}
main().catch(err => getContext().abort(err));
}
With some cleverness you can use webpack to inline node dependencies, including promisify, which lets you do this:
https://github.com/Oblarg/cosmosdb-storedprocs-ts/blob/master/BuildStoredProcs.js
I'm banging my head against the wall to figure out how to push data that is being written on file asynchronously into an array. Writing the data synchronously (and checking if the item is the last on the list) takes too much time so I decided to make it run async. After doing some research, it seems that I could use a callback
I would prefer not to use an external library for doing this, since I'm pretty sure either a callback or a Promise should do the trick. Thanks!
//Iterate through list and make HTTP request to get data
dataDocument.map(function(item, index) {
request(item, function(err, res, html) {
if (err) throw err;
renderData(html, item);
});
});
//Renders data
function renderData(html, item) {
...some calculations here.
writeData(output, id, function() {
pushed(output);
});
};
//Writes the data on file
function writeData(output, id) {
fs.appendFile('./output.json', output);
//SHOULD I USE A CALLBACK HERE TO PUSH INTO AN ARRAY ONCE IT'S COMPLETE?
};
//NEED HELP HERE: Pushed the data into an array and eliminates last comma.
function pushed(data) {
var arr = [];
arr.push(data);
}
With promises it will look cleaner and leaner. Promisify all the involved functions, and use Promise.all to know when you have collected all data:
// Promisify all the involved callback-based functions:
function promiseRequest(item) {
return new Promise(function (resolve, reject) {
request(item, function (err, res, html) {
if (err) {
reject(err);
} else {
resolve(html);
}
})
})
}
//Renders data
function promiseRenderData(html, item) {
//...some calculations here.
return promiseWriteData(output, id).then(function() {
return output;
});
};
//Writes the data on file
function promiseWriteData(output, id) {
return new Promise(function (resolve, reject) {
fs.appendFile('./output.json', output, function (err) {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
//Iterate through list and make HTTP request to get data
Promise.all(dataDocument.map(function(item, index) {
return promiseRequest(item).then(function(html) {
return promiseRenderData(html, item);
};
})).then(function(arr) {
// Do something with `arr` here
});
The code is running well until the Promise.all and then is goes right to the catch saying 'then is not defined'.
I've been trying to figure this out without success for hours :(.
Any help are welcome.
Here is a simplified example of code:
// Save
return new Promise((fulfillSave, rejectSave) => {
// Get
this._getObjects().then((object) => {
var promises = [];
// For each value
object.values.forEach((value) => {
promises.push(
// Create promise
new Promise((fulfill, reject) => {
// Create MDB object + assign value
valueMongoDB.value = value;
// Save
valueMongoDB.save((err, results) => {
if (err) {
reject('Error in saving');
} else {
fulfill();
}
});
})
);
});
// Wait for all promises
Promise.all(promises).then(() => {
// Nothing to do
fulfillSave();
}, then((err) => {
// Err
rejectSave(err);
}));
}
}).catch((err) => {
rejectSave(`Error: ${err.message}`);
});
});
Thanks in advance!
Serge.
This is incorrect:
// Wait for all promises
Promise.all(promises).then(() => {
// Nothing to do
fulfillSave();
}, then((err) => {
// ^^^^--------------------------- error here
// Err
rejectSave(err);
}));
It's trying to call a freestanding function called then and pass its return value into the then on the object returned by Promise.all.
I think you're trying to hook up a failure handler. If so, you don't say then, you just supply a second function:
Promise.all(promises).then(() => {
// Nothing to do
fulfillSave();
}, (err) => {
// Err
rejectSave(err);
}));
But of course, since you're not using the result of that chain and you're just passing the single argument your second function receives into rejectSave, you could just pass rejectSave directly:
Promise.all(promises).then(() => {
// Nothing to do
fulfillSave();
}, rejectSave);
If you told us what your overall code is meant to do and what its inputs are, my suspicion is that that code could be a lot simpler. It's common to create and nest promises unnecessarily, and I suspect that's happening here.
For instance, if you just want to do the saves and get back a promise that will resolve when they're all done successfully or reject on the first failure:
return this._getObjects()
.then(objects => Promise.all(objects.map(value => {
return new Promise((resolve, reject) => {
// Create MDB object + assign value
valueMongoDB.value = value;
// Save
valueMongoDB.save((err, results) => {
if (err) {
reject('Error in saving');
} else {
fulfill();
}
});
});
})));
Or if we give ourselves a helper function for the Mongo bit:
function mongoSavePromise(value) {
return new Promise((resolve, reject) => {
// Create MDB object + assign value
valueMongoDB.value = value;
// Save
valueMongoDB.save((err, results) => {
if (err) {
reject('Error in saving');
} else {
fulfill();
}
});
});
}
then:
return this._getObjects()
.then(objects => Promise.all(objects.map(mongoSavePromise)));
Avoid the Promise constructor antipattern!
Your whole code should be a simple
return this._getObjects().then(object => {
var promises = object.values.map(value => {
// Create MDB object + assign value
valueMongoDB.value = value;
// Save
return valueMongoDB.save().catch(err => {
throw 'Error in saving';
});
});
// Wait for all promises
return Promise.all(promises);
}, err => {
throw `Error: ${err.message}`;
});
No unnecessary callbacks, no room for mistakes. Btw, you shouldn't throw strings.
I have a function that returns a Promise, that accesses the database and pulls a few lines out, assigning them to a Javascript variable.
The issue is that my '.then' clause is being triggered even though I know the Promise hasn't resolved:
app.post("/api/hashtag", function (req, res) {
FindPopularRumours().then(function (resolveVar) {
console.log(resolveVar);
console.log();
res.send(resolveVar);
}).catch(function () {
console.log("DB Error!");
res.send("DB Error!");
});
});
And the Promise function:
function FindPopularRumours() {
return new Promise((resolve, reject) => {
var hashtags = [];
var dbPromise;
db.collection(HASHTAGS).find().forEach(function (doc) {
hashtags.push(doc.hashtag);
console.log(hashtags);
});
resolve(hashtags);
});
}
The result output is:
[ ]
['#test1']
['#test1', '#test2']
['#test1', '#test2', '#test3']
As you can see, the first line ('[ ]') should ONLY be executed AFTER the hashtags have been output. But for some reason my code seems to think the Promise has been resolved before it actually has.
EDIT1
As per Ankit's suggestion, I have amended my function to:
function FindPopularRumours() {
return new Promise((resolve, reject) => {
var hashtags = [];
db.collection(HASHTAGS).find({}, function (err, doc) {
if (!err) {
doc.forEach(function (arg) {
hashtags.push(arg.hashtag);
console.log(hashtags);
});
resolve(hashtags);
} else {
return reject(err);
}
});
});
}
This still returns the same output response as before (e.g the 'then' clause is running before the promise itself).
My POST function is still the same as before.
The db.collection.find() function is async, so you have to resolve the promise inside the callback for that, something like
function FindPopularRumours() {
return db.collection(HASHTAGS).find().toArray().then( (items) => {
return items.map( doc => doc.hashtag);
});
}
takes advantage of the Mongo toArray() method, that returns a promise directly
Please note that db.collection(HASHTAGS).find() is an asynchronous call. So, your promise is resolved before database query returns. To solve this problem, you need to re-write your database query as follows:
function FindPopularRumours() {
return new Promise((resolve, reject) => {
var hashtags = [];
var dbPromise;
db.collection(HASHTAGS).find({}, function(err, doc){
if(!err){
doc.forEach(function (arg) {
hashtags.push(arg.hashtag);
console.log(hashtags);
});
resolve(hashtags);
}else{
return reject(err);
}
});
});
}
Hope the answer helps you!
The second part of the Promise below (inside the then) is never run. When I run the database query without using the Promise(in a node script that I run node myscript.js it returns the data but the console never returns the prompt--the console just hangs and I have to send an interrupt manually. Therefore, when I put it inside a Promise, I think the Promise doesn't know that the database query is complete even though it seems to have returned all the data, therefore the second part of the Promise isn't running ( I think). If that's the problem, how do I write the database query so that it doesn't hang and the Promise can run to completion?
const sqlite = require('/usr/local/lib/node_modules/sqlite3');
const express = require('/usr/local/lib/node_modules/express')
const promise = require('/usr/local/lib/node_modules/promise')
app.get('/', (request, res) => {
var res = [];
function getData() {
return new Promise(function(resolve, reject) {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, rows) => {
var d = {
a: rows['column_a'],
b: rows['column_b']
}
res.push(d)
});
});
}
getData().then(function(data) {
console.log("never run....", res, data) //never run
});
})
You need to resolve a promise by calling one of the functions it provides in the callback through its constructor.
const promise = new Promise((resolve, reject) => {
// you must call resolve() or reject() here
// otherwise the promise never resolves
});
Otherwise it will always stay in Pending state and never call the callbacks(s) you pass into then.
promise.then(() => {
// this never gets called if we don't resolve() or reject()
});
Additionally, promises allow you to resolve with values so there's usually no need to maintain global variables, you can just pass results through.
Finally, the callback in db.each will be called once for each row, so you would need to handle that by resolving the promise after all rows have been obtained
Here's how you could write your code:
function getData() {
const data = [];
return new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
// error reading a row, reject the Promise immediately
// optionally you could accumulate errors here in a similar manner to rows
reject(e);
return;
}
// success reading a row, store the row result
data.push({
a: row['column_a'],
b: row['column_b']
});
}, (e, rowCount) => { // the complete handler called when the operation is done, see docs: https://github.com/mapbox/node-sqlite3/wiki/API#databaseeachsql-param--callback-complete
if (e) {
// operation finished, there was an error
reject(e);
return;
}
// operation succeeded, resolve with rows
resolve(data);
});
});
}
app.get('/', (request, res) => {
getData().then((data) => {
// here `data` is an array of row objects
}, (e) => {
console.error(`Database error: ${e}`);
});
});
Side Note
Not sure why you are redeclaring the parameter res as an [], but there's no need for doing var res = []. Since you already have res, you can just say res = [] to point res to a new array. Of course that will overwrite the response object so I assume that you're doing it just for the purposes of this example. If not, you should probably create a new variable.
You've declared a Promise which means you're responsible for calling one of resolve or reject once and once only.
Here's a cleaned up example:
app.get('/', (request, res) => {
var res = [ ];
new Promise((resolve, reject) => {
db.each('SELECT column_a, column_b FROM trips group by column_a', (e, row) => {
if (e) {
reject(e);
return;
}
res.push({
a: row['column_a'],
b: row['column_b']
});
}, (err) => {
if (err) {
return reject(err);
}
resolve(res);
});
}).then((data) => {
console.log("Running ", res, data)//never run
}
});
If your database layer supports promises that usually makes this sort of code a lot less messy since you can simply chain that in there.
Edit: Since the Sqlite3 API is bizarrely non-standard and the each function has two callbacks you need to handle each row with the first, then the completion handler with the second.
If you design an API like this you're doing it wrong. Don't.
Several points :
resolve/reject must be called, otherwise a new Promise() will remain forever "pending".
always promisify at the lowest level possible, ie promisify db.each() not getData(). This gives you a testable, reusable utility and more comprehensible application code.
db.each() is a challenge to promisify because it has two possible sources of error; one in its iteration callback and one in its complete callback.
the sqlite3 documentation does not state what happens if an iteration error occurs but presumably the iteration continues, otherwise the error would simply appear as a completion error?
Here's a couple of ways to promisify :
1. First iteration error or completion error causes promise rejection - iteration errors are not exposed to your application code.
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, (iterationError, row) => {
if(iterationError) {
reject(iterationError);
} else {
iterationCallback(row);
}
}, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (row) => {
res.push({
a: row['column_a'],
b: row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
2. Only a completion error causes promise rejection - iteration errors are exposed to your application code
// Promisification
db.eachAsync = function(sql, iterationCallback) {
return new Promise(function(resolve, reject) {
db.each(sql, iterationCallback, (completionError, n) => {
if(completionError) {
reject(completionError);
} else {
resolve(n); // the number of retrieved rows.
}
});
});
};
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
// You can choose what to do on iterationError.
// Here, nulls are injected in place of values from the db,
// but you might choose not to push anything.
res.push({
a: iterationError ? null : row['column_a'],
b: iterationError ? null : row['column_b']
});
}).then(n => res);
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
(2) is the better approach because exposing iteration errors affords you more flexibility. For example, you could choose to promisify with (2), and emulate (1) in your application :
// Application
app.get('/', (request, response) => {
function getData() {
var res = [];
var e = null;
return db.eachAsync('SELECT column_a, column_b FROM trips group by column_a', (iterationError, row) => {
if(iterationError && !e) {
// remember the first iteration error
e = iterationError;
} else {
// push only on success
res.push({
a: row['column_a'],
b: row['column_b']
});
}
}).then(n => {
if(e) {
throw e;
} else {
return res;
}
});
}
getData().then(results => {
console.log(results);
}).catch(error => {
console.log(error);
});
});
With (1), by rejecting on first iteration error rather than exposing iteration errors, the same flexibility is not available. (1) could not fully emulate (2).
Fortunately, the preferred approach (2) is the same as would be obtained with Bluebird's .promisify() method :
Promise.promisify(db.each);