I'm trying to use promises with nodejs (I'm trying with node-promise package); however, without any success. See the code below:
var express = require('express'),
request = require('request'),
promise = require('node-promise');
app.get('/promise', function(req, res) {
var length = -1;
new promise.Promise(request(
{uri: "http://www.bing.com"},
function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log("An error occurred when connected to the web site");
return;
}
console.log("I'll return: " + body.length);
length = body.length;
}
)).then(function(result) {
console.log("This is what I got: " + length);
console.log("Done!");
});
res.end();
});
The output of the above code is I'll return: 35857 only and it doesn't go to the then part.
I change the code then to be:
app.get('/promise', function(req, res) {
var length = -1;
promise.when(
request(
{uri: "http://www.bing.com"},
function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log("An error occurred when connected to the web site");
return;
}
console.log("I'll return: " + body.length);
length = body.length;
}
),
function(result) {
console.log("This is what I got: " + length);
console.log("Done!");
},
function(error) {
console.log(error);
}
);
res.end();
});
This time the output is This is what I got: -1 then Done!... looks like the "promise" was not called this time.
So:
What's needed to be done to fix the code above? Obviously I'm not doing it right :)
Is node-promise "the way to go" when I'm doing promises, or is there a better way/package? i.e. simpler and more production-ready.
Thanks.
Try jquery-deferred-for-node.
I'm not an expert but understand that this lib tends to be favoured by programmers who work both server-side and client-side.
Even if you don't already know jQuery's Deferreds, the advantages of going this route are that :
the documentation is excellent (it comprises links to the jQuery docs), though you may struggle to find examples specific to Node.
methods are chainable.
jQuery Callbacks are also included.
when one day you need to do asynchronous stuff client-side, then there's virtually nothing to relearn - the concepts are identical and the syntax very nearly so. See the "Correspondances" section in the github page hyperlinked above.
EDIT
I'm not a node.js person so I'm guessing here but based on your code above, you might want to consider something along the following lines with jquery-deferred-for-node :
var express = require('express'),
request = require('request'),
Deferred = require('JQDeferred');
function fetch(uri, goodCodes) {
goodCodes = (!goodCodes) ? [200] : goodCodes;
var dfrd = Deferred(); // A Deferred to be resolved/rejected in response to the `request()`.
request(uri, function(error, response, body) {
if (!error) {
var isGood = false;
// Loop to test response.statusCode against `goodCodes`.
for (var i = 0; i < goodCodes.length; i++) {
if (response.statusCode == goodCodes[i]) {
isGood = true;
break;
}
}
if (isGood) {
dfrd.resolve(response.statusCode, body);
} else {
dfrd.reject(response.statusCode, "An invalid response was received from " + uri);
}
} else {
dfrd.reject(response.statusCode, "An error occurred attempting to connect to " + uri);
}
});
// Make promise derived from dfrd available to "consumer".
return dfrd.promise();
};
//...
app.get('/promise', function(req, resp) {
fetch("http://www.bing.com").done(function(statusCode, result) {
console.log("Done! This is what I got: " + result.length);
}).fail(function(statusCode, message) {
console.log("Error (" + statusCode + "): " + message);
});
resp.end();
};
Here, I have tried to write a generalized utility for fetching a resource in such a way that the asynchronous response (or error) can be handled externally. I think this is broadly along the lines of what you were trying to achieve.
Out of interest, where do console.log() messages end up with node.js?
EDIT 2
Above, I have given Deferred an initial capital, as is conventional for Constructors
With jQuery Deferreds, there must be any number of ways to fetch() consecutively. The approach below leaves fetch() as it was, and introduces fetch_() to act as its front-end. There may be simpler ways but this allows fetch() to remain a general utility, functionally equivalent to the client-side jQuery.ajax().
function fetch_(uri){
return function(){
return fetch(uri, [200]).then(function(statusCode, result){
console.log("Done! This is what I got: " + result.length);
},function(statusCode, message){
console.log("Error (" + statusCode + "): " + message);
});
};
}
Note that function fetch() returns a function. It has to be like this because where fetch() is called, we want an unexecuted function, not (yet) the result of that function.
Now let's assume an array of uris is available. This can be hard-coded or built dynamically - whatever the application demands.
var uris = [
'http://xxx.example.com',
'http://yyy.example.com',
'http://zzz.example.com'
];
And now, a variety of ways in which fetch_() might be called :
//v1. To call `resp.end()` when the fetching process starts.
app.get('/promise', function(req, resp) {
fetch_(uris[0])().then(fetch_(uris[1])).then(fetch_(uris[2]));
resp.end();
});
//v2. To call `resp.end()` when the fetching process has finished.
app.get('/promise', function(req, resp){
fetch_(uris[0])().then(fetch_(uris[1])).then(fetch_(uris[2])).always(resp.end);
});
//v3. As v2 but building a `.then()` chain of any (unknown) length.
app.get('/promise', function(req, resp){
var dfrd = Deferred().resolve();//
$.each(uris, function(i, uri){
dfrd = dfrd.then(fetch_(uri));
});
dfrd = dfrd.always(resp.end);
});
untested
I have more confidence in v1 and v2. v3 may work.
v2 and v3 should both give exactly the same behaviour but v3 is generalized for any number of uris.
Everything may need debugging.
I would recommend using Q: https://github.com/kriskowal/q. I believe that it's used internally by other frameworks (like jQuery deferred implementation).
I believe that the documentation is "fine"; the syntax is consistent with other promise implementations... and it has a node adapter.
So your deferred style approach:
var deferred = Q.defer();
FS.readFile("foo.txt", "utf-8", function (err, res) {
if (!err) {
deferred.resolve(res);
} else {
deferred.reject(err);
}
});
return deferred.promise;
Can be written more concisely as:
var deferred = Q.defer();
FS.readFile("foo.txt", "utf-8", deferred.makeNodeResolver());
return deferred.promise;
Related
I call getLogs() through a post request and get a list of LogFileID(filename) from a DB and then I pass this LogFileID to do an additional request by calling _getLogFileUrls which gives me a signed url for that ID in response. I push all of them one by one into a global array and return it the end response.
I know it's incorrect to use setTimeout but the problem is not using, it gives me a different result into the array every time. What could I do to resolve this issue? How do I correct this code so that the loop iterates to the next only when the signed url is stored into the global array.
function _getLogFileUrls(logFileId, callback){
var request = require('request'),
config = require('../../config.js');
var fileParams = {
fileName: 'xyzdirectory/' + logFileId
};
request.post({
url: config.filesServiceUrl + 'get-logfile-urls',
json: fileParams
},function(error, response, body) {
if (!error && response.statusCode === 200) {
callback(body);
} else {
res.status(400).send('Error requesting file service for logs:');
}
}).on('error', function(err) {
console.log('File service error for Logs: ' + err);
});
}
function getLogs(req, res){
if(!req.body.id){
return res.status(400).send('Please check the params!');
}
var date;
if(req.body.date){
date = req.body.date;
} else {
date = new Date().toISOString().slice(0,10);
}
var sqlQuery = "SELECT `LogFileID` FROM `logs_data` WHERE `EmpID` = '" + req.body.id + "' AND DATE(`Timestamp`) = '" + date + "'",
resArray= [];
hitThisQueryForMe(sqlQuery, res, function(rows){
if(!rows.length) res.json(rows);
_.each(rows, function(item){
console.log('item: ' + item.LogFileID);
_getLogFileUrls(item.LogFileID, function(response){
resArray.push(response);
});
});
setTimeout(function(){
res.send(resArray);
resArray = [];
}, 4000);
});
}
SQL injection alert
First of all, your code has a serious SQL injection vulnerability. Never use string concatenation to create SQL using user-provided data or otherwise anyone will be able to read, modify and delete anything in your database. This is very serious security issue. For more details see those answers:
cannot use backtick when using nodejs 7.3.0
How to escape mysql special characters with sockets.io/node.js/javascript
The answer
Now to answer your question. To handle what you try to do here you should either stick to callbacks and use a good module to handle concurrency like Async:
https://caolan.github.io/async/
Or you can use promises with a good module to help with concurrency like Q or Bluebird:
http://documentup.com/kriskowal/q/
http://bluebirdjs.com/
Additionally when working with promises you can use generator-based coroutines with tools like co or Bluebird.coroutine:
https://github.com/tj/co
http://bluebirdjs.com/docs/api/promise.coroutine.html
Or you can use ES8 async/await:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
Those are the main ways to handle cases like yours. Reinventing the wheel of concurrency handling can lead (as you can see here) to error-prone, hard to maintain code.
I recommend using the right tool for the job.
Use async/await
Install asyncawait library and its dependency bluebird:
npm install asyncawait --save
npm install bluebird --save
Your edited code should look like:
const async = require('asyncawait/async');
const await = require('asyncawait/await');
const Promise = require('bluebird');
const request = require('request');
const config = require('../../config.js');
function _getLogFileUrls(logFileId) {
return new Promise((resolve, reject) => {
var fileParams = {
fileName: 'xyzdirectory/' + logFileId
};
request.post({
url: config.filesServiceUrl + 'get-logfile-urls',
json: fileParams
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
resolve(body);
} else {
reject('Error requesting file service for logs:');
}
}).on('error', function (err) {
console.log('File service error for Logs: ' + err);
});
});
}
function getLogs(req, res) {
if (!req.body.id) {
return res.status(400).send('Please check the params!');
}
var date;
if (req.body.date) {
date = req.body.date;
} else {
date = new Date().toISOString().slice(0, 10);
}
var sqlQuery = "SELECT `LogFileID` FROM `logs_data` WHERE `EmpID` = '" + req.body.id + "' AND DATE(`Timestamp`) = '" + date + "'",
resArray = [];
hitThisQueryForMe(sqlQuery, res, function (rows) {
if (!rows.length) res.json(rows);
_.each(rows, (async function (item) {
console.log('item: ' + item.LogFileID);
var logFileUrlResponse = await (_getLogFileUrls(item.LogFileID));
resArray.push(logFileUrlResponse);
}));
res.send(resArray);
resArray = [];
});
}
Follow-up to Swap order of arguments to "then" with Bluebird / NodeJS Promises (the posted answer worked, but immediately revealed a new issue)
This is the first time I've ever used promises in NodeJS so I apologize if some conventions are poorly adhered to or the code is sloppy. I'm trying to aggregate data from multiple APIs, put it in a database, then compute some statistics based on similarities and differences in the data. As a starting point I'm trying to get an API token for a single one of the APIs.
Here is my full code:
var Promise = require('bluebird');
var fs = require('fs');
var request = require('request');
Promise.promisifyAll(fs);
Promise.promisifyAll(request);
// tilde-expansion doesn't follow the callback(err, data) convention
var tilde = function(str) {
var _tilde = require('tilde-expansion');
return new Promise(function(resolve, reject) {
try {
_tilde(str, resolve);
} catch(e) {
reject(e);
}
});
}
var getToken = function() {
return request.getAsync(process.env.token_url, {
headers: {
"Content-Type": "applications/x-www-form-urlencoded"
},
form: {
client_id: process.env.client_id,
client_secret: process.env.client_secret,
grant_type: "client_credentials"
}
})
.then(function(resp) { return resp.body; });
}
var tokenFile = tilde(process.env.token_file)
.catch(function(err) {
console.log("Error parsing path to file... can not recover");
});
var token = tokenFile
.then(fs.readFileAsync) //, "utf8")
.then(function(data) {
console.log("Token (from file): " + data);
return data;
})
.then(JSON.parse)
.catch(function(err) {
console.log("Error reading token from file... getting a new one");
return getToken()
.then(function(data) {
console.log("Token (from API): " + data);
return data;
})
.then(JSON.stringify)
.then(fs.writeFileAsync.bind(null, tokenFile.value()));
});
token.then(function(data) {
console.log("Token (from anywhere): " + token.value);
});
This code is currently logging:
Token: undefined
if I fall back to the API. Assuming I did my promise stuff correctly (.catch() can return a promise, right?) then I would assume the issue is occurring because fs.writeFileAsync returns void.
I would like to append a .return() on the end of this promise, but how would I gain access to the return value of getToken()? I tried the following:
.catch(function(err) {
console.log("Error reading token from file... getting a new one");
var token = "nope";
return getToken()
.then(function(data) {
console.log("Token (from API): " + data);
token = data;
return data;
})
.then(JSON.stringify)
.then(fs.writeFileAsync.bind(null, tokenFile.value()))
.return(token);
});
However this logs "nope".
Over the weekend I continued my research on promises and upon making a pivotal realization I was able to develop the solution to this. Posting here both the realization and the solution:
The Realization
Promises were invented so that asynchronous code could be used in a synchronous manner. Consider the following:
var data = processData(JSON.parse(readFile(getFileName())));
This is the equivalent of:
var filename = getFileName();
var fileData = readFile(filename);
var parsedData = JSON.parse(fileData);
var data = processData(parsedData);
If any one of these functions is asynchronous then it breaks, because the value isn't ready on time. So for those asynchronous bits we used to use callbacks:
var filename = getFileName();
var data = null;
readFile(filename, function(fileData){
data = processData(JSON.parse(fileData));
});
This is not only ugly, but breaks a lot of things like stack traces, try/catch blocks, etc.
The Promise pattern fixed this, letting you say:
var filename = getFileName();
var fileData = filename.then(readFile);
var parsedData = fileData.then(JSON.parse);
var data = parsedData.then(processData);
This code works regardless of whether these functions are synchronous or asynchronous, and there are zero callbacks. It's actually all synchronous code, but instead of passing values around, we pass promises around.
The led me to the realization that: for every bit of code that can be written with promises, there is a synchronous corollary
The solution
Realizing this, I tried to consider my code if all of the functions were synchronous:
try {
var tokenFile = tilde(process.env.token_file)
} catch(err) {
throw new Error("Error parsing path to file... can not recover");
}
var token = null;
try {
token = JSON.parse(readFile(tokenFile));
} catch(err) {
token = getToken();
writeFile(tokenFile, JSON.stringify(token));
}
console.log("Token: " + token.value);
After framing it like this, the promise version follows logically:
var tokenFile = tilde(process.env.token_file)
.catch(function(err) {
throw new Error("Error parsing path to file... can not recover");
});
var token = tokenFile
.then(readFile)
.then(JSON.parse)
.catch(function(err) {
var _token = getToken();
_token
.then(JSON.stringify)
.then(writeFile.bind(null, tokenFile.value));
return _token;
});
I'm just starting with NodeJS. I try to do with NodeJS a loop and only then : send my result to an express template.
I tried many lib and promises but none of them worked. Node do "then" before ending the loop...
Here's my last try, can you help me with? Thanks a lot.
[...]
//pveIds contains list of dailies id (object)
var pveIds = body.pve;
//init tab, will contain dailies title
var pveNames = [];
Promise.map(pveIds, function(pveId) {
// Promise.map awaits for returned promises as well.
request.get({
url: 'https://api.guildwars2.com/v2/achievements?id=' + pveId.id,
json: true
},
function(error, response, body) {
console.log('log 1: ' + body.name);
if (response.statusCode == 200) {
return body.name;
}
}).on('data', function(v) {
console.log('log 2: ' + v);
return v;
});
}).then(function(results) {
console.log("done");
console.log(results);
console.log("names tab:" + pveNames);
res.render('pve.ejs', {
names: pveNames
});
});
You need to
return request.get({... instead of just request.get({
The way you have it now your function(pveId) returns undefined so your Promise.map just registers a bunch of undefined's instead of actual promises.
You should also not mix promises with callbacks, use request-promise instead of request.
After recently discovering JS promises, I have been studying them so that I might build a certain functionality that allows me to execute 4 async queries, use the result of each to build an object that I can finally send as a response to a request directed at my node app.
The final object is made up of 3 array properties containing the resulting rows of each query.
It seems that I've done something wrong handling the promises, though, because ultimately, game is not being built. It is sent as an empty object. Here's a JSFiddle.
What is my mistake?
Here's what I have so far:
function sendGame(req, res, sales, settings, categories) {
var game = new Object();
game.sales = sales;
game.settings = settings;
game.categories = categories;
JSONgame = JSON.stringify(game);
res.writeHead(200, {
'Access-Control-Allow-Origin': 'http://localhost',
'Content-Length': JSONgame.length,
'Content-Type': 'application/json'
});
res.write(JSONgame);
res.end();
console.log('Game: ' + JSON.stringify(game, null, 4));
console.log('--------------------------------------');
console.log('User ' + req.body.username + ' successfully retrieved game!');
}
function retrieveSales(req, connection, timeFrame) {
console.log('User ' + req.body.username + ' retrieving sales...');
connection.query('select * from sales_entries where date BETWEEN ? AND ?', timeFrame,
function (err, rows, fields) {
if (err) {
callback(new Error('Failed to connect'), null);
} else {
sales = [];
for (x = 0; x < rows.length; x++) {
sales.push(rows[x]);
}
//console.log('Sales: ' + JSON.stringify(sales, null, 4));
return sales;
}
});
}
retrieveCategories() and retrieveSettings() omitted for readability; they are the same as retrieveSales() mostly.
function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).format("YYYY-MM-DD HH:mm:ss")];
var connection = createConnection();
connection.connect(function (err) {
if (err) return callback(new Error('Failed to connect'), null);
console.log('Connection with the Officeball MySQL database openned for game retrieval...');
var sales = retrieveSales(req, connection, timeFrame);
var settings = retrieveSettings(req, connection);
var categories = retrieveCategories(req, connection);
var all = q.all([sales, settings, categories]);
all.done(function () {
sendGame(req, res, sales, settings, categories);
});
});
}
Your problem is that you're not using promises. All your APIs use callbacks.
A promise is like a closed box:
A promise also has a method that opens the box, works on the value and returns another box on the value (also opening any additional boxes along the way). That method is .then:
In boxes, it does:
=>( . => ) =>
That is, it adds a handler that gets an open box and returns a box. Everything else just combines stuff. All .all does is wait for a list of promises to resolve, it is exactly like .then in the fact it waits for a result. Because promises are boxes, you can pass them around and return them which is very cool.
Generally:
Whenever you return from a promise handler (not a rejection), you are fullfilling it indicating normal flow continuation.
Whenever you throw at a promise handler, you are rejecting indication exceptional flow.
So basically in node speak:
Whenever you returned a null error and a response, you resolve the promise.
Whenever you returned an error and no response, you reject the promise.
So:
function myFunc(callback){
nodeBack(function(err,data){
if(err!== null){
callback(new Error(err),null);
}
callback(data+"some processing");
})
});
Becomes:
function myFunc(){
return nodeBack().then(function(data){ return data+"some processing"; });
}
Which I think is a lot clearer. Errors are propagated across the promise chain just like in synchronous code - it's very common to find synchronous analogs to promise code.
Q.all takes a list of promises and waits for them to complete, instead you want Q.nfcall to transform a callback based API to a promise one and then use Q.all on that.
That is:
var sales = Q.nfcall(retrieveSales,req, connection, timeFrame);
var settings = Q.nfcall(retrieveSettings,req, connection);
var categories = Q.nfcall(retrieveCategories, req, connection);
Q.nfcall takes a nodeback in the err,data convention and converts it to a promise API.
Also, when you do
return sales;
You are not really returning anything, since it returns synchronously. You need to use callback like in your error case or promisify it altogether. If you don't mind, I'll do it with Bluebird since it comes with much better facilities for dealing with these interop cases and does so much much faster, if you'd like you can switch promisifyAll for a bunch of Q.nfcall calls.
// somewhere, on top of file
connection = Promise.promisifyAll(connection);
// note I'm passing just the username - passing the request breaks separation of concerns.
var retrieveSales = Promise.method(username, connection, timeFrame) {
console.log('User ' + username + ' retrieving sales...');
var q = 'select * from sales_entries where date BETWEEN ? AND ?';
return connection.queryAsync(q, timeFrame).then(function(rows, fields){
return rows;
});
}
Note that suddenly you don't need a lot of boilerplate for making a query, you can use queryAsync directly instead if you'd like.
Now the code that wraps it becomes:
var gameSucceed = Promise.method(function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment()....];
var connection = Promise.promisifyAll(createConnection());
return conn.connectAsync().then(function () {
console.log('Connection with the ...');
//sending req, but should really be what they use.
return Promise.all([retrieveSales(req,conn,timeFrame),
retrieveSettings(req,conn),
retrieveCategories(req,conn)]);
});
});
Now you can call sendGame(req, res, sales, settings, categories); outside of gameSucceed which doesn't hide what it does as much -
gameSucceed(req,res).spread(function(sales,settings,cats){
return sendGame(req,res,sales,settings,cats);
});
I have a http server setup which basically needs to look up stuff in the database.
Here is the code snippet :
var sys = require('sys');
var Client = require('mysql').Client;
var client = new Client();
client.host = '_';
client.user = '_';
client.password = '_';
client.database = '_';
var http = require('http');
http.createServer(function(req, res) {
req.on('end', function() {
client.connect(function(error, results) {
if (error) {
console.log('Connection Error:');
return;
}
ClientConnectionReady(client);
});
ClientConnectionReady = function(client) {
var final = '';
client.query('select * from table', function selectCb(error, result, fields) {
if (error) {
console.log('ERROR');
client.end();
return;
}
final += "{" + JSON.stringify(result);
});
client.query("SELECT COUNT(*) from table", function selectCb(error, result, fields) {
if (error) {
console.log('ERROR');
client.end();
return;
}
final += "," + JSON.stringify(result) + "}";
});
res.writeHead(200, {
'Content-Type': 'text/plain'
});
res.write(final);
res.end();
client.end();
};
});
}).listen(8007, "127.0.0.1");
If I print the values of the variable 'final' at the places where I assign them, I see valid values, but at the lines when I do 'res.write(final)', final is still blank.
How do I make this work and why is this failing?? Thanks for the help, I am new to node.js
The Node.js environment is asynchronous. Those statements that modify "final" are inside callbacks that are executed only when the database operations finish. The code immediately after the initiation of the database operations, where you write the result, are executed long before those callbacks run.
You've almost stumbled upon the answer to the problem already: you must not write the result until the operations are finished, which you know will be the case inside the callbacks. If you must wait for both to finish (seems like you do), then you can do something like keep a counter in the outer scope. Each callback can increment the counter, and call the same result-writer function only when the counter indicates that both callbacks are complete. (I have the idea that the Node runtime has a fancier way of doing that sort of thing, but I'm not that familiar with it. In a simple case like this, keeping something like a counter is easy enough to do.)
Also, an unrelated note: that "ClientConnectionReady" variable should probably either be written as a function definition:
function ClientConnectionReady(client) {
// ...
}
or else it should be declared with var. (I'm a little surprised in fact that it's not throwing an error, but again I'm not that familiar with Node.js.)
By the looks of it, you are trying to write final before it is ever assigned a value.
I'm assuming that client.query is asynchronous. Given that, the callback function is most likely being called after the res.writeHead and res.write lines. What you need to do is put other calls and the client.write* lines within the first callback.
This should give you an idea (didn't check if it compiles)
ClientConnectionReady = function(client)
{
var final = '';
//Get the rows
client.query('select * from table',
function selectCb(error, result, fields)
{
if (error)
{
console.log('ERROR');
client.end();
return;
}
final+="{"+JSON.stringify(result);
//Get the count query
client.query("SELECT COUNT(*) from table",
function selectCb(error, result, fields)
{
if (error)
{
console.log('ERROR');
client.end();
return;
}
final+=","+JSON.stringify(result)+"}";
//Return the final results to the client now
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(final);
res.end();
client.end();
});
});
};
What this does is first gets the rows. In that callback, it then gets the count. Finally, when that works, it sends the data to the client within the count callback.