I use github to authenticate in my node application. I have constructed the following code:
var req = request(postOptions, function (res) {
res.on('data', function (d) {
...
var getOptions = parseUrl('https://api.github.com/user?access_token=' + accessToken);
...
var req = request(getOptions, function (resp) {
...
resp.on('data', function (d) {
...
})
.on('end', function () {
...
})
});
req.end();
});
});
req.write(postData);
req.end();
I've removed some code, because the point here is that I have a request in a request. Now, nodejs has deferreds The question is if this can be used to simplify the above code ?
Well, you have no error handling. Promises significantly cleans up code that correctly propagates errors and doesn't leak resources because those become automatic. So it's impossible to make a fair comparison because promise code that doesn't handle errors still propagates them.
var Promise = require("bluebird");
var request = Promise.promisifyAll(require("request"));
function githubAuthenticate() {
return request.postAsync(postOptions, postData)
.spread(function(response, body) {
var accessToken = ...
var getOptions = parseUrl('https://api.github.com/user?access_token=' + accessToken);
return request.getAsync(getOptions);
})
.spread(function(response, body) {
});
}
Now imagine if something failed here? You would add a .catch only once, in one place, and handle it there. Since errors automatically propagate, the code above doesn't need to do anything. The consumer code can just do:
gitHubAuthenticate().then(function() {
}).catch(function(err) {
// Any error that happened with the post, get or your code gets here
// automatically
});
Related
I am still learning node.js, so please be kind.
I am struggling to get my head around some of the basics without having a book open.
I have written a function to go out and get some JSON from a URL. This works but how do I return the data from the function, then pick bits out. here is my code. SerialNumber is one of the JSON elements being returned.
const https = require('https');
function authenticate( uuid , cdcloc ) {
let url = cdcloc + "/api.php?uuid=" + uuid + '&auth';
https.get(url,(res) => {
let body = "";
res.on("data", (chunk) => {
body += chunk;
});
res.on("end", () => {
try {
let cdcResponse = JSON.parse(body);
// do something with JSON
return cdcResponse[0];
} catch (error) {
console.error(error.message);
};
});
}).on("error", (error) => {
console.error(error.message);
});
}
const connection = authenticate( 'DATATOBEPASSED' , 'https://URLHERE');
console.log(connection.SerialNumber);
node-style callbacks
The answer from O.Jones is correct but it goes against Node's convention of error-first callbacks. I think it is also a mistake to reach for https.get every single time you need to make a request. It is a low-level function and because it asks you to connect so many bits and pieces, it is likely you will make easily-avoidable mistakes.
We can write a generic getString function that wraps https.get -
const https = require('https')
function getString(url, options, callback)
{ https
.get(url, options, res => {
let s = "";
res.on("data", d => s += d)
res.on("end", _ => callback(null, s) // error-first callback
})
.on("error", e => callback(e)) // error-first callback
}
Now that we have a generic function to fetch a string, we don't need to write res.on("data, ...) and res.on("end", ...) in every function that makes a request. But don't stop here. You will often want to JSON.parse the result -
function getJSON(url, options, callback)
{ getString(url, options, function(err, data)
{ if (err) callback(err) // getString error
else try { callback(null, JSON.parse(data) } // JSON.parse success
catch (e) { callback(e) } // JSON.parse error
}
}
Now we can write authenticate without having touch the bare https.get or worrying about parsing JSON each time -
function authenticate(uuid, cdcloc, callback) // callback
{ const url = cdcloc + "/api.php?uuid=" + uuid + '&auth'
getJSON(url, {}, function(err, json)
{ if (err)
callback(err) // getJSON error
else if (json.length == 0)
callback(Error("empty response")) // empty response error
else
callback(null, json[0]) // success
}
}
promises
But all of this is pretty painful still, isn't it? Enter Promises. Node-style callbacks were designed at a time when we didn't have access to async control flow primitives. We've come a long way since then. To see how promises work, we will re-implement the functions above, but this time without the need to pass callback and error-check everywhere -
const https = require('https')
function getString(url, options) // no callback
{ return new Promise((resolve, reject) => // return Promise
{ https
.get(url, options, res => {
let s = "";
res.on("data", d => s += d)
res.on("end", _ => resolve(s)) // success, resolve
})
.on("error", e => reject(e)) // failure, reject
}
}
We immediately see the benefits of our new implementation when we rewrite getJSON -
function getJSON(url, options = {}) // no callback
{ return getString(url, options) // return promise
.then(s => JSON.parse(s)) // errors auto bubble up
}
And more benefits when we write authenticate -
function authenticate(uuid, cdcloc) // no callback
{ const url = `${cdcloc}/api.php?uuid=${uuid}&auth`
return getJSON(url) // return promise
.then(data => {
if (data.length == 0)
throw Error("empty response") // local error
else
return data[0] // success
}) // no error-check
}
async/await
Even Promises have been around for a long time and we've learned a lot since their native inclusion in ECMAScript. Remember to return promises and having to sequence all of the data through .then calls is tedious, the same way writing those initial res.on("data", ...) and res.on("end", ...) handlers felt. async and await keywords allows us to work with asynchronous control flow without having to sacrifice synchronous programming style -
async function getJSON(url, options = {}) // async
{ const s = await getString(url, options) // await
return JSON.parse(s) // auto wrapped in Promise
}
Writing authenticate is easy and feels natural -
async function authenticate(uuid, cdcloc) // async
{ const url = `${cdcloc}/api.php?uuid=${uuid}&auth`
const data = await getJSON(url) // await
if (data.length == 0)
throw Error("empty response") // throw if empty
else
return data[0] // return first
}
Using it is easy and feels natural too -
async function connect()
{ const connection = await authenticate( 'DATATOBEPASSED' , 'https://URLHERE')
console.log(connection.SerialNumber)
// ...
return "done" // or whatever
}
connect().then(console.log, console.error) // errors bubble all the way up
URL
I should also mention that building URLs with string concatenation is tedious and prone to a host of errors. You should develop a sense for this pain and know that it means there's room for relief. Take a look at the URL module that can safely build/manipulate URLs in pretty much every way imaginable.
Ah, the joys of learning Javascript's asynchronous programming model!
This line
const connection = authenticate( 'DATATOBEPASSED' , 'https://URLHERE');
returns to its caller before either event handler -- res.on("data", ...) and res.on("data", ...) get called with the results of your get operation.
You need to use a callback from your authenticate() function to deliver the results to its caller.
function authenticate( uuid , cdcloc, callback ) {
let url = cdcloc + "/api.php?uuid=" + uuid + '&auth';
https.get(url,(res) => {
let body = "";
res.on("data", (chunk) => {
body += chunk;
});
res.on("end", () => {
try {
let cdcResponse = JSON.parse(body);
// do something with JSON
callback(cdcResponse[0]);
} catch (error) {
console.error(error.message);
};
});
}).on("error", (error) => {
console.error(error.message);
});
}
authenticate( 'DATATOBEPASSED' , 'https://URLHERE',
function (connection) {
console.log(connection.SerialNumber);
}
);
There are language features known as Promises and async / await to help escape the confusing mess of nested callbacks we get into when we write significant code.
I have a problem with understanding Promises syntax.
So, what I am trying to do:
getPosts() gets some data from a DB then, I want to get some metadata for each row with another promise call, addMetadata(). Then, once all the metadata is fetched, I want to console it out.
See my attempt below:
var getPosts = function(){
return new Promise(function(resolve, reject){
postModel.find()
.exec(function(err, posts) {
resolve(posts);
);
});
};
var addMetadata = function(posts){
var options = {
host: 'localhost',
port: 3000,
path: '',
method: 'GET'
};
var postPromises = posts.map(function(post) {
return new Promise(function(resolve) {
options.path = '/api/user?id=' + post.profileID;
var req = http.get(options, function(res) {
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
var parsedBody = JSON.parse(body);
post.fullname = parsedBody.user.fullname;
post.profilePic = parsedBody.user.profilePic;
// resolve the promise with the updated post
resolve(post);
});
});
});
});
// Is this the right place to put Promise.all???
Promise.all(postPromises)
.then(function(posts) {
//What should I put here
});
};
getPosts()
.then(function(posts){
return addMetadata(posts);
})
.then(function(posts){//I get a little lost here
console.log();//posts is undefined
});
Of course, my understanding is wrong but I thought I was going the right way. Can someone please guide me to the right direction?
Thanks
Change
// Is this the right place to put Promise.all???
Promise.all(postPromises)
.then(function (posts) {
//What should I put here
});
into
// Is this the right place to put Promise.all???
return Promise.all(postPromises);
This way your addMetadata function will return Promise that resolve when all promises from postPromises resolves or reject if any of postPromises rejects.
The key point to understand the async concept of it and what time the content is available.
Reading this will help to put you in the right direction.
For instance:
var promise = new Promise(function(resolve, reject) {
resolve(1);
});
promise
.then(function(val) {
console.log(val); // 1
return val + 2;
})
.then(function(val) {
console.log(val); // 3
})
After as per your scenario, in order to have all the metadata Promise.all is the way to go.
Promise.all(arrayOfPromises).then(function(arrayOfResults) {
// One result per each promise of AddMetadata
})
What you wanna do here, if I am correct, is called streams, as you wanna call multiple paralel promises as your concept of looping through list of posts using map is not going to work this way
Take a look at this short video introducing streams Streams - FunFunFunction, he is using library for workin with streams called Baconjs
Here is a short example on streams
const stupidNumberStream = {
each: (callback) => {
setTimeout( () => callback(1), 3000 )
setTimeout( () => callback(2), 2000 )
setTimeout( () => callback(3), 1000 )
}
}
stupidNumberStream.each(console.log)
Your getPosts function is good in the sense that its only job is to promisfy the database find. (Though, I think if it's mongo, the exec produces a promise for you).
Your addMetadataToAPost is less good, because it mixes up processing an array of posts and "promisifying" the http.get. Use the same pattern you applied correctly in the first function and return a promise to do a single get and add metadata. (It would be even better to just wrap the get, which you can reuse, and build a simple add-metadata function that returns - rather than creates - a promise)
// renamed pedantically
var addMetadataToAPost = function(post) {
return new Promise(function(resolve) {
options.path = '/api/user?id=' + post.profileID;
var req = http.get(options, function(res) {
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
var parsedBody = JSON.parse(body);
post.fullname = parsedBody.user.fullname;
post.profilePic = parsedBody.user.profilePic;
// resolve the promise with the updated post
resolve(post);
});
});
});
}
Now your batching function is simple:
// also renamed pedantically
var addMetadataToAllPosts = function(posts){
var postPromises = posts.map(function(post) {
return addMetadataToAPost(post)
})
return Promise.all(postPromises)
};
Your original code should work...
getPosts().then(function(posts){
return addMetadataToAllPosts(posts);
})
.then(function(posts){
console.log(posts);//posts should be an array of posts with metadata added
});
UPDATE
OK so I worked it out. When using highland.js I needed a .done() to finish the stream.
var requests = [];
_(fs.createReadStream("small.txt", { encoding: 'utf8' }))
.splitBy('-----BEGIN-----\n')
.splitBy('\n-----END-----\n')
.filter(chunk => chunk !== '')
.each(function (x) {
requests.push(function (next) {
Helpers.Authenticate()
.then(function (response1) {
return Helpers.Retrieve();
})
.then(function (response2) {
return Helpers.Retrieve();
})
.then(function () {
next();
});
});
})}).done(function () {
async.waterfall(requests);
});
The request array is now working.
I'm having some issues using electron and chained promises. Heres my code that being run in the main process.
var request = require('request-promise');
request.post(tppAuthenticate)
.then(function (responseFromFirstPost) {
var newoptions = tppRetrieveCertificate(responseFromFirstPost.APIKey)
return request.post(newoptions) // The return is important
})
.then(function (responseFromSecondPost) {
console.log(responseFromSecondPost)
})
The entire code block executes several thousand times as its called by iterating over each line of a file. The first request gets fired continuously, but this seems to significantly block/throttle back the second request which is only getting called periodically.
I was hoping that the entire block would get called in sequence but this does not seem to be happening.
Heres my complete code block including the iteration:
const _ = require('highland');
const request = require('request-promise');
fs.createReadStream(files[0], { encoding: 'utf8' }))
.splitBy('-----BEGIN -----\n')
.splitBy('\n-----END -----\n')
.filter(chunk => chunk !== '')
// .each(_.log);
.each(function (x) {
request.post(tppHelpers.Authenticate)
.then(function (responseFromFirstPost) {
const newoptions = tppHelpers.tppRetrieveCertificate(responseFromFirstPost.APIKey)
console.log(newoptions)
return request.post(newoptions) // The return is important
})
.then(function (responseFromSecondPost) {
console.log(responseFromSecondPost)
event.sender.send('selected-directory', responseFromSecondPost)
})
});
If you don't want to fire every request at once, which seems to be the case reading the comments you made, then instead of running everything in parallel with:
.each(function (x) {
// ...
});
You can instead create an empty array before you run fs.createReadStream:
var requests = [];
and in your each callback create functions to add to your array:
.each(function (x) {
requests.push(function (next) {
// ...
next();
});
});
and then you can run it in series with:
async.series(requests);
using the async module.
Just make sure that the next() is called at the right moment, e.g. in the last .then() callback of the given chain of promises.
Another way would be to use async.queue:
var queue = async.queue(function(x, callback) {
//
callback();
}, 1);
(Here make sure that the callback() is called when it should. Instead of 1 at the end you could use some other number to have a certain number of requests done in parallel.)
And then in your each callback:
.each(function (x) {
queue.push(x);
});
See the async.queue docs for more info. (Thanks to robertklep for mentioning async.queue() in the comments.)
By the way: do you even use the x in your iterations or are you just making a bunch of identical request for each line of your input?
Example
To answer your question from the comments, here is a way to construct the array of functions.
If this was your original code:
yourStream.each(function (x) {
doRequest1()
.then(function (response1) {
return doRequest2();
})
.then(function (response2) {
return doRequest3();
});
});
Then you could consruct that array of functions with something like:
var requests = [];
yourStream.each(function (x) {
requests.push(function (next) {
doRequest1()
.then(function (response1) {
return doRequest2();
})
.then(function (response2) {
return doRequest3();
})
.then(function () {
next();
});
});
});
And you could run them with:
async.series(requests);
Hope it helps.
Follow-up to Swap order of arguments to "then" with Bluebird / NodeJS Promises (the posted answer worked, but immediately revealed a new issue)
This is the first time I've ever used promises in NodeJS so I apologize if some conventions are poorly adhered to or the code is sloppy. I'm trying to aggregate data from multiple APIs, put it in a database, then compute some statistics based on similarities and differences in the data. As a starting point I'm trying to get an API token for a single one of the APIs.
Here is my full code:
var Promise = require('bluebird');
var fs = require('fs');
var request = require('request');
Promise.promisifyAll(fs);
Promise.promisifyAll(request);
// tilde-expansion doesn't follow the callback(err, data) convention
var tilde = function(str) {
var _tilde = require('tilde-expansion');
return new Promise(function(resolve, reject) {
try {
_tilde(str, resolve);
} catch(e) {
reject(e);
}
});
}
var getToken = function() {
return request.getAsync(process.env.token_url, {
headers: {
"Content-Type": "applications/x-www-form-urlencoded"
},
form: {
client_id: process.env.client_id,
client_secret: process.env.client_secret,
grant_type: "client_credentials"
}
})
.then(function(resp) { return resp.body; });
}
var tokenFile = tilde(process.env.token_file)
.catch(function(err) {
console.log("Error parsing path to file... can not recover");
});
var token = tokenFile
.then(fs.readFileAsync) //, "utf8")
.then(function(data) {
console.log("Token (from file): " + data);
return data;
})
.then(JSON.parse)
.catch(function(err) {
console.log("Error reading token from file... getting a new one");
return getToken()
.then(function(data) {
console.log("Token (from API): " + data);
return data;
})
.then(JSON.stringify)
.then(fs.writeFileAsync.bind(null, tokenFile.value()));
});
token.then(function(data) {
console.log("Token (from anywhere): " + token.value);
});
This code is currently logging:
Token: undefined
if I fall back to the API. Assuming I did my promise stuff correctly (.catch() can return a promise, right?) then I would assume the issue is occurring because fs.writeFileAsync returns void.
I would like to append a .return() on the end of this promise, but how would I gain access to the return value of getToken()? I tried the following:
.catch(function(err) {
console.log("Error reading token from file... getting a new one");
var token = "nope";
return getToken()
.then(function(data) {
console.log("Token (from API): " + data);
token = data;
return data;
})
.then(JSON.stringify)
.then(fs.writeFileAsync.bind(null, tokenFile.value()))
.return(token);
});
However this logs "nope".
Over the weekend I continued my research on promises and upon making a pivotal realization I was able to develop the solution to this. Posting here both the realization and the solution:
The Realization
Promises were invented so that asynchronous code could be used in a synchronous manner. Consider the following:
var data = processData(JSON.parse(readFile(getFileName())));
This is the equivalent of:
var filename = getFileName();
var fileData = readFile(filename);
var parsedData = JSON.parse(fileData);
var data = processData(parsedData);
If any one of these functions is asynchronous then it breaks, because the value isn't ready on time. So for those asynchronous bits we used to use callbacks:
var filename = getFileName();
var data = null;
readFile(filename, function(fileData){
data = processData(JSON.parse(fileData));
});
This is not only ugly, but breaks a lot of things like stack traces, try/catch blocks, etc.
The Promise pattern fixed this, letting you say:
var filename = getFileName();
var fileData = filename.then(readFile);
var parsedData = fileData.then(JSON.parse);
var data = parsedData.then(processData);
This code works regardless of whether these functions are synchronous or asynchronous, and there are zero callbacks. It's actually all synchronous code, but instead of passing values around, we pass promises around.
The led me to the realization that: for every bit of code that can be written with promises, there is a synchronous corollary
The solution
Realizing this, I tried to consider my code if all of the functions were synchronous:
try {
var tokenFile = tilde(process.env.token_file)
} catch(err) {
throw new Error("Error parsing path to file... can not recover");
}
var token = null;
try {
token = JSON.parse(readFile(tokenFile));
} catch(err) {
token = getToken();
writeFile(tokenFile, JSON.stringify(token));
}
console.log("Token: " + token.value);
After framing it like this, the promise version follows logically:
var tokenFile = tilde(process.env.token_file)
.catch(function(err) {
throw new Error("Error parsing path to file... can not recover");
});
var token = tokenFile
.then(readFile)
.then(JSON.parse)
.catch(function(err) {
var _token = getToken();
_token
.then(JSON.stringify)
.then(writeFile.bind(null, tokenFile.value));
return _token;
});
So I'm trying to preform a https GET with node.jsand I have the following code
function get(url) {
https.request(url, function(res) {
var data = "";
res.on('data', function (chunk) {
data += chunk;
})
.on('end', function(){
console.log(JSON.parse(data));
});
}).on('error', function(e) {
console.log(e.message);
}).end();
}
This code works fine and dandy except I need this function to return the data its logging
I know the recommended way to do this is to use callbacks, passing a callback function into get and then calling that function in the 'end' listener. But the problem is that this process needs to be synchronized and NOT pipelined because it causes data hazards and uses too much memory. On top of that, its is recursively called and is just one big headache to try and manage.
Basically, I'm trying to return JSON.parse(data) in the get function then the end listener is called, is that possible?
You can't synchronously return data using an asynchronous function to retrieve the data. Your get() function will return long before the https.request() has completed so you just can't do what you asked to do.
The usual design pattern for solving this involves passing in a callback function to your get() function that will be called when the data is available. This will involve restructing the caller of your function to handle an asynchronous response via a callback function.
There are some different choices in how you structure the callback, but here's the general idea:
function get(url, callback) {
https.request(url, function(res) {
var data = "";
res.on('data', function (chunk) {
data += chunk;
})
.on('end', function(){
callback("success", JSON.parse(data));
});
}).on('error', function(e) {
callback("error", e);
}).end();
}
Usage:
get("http://www.example.com/myurl", function(status, data) {
if (status === "success") {
console.log(data);
}
});
May I recommend Q. It is specifically designed to help you fight the famous pyramid of callbacks in JavaScript. I understand that callbacks can lead to less-readable code but you should not try to make synchronous get requests. It kind of defeats the advantages of node.js.
You can convert
step1(function (value1) {
step2(value1, function(value2) {
step3(value2, function(value3) {
step4(value3, function(value4) {
// Do something with value4
});
});
});
});
to this -->
Q.fcall(promisedStep1)
.then(promisedStep2)
.then(promisedStep3)
.then(promisedStep4)
.then(function (value4) {
// Do something with value4
})
.catch(function (error) {
// Handle any error from all above steps
})
.done();