Develop Reference

AWS S3 / Javascript callback issue

So, I'm having a problem with JavaScript asynchronous execution when making an API call to AWS S3.
I have a sequence of nested callbacks that are working fine up until a specific S3 call that my code is not waiting for. Here's my code:
getThumbUrls(contentIndex, function(data) {
console.log('Returning from getThumbUrls');
// let's just display thumbUrls[0] for now...
console.log('The thumbUrls are ' + data[0]);
});
getThumbUrls() looks like this:
function getThumbUrls(contentIndex, callback) {
console.log('Entering getThumbUrls');
var thumbUrls = [];
JSON.parse(contentIndex).forEach(videoKey => {
// get the thumbnail: bucket-name/thumbnails/<first-key>
console.log('videoKey = ' + videoKey);
getThumbFileName(videoKey, function(thumbFileName) {
console.log('Returning from getThumbFileName');
console.log('Returned thumb filename is ' + thumbFileName);
thumbUrls.push(CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName);
});
});
callback(thumbUrls);
}
And getThumbFileName() looks like this:
function getThumbFileName(videoKey, callback) {
console.log('Entering getThumbFileName...');
const s3 = new AWS.S3({
apiVersion: '2006-03-01',
params: {
Bucket: 'my-bucket-name'
}
});
// Get the name of the file.
params = {
Bucket: 'my-bucket-name',
Delimiter: '/',
Prefix: videoKey + '/' + THUMBS_FOLDER,
MaxKeys: 1
};
var urlKey;
//console.log('listObjects params = ' + JSON.stringify(params, null, 4));
s3.listObjectsV2(params, (err, data) => {
if (err) {
console.log(err, err.stack);
callback(err);
return;
}
var thumbsKey = data.Contents;
// MaxKeys was 1 bc first thumbnail key is good enough for now. Therefore, only one iteration.
thumbsKey.forEach(function (keys) {
console.log('thumbKey = ' + keys.Key);
urlKey = keys.Key;
});
});
callback(urlKey);
//callback('20161111-TheWind.jpg');
}
Obviously, what's happening is that execution doesn't wait for the s3.listObjectsV2 call to finish. I've verified that the entire flow works properly when all getThumbFileName() does is callback with the filename.
Would someone kindly show me how to force execution to wait for s3.listObjectsV2 to complete before calling back with undefined?

As discussed, you should avoid callbacks approach when dealing with asynchronous operations over iterations, due their difficulty.
(You can skip this section if you don't want to know motivation behind promises approach).
Just to mention, in a callback approach, you must have to wait for all callbacks to complete in your getThumbUrls(), using a if which will check if all callbacks has been called, then just call callback(thumbUrls); with all responses pushed into your thumbUrls array:
function getThumbUrls(contentIndex, callback) {
const thumbUrls = [];
// counter which will increment by one for every callback
let counter = 0;
JSON.parse(contentIndex).forEach(videoKey => {
getThumbFileName(videoKey, function (thumbFileName) {
thumbUrls.push(CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName);
// for each callback response you must add 1 to a counter and then
counter++;
// check if all callbacks already has been called
if (counter === JSON.parse(contentIndex).length) {
// right here, thumbsUrls are filled with all responses
callback(thumbUrls);
}
});
});
}
So, you can make use of Promises, and a Promise.all will be enough for you to handle all responses from api. You can study over internet and check your code below, which is using a promise approach. I've added some comments to help you understanding what is happening.
// when using promises, no callbacks is needed
getThumbUrls(contentIndex)
.then(function (data) {
console.log('Returning from getThumbUrls');
// let's just display thumbUrls[0] for now...
console.log('The thumbUrls are ' + data[0]);
})
// when using promises, no callbacks is needed
function getThumbUrls(contentIndex) {
console.log('Entering getThumbUrls');
// not needed anymore, Promise.all will return all values
// var thumbUrls = [];
// Promise.all receives an array of promises and returns to next .then() all results
// changing forEach to map to return promises to my Promise.all
return Promise.all(JSON.parse(contentIndex).map(videoKey => {
console.log('videoKey = ' + videoKey);
// returning a promise
return getThumbFileName(videoKey)
.then(function (thumbFileName) {
console.log('Returning from getThumbFileName');
console.log('Returned thumb filename is ' + thumbFileName);
return CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName;
});
}))
}
// when using promises, no callbacks is needed
function getThumbFileName(videoKey) {
console.log('Entering getThumbFileName...');
const s3 = new AWS.S3({
apiVersion: '2006-03-01',
params: {
Bucket: 'my-bucket-name'
}
});
// Get the name of the file.
params = {
Bucket: 'my-bucket-name',
Delimiter: '/',
Prefix: videoKey + '/' + THUMBS_FOLDER,
MaxKeys: 1
};
// urlKey not need anymore
// var urlKey;
// most of AWS functions has a .promise() method which returns a promise instead calling callback funcions
return s3.listObjectsV2(params).promise()
.then(function (data) {
var thumbsKey = data.Contents;
//if you want to return only first one thumbsKey:
return thumbsKey[0];
})
.catch(function (err) {
console.log(err, err.stack);
callback(err);
return;
})
}
Hope this helps you out in your study.

Would someone kindly show me how to force execution to wait
That's the wrong question. You are not trying to get execution to "wait," or, at least, you shouldn't be. You just need to call the callback in the right place -- inside the callback from s3.listObjectsV2(), not outside.
function getThumbFileName(videoKey, callback) {
...
s3.listObjectsV2(params, (err, data) => {
if (err) {
...
}
var thumbsKey = data.Contents;
// MaxKeys was 1 bc first thumbnail key is good enough for now. Therefore, only one iteration.
thumbsKey.forEach(function (keys) {
console.log('thumbKey = ' + keys.Key);
urlKey = keys.Key;
});
callback(urlKey); // right
});
// wrong // callback(urlKey);
}
The way you wrote it, the callback fires after s3.getObjectsV2() begins to run -- not after it finishes (calls its own callback).

JavaScript Promises - Creating an array of promises to be executed together

I'm working with Node.js on async calls to noSQL DynamoDB. I first query to see which 'buddy list(s)' the account belongs. That may return from zero to 'n' new Primary Keys that will contain lists of all of the members of each of those buddy lists. Think of them as clubs to which a person belongs... you may have none or many; each club has several members or even one.
So far (and I am working with Promises for the first time here... though I have used callbacks on prior JA projects) I'm OK with where I am, but I know that I am assembling the array of promises incorrectly. That is, I can see in the console that the .then function executes before both promises resolve. For the record, I do expect that... it seems reasonable that any .then may be satisfied with a single promise resolving.
Anyways, Can someone offer up some tips as to how to structure? I'd like to end up with pseudo:
getBuddyLists
.then(getAllTheBuddiesInTheLists)
.then(function(){//doSomething});
my Node.js code:
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:'us-east-1'});
var buddyGroups = [];
exports.handler = function(event, context, callback) {
var params = {
TableName: 'USER_INFORMATION',
Key: {
"DEVICE_ID": event.DEVICE_ID
}
};
var getBuddyList = new Promise(function(resolve, reject){
docClient.get(params, function(err, data){
if(err){
reject(err);
}
else{
var dataJSON = JSON.parse(JSON.stringify(data));
dataJSON.Item.my_buddies.values.forEach(function(value){
buddyGroups.push(value);
});
console.log('Got buddy groups: ' + buddyGroups);
resolve(buddyGroups);
}
});
});
getBuddyList.then(function(capturedData){ // capturedData => an array of primary keys in another noSQL document
console.log('groups: ' + capturedData);
var myPromises = [];
capturedData.forEach(function(value){
var reqParams = {
TableName: "buddy_list",
Key: {"BUDDY_ID": value}
};
myPromises.push(new Promise (function(resolve, reject){
docClient.get(reqParams, function(err, data){
if(err){
//console.log(err, data);
reject(err);
}else{
var returnedJSON = JSON.parse(JSON.stringify(data));
console.log(returnedJSON);
resolve(returnedJSON);
}
});
}));
});
//
Promise.all(myPromises); // ADDED IN EDIT <<<<<<<<<<<<<<<<<<<<<<
// how to make sure all of myPromises are resolved?
//
}).then(function(){
console.log("done");
})
.catch(function(err){
console.log("error message:" + error);
});
};
EDIT: Added location of Promise.all(myPromises);

Just to clarify how you would use Promise.all, even though you accepted an answer, you edited the question, but still are not using it correctly
Also, the discussion of .map vs .forEach - this is how you would use .map
getBuddyList.then(function(capturedData){ // capturedData => an array of primary keys in another noSQL document
console.log('groups: ' + capturedData);
// changes to use .map rather than .forEach
// also, you need to RETURN a promise - your edited code did not
return Promise.all(capturedData.map(function(value){
return new Promise (function(resolve, reject){
var reqParams = {
TableName: "buddy_list",
Key: {"BUDDY_ID": value}
};
docClient.get(reqParams, function(err, data){
if(err){
//console.log(err, data);
reject(err);
}else{
var returnedJSON = JSON.parse(JSON.stringify(data));
console.log(returnedJSON);
resolve(returnedJSON);
}
});
});
}));
}).then(function(){
console.log("done");
})
.catch(function(err){
console.log("error message:" + error);
});

You're looking for Promise.all(myPromises), which returns a single promise of an array of the results of the promises.

Calling done on an array of http.get requests in Node.js

I have an array of URLs that I'm using a for loop to call http.get requests. Since this is an async process, I'd like to call done after ALL requests have returned.
Here is my current attempt:
grunt.registerTask('verify', function() {
var done = this.async();
var promises = [];
var urlPrefix = 'http://example.com/';
for(var i = 0; i < deployableFiles.length; i++) {
(function(i) {
var deferred = Q.defer();
promises.push(deferred);
var file = deployableFiles[i];
var path = file.filetype + '/' + getVersionedFileName(file.basename, file.filetype);
http.get(urlPrefix + path, function(res) {
deferred.resolve();
if(res.statusCode === 200) {
grunt.log.oklns(path + ' was found on production server.');
} else {
grunt.log.error('Error! ' + path + ' was not found on production server!');
}
}).on('error', function(e) {
grunt.log.error("Got error: " + e.message);
done();
});
})(i);
}
Q.all(promises)
.done(function() {
// Everything executed correctly
return done();
}, function(reason) {
// There was an error somewhere
return done(false);
});
});
I'm sure it's just me not wrapping my head around the whole async nature of node correctly, but is there anything glaringly obvious to anyone else?
I've searched about using http with the Q library, and it appears it might be required to use Q.nfcall to get this to work. I'm just having trouble seeing WHY I'd have to do that. (I'm not adverse to actually doing that, I'm more curious than anything else)
Thanks!

If this is not a typo, promises.push(deferred) should be pushed the promise promises.push(deferred.promise).
function foo() {
...
return defer.promise;
}
// => foo().then(function() ...);
Q.all([
foo(),
foo(),
...
]).done(function() ...);
Q.all expects an array of promises. https://github.com/kriskowal/q#combination
Q.nfcall is just sugar around that if
working with functions that make use of the Node.js callback pattern, where callbacks are in the form of function(err, result)
https://github.com/kriskowal/q#adapting-node

You should always perform promisification at the lowest level possible. That makes reasoning about concurrency a lot easier.
function getPing(url){
return new Q.Promise(function(resolve,reject){
http.get(url,function(res){
// note this will _not_ wait for the whole request
// but just the headers.
if(res.statusCode === 200) resolve();
else reject();
});
});
}
This would let you do:
grunt.registerTask('verify', function() {
var done = this.async();
var urlPrefix = 'http://example.com/';
var pings = deployableFiles.map(function(file){
var path = file.filetype + '/' +
getVersionedFileName(file.basename, file.filetype);
return getPing(urlPrefix + path);
});
Q.all(pings).then(done).catch(function(reason) {
// There was an error somewhere
// this will happen as soon as _one_ promise rejected
return done(false);
});
});
This can be further shortened by using a better promise library like Bluebird.

You can also do this with async:
var urlPrefix = 'http://example.com/';
async.each(deployableFiles, function(file, cb) {
var path = file.filetype
+ '/'
+ getVersionedFileName(file.basename, file.filetype);
http.get(urlPrefix + path, function(res) {
if (res.statusCode === 200)
grunt.log.oklns(path + ' was found on production server.');
else
grunt.log.error('Error! ' + path + ' was not found on production server!');
cb();
}).on('error', function(e) {
grunt.log.error("Got error: " + e.message);
cb(e);
});
}, function(err) {
// all done
if (err) throw err;
// all successful
});

Using JS Promises to Execute Multiple Async Queries to Build an Object

After recently discovering JS promises, I have been studying them so that I might build a certain functionality that allows me to execute 4 async queries, use the result of each to build an object that I can finally send as a response to a request directed at my node app.
The final object is made up of 3 array properties containing the resulting rows of each query.
It seems that I've done something wrong handling the promises, though, because ultimately, game is not being built. It is sent as an empty object. Here's a JSFiddle.
What is my mistake?
Here's what I have so far:
function sendGame(req, res, sales, settings, categories) {
var game = new Object();
game.sales = sales;
game.settings = settings;
game.categories = categories;
JSONgame = JSON.stringify(game);
res.writeHead(200, {
'Access-Control-Allow-Origin': 'http://localhost',
'Content-Length': JSONgame.length,
'Content-Type': 'application/json'
});
res.write(JSONgame);
res.end();
console.log('Game: ' + JSON.stringify(game, null, 4));
console.log('--------------------------------------');
console.log('User ' + req.body.username + ' successfully retrieved game!');
}
function retrieveSales(req, connection, timeFrame) {
console.log('User ' + req.body.username + ' retrieving sales...');
connection.query('select * from sales_entries where date BETWEEN ? AND ?', timeFrame,
function (err, rows, fields) {
if (err) {
callback(new Error('Failed to connect'), null);
} else {
sales = [];
for (x = 0; x < rows.length; x++) {
sales.push(rows[x]);
}
//console.log('Sales: ' + JSON.stringify(sales, null, 4));
return sales;
}
});
}
retrieveCategories() and retrieveSettings() omitted for readability; they are the same as retrieveSales() mostly.
function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).format("YYYY-MM-DD HH:mm:ss")];
var connection = createConnection();
connection.connect(function (err) {
if (err) return callback(new Error('Failed to connect'), null);
console.log('Connection with the Officeball MySQL database openned for game retrieval...');
var sales = retrieveSales(req, connection, timeFrame);
var settings = retrieveSettings(req, connection);
var categories = retrieveCategories(req, connection);
var all = q.all([sales, settings, categories]);
all.done(function () {
sendGame(req, res, sales, settings, categories);
});
});
}

Your problem is that you're not using promises. All your APIs use callbacks.
A promise is like a closed box:
A promise also has a method that opens the box, works on the value and returns another box on the value (also opening any additional boxes along the way). That method is .then:
In boxes, it does:
=>( . => ) =>
That is, it adds a handler that gets an open box and returns a box. Everything else just combines stuff. All .all does is wait for a list of promises to resolve, it is exactly like .then in the fact it waits for a result. Because promises are boxes, you can pass them around and return them which is very cool.
Generally:
Whenever you return from a promise handler (not a rejection), you are fullfilling it indicating normal flow continuation.
Whenever you throw at a promise handler, you are rejecting indication exceptional flow.
So basically in node speak:
Whenever you returned a null error and a response, you resolve the promise.
Whenever you returned an error and no response, you reject the promise.
So:
function myFunc(callback){
nodeBack(function(err,data){
if(err!== null){
callback(new Error(err),null);
}
callback(data+"some processing");
})
});
Becomes:
function myFunc(){
return nodeBack().then(function(data){ return data+"some processing"; });
}
Which I think is a lot clearer. Errors are propagated across the promise chain just like in synchronous code - it's very common to find synchronous analogs to promise code.
Q.all takes a list of promises and waits for them to complete, instead you want Q.nfcall to transform a callback based API to a promise one and then use Q.all on that.
That is:
var sales = Q.nfcall(retrieveSales,req, connection, timeFrame);
var settings = Q.nfcall(retrieveSettings,req, connection);
var categories = Q.nfcall(retrieveCategories, req, connection);
Q.nfcall takes a nodeback in the err,data convention and converts it to a promise API.
Also, when you do
return sales;
You are not really returning anything, since it returns synchronously. You need to use callback like in your error case or promisify it altogether. If you don't mind, I'll do it with Bluebird since it comes with much better facilities for dealing with these interop cases and does so much much faster, if you'd like you can switch promisifyAll for a bunch of Q.nfcall calls.
// somewhere, on top of file
connection = Promise.promisifyAll(connection);
// note I'm passing just the username - passing the request breaks separation of concerns.
var retrieveSales = Promise.method(username, connection, timeFrame) {
console.log('User ' + username + ' retrieving sales...');
var q = 'select * from sales_entries where date BETWEEN ? AND ?';
return connection.queryAsync(q, timeFrame).then(function(rows, fields){
return rows;
});
}
Note that suddenly you don't need a lot of boilerplate for making a query, you can use queryAsync directly instead if you'd like.
Now the code that wraps it becomes:
var gameSucceed = Promise.method(function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment()....];
var connection = Promise.promisifyAll(createConnection());
return conn.connectAsync().then(function () {
console.log('Connection with the ...');
//sending req, but should really be what they use.
return Promise.all([retrieveSales(req,conn,timeFrame),
retrieveSettings(req,conn),
retrieveCategories(req,conn)]);
});
});
Now you can call sendGame(req, res, sales, settings, categories); outside of gameSucceed which doesn't hide what it does as much -
gameSucceed(req,res).spread(function(sales,settings,cats){
return sendGame(req,res,sales,settings,cats);
});

Having trouble with promises in nodejs

I'm trying to use promises with nodejs (I'm trying with node-promise package); however, without any success. See the code below:
var express = require('express'),
request = require('request'),
promise = require('node-promise');
app.get('/promise', function(req, res) {
var length = -1;
new promise.Promise(request(
{uri: "http://www.bing.com"},
function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log("An error occurred when connected to the web site");
return;
}
console.log("I'll return: " + body.length);
length = body.length;
}
)).then(function(result) {
console.log("This is what I got: " + length);
console.log("Done!");
});
res.end();
});
The output of the above code is I'll return: 35857 only and it doesn't go to the then part.
I change the code then to be:
app.get('/promise', function(req, res) {
var length = -1;
promise.when(
request(
{uri: "http://www.bing.com"},
function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log("An error occurred when connected to the web site");
return;
}
console.log("I'll return: " + body.length);
length = body.length;
}
),
function(result) {
console.log("This is what I got: " + length);
console.log("Done!");
},
function(error) {
console.log(error);
}
);
res.end();
});
This time the output is This is what I got: -1 then Done!... looks like the "promise" was not called this time.
So:
What's needed to be done to fix the code above? Obviously I'm not doing it right :)
Is node-promise "the way to go" when I'm doing promises, or is there a better way/package? i.e. simpler and more production-ready.
Thanks.

Try jquery-deferred-for-node.
I'm not an expert but understand that this lib tends to be favoured by programmers who work both server-side and client-side.
Even if you don't already know jQuery's Deferreds, the advantages of going this route are that :
the documentation is excellent (it comprises links to the jQuery docs), though you may struggle to find examples specific to Node.
methods are chainable.
jQuery Callbacks are also included.
when one day you need to do asynchronous stuff client-side, then there's virtually nothing to relearn - the concepts are identical and the syntax very nearly so. See the "Correspondances" section in the github page hyperlinked above.
EDIT
I'm not a node.js person so I'm guessing here but based on your code above, you might want to consider something along the following lines with jquery-deferred-for-node :
var express = require('express'),
request = require('request'),
Deferred = require('JQDeferred');
function fetch(uri, goodCodes) {
goodCodes = (!goodCodes) ? [200] : goodCodes;
var dfrd = Deferred(); // A Deferred to be resolved/rejected in response to the `request()`.
request(uri, function(error, response, body) {
if (!error) {
var isGood = false;
// Loop to test response.statusCode against `goodCodes`.
for (var i = 0; i < goodCodes.length; i++) {
if (response.statusCode == goodCodes[i]) {
isGood = true;
break;
}
}
if (isGood) {
dfrd.resolve(response.statusCode, body);
} else {
dfrd.reject(response.statusCode, "An invalid response was received from " + uri);
}
} else {
dfrd.reject(response.statusCode, "An error occurred attempting to connect to " + uri);
}
});
// Make promise derived from dfrd available to "consumer".
return dfrd.promise();
};
//...
app.get('/promise', function(req, resp) {
fetch("http://www.bing.com").done(function(statusCode, result) {
console.log("Done! This is what I got: " + result.length);
}).fail(function(statusCode, message) {
console.log("Error (" + statusCode + "): " + message);
});
resp.end();
};
Here, I have tried to write a generalized utility for fetching a resource in such a way that the asynchronous response (or error) can be handled externally. I think this is broadly along the lines of what you were trying to achieve.
Out of interest, where do console.log() messages end up with node.js?
EDIT 2
Above, I have given Deferred an initial capital, as is conventional for Constructors
With jQuery Deferreds, there must be any number of ways to fetch() consecutively. The approach below leaves fetch() as it was, and introduces fetch_() to act as its front-end. There may be simpler ways but this allows fetch() to remain a general utility, functionally equivalent to the client-side jQuery.ajax().
function fetch_(uri){
return function(){
return fetch(uri, [200]).then(function(statusCode, result){
console.log("Done! This is what I got: " + result.length);
},function(statusCode, message){
console.log("Error (" + statusCode + "): " + message);
});
};
}
Note that function fetch() returns a function. It has to be like this because where fetch() is called, we want an unexecuted function, not (yet) the result of that function.
Now let's assume an array of uris is available. This can be hard-coded or built dynamically - whatever the application demands.
var uris = [
'http://xxx.example.com',
'http://yyy.example.com',
'http://zzz.example.com'
];
And now, a variety of ways in which fetch_() might be called :
//v1. To call `resp.end()` when the fetching process starts.
app.get('/promise', function(req, resp) {
fetch_(uris[0])().then(fetch_(uris[1])).then(fetch_(uris[2]));
resp.end();
});
//v2. To call `resp.end()` when the fetching process has finished.
app.get('/promise', function(req, resp){
fetch_(uris[0])().then(fetch_(uris[1])).then(fetch_(uris[2])).always(resp.end);
});
//v3. As v2 but building a `.then()` chain of any (unknown) length.
app.get('/promise', function(req, resp){
var dfrd = Deferred().resolve();//
$.each(uris, function(i, uri){
dfrd = dfrd.then(fetch_(uri));
});
dfrd = dfrd.always(resp.end);
});
untested
I have more confidence in v1 and v2. v3 may work.
v2 and v3 should both give exactly the same behaviour but v3 is generalized for any number of uris.
Everything may need debugging.

I would recommend using Q: https://github.com/kriskowal/q. I believe that it's used internally by other frameworks (like jQuery deferred implementation).
I believe that the documentation is "fine"; the syntax is consistent with other promise implementations... and it has a node adapter.
So your deferred style approach:
var deferred = Q.defer();
FS.readFile("foo.txt", "utf-8", function (err, res) {
if (!err) {
deferred.resolve(res);
} else {
deferred.reject(err);
}
});
return deferred.promise;
Can be written more concisely as:
var deferred = Q.defer();
FS.readFile("foo.txt", "utf-8", deferred.makeNodeResolver());
return deferred.promise;