Develop Reference

AWS S3 / Javascript callback issue

So, I'm having a problem with JavaScript asynchronous execution when making an API call to AWS S3.
I have a sequence of nested callbacks that are working fine up until a specific S3 call that my code is not waiting for. Here's my code:
getThumbUrls(contentIndex, function(data) {
console.log('Returning from getThumbUrls');
// let's just display thumbUrls[0] for now...
console.log('The thumbUrls are ' + data[0]);
});
getThumbUrls() looks like this:
function getThumbUrls(contentIndex, callback) {
console.log('Entering getThumbUrls');
var thumbUrls = [];
JSON.parse(contentIndex).forEach(videoKey => {
// get the thumbnail: bucket-name/thumbnails/<first-key>
console.log('videoKey = ' + videoKey);
getThumbFileName(videoKey, function(thumbFileName) {
console.log('Returning from getThumbFileName');
console.log('Returned thumb filename is ' + thumbFileName);
thumbUrls.push(CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName);
});
});
callback(thumbUrls);
}
And getThumbFileName() looks like this:
function getThumbFileName(videoKey, callback) {
console.log('Entering getThumbFileName...');
const s3 = new AWS.S3({
apiVersion: '2006-03-01',
params: {
Bucket: 'my-bucket-name'
}
});
// Get the name of the file.
params = {
Bucket: 'my-bucket-name',
Delimiter: '/',
Prefix: videoKey + '/' + THUMBS_FOLDER,
MaxKeys: 1
};
var urlKey;
//console.log('listObjects params = ' + JSON.stringify(params, null, 4));
s3.listObjectsV2(params, (err, data) => {
if (err) {
console.log(err, err.stack);
callback(err);
return;
}
var thumbsKey = data.Contents;
// MaxKeys was 1 bc first thumbnail key is good enough for now. Therefore, only one iteration.
thumbsKey.forEach(function (keys) {
console.log('thumbKey = ' + keys.Key);
urlKey = keys.Key;
});
});
callback(urlKey);
//callback('20161111-TheWind.jpg');
}
Obviously, what's happening is that execution doesn't wait for the s3.listObjectsV2 call to finish. I've verified that the entire flow works properly when all getThumbFileName() does is callback with the filename.
Would someone kindly show me how to force execution to wait for s3.listObjectsV2 to complete before calling back with undefined?

As discussed, you should avoid callbacks approach when dealing with asynchronous operations over iterations, due their difficulty.
(You can skip this section if you don't want to know motivation behind promises approach).
Just to mention, in a callback approach, you must have to wait for all callbacks to complete in your getThumbUrls(), using a if which will check if all callbacks has been called, then just call callback(thumbUrls); with all responses pushed into your thumbUrls array:
function getThumbUrls(contentIndex, callback) {
const thumbUrls = [];
// counter which will increment by one for every callback
let counter = 0;
JSON.parse(contentIndex).forEach(videoKey => {
getThumbFileName(videoKey, function (thumbFileName) {
thumbUrls.push(CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName);
// for each callback response you must add 1 to a counter and then
counter++;
// check if all callbacks already has been called
if (counter === JSON.parse(contentIndex).length) {
// right here, thumbsUrls are filled with all responses
callback(thumbUrls);
}
});
});
}
So, you can make use of Promises, and a Promise.all will be enough for you to handle all responses from api. You can study over internet and check your code below, which is using a promise approach. I've added some comments to help you understanding what is happening.
// when using promises, no callbacks is needed
getThumbUrls(contentIndex)
.then(function (data) {
console.log('Returning from getThumbUrls');
// let's just display thumbUrls[0] for now...
console.log('The thumbUrls are ' + data[0]);
})
// when using promises, no callbacks is needed
function getThumbUrls(contentIndex) {
console.log('Entering getThumbUrls');
// not needed anymore, Promise.all will return all values
// var thumbUrls = [];
// Promise.all receives an array of promises and returns to next .then() all results
// changing forEach to map to return promises to my Promise.all
return Promise.all(JSON.parse(contentIndex).map(videoKey => {
console.log('videoKey = ' + videoKey);
// returning a promise
return getThumbFileName(videoKey)
.then(function (thumbFileName) {
console.log('Returning from getThumbFileName');
console.log('Returned thumb filename is ' + thumbFileName);
return CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName;
});
}))
}
// when using promises, no callbacks is needed
function getThumbFileName(videoKey) {
console.log('Entering getThumbFileName...');
const s3 = new AWS.S3({
apiVersion: '2006-03-01',
params: {
Bucket: 'my-bucket-name'
}
});
// Get the name of the file.
params = {
Bucket: 'my-bucket-name',
Delimiter: '/',
Prefix: videoKey + '/' + THUMBS_FOLDER,
MaxKeys: 1
};
// urlKey not need anymore
// var urlKey;
// most of AWS functions has a .promise() method which returns a promise instead calling callback funcions
return s3.listObjectsV2(params).promise()
.then(function (data) {
var thumbsKey = data.Contents;
//if you want to return only first one thumbsKey:
return thumbsKey[0];
})
.catch(function (err) {
console.log(err, err.stack);
callback(err);
return;
})
}
Hope this helps you out in your study.

Would someone kindly show me how to force execution to wait
That's the wrong question. You are not trying to get execution to "wait," or, at least, you shouldn't be. You just need to call the callback in the right place -- inside the callback from s3.listObjectsV2(), not outside.
function getThumbFileName(videoKey, callback) {
...
s3.listObjectsV2(params, (err, data) => {
if (err) {
...
}
var thumbsKey = data.Contents;
// MaxKeys was 1 bc first thumbnail key is good enough for now. Therefore, only one iteration.
thumbsKey.forEach(function (keys) {
console.log('thumbKey = ' + keys.Key);
urlKey = keys.Key;
});
callback(urlKey); // right
});
// wrong // callback(urlKey);
}
The way you wrote it, the callback fires after s3.getObjectsV2() begins to run -- not after it finishes (calls its own callback).

NodeJS Change variable from within a function [duplicate]

This question already has answers here:
Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference
(7 answers)
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
Long story short, im newish to NodeJS and im trying to change a variable after a function has been executed, however I cant return a result like I thought I could.
app.get('/:steamid', function(req, res) {
var steamID;
var steamAvatarUrl;
var steamRegisterDate;
steamAPI.ResolveVanityURL(req.params.steamid, function(err, res) {
console.log('Steam64: ' + res); // <- this prints out my result
steamID = res;
});
steamAPI.getAvatarUrl(req.params.steamid, function(err, res) {
console.log('Avatar URL: ' + res); // <- this prints out my result
steamAvatarUrl = res;
});
steamAPI.memberSince(req.params.steamid, function(err, res) {
console.log('Registered: ' + res); // <- this prints out my result
steamRegisterDate = res;
});
console.log(steamID); //<--- This returns undefined
res.render('steamid.hbs', {
"title": steamID,
"avatarURL": steamAvatarUrl,
"registerDate": steamRegisterDate
});
});
An help with an example/resolution of how I accomplish this would be amazing. I am a visual learner so please dont say "use this or do that" without giving some sort of example to it or I honestly wont get it lol.
Thanks ahead of time.

Javascript is Async in nature, so before it fetches the data you are trying to print it. You can write console.log() inside your callback function.
or please use promises.

You can solve this problem using Promise. Your api call is async. So it will not wait to complete the api call. it just executes console.log(). That's why you are getting undefined. I'm not sure about your node version. if promises are not available you could use bluebird library.
app.get('/:steamid', function(req, res) {
var steamID;
new Promise(function(resolve, reject){
steamAPI.ResolveVanityURL(req.params.steamid, function(err, res) {
resolve(res);
});
}).then(function(res){
steamID = res;
console.log('Steam64: ' + steamId);
}
});

Javascript is asynchronous by default. You can't return a value inside of a callback function neither can you assign to a variable outside as it is returned immediately.
app.get('/:steamid', function(req, res) {
steamAPI.ResolveVanityURL(req.params.steamid, function(err, res) {
console.log('Steam64: ' + res); // <- this prints out my result
var steamID = res;
console.log(steamID); // <-- defined
});
});
You could learn about Promises and how they work as they are preferred to using callbacks.

Javascript is asynchronous in nature. The function 'steamAPI.ResolveVanityURL()' is taking time to execute and therefore next statement 'console.log(steamID)' is getting executed before waiting for the result.
You can either use callbacks or below is an example using 'q' promise library to solve your problem -
var Q = require('q');
app.get('/:steamid', function(req, res) {
var steamID;
resolveUrl(req.params.steamid).then(function(steamID) {
console.log(steamID); //<--- This will return steamID
(...)
});
});
function resolveUrl(id) {
var deferred = Q.defer();
steamAPI.ResolveVanityURL(id, function(err, res) {
console.log('Steam64: ' + res); // <- this prints out my result
deferred.resolve(res);
});
return deferred.promise;
}
Also here is a link to get more information about promises in node.js -
http://www.dotnetcurry.com/nodejs/1242/promises-nodejs-using-q-goodbye-callbacks
Hope this solves your problem.

#Zenderx let breif this .
Node.js is Async programming , if you do any I/O event like database call ,http , timeout call .
It send then in event pool and execute in sequence .
See the comment what happens step by step.
What is happing in your code it this.
This is your api call
app.get('/:steamid', function(req, res) {
// excution comes inside
var steamID;
// then you call an I/O event and iniallized streamID but what happen here is this ,
//this is I/O event so went to event loop for excution
steamAPI.ResolveVanityURL(req.params.steamid, function(err, res) {
console.log('Steam64: ' + res); // <- this prints out my result
steamID = res;
});
// while the block is in event loop or excution this statement run because of Async nature
console.log(steamID); //<--- This returns undefined
(...)
});
if you want to print StreamID print it inside the block only .
steamAPI.ResolveVanityURL(req.params.steamid, function(err, res) {
console.log('Steam64: ' + res); // <- this prints out my result
steamID = res;
console.log(steamID);
});
Too study in depth of async , you can look to promise and async library in node.js .
The simplest thing to do for newbie is this .
app.get('/:steamid', function(req, res) {
var steamID;
var steamAvatarUrl;
var steamRegisterDate;
steamAPI.ResolveVanityURL(req.params.steamid, function(err, resu) {
console.log('Steam64: ' + resu); // <- this prints out my result
steamID = resu;
steamAPI.getAvatarUrl(req.params.steamid, function(err, resu) {
console.log('Avatar URL: ' + resu); // <- this prints out my result
steamAvatarUrl = resu;
steamAPI.registerDate(req.params.steamid, function(err, resu) {
console.log('Registered: ' + resu); // <- this prints out my result
steamRegisterDate = resu;
console.log(steamID); //<--- This returns undefined
res.render('steamid.hbs', {
"title": steamID,
"avatarURL": steamAvatarUrl,
"registerDate": steamRegisterDate
});
});
});
});
});
If you need it to be solve by promise and async module ping.

How to assign a variable in callback function in a callback function in javascript

So I have found this question which seems pretty similar but I do not understand the answer at all I tried to implement it but I do not recognize the patterns of the answer in my code. similar question
Now here is my problem, I have this piece of code :
var fs = require('fs');
var index = JSON.parse(fs.readFileSync('../data/7XXX7/index.json', 'utf8'));
window = {};
var indicators = require('./indicators');
var parser = new window.patient.Indicator('tes', 'test');
var i = 0;
function create_indicators() {
var result = [];
fs.readdirSync('../data/7XXX7/files/').forEach(file => {
fs.readFile('../data/7XXX7/files/' + file, 'utf8', function (err, data) {
if (err)
throw err;
let $ = {};
$.poids = parser.poids(data);
$.taille = parser.taille(data);
$.temperature = parser.temperature(data);
$.tension = parser.tension(data);
$.pouls = parser.pouls(data);
$.ps = parser.ps(data);
$.saturation = parser.saturation(data);
for (var j in index.files)
{
if (index.files[j].name === file)
{
$.id = index.files[j].name;
$.date = index.files[j].date;
$.name = index.files[j].IntituleSession;
break;
}
}
if ($.poids || $.taille || $.temperature || $.tension || $.pouls || $.ps || $.saturation)
{
result.push($);
console.log(result); // print the actual state of result
// console.log(i); prints 0 then 1 then ...
i++;
}
});
console.log(i); // prints 0
});
console.log(result); // prints []
return result;
}
let result = create_indicators();
console.log(result); // prints []
And it displays :
[]
Why does the callback function in readFile has it's own variables ? Cause it's asynchronous ? But when I use readFileSync it doesn't work too.
How to make result get all the values I put into it ? when I console log result after result.push($); it works so that's not my parser, i is also properly indented each time.

Your code doesn't wait for the files to get read and have the result pushed to result before moving on. Where you're doing asynchronous operations on items in an array, I would recommend using promises and using Promise.all() to wait for each file to get read and processed before you try using the result. You could do something like this:
function create_indicators() {
const result = fs.readdirSync('../data/7XXX7/files/').map(file =>
new Promise((resolve, reject) => {
fs.readFile('../data/7XXX7/files/' + file, 'utf8', (err, data) => {
if (err) reject(err);
// do whatever
if ($.poids || /* ... */ $.saturation) {
// ...
resolve($); // instead of `result.push($);`
} else {
resolve(); // can't reject for `Promise.all()` to work
}
})
}));
return Promise.all(result).then(items => items.filter(item => item));
}
create_indicators().then(indicators => {
// do something with your list of indicators
}).catch(err => {
// handle error
});
It creates a promise for each file in your directory that resolves when the file has been processed. It resolves with the item if there is one or nothing if your condition is not met, rejecting if there's an error (promise equivalent to throw). Since you only want the items that meet your condition, you can then do a filter on the result of Promise.all() to get rid of any undefined in the array (you could also get rid of the condition checking in the fs.readFile callback and do it instead in the filter if you'd like). This returns a promise that resolves with your filtered list.

Here's your problem:
fs.readFileSync('../data/7XXX7/files/' + file, 'utf8', function (err, data) {
The readFileSync doesn't take a callback as an argument. It returns the data or raises an exception. It is synchronous (as the "Sync" in the name suggests) and you're using it as if it was asynchronous.
See the docs:
https://nodejs.org/api/fs.html

readFileSync doesn't callback. It is synchronous.
use fs.readdir to get the list of files you want to read. See How do you get a list of the names of all files present in a directory in Node.js?
Need to understand how callback works.
readFileSync doesn't callback. It might be helpful to explain how callback works in asynchronous fs.readFile and fs.readdir
When you are doing asynchronous operations, because you don't know when it is going to be finished, you pass in a function (callback) in the parameter, and run it at the end of the operation.
fs.readFile('/etc/passwd', function (err, data) {
if (err) throw err;
console.log(data);
});
fs.readFile in the above code will run the function (err, data) when it finishes executing and pass in the data as the second parameter. If error occurs it will pass in the error as the first parameter.
You can also get a callback function defining what to do when the parsing is over. The callback will need to take error and result. (if you need the error)
Read:
http://fredkschott.com/post/2014/03/understanding-error-first-callbacks-in-node-js/
So your create_indicators function should take a callback function.
fs = require("fs")
function create_indicators(folderPath, callback) {
let result = [];
fs.readdir(folderPath, (err, files) => {
if (err)
callback(err, null); //pass the error to callback if there is any
else {
files.forEach((file, index, filesArray) => {
fs.readFile(file, (err, data) => {
if (err)
callback(err, null); //pass the error to callback if there is any
else {
//.....parse....
result.push(data);
// pass data to callback function when it is the last result
if (result.length == filesArray.length)
callback(null, result);
}
});
});
}
})
}
When you call it, pass in what you want to do with the result and error as a function.
create_indicators(".", function(err,result){
if (err)
console.error("Got error:", err);
else
console.log("Got result:", result);
//do what you want with the final result
})
Once you got the callback working, look into Promise which will make this procedure cleaner and easier. Read: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise

Returned object undefined from module function

This has to be a scope issue that I'm not familiar with. I have a small module I've written as so:
(function () {
var getPlanInfo = function (id, conn) {
conn.query('SELECT * FROM `items` WHERE `id` = ?', [id], function (error, result) {
if (error) console.error('Query error: ' + error.stack);
console.log(result[0]); // Everything is great
return result[0];
});
};
modules.exports.getPlanInfo = function (id, conn) { return getPlanInfo(id, conn); // Typo }
})();
Here comes the problem. When I call it from anywhere (inside the module itself or another file), the return value is always undefined. I checked from within the function, the query returns the result as expected.
var backend = require('./module.js');
var t = backend.getPlanInfo();
t is undefined. This is the same if I call that method from inside the module itself (another function within that module).
I'm familiar with the callback principle in javascript and how objects and functions have to be passed around as an argument to remain in scope. Is this the issue here or is this a node.js particularity?
I tried in in the Developer Console (Chrome), works as expected.

conn.query() looks like it is async. Thus, you can't return its result from getPlanInfo() because getPlanInfo() returns long before the result is available. Returning result[0] from the conn.query() callback just returns a piece of data back into the conn.query() infrastructure. getPlanInfo() has long before already returned.
If you want an async result, then you will have to change getPlanInfo() to use a mechanism that supports getting async results such as a direct callback or a promise or something like that.
Here's a plain callback way:
var getPlanInfo = function (id, conn, callback) {
conn.query('SELECT * FROM `items` WHERE `id` = ?', [id], function (error, result) {
if (error) {
console.error('Query error: ' + error.stack);
callback(error);
return;
}
console.log(result[0]); // Everything is great
callback(0, result[0]);
});
};
modules.exports.getPlanInfo = getPlanInfo;
Then, the caller of that module would look like this:
var m = require('whatever');
m.getPlanInfo(id, conn, function(err, result) {
if (err) {
// error here
} else {
// process result here
}
});

You don't return anything from getPlanInfo. Probably you wanted to write modules.exports.getPlanInfo = function (id, conn) { return getPlanInfo; }
(with return getPlanInfo; instead of return getPlanInfo();)

Returning from anonymous function passed as parameter

Consider the following code:
function dbTask(q) {
mysql = new MySQL();
mysql.host = sqlHost;
mysql.user = sqlUser;
mysql.password = sqlPassword;
mysql.query("USE stock");
return mysql.query(q, function(err, results, fields) {
if(err) {
console.log("MySQL Error: " + err + ", Query: " + q);
return false;
} else {
return results; //here
}
});
};
var r = dbTask("SELECT * FROM user;");
console.log(r);
Whereas, I want the results to be returned from the inner anonymous function when calling dbTask() in the second last line, I am getting different output which seems like some internal construct of the mysql library under use.
How can I get dbTask to return the results when called?

Since mysql.query is asynchronous, then you'll have to re-think your architecture.
Instead of having your function return true or false, you'll have to pass in handlers to be called if the query returned true or false.
Something like this:
function dbTask(q, success, failure) {
mysql = new MySQL();
mysql.host = sqlHost;
mysql.user = sqlUser;
mysql.password = sqlPassword;
mysql.query("USE stock");
mysql.query(q, function(err, results, fields) {
if(err) {
console.log("MySQL Error: " + err + ", Query: " + q);
failure(err, results, fields);
} else {
success(results, fields);
}
});
};
Which you'd call like this:
dbTask(q,
function(results, fields) { /* ... */ },
function(err, results, fields) { /* ... */ });

You can't because dbTask returns once the mysql.query call completes to initiate the query. So by the time mysql.query's callback is called, console.log(r) has already executed.
This is the crux of the asynchronous nature of node.js.
What you can do is have dbTask accept a callback parameter that the function calls once results is available so that it can be provided to the caller asynchronously.