Develop Reference

nodejs - Help "promisifying" a file read with nested promises

So I've recently delved into trying to understand promises and the purpose behind them due to javascripts asynchronous behavior. While I "think" I understand, I still struggle with how to promisify something to return the future value, then execute a new block of code to do something else. Two main node modules I'm using:
pg-promise
exceljs
What I'd like to do is read a file, then once fully read, iterate of each worksheet executing DB commands. Then once all worksheets are processed, go back and delete the original file I read. Here is the code I have. I have it working to the point everything writes into the database just fine, even when there are multiple worksheets. What I don't have working is setting it up to identify when all the worksheets have been fully processed, then to go remove the file
workbook.csv.readFile(fileName)
.then(function () {
// this array I was going to use to somehow populate a true/false array.
// Then when done with each sheet, push a true into the array.
// When all elements were true could signify all the processing is done...
// but have no idea how to utilize this!
// So left it in to take up space because wtf...
var arrWorksheetComplete = [];
workbook.eachSheet(function (worksheet) {
console.log(worksheet.name);
db.tx(function (t) {
var insertStatements = [];
for (var i = 2; i <= worksheet._rows.length; i++) {
// here we create a new array from the worksheet, as we need a 0 index based array.
// the worksheet values actually begins at element 1. We will splice to dump the undefined element at index 0.
// This will allow the batch promises to work correctly... otherwise everything will be offset by 1
var arrValues = Array.from(worksheet.getRow(i).values);
arrValues.splice(0, 1);
// these queries are upsert. Inserts will occur first, however if they error on the constraint, an update will occur instead.
insertStatements.push(t.one('insert into rq_data' +
'(col1, col2, col3) ' +
'values($1, $2, $3) ' +
'ON CONFLICT ON CONSTRAINT key_constraint DO UPDATE SET ' +
'(prodname) = ' +
'($3) RETURNING autokey',
arrValues));
}
return t.batch(insertStatements);
})
.then(function (data) {
console.log('Success:', 'Inserted/Updated ' + data.length + ' records');
})
.catch(function (error) {
console.log('ERROR:', error.message || error);
});
});
});
I would like to be able to say
.then(function(){
// everything processed!
removeFile(fileName)
// this probably also wouldn't work as by now fileName is out of context?
});
But I'm super confused when having a promise inside a promise.. I have the db.tx call which is essentially a promise nested inside the .eachSheet function.
Please help a dumb programmer understand! Been beating head against wall for hours on this one. :)

If i understand correctly, you're trying to chain promises.
I suggest you to read this great article on Promises anti-pattern (see 'The Collection Kerfuffle' section)
If you need to execute promises in series, this article suggests to use reduce.
I'll rewrite your snippet to:
workbook.csv.readFile(fileName).then(function () {
processWorksheets().then(function() {
// all worksheets processed!
});
});
function processWorksheets() {
var worksheets = [];
// first, build an array of worksheet
workbook.eachSheet(function (worksheet) {
worksheets.push(worksheet);
});
// then chain promises using Array.reduce
return worksheets.reduce(function(promise, item) {
// promise is the the value previously returned in the last invocation of the callback.
// item is a worksheet
// when the previous promise will be resolved, call saveWorksheet on the next worksheet
return promise.then(function(result) {
return saveWorksheet(item, result);
});
}, Promise.resolve()); // start chain with a 'fake' promise
}
// this method returns a promise
function saveWorksheet(worksheet, result) {
return db.tx(function (t) {
var insertStatements = [];
for (var i = 2; i <= worksheet._rows.length; i++) {
// here we create a new array from the worksheet, as we need a 0 index based array.
// the worksheet values actually begins at element 1. We will splice to dump the undefined element at index 0.
// This will allow the batch promises to work correctly... otherwise everything will be offset by 1
var arrValues = Array.from(worksheet.getRow(i).values);
arrValues.splice(0, 1);
// these queries are upsert. Inserts will occur first, however if they error on the constraint, an update will occur instead.
insertStatements.push(t.one('insert into rq_data' +
'(col1, col2, col3) ' +
'values($1, $2, $3) ' +
'ON CONFLICT ON CONSTRAINT key_constraint DO UPDATE SET ' +
'(prodname) = ' +
'($3) RETURNING autokey',
arrValues));
}
return t.batch(insertStatements);
})
// this two below can be removed...
.then(function (data) {
return new Promise((resolve, reject) => {
console.log('Success:', 'Inserted/Updated ' + data.length + ' records');
resolve();
});
})
.catch(function (error) {
return new Promise((resolve, reject) => {
console.log('ERROR:', error.message || error);
reject();
});
});
}
Don't forget to include the promise module:
var Promise = require('promise');
I haven't tested my code, could contains some typo errors.

Array returning Undefined because of asynchrony

I am accessing the API Trello, but I came across the following problem:
Trello access the information, getting the id of each existing row, the code is as follows:
var x;
var numberCardsByList = [];
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Number of list: " + data.length);
for(var i=0; i<data.length; i++){
x = data[i];
findNumberCards(x);
}
});
As you can see, after getting the size, I walk all these queues with is, within the loop, attach each row in a variable x and call a function that aims to get the number of cards that queue. The code for the number of cards is as follows:
function findNumberCards(x){
trello.get("/1/lists/"+x.id+"/cards", function(err, dados){
if(err) throw err;
console.log("Name List: " + x.name + " have " + dados.length + " cards");
numberCardsByList[x.name] = dados.length;
});
}
Until then all right, but when I try to access the vector numberCardsByList after the end of the search in Trello, it returns undefined:
var x;
var numberCardsByList = [];
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Quantidade de Filas: " + data.length);
for(var i=0; i<data.length; i++){
x = data[i];
findNumberCards(x);
}
});
console.log(numberCardsByList);
I am aware that it is because of asynchrony, however, can not solve.

The problem you're facing has been solved many times before. If you want to know more, search for the keyword "Promise". If you're familiar with jQuery, try and look up: $.whenAll, $.ajax().done, $.ajax().always, etc.
If you want to come up with a light weight solution yourself, here's a pointer:
By the time you get to your console.log(numberCardsByList), your requests triggered by findNumberCards haven't yet completed, making the Array empty. You'll need to make sure you know when all findNumberCards requests have completed and then log them. Alternatively, you could log them every time one of them completes.
There are roughly two approaches:
Keep track of your open requests and call a function when a request is handled.
Observe your numberCardsByList object and call a function whenever items are added (you won't know if they were added async or synchronously)
I'd suggest going with the first approach. Check out this example code and the comments:
var numberCardsByList = {};
// This array will store the url for every open request
var openRequests = [];
var removeRequest = function(url) {
var index = openRequests.indexOf(url);
if (index === -1) return;
// Remove url from array
openRequests = openRequests
.slice(0, index)
.concat(openRequests
.slice(index + 1));
};
// This will be called whenever one request completes
var onComplete = function(url) {
removeRequest(url);
// When all have completed, we can call our callback
if (openRequests.length === 0) {
onAllComplete();
}
});
// This will be called when there are no open requests left
var onAllComplete = function(data) {
console.log(numberCardsByList);
}
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Number of list: " + data.length);
for (var i = 0; i < data.length; i++) {
x = data[i];
findNumberCards(x);
}
});
function findNumberCards(x) {
var url = "/1/lists/" + x.id + "/cards";
// Before we make the request, we register it:
openRequests.push(url);
trello.get(url, function(err, dados) {
numberCardsByList[x.name] = dados.length;
// When it is completed, we call onComplete
onComplete(url);
});
};
Note that this onAllComplete isn't 100% safe: it might be called multiple times if a request finishes before the next one is started.
Concluding:
If you can, I'd use a library to handle promises. If you want to try and build something yourself, you could try and keep track of the requests and execute a callback when they've all completed.

Keep in mind my above code most likely wont work for you as i dont know whats going on in your code so this is an example / explanation how to deal with your problem.
Since you are unfamiliar with async operation i will assume you dont have a prior knowledge of promises and therefore give you a less optimal solution - however promises are alot better and you should defintely learn them.
You need to execute sequence procedures inside the result of the async code.
First you'll create a function for the second operation for example:
function numberCardsByList (param1,param2){.....}
You will then change fineNumberCards to also accept a callback:
function findNumberCards(x, callback){
trello.get("/1/lists/"+x.id+"/cards", function(err, dados){
if(err) throw err;
console.log("Name List: " + x.name + " have " + dados.length + " cards");
numberCardsByList[x.name] = dados.length;
});
// pass in any params you need.
callback();
}
And then you will pass the newly created function numberCardsByList to findNumberCards or wherever you want it.
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Number of list: " + data.length);
for(var i=0; i<data.length; i++){
x = data[i];
// and here we are calling findNumberCards and passing in the callback..
findNumberCards(x, numberCardsByList);
}
});
That is generally how you will deal with async operation, you will pass a callback for the next operation to be executed.
update
here is an example of how this is done with another scenario just to demonstrate the point farther.
we start by getting user
service.getUser(userName, function(err,user){
if(user) {
// we get user picture passing getPictureSize as callback
getUserPicture(user.picture, getPictureSize)
}
})
we get the pictureURL
function getUserPicture(picName, cb){
service.getPictureURL(picName, function(err, pictureURL){
if(pictureURL) {
// we then call the callback - the next async operation we want.
cb(pictureURL);
}
});
}
we get picture size - which is the last operation
function getPictureSize(pictureURL){
service.getPictureSize(pictureURL, function(err, pictureSize){
$('.picName').attr('src', picName);
$('.picName').width(pictureSize.width);
$('.picName').height(pictureSize.height);
});
}
I hope that clarify things a little.

Why does Node.js appear to pass arguments differently in these two uses?

Check out this super-simple node.js program:
var g = { a : 1, b : 2 }
function callBack (key, value) {
console.log("Callback called with key: " + key + "\nAnd value: " + value) ;
}
function doNothing (key, value, cb) {
true ;
console.log(key + ": doing nothing") ;
cb() ;
}
function doLoop () {
for (k in g) {
f = function () {
callBack(k, g[k]) ;
}
doNothing(k, g[k], f) ;
}
}
doLoop() ;
When run, it produces this output:
a: doing nothing
Callback called with key: a
And value: 1
b: doing nothing
Callback called with key: b
And value: 2
OK. That makes sense - each time the callback is called, it has the correct arguments.
Now look at this program:
var mysql = require('mysql') ;
var dbClient = undefined ;
var db_uri = "mysql://xxx:xxx#127.0.0.1/xxx" ;
var schema = {
redirects : "(id int AUTO_INCREMENT, key VARCHAR(50), url VARCHAR(2048))",
clicks : "(ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, IP VARBINARY(16))"
} ;
function createOnEmpty(err, results, fields, tableName, create_def) {
console.log("createOnEmpty called on " + tableName) ;
if (err) {
console.error(err) ;
process.exit(1) ;
} else {
if (0 == results.length) {
dbClient.query(["create table ", tableName, create_def].join(" "),
function (err, results, fields) {} ) ;
} else {
console.log(tableName + " table already exists.") ;
}
}
console.log("\n\n") ;
}
function setupSchema() {
for (table in schema) {
console.log("Checking for table: " + table) ;
// FIXME: Why does this always seem to pass clicks as tablename?!
dbClient.query(
"show tables LIKE '" + table + "'",
function (err, results, fields) {
createOnEmpty(err, results, fields, table, schema[table])
}
);
}
}
function handleDBConnect(err) {
if (err) {
console.error("ERROR: problem connecting to DB: " + err.code) ;
process.exit(1) ;
} else {
console.log("Connected to database.") ;
// Automatically set up the schema, if the tables don't exist
setupSchema() ;
}
}
function MySQLConnect() {
dbClient = mysql.createConnection(db_uri) ;
dbClient.connect(handleDBConnect) ;
}
MySQLConnect() ;
It outputs:
Connected to database.
Checking for table: redirects
Checking for table: clicks
createOnEmpty called on clicks
createOnEmpty called on clicks
The loop seems to be giving the argument 'clicks' as the argument 'table' both times, even though the variable has clearly been switched to 'redirects'.
I figure I must have some fundamental misunderstanding of how JavaScript/Node works here.

To understand this behaviour you need to understand this 2 core js concepts:
Closures
Eventloop
Let say we have global variable a, log function which outputs a to console, and main function which calls log twice, first time with timeout (asynchronously), second time just simple function call
var a = 42;
function log() {
console.log(`a is ${a}`);
}
function main() {
setTimeout(log, 100);
a = 13;
log();
}
main();
This code produces the following output:
a is 13
a is 13
Why the hell the first time a is 13?
When you call setTimeout, it doesn't block main js thread for 100ms, it just adds log function to callback queue. The next line is a = 13. As a wasn't declared inside function body with var keyword, 13 assigned to a which was declared on the first line of code. Then we have the first line of output as a result of the last line of main function. Now we have empty callstack, nothing else happening in our code, but we still have log function in callback queue. After 100ms passed, if and only if callstack is empty (which is our case), log function could be called second time. It logs 'a is 13' once again, as a-s value was already reassigned.
This is a short explanation of how async callbacks work in javascript, and this is the reason why createOnEmpty called on clicks twice. dbClient.query is asynchronous and by the time it was called first time, your for loop finished its execution and table value is clicks.
Quick and dirty solution of your problem will be
for (table in schema) {
console.log("Checking for table: " + table) ;
(function (table) {
dbClient.query("show tables LIKE '" + table + "'",
function (err, results, fields) {
createOnEmpty(err, results, fields, table,
schema[table]) } );
)(table);
}
This immediately called function memorizes table value on each loop iteration in scope
Check out this excellent talk about eventloop and how async stuff works in js
How closures work

By the time the callback of dbClient.query is called the loop has already finished, leaving your (implicitly global) table variable at the last key in schema.
You need to scope it using an anonymous function(see the various answers there) or use a callback based iteration like so:
function setupSchema() {
Object.keys(schema).forEach(function (table) {
console.log("Checking for table: " + table) ;
// FIXME: Why does this always seem to pass clicks as tablename?!
dbClient.query(
"show tables LIKE '" + table + "'",
function (err, results, fields) {
createOnEmpty(err, results, fields, table, schema[table])
}
);
});
}

Node JS How to get the query data outside

I am new from Node js , just i am trying implementing this functionality last few days but i am unable to fix
exports.get_exercises_for_trainer = function(req, res)
{
connection.query('SELECT * FROM ag_exercise', function(err, exercise)
{
console.log('------------------------------before add fields ----------------------------------');
console.log(exercise);
for (var i in exercise)
{
fields(exercise[i].wt_id, function(result1) {
exercise[i].wt_fields = result1; //here i am adding result set of other query but i am not geting this fields data
console.log(result1) //but i printed here working fine but i need this result1 data out side query
});
}
console.log('------------------------------after add fields ----------------------------------');
console.log(exercise);
res.render('pages/trainer_home.ejs',{page_title:"Exercise Create",exercise:exercise});
});
}
function fields(wt_id,callback)
{
connection.query('SELECT * FROM ag_workout_type_fields WHERE wt_id = "'+wt_id+'"', function( err1, result1){
callback(result1);
});
}
I have one more query ? in node js : If table having users , users having different relation tables like orders , profiles , address
How to implement this
First i am getting users
User loop
getting every user profiles ,address,orders
end user loop
but above scenario i am unable to implement in node js but in php very simple like this
$get_users = ... //users data
foreach($getusers as $usr)
{
$usr->orders = //orders data
.... like etc
}

There are three main questions here, I will adress each seperately.
Question 1: When making an async function, how do I then access my data outside that function?
All data from async calls are accessed via callback, event listeners or promises (a fancy callback and event listener handler). For the most part, you are going to just be using callbacks. So, instead of :
get_user = function(user_id){
//Do some stuff to get the user
return the_user;
};
var user = get_user('1234');
//do whatever you want with user
You will see :
get_user = function(user_id,callback){
//Do some stuff to get the user
callback(null,the_user);
}
get_user('1234',function(err,user){
//do whatever you want with user
});
When we get to Question 3, you will see the more complicated use case you were speaking of.
Question 2: How do I loop through my data, perform a subsiquent query on each row, and append that data to my current data?
There are a couple of issues here.
Every time you query the database, you are performing an asynchronous function, so you need to manage all of those callbacks accordingly. Fortunately there are some great tools to do that for you, and we will be using async.
Every time you call an asynchronous function in a for loop, the for loop continues, thus your iterator is overwritten, but your asynchronous function is not done with it yet, so you will get all sorts of unexpected results like vanishing variables, or missmapped results. You can handle this with JavaScript closures, or, you can rely again on libraries like async which handle it all for you.
Instead of running a for loop over your queries results, we're going to pass it to async.forEachOf, which we will use to modify the existing array and append the results of the subsequent queries to the primary query's rows. It is important to note that forEachOf will run the subsequent queries in parallel, so you should not be using a single database connection, but a pool. If you MUST use a single database connection, use forEachOfSeries instead.
async = require('async');
exports.get_exercises_for_trainer = function(req, res){
connection.query('SELECT * FROM ag_exercise', function(err, exercises)
{
console.log('------------------------------before add fields ----------------------------------');
console.log(exercises);
async.forEachOf(exercises,function(exercise,index,callback){
connection.query('SELECT * FROM ag_workout_type_fields WHERE wt_id = "' + exercise.wt_id + '"', function( err, result1){
if(err)return callback(err1);
exercises[index].wt_fields = result1; //Modify original array
return callback();
});
},function(err){
if(err){return;} //do some error handling
console.log('------------------------------after add fields ----------------------------------');
console.log(exercises);
res.render('pages/trainer_home.ejs',{page_title:"Exercise Create",exercise:exercises});
});
});
};
Question 3: How do I perform many related but different queries so that I can populate information about my object?
This is another great usage of the async library. In this case since the queries are all different, we'll use parallel instead of forEachOf.
async = require('async');
populate_user = function(user,_callback){
async.paralell({
profile: function(callback){
var sql = "SELECT * FROM profiles WHERE user_id = " + user.id + " LIMIT 1 ";
var connection.query(sql,function(err,rows,fields){
if(err)return callback(err);
if(rows.length === 1)return callback(null,rows[0]);
return callback(null,[]);
});
},
address: function(callback){
var sql = "SELECT * FROM addresses WHERE user_id = " + user.id + " LIMIT 1 ";
var connection.query(sql,function(err,rows,fields){
if(err)return callback(err);
if(rows.length === 1)return callback(null,rows[0]);
return callback(null,[]);
});
},
orders: function(callback){
var sql = "SELECT * FROM orders WHERE user_id = " + user.id;
var connection.query(sql,function(err,rows,fields){
if(err)return callback(err);
if(rows.length > 0)return callback(null,rows); //notice how this one could have multiple results so we're returning them all
return callback(null,[]);
});
},
},
function(err,result){
if(err)return _callback(err);
for(var att in result){user[att] = result[att];}
callback(null,user);
}
}
user = {id:1234};
populate_user(user,function(err,populated_user)){
console.log(user); //wow notice how it's populated too!
console.log(populated_user); //this is really just a new handle for the same object
});
I want to note that NONE of this was tested, not even for syntax, so it may take a little reworking.

JavaScript Callbacks and Splunk

I am in the process of using the Splunk Javascript API to gain access to some of its functionality, but I'm having trouble understanding JavaScript concepts behind callbacks.
An example from the docs:
var http = new splunkjs.ProxyHttp("/proxy");
var service = new splunkjs.Service(http, {
username: username,
password: password,
scheme: scheme,
host: host,
port: port,
version: version
});
Async.chain([
// First, we log in
function(done) {
service.login(done);
},
// Perform the search
function(success, done) {
if (!success) {
done("Error logging in");
}
service.search("search index=_internal | head 3", {}, done);
},
// Wait until the job is done
function(job, done) {
Async.whilst(
// Loop until it is done
function() { return !job.properties().isDone; },
// Refresh the job on every iteration, but sleep for 1 second
function(iterationDone) {
Async.sleep(1000, function() {
// Refresh the job and note how many events we've looked at so far
job.fetch(function(err) {
console.log("-- fetching, " + (job.properties().eventCount || 0) + " events so far");
iterationDone();
});
});
},
// When we're done, just pass the job forward
function(err) {
console.log("-- job done --");
done(err, job);
}
);
},
// Print out the statistics and get the results
function(job, done) {
// Print out the statics
console.log("Job Statistics: ");
console.log(" Event Count: " + job.properties().eventCount);
console.log(" Disk Usage: " + job.properties().diskUsage + " bytes");
console.log(" Priority: " + job.properties().priority);
// Ask the server for the results
job.results({}, done);
},
// Print the raw results out
function(results, job, done) {
// Find the index of the fields we want
var rawIndex = utils.indexOf(results.fields, "_raw");
var sourcetypeIndex = utils.indexOf(results.fields, "sourcetype");
var userIndex = utils.indexOf(results.fields, "user");
// Print out each result and the key-value pairs we want
console.log("Results: ");
for(var i = 0; i < results.rows.length; i++) {
console.log(" Result " + i + ": ");
console.log(" sourcetype: " + results.rows[i][sourcetypeIndex]);
console.log(" user: " + results.rows[i][userIndex]);
console.log(" _raw: " + results.rows[i][rawIndex]);
}
// Once we're done, cancel the job.
job.cancel(done);
}
],
function(err) {
callback(err);
}
);
Async.chain is defined here as being root.chain = function(tasks, callback). My understanding is that there are 5 functions in the tasks array which are executed one after the other, and pass the results from one to the other.
However I do not understand how and where "done", "success","job" and "results" are defined, or how it is that they are used as arguments within the body of their functions?
function(success, done) {
if (!success) {
done("Error logging in");
}
service.search("search index=_internal | head 3", {}, done);
}
here, how is it testing against success, and passing a string to done()?
and how does the two functions
function(job, done) {// Print out the statics ..}
&
function(results, job, done) { .. }
pass the results data from the first function to the second?
Apologies for the long question.

In Javascript, functions create new scope. That means it does not matter what the passed arguments were named before they were passed to the function.
var awesomeName = 'bob';
hi(awesomeName);
// name === undefined
function hi(name) {
// name === 'bob';
console.log('hi', name); // Outputs: 'hi bob' in console
}
// name === undefined
As you said, each task calls the next task as a callback. The last argument is always the next task function/callback. That means that Async.chain probably automagically adds the callbacks to the end of the arguments before calling each task function. done is just a conventional name to assign to the callback. Similarly, the other arguments are just descriptive names for the arguments passed by the previous task. In order to see why they are named that way, you should look at the function that is calling the callback.
For example:
service.login(done) probably has some kind of code in it that does something like this:
login: function(callback) {
var successful;
// Do Login logic here and assign true/false to successful
callback(successful);
}
The callback is the next task in the chain and has two arguments, success and done. success is just the first argument that login passed to it. Async.chain always passes another argument as the last argument: the next task function, which is just assigned the name done by convention. You could name it whatever you want within each function, as long as you refer to it as the same name within the function.
function cb(success, fuzzyCallback) {
if (!success) {
fuzzyCallback('Error!');
}
fuzzyCallback(null);
}