Request in NodeJS async waterfall return undefined - javascript

I'm pretty new to async on node js. I use the waterfall method while parsing a xml file like this:
$('situation').each( function(){
var situation = [];
$(this).find('situationRecord').each( function(i){
var record = this;
async.waterfall([
function (callback){
var filter = {
startLocationCode: $(record).find('alertCMethod2SecondaryPointLocation').find('specificLocation').text(),
endLocationCode: $(record).find('alertCMethod2PrimaryPointLocation').find('specificLocation').text(),
overallStartTime: $(record).find('overallStartTime').text(),
overallEndTime: $(record).find('overallEndTime').text()
}
callback(null, filter, record);
},
function (filter, record, callback){
var timestamp = new Date().toDateInputValue();
var startDbResponse = 0;
var endDbResponse = 0;
if((filter.startLocationCode != '') && new Date(timestamp) >= new Date(filter.overallStartTime) && new Date(timestamp) <= new Date(filter.overallEndTime) ){
startDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.startLocationCode);
endDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.endLocationCode);
}
console.log("startDbResponse: ", startDbResponse);
console.log("endDbResponse: ", endDbResponse);
callback(null, filter, record, startDbResponse, endDbResponse);
},
function (filter, record, startDbResponse, endDbResponse, callback){
console.log("startDbResponse: ", startDbResponse);
console.log("endDbResponse: ", endDbResponse);
var situationRecord = createSituationRecord($, record, filter.startLocationCode, filter.endLocationCode, startDbResponse, endDbResponse);
console.log(situationRecord);
},
function (situationRecord, callback){
situation[i] = { situationRecord };
}
],
function(err, results){
console.error("There was an error by filtering the xml file");
console.error(err);
});
})
if(situation.length > 0){ //if situation is not empty
locations.push(situation);
}
})
console.log(locations);
}
In this part of the waterfall I make a request to my database with locationCodeToGeodataRequst.geodataByLocationcode(filter.startLocationCode); but startDbResponse and endDbResponse is undefined :
....
function (filter, record, callback){
var timestamp = new Date().toDateInputValue();
var startDbResponse = 0;
var endDbResponse = 0;
if((filter.startLocationCode != '') && new Date(timestamp) >= new Date(filter.overallStartTime) && new Date(timestamp) <= new Date(filter.overallEndTime) ){
startDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.startLocationCode);
endDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.endLocationCode);
}
console.log("startDbResponse: ", startDbResponse);
console.log("endDbResponse: ", endDbResponse);
callback(null, filter, record, startDbResponse, endDbResponse);
},
....
The request by it self works because a console.log in the request module show the correct data. So I don't understand why its undefined.
This is the request module:
exports.geodataByLocationcode = function geodataByLocationcode(locationcode){
let sql = "SELECT * FROM tmc WHERE LOCATION_CODE = " + locationcode;
let query = db.query(sql, (err, result) =>{
if(err == null){
//console.log(result);
return result;
}else{
console.log("Error by getting item from db: " + err);
throw err;
}
});
}
This is a snipped of the console.log:
....
startDbResponse: undefined
endDbResponse: undefined
startDbResponse: undefined
endDbResponse: undefined
startDbResponse: 0
endDbResponse: 0
startDbResponse: 0
endDbResponse: 0
[]
//here comes the output of the requests as json objects
....

move the console log to the last function of async. As Manjeet pointed out async takes time and console prints early.
function (situationRecord, callback){
situation[i] = { situationRecord };
if(situation.length > 0){ //if situation is not empty
locations.push(situation);
}
})
console.log(locations);
}
}

Related

how to call an array which is in a function from the another function in javascript?

This is the async function:
async function getpackages(conn, childId, callback) {
var childId = childId;
var request6 = new sql.Request(conn);
var packageQuery = "select OrderId,ChildID from dbo.tbl_Scheduler where NoOfMealsLeft>0 and ChildId=" + childId;
await request6.query(packageQuery, function (err, packagelist) {
if (!err && packagelist.recordsets.length > 0) {
console.log("Error:" + err + "Result:" + util.inspect(packagelist.recordsets[0]));
var orderdetail_ = [];
for (i = 0; i < packagelist.recordsets[0].length; i++) {
orderdetail_.push(packagelist.recordsets[0][i].OrderId);
}
console.log("-->" + orderdetail_);
callback(null, packagelist.recordsets[0]);
} else if (packagelist.recordsets.length < 1) {
callback("Not a valid id input", null);
}
});
};
I need to call the orderdetails_ array in the query. The array contains four data and I need to iterate over 4 data one by one, using the or in the SQL query.
module.exports.newscheduledmeal = function (req, res, next, callback) {
let entered_date = req.query.date;
let childId = req.query.id;
let current_date = new Date().toISOString().slice(0, 10);
if (entered_date < current_date) {
return callback('Please enter date more than or equal to current date.', null);
} else
var conn = new sql.ConnectionPool(dbConfig);
try {
conn.connect().then(function () {
var request = new sql.Request(conn);
getpackages(conn, childId, function (err, orderid) {
if (err) {
callback(err, null);
} else
var PackageidQuery = "select PackageId from dbo.tbl_Order where OrderId=";
request.query(PackageidQuery, function (err, packagelist) {
if (!err) {
conn.close();
callback(null, packagelist.recordsets);
} else {
conn.close();
callback("Error", null);
}
});
});
});
} catch (err) {
console.log("Exception occured:" + err);
conn.close();
callback(err, null);
}
};
I want to get the details of the array which is in getpackages to be used in the module section and specifically in the SQL query section.

Why is my code not waiting for the completion of the function?

I am trying to read some data from a file and store it in a database.
This is part of a larger transaction and I need the returned ids for further steps.
async parseHeaders(mysqlCon, ghID, csv) {
var self = this;
var hIDs = [];
var skip = true;
var idx = 0;
console.log("Parsing headers");
return new Promise(async function(resolve, reject) {
try {
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(csv)
});
await lineReader.on('close', async function () {
console.log("done: ", JSON.stringify(hIDs));
resolve(hIDs);
});
await lineReader.on('line', async function (line) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx++] = await self.getParamID(mysqlCon, ghID, v, c, data);//, function(hID,sidx) { //add data in case the parameter is not in DB, yet.
}
});
} catch(e) {
console.log(JSON.stringify(e));
reject("some error occured: " + e);
}
});
}
async getParamID(mysqlCon,ghID,variable,category,data) {
return new Promise(function(resolve, reject) {
var sql = "SELECT ID FROM Parameter WHERE GreenHouseID="+ghID+" AND Variable = '" + variable + "' AND Category='" + category + "'";
mysqlCon.query(sql, function (err, result, fields) {
if(result.length === 0 || err) { //apparently not in DB, yet ... add it (Acronym and Machine need to be set manually).
sql = "INSERT INTO Parameter (GreenHouseID,Variable,Category,Control) VALUES ("+ghID+",'"+variable+"','"+category+"','"+data[3]+"')";
mysqlCon.query(sql, function (err, result) {
if(err) {
console.log(result,err,this.sql);
reject(err);
} else {
console.log("Inserting ",variable," into DB: ",JSON.stringify(result));
resolve(result.insertId); //added, return generated ID.
}
});
} else {
resolve(result[0].ID); //found in DB .. return ID.
}
});
});
}
The functions above are in the base class and called by the following code:
let headerIDs = await self.parseHeaders(mysqlCon, ghID, filePath);
console.log("headers:",JSON.stringify(headerIDs));
The sequence of events is that everything in parseHeaders completes except for the call to self.getParamID and control returns to the calling function which prints an empty array for headerIDs.
The console.log statements in self.getParamID are then printed afterward.
What am I missing?
Thank you
As you want to execute an asynchronous action for every line we could define a handler to do right that:
const once = (target, evt) => new Promise(res => target.on(evt, res));
function mapLines(reader, action) {
const results = [];
let index = 0;
reader.on("line", line => results.push(action(line, index++)));
return once(reader, "close").then(() => Promise.all(results));
}
So now you can solve that easily:
let skip = false;
const hIDs = [];
await mapLines(lineReader, async function (line, idx) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx] = await self.getParamID(mysqlCon, ghID, v, c, data);
}
});

JS Asynchronous execution is causing pain

I am developing a backend server code with NodeJS. What the code does is, periodically connect to a REST API, request updates and then write to my database.
I have no way of getting delta of the data from the API, so I drop my collection from MongoDB and then just insert the newly pulled data.
I implemented promises to make sure that the dependent methods are executed only after the previous methods resolve. This however doesn't seem to work as I anticipated.
So, I drop the collection and insert, this works. But the following method seems to execute before the new data is populated. It sometime works, when I have some new console.log statements which seems to induce a slight delay ever so slightly to make it all work.
setTimeout function didn't seem to help. Any suggestions?
Here is a sanitized version of the code: https://jsfiddle.net/ppbfrozg/
var request = require("request");
var q = require('q');
function authenticate() {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function getData(token) {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData);
console.log("Records Inserted");
} else {
db.collection('Details').drop();
db.collection('Details').insert(JSON.parse(a[0]));
console.log("Records Inserted");
}
deferred.resolve(token);
});
return deferred.promise;
}
function getMedia(dataContext) {
var deferred = q.defer();
var cursor = dataContext[0];
var token = dataContext[1];
if (cursor !== null) {
console.log("Inside cursor not null");
cursor.forEach(function(data) {
insertDataInMongo(data);
})
}
return deferred.promise;
}
function check(array, attr, value) {
for (var i = 0; i < array.length; i += 1) {
if (array[i][attr] === value) {
return false;
}
}
return true;
}
function get_value(array, attr) {
for (var i = 0; i < array.length; i += 1) {
if (array[i].hasOwnProperty(attr)) {
return array[i][attr];
}
}
}
function getNames(token) {
var deferred = q.defer();
var MongoClient2 = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
console.log("going to get Data");
MongoClient2.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err));
console.log("connected for select");
var data = db.collection('Details').find();
var dataContext = [data, token, 0, 0, 0, 0, 0, 0, 0, null];
deferred.resolve(dataContext);
});
return deferred.promise;
}
function convertDate(date) {
var yyyy = date.getFullYear().toString();
var mm = (date.getMonth() + 1).toString();
var dd = (date.getDate() - 3).toString();
var mmChars = mm.split('');
var ddChars = dd.split('');
return yyyy + '-' + (mmChars[1] ? mm : "0" + mmChars[0]) + '-' + (ddChars[1] ? dd : "0" + ddChars[0]);
}
authenticate()
.then(getData)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
This should work. Let me know if any issue.
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData, function(){
console.log("Records Inserted");
return deferred.resolve(token);
});
} else {
db.collection('Details').drop(function(error, result){//Callback function that executes after drop operation has completed.
if(error){
return deferred.reject(error);//Reject the promise if there was an error
}
db.collection('Details').insert(JSON.parse(a[0]), function(err, res){//Callback function that executes after insert operation has completed.
if(err){
return deferred.reject(err);//Reject the promise if there was an error
}
console.log("Records Inserted");
return deferred.resolve(token);
});
});
}
});
return deferred.promise;
}
From what I see from nodeJS driver API for MongoDB :
https://mongodb.github.io/node-mongodb-native/api-generated/collection.html
hint : db.collection.drop is asynchronous, so you have to use a callback with it
db.collection('Details').drop(function(err, result){
// Do anything AFTER you dropped your collection
}
but since you are using promises, you should use something like that:
authenticate()
.then(getData)
.then(db.collection('Details').drop)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
or if you want to really keep the same code format :
function dropCollectionInMongo() {
db.collection('Details').drop()
}
authenticate()
.then(getData)
.then(dropCollectionInMongo)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);

Javascript for loop wait for callback

I have this function:
function tryStartLocalTrendsFetch(woeid) {
var userIds = Object.keys(twitClientsMap);
var isStarted = false;
for (var i = 0; i < userIds.length; i++) {
var userId = userIds[i];
var twitClientData = twitClientsMap[userId];
var isWoeidMatch = (woeid === twitClientData.woeid);
if (isWoeidMatch) {
startLocalTrendsFetch(woeid, twitClientData, function (err, data) {
if (err) {
// Couldn't start local trends fetch for userId: and woeid:
isStarted = false;
} else {
isStarted = true;
}
});
// This will not obviously work because startLocalTrendsFetch method is async and will execute immediately
if (isStarted) {
break;
}
}
}
console.log("No users are fetching woeid: " + woeid);
}
The gist of this method is that I want the line if (isStarted) { break; } to work. The reason is that if it's started it should not continue the loop and try to start another one.
I'm doing this in NodeJS.
try to use a recursive definition instead
function tryStartLocalTrendsFetch(woeid) {
var userIds = Object.keys(twitClientsMap);
recursiveDefinition (userIds, woeid);
}
function recursiveDefinition (userIds, woeid, userIndex)
var userId = userIds[userIndex = userIndex || 0];
var twitClientData = twitClientsMap[userId];
var isWoeidMatch = (woeid === twitClientData.woeid);
if (isWoeidMatch && userIndex<userIds.length) {
startLocalTrendsFetch(woeid, twitClientData, function (err, data) {
if (err) {
recursiveDefinition(userIds, woeid, userIndex + 1)
} else {
console.log("No users are fetching woeid: " + woeid);
}
});
} else {
console.log("No users are fetching woeid: " + woeid);
}
}
You may also use async (npm install async):
var async = require('async');
async.forEach(row, function(col, callback){
// Do your magic here
callback(); // indicates the end of loop - exit out of loop
}, function(err){
if(err) throw err;
});
More material to help you out: Node.js - Using the async lib - async.foreach with object

Recursions on the tree structure in node.js

The task is to recursively build a tree-structure in node.js. Namely, I work with a deck of slides that might include also other subdecks. I am new to functional programming and I am stuck in understanding what I need to do. I know, there are much more advanced ways to control the flows, but I want for the beginning try to do it in a nested way, if it is possible. What I have for now:
The server.js calls an api and responds with a json object:
router.get('/deck/tree/:rev_id', function(req, res) {
var Deck = require('./models/deck');
var deck = new Deck(connection);
deck.getTree(req.params.rev_id ,function (tree) {res.json(tree)});
});
In the deck module I have several functions:
2.a Function to get the title of a deck:
this.getTitle = function(rev_id, callback){
var sql = "SELECT title FROM ?? WHERE ?? = ?";
var inserts = ['deck_revision', 'id', rev_id];
sql = mysql.format(sql, inserts);
connection.query(sql, function(err, results) {
if (err) throw err;
callback(results[0].title);
});
};
2.b Function to get the "direct" children of a deck:
this.getChildren = function(acc, callback){
acc.children = [];
var sql = "SELECT item_id AS id, item_type AS type FROM ?? WHERE ?? = ? ORDER BY position";
var inserts = ['deck_content', 'deck_revision_id', acc.id];
sql = mysql.format(sql, inserts);
connection.query(sql, function(err, results) {
if (err) throw err;
results.forEach(function(res_item){
acc.children.push({'id': res_item.id, 'type': res_item.type});
if (acc.children.length === results.length){
callback(acc);
}
});
});
};
Function that should buid the tree (does not work for now):
var results = [];
this.getTree = function(id, callback) {
var acc = {};
var deck = this;
var new_slide = new Slide(connection);
deck.getTitle(id, function (title_str) {
acc.title = title_str;
acc.id = id;
acc.type = 'deck';
deck.getChildren(acc, function(new_acc){
new_acc.children.forEach(function(element){
if (element.type === 'deck'){
// I suppose the help is needed here!
deck.getTree(element.id, function(new_element){
results.push(new_element);
if(results.length === new_acc.children.length) {
callback(results);
}
});
}else{
new_slide.getTitle(element.id, function(title_str){
element.title = title_str;
results.push(element);
if(results.length === new_acc.children.length) {
callback(results);
}
});
}
If somebody is interested, here how I/ve made it work:
this.getChildren = function(id, callback){
var sql = "SELECT item_id AS id, item_type AS type, position FROM ?? WHERE ?? = ? ORDER BY position";
var inserts = ['deck_content', 'deck_revision_id', id];
sql = mysql.format(sql, inserts);
connection.query(sql, function(err, results) {
if (err) throw err;
callback(results);
});
};
this.getTree = function(id, acc, callback) {
deck = new Deck(connection);
var new_slide = new Slide(connection);
deck.getTitle(id, function (title_str) {
acc.title = title_str;
acc.id = id;
acc.type = 'deck';
acc.children = [];
deck.getChildren(acc.id, function(children){
children.forEach(function(element){
if (element.type === 'deck'){
deck.getTree(element.id, element, function(new_element){
acc.children[new_element.position - 1] = new_element;
var size = acc.children.filter(function(value) { return value !== null }).length;
if (size === children.length){
callback(acc);
}
});
}
else{
new_slide.getTitle(element.id, function(title_str){
element.title = title_str;
acc.children[element.position - 1] = element;
var size = acc.children.filter(function(value) { return value !== null }).length;
if(size === children.length) {
callback(acc);
}
});
}
});
});
});
};

Categories

Resources