On a Form Action of type POST, we fetch all the values in Node.JS/Express and try saving it into MongoDB.
A hidden field determines the length of a property from the frontend javascript and it's value is updated as the hidden field's value.
This length is used in the backend (Node) to iterate over a list of items.
I have a async.waterfall function and a for loop running inside it like this.
async.waterfall([
function(callback){
var itemLength = req.body.itemLength;
var itemProp,itemComponent;
var destination;
var destinationsArray =[];
for(var k=1; k<=itemLength; k++){
destination = new Destination({
name: req.body['destinationName'+k],
});
itemComponent = {
"itemCompProp" : req.body['itemCompProp'+k]
};
itemProp = new ItemProp({
itemComponent: itemComponent
});
itemProp.save(function(err,itemPropSaved){
destination.newProperty = itemPropSaved._id
destination.save(function(err,destinationSaved){
if(err){
console.log("Error== " + err);
}
else{
destinationsArray.push(destinationSaved._id);
}
});
});
}// End of For
callback(null,destinationsArray);
},
function(destinationsArray,callback){
var brand = new Brand({
name : req.body.brandName,
});
brand.save(function(err,brandSaved){
if(err){
console.log("Error== " + err);
}else{
console.log('Brand Saved');
}
});
callback(null);
}
], function (err, status) {
if(err){
req.flash('error', {
msg: 'Error Saving Brands'
});
console.log("Error : " + err);
}
else{
console.log("Brand Saved.");
req.flash('success', {
msg: 'Brand Successfully Added!'
});
}
});
res.redirect('/redirectSomewhere');
When we run this, The destinationsArray is returned first as null, as opposed to going through the for loop and then returning the proper value of destinationsArray over a length (itemLength) of destinations.
We want the process to be synchronous. We also tried using a closure wrapping the for Loop but to no avail.
We can't use a async.eachSeries instead of the for Loop as I am just iterating over a numeric property and we don't have any documents to iterate over
Any feasible solution to run a for Loop inside a async.waterfall?
Cheers and Thanks in Advance.
There are few problems with the code you have there:
where the callbacks got called.
where res.redirect() got call.
the for loop.
save() is asynchronous. Regular for loop will just continue without waiting for all save() calls to finish. That's why destinationsArray is empty. As you said, you cannot use async.eachSeries() since you're iterating through numeric property. However, you're on the right track there. Async.whilst() does just that. Here is the revised code with Async.whilst() and proper calling locations of the callbacks:
async.waterfall([
function(callback){
var itemLength = req.body.itemLength;
var itemProp,itemComponent;
var destination;
var destinationsArray =[];
var k = 1; // 1st part of for loop: for(k=1; k<=itemLength; k++)
async.whilst(
function() {
return k <= itemLength; // 2nd part of for loop: for(k=1; k<=itemLength; k++)
},
function(whilstCb) {
destination = new Destination({
name: req.body['destinationName'+k]
});
itemComponent = {
"itemCompProp" : req.body['itemCompProp'+k]
};
itemProp = new ItemProp({
itemComponent: itemComponent
});
itemProp.save(function(err,itemPropSaved){
destination.newProperty = itemPropSaved._id
destination.save(function(err,destinationSaved){
if(err){
console.log("Error== " + err);
} else {
destinationsArray.push(destinationSaved._id);
}
k++; // 3rd part of for loop: for(k=1; k<=itemLength; k++)
whilstCb(null);
});
});
},
function(err) {
// It gets here once the loop is done
console.log(destinationsArray); // This array should have all the values pushed
callback(null, destinationsArray);
}
);
},
function(destinationsArray,callback){
var brand = new Brand({
name : req.body.brandName
});
brand.save(function(err,brandSaved){
if(err){
console.log("Error== " + err);
} else {
console.log('Brand Saved');
}
callback(null);
});
}
], function (err, status) {
if(err){
req.flash('error', {
msg: 'Error Saving Brands'
});
console.log("Error : " + err);
} else {
console.log("Brand Saved.");
req.flash('success', {
msg: 'Brand Successfully Added!'
});
}
res.redirect('/redirectSomewhere');
});
Its not so much the for loop that is causing you problems but that save is an asynchronous operation. The for loop completes and the callback is executed before any of the save callbacks have had chance to complete.
What you want to do is call the async.waterfall callback after all the destination save callbacks have been executed. Something like:
destination.save(function(err,destinationSaved){
if(err){
console.log("Error== " + err);
} else {
destinationsArray.push(destinationSaved._id);
if (k === itemLength) {
// all destination callbacks have completed successfully
callback(null, destinationsArray);
}
}
});
The issue has to do with callback(null, destinationsArray); getting called outside the for loop without checking first to see the loop has been finished.
Try replacing callback(null, destinationsArray); with something like this:
if (itemLength > 0 && destinationsArray.length === k - 1) {
callback(null, destinationsArray);
} else {
callback(true);
}
The above checks to make sure the destination.save() gets completed the proper number of times successfully.
I actually prefer the method proposed by djskinner. However, because of the console.log() that occurs when there is a save() error, the callbacked destinationsArray could possibly hold the incorrect number of items. To fix this, you could make sure to replace the console.log("Error== " + err); with something like callback(err) to end the waterfall with the error returned. In addition, the k === itemLength check doesn't properly account for the correct number of items that should be saved. This should be replaced with k === destinationsArray.length.
I made modifications to fix this and posted an updated version below.
destination.save(function(err, destinationSaved){
if (err) {
callback(err);
}
else {
destinationsArray.push(destinationSaved._id);
if (k === destinationsArray.length) {
callback(null, destinationsArray);
}
}
});
--EDIT-- I really like the solution that Ben posted using whilst(). This allows the creation of a loop where the iterations runs serially. For more info, view the npm page here.
Related
I am new to NodeJs and I'm finding the Non Blocking and Asynchronous nature of JS extremely difficult to understand and handle,
I have a piece of code which is supposed to Iterate an array
and for every iteration, I'm supposed to make a DB update.
Can someone provide the correct implementation of Async library functions and help fix my code?
Code example -
function updateFunction(conn, requestBody, callback) {
let arr = [];
async.each(requestBody.arr, function(item, callback) {
let sqlData = []
let columns = "";
if(item.columnData != null){
sqlData.push(item.columnData);
columns += "`columnName` = ?,";
}
if(columns != ''){
columns = columns.substring(0,columns.length-1);
let sqlQuery = 'UPDATE someTable SET '+columns
+' WHERE id = "' + item.id + '"';
conn.query(sqlQuery, sqlData, function (err, result) {
if (err) {
return callback(err, false);
}
})
}
else{
return callback(null, false);
}
columns = "";
sqlData = [];
},
function(err, results) {
//Code never reaches here, don't know why
if (err) {
return callback(err, false);
}
else{
return callback(null, true);
}
});
} // END
During your database query call, on a successful query your callback is not called, therefore causing your code to never reach the final callback.
You will want to add another return statement after your if (err) { return callback(err); } to let async know your database query is finished.
And another thing, according to the docs, the async each method's final callback does not invoke with results in its callback.
A callback which is called when all iteratee functions have finished, or an error occurs. Invoked with (err).
Therefore, it is not required for you to pass a value into the callback statement within your iteratee function.
Modify your code to do this and it will work.
conn.query(sqlQuery, sqlData, function (err, result) {
if (err) {
return callback(err);
}
return callback(null);
})
Hope this helps.
conn.query(sqlQuery, sqlData, async function (err, result) {
if (err) {
return await callback(err, false);
}
})
Something like this.. so the function callback is async here and we gave await which actually waits until the return call is finished..
this is a follow up question to Asynchron Errorhandling inside $.each. As mentioned in the comments there, i want to handle data after the last async job from a $.each loop.
So for instance:
var errors = 0;
var started = 0;
var successful = 0;
$.each(..., function(){
started++;
connection.query('INSERT INTO tableName SET ?', post, function(err, result)
{
if (err) {
if (err.code === 'ER_DUP_ENTRY')
{ errors++; }
else
{ throw err; }
} else { successful++;}
if (started == successful + errors) {
// all done
console.log(errors + " errors occurred");
}
});
});
In this case everything logs out properly when the // all done comment is reached. But what if i want to use this data later on instead of just logging it out.
Is there a way to wait for this data outside of the $.each scope? Or do i always have to handle everything in the nested function?
You can use promises instead
var promises = [];
$.each(..., function() {
var promise = new Promise(function(resolve, reject) {;
connection.query('INSERT INTO tableName SET ?', post, function(err, result) {
if (err) {
resolve(err.code);
} else {
resolve(result);
}
});
});
promises.push(promise);
});
var result = Promise.all(promises);
And then when you want to use the data, you do
result.then(function(data) {
// use data array looking like ["result data", "result data", "ER_DUP_ENTRY" .. etc]
})
I am using Javascript, webdriverio (v2.1.2) to perform some data extraction from an internal site. So the idea is
Authenticate
Open the required URL, when authenticated
In the new page, search for an anchor tag having specific keyword
Once found, click on the anchor tag
Below is what I have tried and it works (last two points). I had to use Q and async to achieve it. I was hoping to use only Q to achieve it. Can someone help me, on how to achieve it using Q only ??
var EmployeeAllocationDetails = (function () {
'use stricy';
/*jslint nomen: true */
var Q = require('Q'),
async = require('async'),
_ead_name = 'Employee Allocation Details',
goToEadFromHome;
goToEadFromHome = function (browserClient) {
browserClient.pause(500);
var deferred = Q.defer();
browserClient.elements('table.rmg td.workListTD div.tab2 div.contentDiv>a', function (err, results) {
if (err) {
deferred.reject(new Error('Unable to get EAD page. ' + JSON.stringify(err)));
} else {
async.each(results.value, function (oneResult, callback) {
console.log('Processing: ' + JSON.stringify(oneResult));
browserClient.elementIdText(oneResult.ELEMENT, function (err, result) {
if (err) {
if (err.message.indexOf('referenced element is no longer attached to the DOM') > -1 ){
callback();
} else {
callback('Error while processing :' + JSON.stringify(oneResult) + '. ' + err);
}
} else if(!result){
console.log('result undefined. Cannot process: ' + JSON.stringify(oneResult));
callback();
} else if(result.value.trim() === _ead_name){
deferred.resolve(oneResult);
callback();
}
});
}, function (err) {
// if any of the processing produced an error, err would equal that error
if( err ) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('A processing failed to process. ' + err);
} else {
console.log('All results have been processed successfully');
}
}); //end of async.each
}
});
return deferred.promise;
};
return {
launchEad : goToEadFromHome
}
})();
module.exports = EmployeeAllocationDetails;
Related Github Issue link https://github.com/webdriverio/webdriverio/issues/123
I think you should use async. I think your code is great. It runs everything in parallel and it handles error well.
If
If you want to remove async, there are several options:
use Q flow control
copy paste async's implementation
implement it yourself
If you try to use Q's flow control it will look something like this (pseudo-code):
var getTextActions = [];
function createAction(element){
return function(){
return element.getText();
}
}
each(elements, function(element){ getTextActions.push( createAction(element) ) });
Q.all(getTextActions).then(function(results) {
... iterate all results and resolve promise with first matching element..
} );
note this implementation has worse performance. It will first get the text from all elements, and only then try to resolve your promise. You implementation is better as it all runs in parallel.
I don't recommend implementing it yourself, but if you still want to, it will look something like this (pseudo-code):
var elementsCount = elements.length;
elements.each(function(element){
element.getText(function(err, result){
elementsCount --;
if ( !!err ) { logger.error(err); /** async handles this much better **/ }
if ( isThisTheElement(result) ) { deferred.resolve(result); }
if ( elementsCount == 0 ){ // in case we ran through all elements and didn't find any
deferred.resolve(null); // since deferred is resolved only once, calling this again if we found the item will have no effect
}
})
})
if something is unclear, or if I didn't hit the spot, let me know and I will improve the answer.
This question already has answers here:
JavaScript closure inside loops – simple practical example
(44 answers)
Closed 8 years ago.
Take the code below, in which an object data is initialized with some values that are then processed by some time-intensive function, like database access. If the function succeeds, the name of the successful data item is printed to the console. Otherwise, a failure notice is printed:
data = {first: 'someinfo', second: 'somemoreinfo', third: 'evenmoreinfo'};
for (var item in data) {
timeIntensiveFunction(item, data[item], function(err) {
if (!err) {
console.log(item + ' processed successfully');
} else {
console.log(item + ' failed');
}
});
}
You would expect the console to show this, assuming the function succeeds for all three data items:
first processed successfully
second processed successfully
third processed successfully
Instead it will show this, assuming the first database access takes longer than the for loop:
third processed successfully
third processed successfully
third processed successfully
This is because the console logging is done in a callback, which would reasonably only be called after the for loop is done, because timeIntensiveFunction() takes so long. By the time the first callback is called, item already has its last value, third.
How do you pass the 'current' value of item into the callback?
Problem is because it's calling the callback with the last item only.
You can bind each of your item with a function like bellow.
var printStatus = function(item){
return function(err) {
if (!err) {
console.log(item + ' processed successfully');
} else {
console.log(item + ' failed');
}
}
}
for (var item in data) {
timeIntensiveFunction(item, data[item], printStatus(item));
}
This is a common "gotcha" with closures in javascript. One way around it is to wrap your function call in an anonymous function and rescope item. Like so:
for (var item in data) {
(function(item){
timeIntensiveFunction(item, data[item], function(err) {
if (!err) {
console.log(item + ' processed successfully');
} else {
console.log(item + ' failed');
}
});
})(item);
}
If you're looking for a library to make working with async tasks more easy, check out caolan/async.
var async = require("async");
var data = [{id: "first"}, {id: "second"}, {id: "third"}];
function timeIntensiveFunction(item, done) {
// do something
console.log("time intensive task started:", item.id);
// err?
// if (err) return done(err);
done();
}
function processItem(item, done) {
timeIntensiveFunction(item, function(err) {
if (err) return done(err);
console.log("task complete:", item.id);
done();
});
}
async.map(data, processItem);
Output
time intensive task started: first
task complete: first
time intensive task started: second
task complete: second
time intensive task started: third
task complete: third
For users looking to learn how to do this without a library, you can see the revision history of this answer.
So i have a csv file containing my information, i need to do a mass add/update
exports.add_questions_from_file = function (file_path, surveyid, callback)
{
var U = [{}];
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
console.log(data);
d = data.split(/\r\n|\n/);
for (x=0;x <d.length;x++)
{
line = d[x].split(',');
if (line[0] == "") {return};
RQuestion.add_by_line (line,function (err, question)
{
U.push({id:question.id});
console.log(U);
});
}
}
});
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":U}},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
};
But even though im using callback functions the update seems to happen with the same object always, even the console.log here
U.push({id:question.id});
console.log(U);
returns the same object (even that all the other were created)
Im doing something wrong?
I see a few issues.
First for:
if (line[0] == "") {return};
Don't you mean to use a break or continue instead? Otherwise the entire function will quit if there is a blank line anywhere in the file. This is very important because Survey.update won't get called either.
Second: I assumed that RQuestion.add_by_line and Survey.update are doing something async like updating a database. Your code needs to be restructured to wait for those async items to complete before moving on to the next step. I'd recommend an npm package named async for that.
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
d = data.split(/\r\n|\n/);
async.map(d, function(line, callback) {
//this function is called for each line
add_by_line (line,function (err, question)
{
callback(err,{id:question.id});
});
}, function(err, results) {
//this function is called when all of the items are done
console.log("done with async");
console.dir(results);
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":results},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
});
}
});