I am doing a for loop to find the result from mongodb, and concat the array. But I am not getting the final results array when the loop is finished. I am new to node.js, and I think it's not working like objective-c callback.
app.get('/users/self/feed', function(req, res){
var query = Bill.find({user: req.param('userId')});
query.sort('-createdAt');
query.exec(function(err, bill){
if (bill) {
var arr = bill;
Following.findOne({user: req.param('userId')}, function(err, follow){
if (follow) {
var follows = follow.following; //this is a array of user ids
for (var i = 0; i < follows.length; i++) {
var followId = follows[i];
Bill.find({user: followId}, function(err, result){
arr = arr.concat(result);
// res.send(200, arr);// this is working.
});
}
} else {
res.send(400, err);
}
});
res.send(200, arr); //if put here, i am not getting the final results
} else {
res.send(400, err);
}
})
});
While I'm not entirely familiar with MongoDB, a quick reading of their documentation shows that they provide an asynchronous Node.js interface.
That said, both the findOne and find operations start, but don't necessarily complete by the time you reach res.send(200, arr) meaning arr will still be empty.
Instead, you should send your response back once all asynchronous calls complete meaning you could do something like:
var billsToFind = follows.length;
for (var i = 0; i < follows.length; i++) {
var followId = follows[i];
Bill.find({user: followId}, function(err, result){
arr = arr.concat(result);
billsToFind -= 1;
if(billsToFind === 0){
res.send(200, arr);
}
});
}
The approach uses a counter for all of the inner async calls (I'm ignoring the findOne because we're currently inside its callback anyway). As each Bill.find call completes it decrements the counter and once it reaches 0 it means that all callbacks have fired (this works since Bill.find is called for every item in the array follows) and it sends back the response with the full array.
That's true. Your codes inside for will be executed in parallel at the same time (and with the same value of i I think). If you added console.log inside and after your for loop you will found the outside one will be printed before inside one.
You can wrap the code that inside your for into array of functions and execute them by using async module (https://www.npmjs.org/package/async) in parallel or series, and retrieve the final result from async.parallel or async.series's last parameter.
Related
I have some code that takes an uploaded file, executes it and gets the output. It then compares this output against expected output to check if the script did as expected.
I am now trying to improve this functionality so that an uploaded file will be run several times, each time being checked against a different expected output, or "test case". I then want to push "correct" or "incorrect" onto a results array, so that I can go through that array at the end and check whether there are any "incorrect" (whether the file failed any test case).
I have tried just callbacks within each function.
I have tried using await and async on the getArray as seen below
Using both callbacks and async together.
This is the parent function code that calls for the array to be created, and wants to iterate through it after it has been created.
var resultsArr = await getResults(file.name, files.length, markerDir);
//file.name is the name from the uploaded file object
//files.length is the number of subdirectories (number of testcases to run against)
//markerDir is the str path to where these testcases are stored
if (resultsArr){
for(var i=0;i<resultsArr.length;i++) {
if (resultsArr[i] == "incorrect"){
checkForMarkerCb('incorrect'); //Calls back to frontend to
break; //display result on web app
}
else if (i+1 == resultsArr.length) {
checkForMarkerCb('correct');
}
}
}
The following is inside the getResults function that is called above
for(var i=1; i<=fileLength; i++) {
var sampleOut = markerDir + '/test' + i + '/stdout.txt';
//Grab expected stdout.txt
var markerOut = fs.readFileSync(sampleOut, 'utf-8', function(err){
if (err){
throw err;
};
});
//Run the file and grab the output
executeFile(filename, function(fileOut){
//Compare output with sample stdout
if (markerOut == fileOut){
resultsArr.push('correct');
}
else {
resultsArr.push('incorrect');
}
});
}
//If results array has a response for each testcase
if (resultsArr.length == fileLength) {
return resultsArr;
}
Implementation of executeFile() as requested:
function executeFile(filename, execFileCb){
//pathToUpload is a str path to where the upload is stored
const child = execFile('python', [pathToUpload], (err,stdout,stderr) => {
if (err) {
throw err;
}
execFileCb(stdout); //Callback with output of file
});
}
function executeFileAsync(filename) {
return new Promise(function(resolve,reject){
executeFile(filename, function(err, data){
if (err !== null) reject(err);
else resolve(data);
});
});
}
which was called inside getResults() using
var fileOut = await executeFileAsync(filename)
The initial function that calls getResults().
getResults(): which gets the path to each directory and calls pushes the results of comparing outputs onto a results array.
executeFile(): uses 'child_process' to run a file and calls back with the output.
I expect the code to wait for getResults to return with the resultsArr so that the for loop can iterate through and check for any "incorrect". Instead, getResults returns before resultsArr is populated.
Using some logging, I see that the code for checking markerOut == fileOut is executed at the end after the getResults() for loop has already completed. I tried setting up the call to executeFile() to also be an async/await similar to how getResults() is called but still no change.
I may not be using async/callbacks correctly, any help is greatly appreciated.
Your executeFileAsync function currently calls executeFile with a callback that is expecting two arguments, but executeFile then does call this execFileCb always with only one argument which is interpreted as an error. It also should not use throw in an asynchronous callback.
Instead, merge them into one function:
function executeFile(filename) {
return new Promise(function(resolve,reject){
//pathToUpload is a str path to where the upload is stored
const child = execFile('python', [pathToUpload], (err,stdout,stderr) => {
if (err) reject(err);
else resolve(stdout); //Callback with output of file
});
});
}
Maybe this is a general issue, and i need a solution to my case : due to the non blocking aspect of javascript , I don't find how can I execute my function with all iteration in for loop , and here is my example ,
var text_list=[]
for (var i = 0; i < 10; i++) {
var element = array[index];
tesseract.process("img"+i+".jpg", options, function (err, text) {
if (err) {
return console.log("An error occured: ", err);
}
text_list.push(text)
});
}
console.log(text_list) //
And the result as if I do :
tesseract.process("img"+9+".jpg"...
tesseract.process("img"+9+".jpg"...
tesseract.process("img"+9+".jpg"...
.
.
.
and what i need is :
tesseract.process("img"+0+".jpg"...
tesseract.process("img"+1+".jpg"...
tesseract.process("img"+2+".jpg"...
.
.
.
Your question does not really explain what result you are getting and your code looks like it's missing parts of the code. So, all I can really do here to help is to explain generically (using your code where possible) how to solve this class of problem.
If you are ending up with a lot of results that all reference the last value of i in your loop, then you are probably trying to reference i in an async callback but because the callback is called sometime later, the for loop has already finished long before the callback executes. Thus, your value of i is sitting on the last value it would have in the for loop. But, your question doesn't actually show code that does that, so this is just a guess based on the limited result you describe. To solve that type of issue, you have make sure you're separately keeping track of i for each iteration of the loop. There are many ways to do that. In ES6, using let in the for loop definition will solve that entire issue for you. One can also construct a closure, use .forEach(), etc...
Async operations with a loop require extra work and coding to deal with. The modern solution is to convert your async operations to use promises and then use features such as Promise.all() to both tell you when all the async operations are done and to keep the results in order for you.
You can also code it manually without promises. Here's a manual version:
const len = 10;
let text_list = new Array(10);
let doneCnt = 0;
let errFlag = false;
// using let here so each invocation of the loop gets its own value of i
for (let i = 0; i < len; i++) {
tesseract.process("img"+i+".jpg", options, function (err, text) {
if (err) {
console.log("An error occured: ", err);
// make sure err is wrapped in error object
// so you can tell errors in text_list array from values
if (!(err instanceof Error)) {
err = new Error(err);
}
text_list[i] = err;
errFlag = true;
} else {
text_list[i] = text;
}
// see if we're done with all the requests
if (++doneCnt === len) {
if (errFlag) {
// deal with situation where there were some errors
} else {
// put code here to process finished text_list array
}
}
});
}
// you can't process results here because async operations are not
// done yet when code here runs
Or, using promises, you can make a "promisified" version of tesseract.process() and then use promise functionality to track multiple async operations:
// make promisified version of tesseract.process()
tesseract.processP = function(img, options) {
return new Promise(function(resolve, reject) {
tesseract.process(img, options, function(err, text) {
if (err) {
reject(err)
} else {
resolve(text);
}
});
});
}
const len = 10;
let promises = [];
for (let i = 0; i < len; i++) {
promises.push(tesseract.processP("img"+i+".jpg", options));
}
Promise.all(promises).then(function(results) {
// process results array here (in order)
}).catch(function(err) {
// handle error here
});
I am accessing the API Trello, but I came across the following problem:
Trello access the information, getting the id of each existing row, the code is as follows:
var x;
var numberCardsByList = [];
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Number of list: " + data.length);
for(var i=0; i<data.length; i++){
x = data[i];
findNumberCards(x);
}
});
As you can see, after getting the size, I walk all these queues with is, within the loop, attach each row in a variable x and call a function that aims to get the number of cards that queue. The code for the number of cards is as follows:
function findNumberCards(x){
trello.get("/1/lists/"+x.id+"/cards", function(err, dados){
if(err) throw err;
console.log("Name List: " + x.name + " have " + dados.length + " cards");
numberCardsByList[x.name] = dados.length;
});
}
Until then all right, but when I try to access the vector numberCardsByList after the end of the search in Trello, it returns undefined:
var x;
var numberCardsByList = [];
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Quantidade de Filas: " + data.length);
for(var i=0; i<data.length; i++){
x = data[i];
findNumberCards(x);
}
});
console.log(numberCardsByList);
I am aware that it is because of asynchrony, however, can not solve.
The problem you're facing has been solved many times before. If you want to know more, search for the keyword "Promise". If you're familiar with jQuery, try and look up: $.whenAll, $.ajax().done, $.ajax().always, etc.
If you want to come up with a light weight solution yourself, here's a pointer:
By the time you get to your console.log(numberCardsByList), your requests triggered by findNumberCards haven't yet completed, making the Array empty. You'll need to make sure you know when all findNumberCards requests have completed and then log them. Alternatively, you could log them every time one of them completes.
There are roughly two approaches:
Keep track of your open requests and call a function when a request is handled.
Observe your numberCardsByList object and call a function whenever items are added (you won't know if they were added async or synchronously)
I'd suggest going with the first approach. Check out this example code and the comments:
var numberCardsByList = {};
// This array will store the url for every open request
var openRequests = [];
var removeRequest = function(url) {
var index = openRequests.indexOf(url);
if (index === -1) return;
// Remove url from array
openRequests = openRequests
.slice(0, index)
.concat(openRequests
.slice(index + 1));
};
// This will be called whenever one request completes
var onComplete = function(url) {
removeRequest(url);
// When all have completed, we can call our callback
if (openRequests.length === 0) {
onAllComplete();
}
});
// This will be called when there are no open requests left
var onAllComplete = function(data) {
console.log(numberCardsByList);
}
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Number of list: " + data.length);
for (var i = 0; i < data.length; i++) {
x = data[i];
findNumberCards(x);
}
});
function findNumberCards(x) {
var url = "/1/lists/" + x.id + "/cards";
// Before we make the request, we register it:
openRequests.push(url);
trello.get(url, function(err, dados) {
numberCardsByList[x.name] = dados.length;
// When it is completed, we call onComplete
onComplete(url);
});
};
Note that this onAllComplete isn't 100% safe: it might be called multiple times if a request finishes before the next one is started.
Concluding:
If you can, I'd use a library to handle promises. If you want to try and build something yourself, you could try and keep track of the requests and execute a callback when they've all completed.
Keep in mind my above code most likely wont work for you as i dont know whats going on in your code so this is an example / explanation how to deal with your problem.
Since you are unfamiliar with async operation i will assume you dont have a prior knowledge of promises and therefore give you a less optimal solution - however promises are alot better and you should defintely learn them.
You need to execute sequence procedures inside the result of the async code.
First you'll create a function for the second operation for example:
function numberCardsByList (param1,param2){.....}
You will then change fineNumberCards to also accept a callback:
function findNumberCards(x, callback){
trello.get("/1/lists/"+x.id+"/cards", function(err, dados){
if(err) throw err;
console.log("Name List: " + x.name + " have " + dados.length + " cards");
numberCardsByList[x.name] = dados.length;
});
// pass in any params you need.
callback();
}
And then you will pass the newly created function numberCardsByList to findNumberCards or wherever you want it.
trello.get("/1/boards/[idBoard]/lists/all", function(err, data) {
if (err) throw err;
console.log("Number of list: " + data.length);
for(var i=0; i<data.length; i++){
x = data[i];
// and here we are calling findNumberCards and passing in the callback..
findNumberCards(x, numberCardsByList);
}
});
That is generally how you will deal with async operation, you will pass a callback for the next operation to be executed.
update
here is an example of how this is done with another scenario just to demonstrate the point farther.
we start by getting user
service.getUser(userName, function(err,user){
if(user) {
// we get user picture passing getPictureSize as callback
getUserPicture(user.picture, getPictureSize)
}
})
we get the pictureURL
function getUserPicture(picName, cb){
service.getPictureURL(picName, function(err, pictureURL){
if(pictureURL) {
// we then call the callback - the next async operation we want.
cb(pictureURL);
}
});
}
we get picture size - which is the last operation
function getPictureSize(pictureURL){
service.getPictureSize(pictureURL, function(err, pictureSize){
$('.picName').attr('src', picName);
$('.picName').width(pictureSize.width);
$('.picName').height(pictureSize.height);
});
}
I hope that clarify things a little.
I'm trying to use Node.js to get a response from an API, I want to clean the API response and use the result.
So to access the first API I have the following code.
To store and use the result I believe I need to store the JSON output globally.
However, I can't work out how to do this.
Example -
var request = require('request');
request({url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true}, function(err, res, json) {
if (err) {
throw err;
}
car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
console.log(car2go);
});
This prints the desired result however I know that this is because my variable is defined within the function.
I want to access the variable outside of the function.
To test if I could do this I changed the code to -
var request = require('request');
request({url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true}, function(err, res, json) {
if (err) {
throw err;
}
car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
});
console.log(car2go);
But if I do this I get
ReferenceError: car2go is not defined
I am running Node v0.12.2 on Mac OS Yosemite (10.10.3).
Admittedly I am very new to node and I am more familiar with R, Python and PL SQL.
There is no way to get reference to it outside of the callback function because the console.log line runs before the callback function is invoked. The reason you have to pass a callback function into the request API is because the request library needs to invoke that function when it's done making the request. Meanwhile, your app moves on and does other things (such as running that console.log line) while it waits for the callback function to fire.
That said, there are a number of ways to deal with asynchronous code. My favorite way is with promises. I use a library called bluebird for handling promises.
var request = require('request');
var Promise = require('bluebird');
var requestP = Promise.promisify(request);
The call to Promise.promisify(request) returns a new function that doesn't take a callback function, but instead returns a promise.
requestP({ url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true })
.spread(function(res, json) {
var car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
})
.then(function (car2go) {
console.log(car2go);
})
.catch(function (err) {
console.error(err);
});
Note: .spread is the same as .then except if the resolved value is an array (which it will be because the callback passed to the request library accepts 2 arguments, which bluebird will translate into an array that the promise resolves to) .spread will split up the array back into multiple arguments passed into the function you give to .spread.
Promise.resolve(['hi', 'there']).then(function (result) {
console.log(result); // "['hi', 'there']"
});
Promise.resolve(['hi', 'there']).spread(function (str1, str2) {
console.log(str1); // 'hi'
console.log(str2); // 'there'
});
You're not going to be able to return that value all the way back out to the same context from which you began the asynchronous call, but you can at least write code that looks somewhat synchronous when using promises.
Without promises you'll be forced to call functions from within functions from within functions from within functions ;)
The response is asynchronous. That means the callback function gets called sometime LATER in the future so your console.log(car2go) is executing BEFORE the callback has even been called.
The only place you can reliably use the response is inside the callback or in a function called from the callback. You cannot use it the way you are trying to. Using asynchronous responses in Javascript requires programming in an asynchronous fashion which means processing results and using results IN the asynchronous callbacks only.
Here's where the console.log() should be:
var request = require('request');
request({url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true}, function (err, res, json) {
if (err) {
throw err;
}
car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
// here is where the result is available
console.log(car2go);
});
I want to query some data from the database, from a for loop. The problem is, the query is processed after the end of the loop.
This code:
ret = [];
first = rows[0];
first.device_firsttime = first.device_lasttime;
first.alerts = [];
for(var i = 0; i < 5; i++)
{
console.log("!"+i);
(function(i) {
query("SELECT EXTRACT('epoch' FROM alert_time)::integer alert_time, alert_id, alert_lat, alert_lon, alert_str, alert_cc, alert_distance FROM blitz_device_former_alerts WHERE alert_locid = $1", [first.device_locid], function(error_a,rows_a,result_a)
{
console.log(i+"!");
ret.push(i);
});
})(i);
}
console.log("-END---"+JSON.stringify(ret));
ret.push(first);
res.end(JSON.stringify(ret));
Writing this to the console:
!0
!1
!2
!3
!4
-END---[]
POST /userlogs 200 140.110 ms - -
0!
1!
2!
3!
4!
The correct output should be
!0
0!
!1
1!
!2
2!
!3
3!
!4
4!
-END---[0,1,2,3,4]
POST /userlogs 200 xxx.xxx ms - -
The query is asynchronous. The loop doesn't wait for those queries to finish. It's easier to use some flow control library such as 'async'.
var async = require('async');
async.eachSeries([0, 1, 2, 3, 4], function(i, callback) {
console.log("!"+i);
query("SELECT EXTRACT('epoch' FROM alert_time)::integer alert_time, alert_id, alert_lat, alert_lon, alert_str, alert_cc, alert_distance FROM blitz_device_former_alerts WHERE alert_locid = $1", [first.device_locid], function(error_a,rows_a,result_a) {
console.log(i+"!");
ret.push(i);
callback(null); // null -> no error
});
}, function(err) {
console.log("-END---"+JSON.stringify(ret));
ret.push(first);
res.end(JSON.stringify(ret));
});
You may want to use the async library for that job: essentially it will run your functions in sequence. It is also a good practice to use named functions, in this case a second-order function getExtractor(): a function that returns another function, usable by async. async likes to have functions that accept callbacks, which is where the result of each execution is added.
The code would be something like this:
var async = require('async');
first = rows[0];
first.device_firsttime = first.device_lasttime;
first.alerts = [];
var tasks = [];
for(var i = 0; i < 5; i++)
{
tasks.push(getExtractor(i));
}
async.series(function(result) {
console.log("-END---"+JSON.stringify(result));
result.push(first);
res.end(JSON.stringify(result));
});
function getExtractor(i) {
return function(callback) {
console.log("!"+i);
query("SELECT EXTRACT('epoch' FROM alert_time)::integer alert_time, alert_id, alert_lat, alert_lon, alert_str, alert_cc, alert_distance FROM blitz_device_former_alerts WHERE alert_locid = $1", [first.device_locid], function(error_a,rows_a,result_a)
{
console.log(i+"!");
return callback(null, i);
});
});
}
A few notes:
Each function pushed to the tasks array accepts a callback of the form function(error, result). The first parameter is an optional error.
I have not dealt with errors in the code, for simplicity, but it is important to do it.
If for any reason you want to run all functions in parallel you can do it just using async.parallel().
The final bit (when all pieces have been gathered) is executed in the callback passed to async.series().
When async receives an array of functions, it sends to its callback an array of results. That is why we get the result automatically constructed by async.
You can do the same without async but frankly, the code is quite complex and I don't recommend it.