Develop Reference

sails.js node.js Parse JSON on controller

In my controller called MapController I'm doing a function to do a parse of remote json files, and from an if-else structure add some values in an array called "parsewebservice", apparently everything is working fine but console.log ( parsewebservice); is not returning the values that were passed to the array "parsewebservice" in the place where it is returning it empty. But when I put it inside the forEach it returns, but everything cluttered and repeated then is not the right way.
I wanted to know why the values that were passed to the array "parsewebservice" are not going along with the variable after populada and what would be the correct way to do it?
Here is my code below:
/**
* MapController
*
* #description :: Server-side logic for managing Maps
* #help :: See http://sailsjs.org/#!/documentation/concepts/Controllers
*/
module.exports = {
index: function(req, res, next) {
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
datas.forEach(function(data, index) {
var req = require("request");
var url = data.address + "?f=pjson";
req(url, function(err, res, retorno) {
if (err) {
console.log(err);
} else {
var camadas = JSON.parse(retorno);
if (camadas.mapName) {
camadas.layers.forEach(function(campo, i) {
if (campo.subLayerIds != null) {
} else if (campo.subLayerIds == null) {
parsewebservice.push([i, "dynamicMapLayer", campo.name, data.address]);
}
});
} else if (camadas.serviceDataType) {
parsewebservice.push([null, "imageMapLayer", camadas.name, data.address]);
} else if (camadas.type) {
parsewebservice.push([null, "featureLayer", camadas.name, data.address]);
}
}
});
});
console.log(parsewebservice);
});
},
};

My first comment has to be that you should not combine function(req, res) with var req = require('request')... you lose your access to the original req object!
So, you need to run a list of async tasks, and do something when they are all complete. That will never be entirely easy, and no matter what, you will have to get used to the idea that your code does not run from top to bottom as you've written it. Your console.log at the bottom runs before any of the callbacks (functions you pass in) you pass to your external requests.
The right way to do this is to use promises. It looks like you are using this request library, whose returned requests can only accept callbacks, not be returned as promises. You can create your own promise wrapper for them, or use an alternative library (several are recommended on the page).
I don't want to write a whole intro-to-promises right here, so what I will do is give you a less pretty, but maybe more understandable way to run some code at the completion of all your requests.
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
// here we will write some code that we will run once per returned data
var processResponse = function(resp) {
parsewebservice.push(resp);
if(parsewebservice.length >= datas.length) {
// we are done, that was the final request
console.log(parsewebservice);
return res.send({data: parsewebservice)}); // or whatever
}
};
datas.forEach(function(data, index) {
var request = require("request");
var url = data.address + "?f=pjson";
request(url, function(err, res, retorno) {
// do some processing of retorno...
// call our function to handle the result
processResponse(retorno);
});
});
console.log(parsewebservice); // still an empty array here
});

I solved the problem.
the "request" module is asynchronous so we need to wait for it to respond and then send the response to the view.
To do this we created a function called "foo" to contain the foreach and the request, we made a callback of that function and finally we made the response (res.view) within that function, so that the controller response would only be sent after the response of the "foo" function to the callback. So we were able to parse.json the data from the "data" collection using foreach and the "request" module and send the objects to the view.
Many thanks to all who have helped me, my sincere thanks.

NodeJS creating a callback inside a request function

I'm currently scraping data from a webpage and then pushing it to an array, the code currently looks like this
url = //url
let data = [];
request(url, function (err, response, html) {
if (!err) {
var $ = cheerio.load(html);
$('#id').each(function (i, element) {
data_element = //(this).find...
data.push(data_element);
});
}
console.log(data); //console logs the data inside the request
})
console.log(data); //logs empty array outside of request
The data is logged when I call console log inside the request, but if I call it outside of the request function then it returns an empty array. I know I need to use a callback function but I was wondering what the best way to go about this is, as I will be making multiple requests inside my function.

You should use promises instead of callback functions.
A good promise library is Bluebird
You can find an example of using Promise instead of callback here
Each function should return a promise. You need to "wait" for all promises to finish by using Promise.all (link to documentation).
Then, you can write it all to your log
Here's an example:
const rp = require('request-promise'); // A package for Request with Bluebird promises
const Promise = require('bluebird');
const url = //url
let data = [];
const options = {
uri: url
};
const p1 = rp(options).then((response) => {
var $ = cheerio.load(response.body);
$('#id').each(function (i, element) {
data_element = //(this).find...
data.push(data_element);
});
console.log(data); //console logs the data inside the request
return data; // this data will be available on results parameter on Promise.all
});
const p2 = // another request
Promise.all([p1, p2]).then((results) => {
console.log(results) // print whatever you want
})

I think its because "let" type of variable stops it. change it to var and it should be working.

How to return from a looped asynchronous function with callback in Node

I am trying to write a function that:
Takes an array of URLs
Gets files from URLs in parallel (order's irrelevant)
Processes each file
Returns an object with the processed files
Furthermore, I don't need for errors in #2 or #3 to affect the rest of the execution in my application in any way - the app could continue even if all the requests or processing failed.
I know how to fire all the requests in a loop, then once I have all the data, fire the callback to process the files, by using this insertCollection pattern.
However, this is not efficient, as I shouldn't need to wait for ALL files to download before attempting to process them - I would like to process them as each download finishes.
So far I have this code:
const request = require('request');
const urlArray = [urlA, urlB, urlC];
const results = {};
let count = 0;
let processedResult;
const makeRequests = function (urls, callback) {
for (let url of urls) {
request(url, function(error, response, body) {
if (error) {
callback(error);
return;
}
processedResult = callback(null, body)
if (processedResult) {
console.log(processedResult); // prints correctly!
return processedResult;
}
})
}
};
const processResult = function(error, file) {
if (error) {
console.log(error);
results.errors.push(error);
}
const processedFile = file + `<!-- Hello, Dolly! ${count}-->`;
results.processedFiles.push(processedFile);
if (++count === urlArray.length) {
return results;
}
};
const finalResult = makeRequests(urlArray, processResult);
console.log(finalResult); // undefined;
In the last call to processResult I manage to send a return, and makeRequests captures it, but I'm failing to "reign it in" in finalResult after that.
My questions are:
Why is this not working? I can print a well-formed processedResult
on the last iteration of makeRequests, but somehow I cannot return
it back to the caller (finalResult)
How can this be solved, ideally "by hand", without promises or the
help of libraries like async?

The makeRequests function returns undefined to finalResult because that is a synchronous function. Nothing stops the code executing, so it gets to the end of the function and, because there is no defined return statement, it returns undefined as default.

Where is future necessary to run a sequence of async functions in node?

Previously, I have used future.wait() to wait for a value before I return it to the next function. (I am coding in Meteor by the way)
After reading node's non-blocking I/O architecture, doesn't my following method (which does work) defeat the purpose entirely?
What I am doing is passing the returned result from the following 'requesting' function into another function. Instead, is the callback method the best convention?
The only reason why I used future was because using the return gets executed immediately. I want to learn the best practice as I think I am adding extra layers of unnecessary code using future.
If I use the callback convention, callback(null, result), does 'callback' wait for objects to be passed into the arguments?
My code using future to wait for results:
function requesting(perm_data) {
var f = new future();
request.get('example.com/api/auth_user', {
oauth: {consumer_key:'somekey',
token: perm_data.oauth_token,
token_secret: perm_data.oauth_token_secret
}
}, function response(error, response, body) {
if (!error && response.statusCode === 200) {
var bodyToJson = parser.toJson(body, options)
var userId = bodyToJson.user.id
return f.return(userId)
}
else {
f.throw(error)
}
})
return f.wait()
}
function nextFunction(data) {
//do some thing with data
}
Meteor.methods({
sequenceAsyncFunctions: function() {
try {
var resultOne = requesting()
var resultTwo = nextFuntion(resultOne)
} catch (e) {
//handling my errors
}
}
})

'callback' wait for objects to be passed into the arguments
in fact the callback does not wait for the object. The callback is called with the argument by the function you give it has argument.
You could use one of the promise library such as Q to handle this
Here is some article describing it in more depth
https://strongloop.com/strongblog/promises-in-node-js-with-q-an-alternative-to-callbacks/
Nodeschool tutorial: https://github.com/stevekane/promise-it-wont-hurt
Some alternatives are Kew and Async.
Promises have been specified, you can see more details and library here: http://wiki.commonjs.org/wiki/Promises/A

I noticed you tagged this question meteor. Meteor uses Fibers on the server. This package gives you co-operative multitasking using a sort of lightweight thread (called a fiber). This allows you to write "blocking" functions like your requesting without actually blocking the event loop. Future lets you use callback-based node.js functions in a blocking way using Fibers.
Without Fibers, it is impossible to write a function like your requesting which does I/O and returns the result directly. Instead you'd have to pass a callback (or use promises).
Idiomatic Meteor code should make use of Fibers on the server to make "blocking" functions. This is how the core Meteor APIs work, and is necessary inside Meteor methods. Rather than using future, you might find Meteor.wrapAsync more convenient.
In regular node.js code you could use Fibers, but it's more common to use function (err, result) {...} callbacks or promises. In the browser (including Meteor client code) you can't use Fibers, so you have to use callbacks or promises.
Here's how you would sequence operations using future, regular callbacks and promises:
// Future
function doTwoThings() {
var future = new Future();
asyncFunction(argument, function (err, result) {
if (err) future.throw(err);
else future.return(result);
});
var result = future.wait();
var future2 = new Future();
anotherAsyncFunction(result, function (err, result2) {
if (err) future2.throw(err);
else future2.return(result2);
});
var result2 = future2.wait();
return result2 + 2;
}
// wrapAsync
var wrappedAsyncFunction = Meteor.wrapAsync(asyncFunction);
var wrappedAnotherAsyncFunction = Meteor.wrapAsync(anotherAsyncFunction);
function doTwoThings() {
var result = wrappedAsyncFunction(argument);
var result2 = wrappedAnotherAsyncFunction(anotherAsyncFunction);
return result2 + 2;
}
// Regular callbacks
function doTwoThings(callback) {
asyncFunction(argument, function (err, result) {
if (err) return callback(err);
anotherAsyncFunction(result, function (err, result2) {
if (err) return callback(err);
callback(null, result2 + 2);
});
});
}
// Promises
function doTwoThings() {
return asyncFunctionReturningPromise(argument).then(function (result) {
return anotherAsyncFunctionReturningPromise(result);
}).then(function (result2) {
return result2 + 2;
});
}

node.js nested callback, get final results array

I am doing a for loop to find the result from mongodb, and concat the array. But I am not getting the final results array when the loop is finished. I am new to node.js, and I think it's not working like objective-c callback.
app.get('/users/self/feed', function(req, res){
var query = Bill.find({user: req.param('userId')});
query.sort('-createdAt');
query.exec(function(err, bill){
if (bill) {
var arr = bill;
Following.findOne({user: req.param('userId')}, function(err, follow){
if (follow) {
var follows = follow.following; //this is a array of user ids
for (var i = 0; i < follows.length; i++) {
var followId = follows[i];
Bill.find({user: followId}, function(err, result){
arr = arr.concat(result);
// res.send(200, arr);// this is working.
});
}
} else {
res.send(400, err);
}
});
res.send(200, arr); //if put here, i am not getting the final results
} else {
res.send(400, err);
}
})
});

While I'm not entirely familiar with MongoDB, a quick reading of their documentation shows that they provide an asynchronous Node.js interface.
That said, both the findOne and find operations start, but don't necessarily complete by the time you reach res.send(200, arr) meaning arr will still be empty.
Instead, you should send your response back once all asynchronous calls complete meaning you could do something like:
var billsToFind = follows.length;
for (var i = 0; i < follows.length; i++) {
var followId = follows[i];
Bill.find({user: followId}, function(err, result){
arr = arr.concat(result);
billsToFind -= 1;
if(billsToFind === 0){
res.send(200, arr);
}
});
}
The approach uses a counter for all of the inner async calls (I'm ignoring the findOne because we're currently inside its callback anyway). As each Bill.find call completes it decrements the counter and once it reaches 0 it means that all callbacks have fired (this works since Bill.find is called for every item in the array follows) and it sends back the response with the full array.

That's true. Your codes inside for will be executed in parallel at the same time (and with the same value of i I think). If you added console.log inside and after your for loop you will found the outside one will be printed before inside one.
You can wrap the code that inside your for into array of functions and execute them by using async module (https://www.npmjs.org/package/async) in parallel or series, and retrieve the final result from async.parallel or async.series's last parameter.