Develop Reference

sails.js node.js Parse JSON on controller

In my controller called MapController I'm doing a function to do a parse of remote json files, and from an if-else structure add some values in an array called "parsewebservice", apparently everything is working fine but console.log ( parsewebservice); is not returning the values that were passed to the array "parsewebservice" in the place where it is returning it empty. But when I put it inside the forEach it returns, but everything cluttered and repeated then is not the right way.
I wanted to know why the values that were passed to the array "parsewebservice" are not going along with the variable after populada and what would be the correct way to do it?
Here is my code below:
/**
* MapController
*
* #description :: Server-side logic for managing Maps
* #help :: See http://sailsjs.org/#!/documentation/concepts/Controllers
*/
module.exports = {
index: function(req, res, next) {
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
datas.forEach(function(data, index) {
var req = require("request");
var url = data.address + "?f=pjson";
req(url, function(err, res, retorno) {
if (err) {
console.log(err);
} else {
var camadas = JSON.parse(retorno);
if (camadas.mapName) {
camadas.layers.forEach(function(campo, i) {
if (campo.subLayerIds != null) {
} else if (campo.subLayerIds == null) {
parsewebservice.push([i, "dynamicMapLayer", campo.name, data.address]);
}
});
} else if (camadas.serviceDataType) {
parsewebservice.push([null, "imageMapLayer", camadas.name, data.address]);
} else if (camadas.type) {
parsewebservice.push([null, "featureLayer", camadas.name, data.address]);
}
}
});
});
console.log(parsewebservice);
});
},
};

My first comment has to be that you should not combine function(req, res) with var req = require('request')... you lose your access to the original req object!
So, you need to run a list of async tasks, and do something when they are all complete. That will never be entirely easy, and no matter what, you will have to get used to the idea that your code does not run from top to bottom as you've written it. Your console.log at the bottom runs before any of the callbacks (functions you pass in) you pass to your external requests.
The right way to do this is to use promises. It looks like you are using this request library, whose returned requests can only accept callbacks, not be returned as promises. You can create your own promise wrapper for them, or use an alternative library (several are recommended on the page).
I don't want to write a whole intro-to-promises right here, so what I will do is give you a less pretty, but maybe more understandable way to run some code at the completion of all your requests.
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
// here we will write some code that we will run once per returned data
var processResponse = function(resp) {
parsewebservice.push(resp);
if(parsewebservice.length >= datas.length) {
// we are done, that was the final request
console.log(parsewebservice);
return res.send({data: parsewebservice)}); // or whatever
}
};
datas.forEach(function(data, index) {
var request = require("request");
var url = data.address + "?f=pjson";
request(url, function(err, res, retorno) {
// do some processing of retorno...
// call our function to handle the result
processResponse(retorno);
});
});
console.log(parsewebservice); // still an empty array here
});

I solved the problem.
the "request" module is asynchronous so we need to wait for it to respond and then send the response to the view.
To do this we created a function called "foo" to contain the foreach and the request, we made a callback of that function and finally we made the response (res.view) within that function, so that the controller response would only be sent after the response of the "foo" function to the callback. So we were able to parse.json the data from the "data" collection using foreach and the "request" module and send the objects to the view.
Many thanks to all who have helped me, my sincere thanks.

NodeJS Variable Collision? Async / Sync with Request

Issue : Data INFO_X sometimes and randomly become Null.
Question ,
Does the variable overwrites each other for INFO_1 INFO_2 INFO_3 since Nodejs run fast unlike PHP it follows a sequence/step by step.
I checked for NULLS before doing a request but the debug shows it's NOT NULL before executing the 2nd request, at random, random variables will become null upon submitting the 2nd request.
I also checked my source is definitely not returning any null.
Is the variable being overwritten before the 2nd request is sent or what? Please advice.
var request = require('request');
var urls = [ 'URL',
'URL',
'URL'];
urls.forEach(processUrl);
function processUrl(url) {
request(url, function (error, response, body) {
if (!error) {
var obj = JSON.parse(body);
for (var i = 0, len = obj['products'].length; i < len; ++i) {
var data = obj['products'][i];
var INFO_1 = data.INFO_1
var INFO_2 = data.INFO_2
var INFO_3 = data.INFO_3
request("URL/POSTINFO?INFO_1="+INFO_1+"&INFO_2="+INFO_2+"&INFO_3="+INFO_3+"&seller_id=", function(error, response, body) {
console.log(body);
});
}
}
});
}

Yes, that is the case, because request function is asynchronous. I wouldn't call nodejs "faster" than PHP, it just runs asynchronous request methods, while PHP is generally synchronous.
You could resolve the issue with promises, e.g. Promise.all([]) and provide an array of request functions (see here: https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Promise/all), or in your case, use this library https://github.com/caolan/async#forEach
Since you're using callbacks in the request function, your best option is to use async as provided in the link above. Here's an example code:
function request(url, cb) {
setTimeout(() => {
cb(url + ' accessed at ' + new Date());
}, 2000);
}
var urls = ['URL1', 'URL2', 'URL3'];
async.each(urls, (item) => {
console.log(item);
request(item, (value) => {
request(value, (newValue)=>{
console.log(newValue);
});
});
}, (err) => {
console.log(err);
});
Here's a working example: https://plnkr.co/edit/Q5RAvKdaLxV9cUT4GP4w?p=preview

Return object from within Callback function

This question might sound duplicate of the one here but my scenario is quiet different. I have getSystemIds.js file like this:
var system_ids_list = {};
var getSystemIDs = function(req, callback) {
var client = //creating an object using internally developed node library
client.request('sql_parameter', {'Query.ID' : query_number},
function(err, req, res){
//Do some stuff to calculate necessary values
system_ids_list[key_value] = value_array;
i+= 1;
}
return(req, callback)
}
)
};
module.exports = getSystemIDs;
Now, as shown in the answer in the link above, I am doing this in app.js
appSSL.get('/sysids', function(req, res) {
var sys_ids = system_ids_list(req, function(err, sys_ids) {
res.render('sysids', sys_ids);
})
});
I am not getting any specific error but the web page never loads as if something is stuck in the process or it does not know where to go next. Can someone help me figure out what would be the best way to do this?

Your getSystemIds() function is never calling the callback that was passed to it so the caller of getSystemIds() never gets a result - thus nothing ever happens on the request.
Change it to this:
var system_ids_list = {};
var getSystemIDs = function (req, callback) {
var client = //creating an object using internally developed node library
client.request('sql_parameter', {'Query.ID': query_number}, function (err, req, res) {
//Do some stuff to calculate necessary values
system_ids_list[key_value] = value_array;
i += 1;
// call the callback now to communicate back the async results
callback(null, system_ids_list);
});
};
module.exports = getSystemIDs;
The way you have your code structured, system_ids_list will accumulate more and more values each time getSystemIDs() is called. That seems a bit of an odd way to structure things so I'm pointing that out in case that is not really what you intend.
Also, your getSystemIDs() function does not return anything so you should change this:
appSSL.get('/sysids', function(req, res) {
var sys_ids = system_ids_list(req, function(err, sys_ids) {
res.render('sysids', sys_ids);
});
});
to this to make it less missleading about what is going on:
appSSL.get('/sysids', function(req, res) {
system_ids_list(req, function(err, sys_ids) {
res.render('sysids', sys_ids);
});
});
And, if res.render() is from a system like ExpressJS, then you probably want to be passing an object and naming a template:
res.render('sometemplate.html', {sysids: sys_ids});
If you want system_ids_list to not accumulate values, but to return a fresh value each time, you can define it within your function like this:
var getSystemIDs = function (req, callback) {
var system_ids_list = {};
var client = //creating an object using internally developed node library
client.request('sql_parameter', {'Query.ID': query_number}, function (err, req, res) {
//Do some stuff to calculate necessary values
system_ids_list[key_value] = value_array;
i += 1;
// call the callback now to communicate back the async results
callback(null, system_ids_list);
});
};
module.exports = getSystemIDs;

Returning values from Promise(Bluebird) in Node.js

I am learning Node.js and was looking at the Promise module. I am very new at this so please bare with bad code conventions and stuff. I am trying to build a REST Client API using restify. I have attached my Client Code below:
Here is my Client:
// local functions
function loadResult(err, req, res, obj) {
//console.log(err, req, res, obj);
var result = {
'err': err,
'text': obj
};
console.log(result);
return result;
}
function getClient() {
if (client) {
return client;
} else {
client = restify.createJsonClient(options);
return client;
}
};
function RestClient(headers) {
if (headers) {
global_headers.Authorization = headers.Authorization || global_headers.Authorization;
options.headers = global_headers;
}
};
RestClient.prototype.getClient = getClient;
RestClient.prototype.doGET = function doGET(path) {
return new Promise(function(resolve) {
client.get(path, loadResult);
}).then(function(result) {
console.log("before final return:" + result);
return result;
});
};
// exports
module.exports = RestClient;
In the console when I run the following lines:
var Rest = require('./testRest.js');
var restClient = new Rest();
var client = restClient.getClient();
var data = restClient.doGET('/hello/mark');
I can see the result printed just once on the console. But the Data is just what I believe a Promise. Is there a way for me to get the result in data? Also trying to figure out why the result never got printed the second time.
P.S.: I tried using Promise.promisifyAll(require('restify')) but it kept throwing some error saying fn is not a function which force use Promise in this way.

First of all, there is no such error when I do:
var Promise = require("bluebird");
Promise.promisifyAll(require("restify"));
Secondly, restify unfortunately uses some weird getters when exporting the module, as getters can (and do) have side effects, promisifyAll will not visit them, so the one line magic won't work. You need to promisify the classes manually, e.g.
Promise.promisifyAll(require("restify").JsonClient.prototype);
Then you would call:
client.getAsync(path).spread(function(req, res, obj) {
});

Can someone explain this : EventEmitter causes failure when global variable

I spent a while trying to diagnose this error.
First I had created a subclass of EventEmitter
File Client.js
var bindToProcess = function(fct) {
if (fct && process.domain) {
return process.domain.bind(fct)
}
return fct
};
function Client(){
EventEmitter.call(this);
}
util.inherits(Client, EventEmitter);
Client.prototype.success =
function(fct) {
this.on('success', bindToProcess(fct))
return this;
}
Client.prototype.login = function(username, password) {
body = {
username : username,
password : password
};
var self = this;
request.post(url, { json:true, body: body }, function (error, response, body) {
if (error ||response.statusCode != HTTPStatus.OK ) {
return self.emit('error', error);
}
return self.emit('success', body);
});
return this;
}
module.exports = Client
Then in another file in my Express App
File user.js
var Client = require('../utils/client');
var client = new Client();
// GET '/login'
exports.login = function(req, res){
client.login(username, password).success( function(user) {
res.redirect('/');
}).error( function(error) {
res.redirect('login');
});
}
The thing is though on the second request, the server crashes with the error:
Error: Can't set headers after they are sent.
In the interim I've solved the problem by created the Client inside the middleware rather than having it a global variable. I'm just curious why this is happening ?
Thanks,
(hopefully there is enough information)

What happens here is the call of the event handling function from the first request during second request because the variable client is shared between the requests.
At first, the client is created in the global scope. Then two handlers are attached to its events and then the request is actually performed and corresponding handler is called.
On the second request, two more handlers are attached to the same object and then on either success or fail two handlers (from previous call and from the current) are notified.
So you need to move the client creation to the action method or change how the code responds on the events - I can suggest promises-like approach: pass two callbacks as parameters to one method; or just the standard callback approach: pass the error result as first argument of the callback.