i have an application which needs a data.json file in order to draw a d3-graph. However i need to update that file on an onClick-Event:
d3.select("#updatebutton").on("click", function(e) {
try{
$.get('https://localhost:4444/data', function(data) {
});
}
catch (e) {
alert('Error: ' + e);
}
});
Above is the update-Button with the jquery-call. In my app.js File I am using it like this:
app.get('/data', function(req, res, next) {
try{
getJSON();
}
catch(e) {
alert('Error');
}
});
The getJSON()-Function is received Data over an https-Request, processes that data and saves it to data.json:
function getJSON() {
var req = https.get(options, function(response) {
// handle the response
var res_data = '';
response.on('data', function(chunk) {
res_data += chunk;
});
response.on('end', function() {
//process data
// save to file
fs.writeFile(filePath, JSON.stringify(finalJson), function(err) {
if (err)
throw err;
});
});
});
}
However if i click on my updateButton repeatedly after seconds, it seems that data.json is not overwritten but the file gets bigger and bigger, means that data is added to the file instead of overwritten.
What am I doing wrong?
Thanks for help.
Since you use app.get as your route, I guess you are using express.
In your routes definition:
var getData = (function() {
var callbacks = [];
function executeCallbacks(err, data) {
for (var i = 0; i < callbacks.length; i++) {
callbacks[i](err, data);
}
callbacks = [];
}
return function(cb) {
callbacks.push(cb);
if( callbacks.length === 1 ) {
var req = https.get(options, function(response) {
// handle the response
var res_data = '';
response.on('data', function(chunk) {
res_data += chunk;
});
response.once('end', function() {
// process data here
// save to file
fs.writeFile(filePath, JSON.stringify(finalJson), function(err) {
if (err) {
// call error handler
return executeCallbacks(err);
}
executeCallbacks(null, body);
});
});
response.once('error', function() {
return executeCallbacks(err);
});
}
req.end();
}
};
})();
app.get('/data', function(req, res, next) {
getData(function(err, data) {
if(err) {
return next(err);
}
return data;
});
});
In your browser js file:
d3.select("#updatebutton").on("click", function(e) {
$.get( 'https://localhost:4444/data', function(data) {
alert( "success" );
var json = JSON.parse(data);
})
.fail(function() {
alert( "error" );
});
});
I see you use try / catch around callback functions. The callback function fires after the original function completes. So don't use Try / Catch around callback function.
Read: https://strongloop.com/strongblog/async-error-handling-expressjs-es7-promises-generators/
Related
My node js app is making multiple http request if there is delay in response of say 20 secs. Below is the sample code for the same.
First I make call to the getAPI function from browser. getApi function calls the getAccessToken API, after receiving the accesstoken I am calling the testApi. Now if there is a delay of 20 secs in response from testApi then getAccessToken Api is getting called again. I don't want to use promise. Can anyone point out what I am missing or doing wrong here?
shttp = require('http-https');
exports.getAPI = function(typeObj, request, response, callback) {
var userConf; //contains info such as port, host, url etc
_this.getAccessToken(function(tokenResponse) {
var tokenInfo = JSON.parse(tokenResponse);
var accessToken = JSON.parse(tokenInfo.response);
accessToken = accessToken.access_token;
if(accessToken) {
_this.testApi(userConf,accessToken,function(sjmResponse) {
callback(sjmResponse);
}
} else {
callback(JSON.stringify({"payLoad":null,"reasonCode":"fail","status":null}));
}
});
};
exports.getAccessToken = function(cb) {
var tokenConf; //contains info such as port, host, url etc
var httpReq = shttp.request(tokenConf, function(res) {
res.setEncoding('utf8');
if (res.statusCode == 200) {
var body = "";
res.on('data', function (result) {
body += result;
});
res.on('end', function (){
cb(JSON.stringify({error: '', response: (body)}));
});
} else {
cb(JSON.stringify({error: 'Failed to get user access token '+res.statusCode, response:''}));
}
});
httpReq.on('error', function(e) {
cb(JSON.stringify({error: 'Failed to get user access token'+e, response:''}));
});
httpReq.end();
};
exports.testApi = function(userConf,accessToken,sjmCallback) {
var userConf; //contains info such as port, host, url etc
var httpSubmitReq = shttp.request(userConf, function(res) {
res.setEncoding('utf8');
if (res.statusCode == 200) {
var body = "";
res.on('data', function (result) {
body += result;
});
res.on('end', function () {
sjmCallback(body);
});
} else {
sjmCallback(JSON.stringify({"payLoad":null,"reasonCode":"fail","status":null}));
}
});
httpSubmitReq.on('error', function(e) {
sjmCallback(JSON.stringify({"payLoad":null,"reasonCode":"fail","status":null}));
});
httpSubmitReq.end();
};
app.get('/testApi', function (req, res) {
var typeObj = {};
typeObj.apiType= 'testApi';
try {
getAPI(JSON.stringify(typeObj), req, res, function(response) {
res.end(response);
});
} catch(err) {
res.end(err);
}
});
I am trying to use a callback to indicate when all the async workers are complete, but I am getting the dreaded
TypeError: callback is not a function.
I would like to individually process each element in data, and on completion, have queue.drain to send the callback(data) to refresh Data on completion. I have been readying the async documentation, but clearly i am not getting something.
function refreshData(postData, callback) {
var options = {
host: 'www.myhost.com',
port: 443,
path: '/pulldata,
method: 'POST',
headers: {
"Content-Type": "application/json"
}
};
var req = https.request(options, function(res) {
var headers = res.headers
var d = '';
res.setEncoding('utf8');
res.on('data', function (chunk) {
d = d + chunk;
});
res.on('end', function() {
if (res.statusCode == '200') {
data = JSON.parse(d);
queue = async.queue(function (task, cb) {
processData(task,cb);
},1);
//this is were the errors are
queue.drain = function() {
callback(data)
};
for(i=0; i<data.length; i++) {
queue.push(data[i],'');
}
} else {
callback(false)
}
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write(postData);
req.end();
}
Any assistance would be greatly appreciated!
Edit, added some pseudo code to demonstrate how refreshData is being used:
Node https.createServer(req,res) {
req.on(){
read userData
}
req.end(){
validateUser(userData, function(callbackData) {
if(callbackData==false) {
//bad user or error with request
res.writeHead(404);
res.end('bye');
} else {
//good user and responses
res.writeHead(200);
res.end(callbackData);
}
})
}
}
function validateUser(userData,callback) {
//do some stuff to validate
if(userData is good) {
//call refreshData if user
refreshData(userData,callback)
} else {
callback(false)
}
}
[EDIT] Added a callback
As given in the documentation you pointed to , change this line
queue.push(data[i],'');
to
queue.push(data[i], function(err){
// handle error
});
Try it here async-queue-callback
First Node/Express app.
I'm having a hard time wrapping my head around on how to retrieve data from an endpoint and rendering it in the browser.
I have a dataservice.js that gets a JSON object from an endpoint like this:
const http = require('http');
getFinhockeyData = function() {
http.get('http://tilastopalvelu.fi/ih/modules/mod_standings/helper/standings.php?statgroupid=3545', (res) => {
console.log(`Got response: ${res.statusCode}`);
var body = "";
res.on('data', function (chunk) {
body += chunk;
})
res.on('end', function () {
var data = JSON.parse(body);
console.log('data parsed.');
console.log('first team name: ' + data.teams[0].TeamName);
console.log(typeof data);
return data;
})
}).on('error', (e) => {
console.log(`Got error from Finhockey: ${e.message}`);
});
}
module.exports.getFinhockeyData = getFinhockeyData;
Up until now things work and the data object can be console.logged and its content is usable.
The router.js looks currently like this:
'use strict';
const express = require('express');
const async = require('async');
const router = express.Router();
const dataservice = require('./dataservice.js')
router.get('/', function(req, res) {
async.series([
function(callback) {
getFinhockeyData(callback)
}
],
function(err, results) {
console.log('start rendering');
res.render('index', { data: data });
})
});
module.exports = router;
When I run the app and refresh the / route, I can see from the console that the getFinhockeyData is called and the data object's content is available in dataservice.js's console.logs, but the browser window hangs and the res.render part is never reached.
I understand that the rendering should be done only after the endpoint data request has finished (async.series usage), but it seems that I lack a fundamental understanding on how to actually use the result data from the getFinhockeyData function in the main route.
Any advice on this? I'll be happy to provide more info if necessary.
Firstly, doing the request is asynchronous, so you'll have to use either a callback or a promise.
Even the async middleware won't let you just return data from an asynchronous call, it requires a callback, but using native promises seems easier here
const http = require('http');
getFinhockeyData = function() {
return new Promise( (resolve, reject) => {
http.get('http://tilastopalvelu.fi/ih/modules/mod_standings/helper/standings.php?statgroupid=3545', (res) => {
var body = "";
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
resolve( JSON.parse(body) );
});
}).on('error', reject);
});
}
module.exports.getFinhockeyData = getFinhockeyData;
Also note that you're exporting as a module with a property
module.exports.getFinhockeyData = getFinhockeyData;
when you're going to use that in the routes, you have to use the property
const dataservice = require('./dataservice.js');
router.get('/', function(req, res) {
dataservice.getFinhockeyData().then(function(data) {
res.render('index', { data: JSON.stringify(data) });
}).catch(function(err) {
// epic fail, handle error here
});
});
You are responding to your route call with
res.render('index', { data: data });
But there is no data variable. It should be
res.render('index', { data: results });
Which is the variable where you are storing your data when it comes from the callback
The reason for res.render() not being called is, http requests are async. To get the response a callback must be passed, which you did but forgot to call it in the dataservice.js
This should help...
Change your dataservice.js like the following...
const http = require('http');
getFinhockeyData = function(callback) {
http.get('http://tilastopalvelu.fi/ih/modules/mod_standings/helper/standings.php?statgroupid=3545', (res) => {
console.log(`Got response: ${res.statusCode}`);
var body = "";
res.on('data', function (chunk) {
body += chunk;
})
res.on('end', function () {
var data = JSON.parse(body);
console.log('data parsed.');
console.log('first team name: ' + data.teams[0].TeamName);
console.log(typeof data);
callback(null, data); //returning the data to the callback
})
}).on('error', (e) => {
console.log(`Got error from Finhockey: ${e.message}`);
callback(e, null);
});
}
module.exports.getFinhockeyData = getFinhockeyData;
Change your router.js like the following...
router.get('/', function(req, res) {
async.series([
function(callback) {
getFinhockeyData(callback)
}
],
function(err, results) {
if(err === null){
console.log('start rendering');
res.render('index', { data: results[0] });
}
})
});
I am new to node js and I am trying to use async module to eliminate the setTimeouts. Here I am facing a problem. It is not working as expected. It calls the second function even before the first function completes execution. I searched for answers and tried multiple ways. But it doesn't seem to work. It prints "Inside db insert in async series" even before the async.forEach finishes. Can anyone please check the code and tell me where I'm going wrong?
setTimeout(function() {
async.series([function(callback1) {
console.log("Inside async series");
try {
var msg = "";
var datas = [];
for (var i = 0; i < service_name.length; i++) {
console.log("Inside for loop service names");
var child = {
"space_guid": space_guid,
"name": service_name[i],
"service_plan_guid": service_plan_guid[i]
};
datas.push(child);
console.log("datas array===" + JSON.stringify(datas))
}
async.forEach(datas, function(data1, callback) {
console.log("Inside async task");
var data = JSON.stringify(data1);
console.log("data value===" + JSON.stringify(data));
var options = {
host: 'api.ng.bluemix.net',
path: '/v2/service_instances' +
'?accepts_incomplete=true',
method: 'POST',
headers: {
'Authorization': full_token_new
}
};
console.log("options is" + JSON.stringify(options));
var reqst = http.request(options, function(res) {
console.log("Sent for request");
res.setEncoding('utf8');
res.on('data', function(chunk) {
msg += chunk;
});
res.on('end', function() {
try {
console.log("message =======", msg);
console.log("-----------------------------------------");
msg = JSON.stringify(msg);
msg1 = JSON.parse(msg);
console.log("printing msg--" + msg1);
console.log("-----------------------------------------");
console.log("here i am", i);
console.log(service_name.length - 1);
callback();
} catch (err) {
console.log(err);
}
});
});
reqst.on('error', function(e) {
console.log(e);
});
reqst.write(data);
reqst.end();
}, function(err) {
console.log("for each error" + err);
});
callback1(null, null);
} catch (err) {
console.log(err);
}
},
function(callback1) {
console.log("Inside db insert in async series")
db_insert(service_name, solnName, full_token_new, uname, version);
callback1(null, null);
}
],
function(err, results) {
if (err) {
console.log("There's an error" + err);
} else {
console.log("result of async", results);
}
})
}, 3000)
You are mixing try...catch with asynchronous code, this is bad practice and almost impossible to do right.
Also, your error stem from the fact you are calling callback just after async.forEach, which don't finish, and go to the next step.
Also, what do you mean by "eliminate the timeout"? Your whole code is in it, you can remove it whenever you want.
'use strict';
async.series([
(callback) => {
let msg = "",
datas = [],
i = 0;
while(i < service_name.length) {
let child = {
"space_guid": space_guid,
"name": service_name[i],
"service_plan_guid": service_plan_guid[i]
};
datas.push(child);
i = i + 1;
}
async.forEach(datas, (data1, callback) => {
let data = JSON.stringify(data1),
options = {
host: 'api.ng.bluemix.net',
path: '/v2/service_instances?accepts_incomplete=true',
method: 'POST',
headers: {
'Authorization': full_token_new
}
},
reqst = http.request(options, (res) => {
res.setEncoding('utf8');
res.on('data', (chunk) => {
msg += chunk;
});
res.on('end', () => {
msg = JSON.stringify(msg);
msg1 = JSON.parse(msg);
callback();
});
});
reqst.on('error', (error) => {
callback(error);
});
reqst.write(data);
reqst.end();
}, (error) => {
callback(error);
});
},
(callback) => {
db_insert(service_name, solnName, full_token_new, uname, version);
callback();
}
],
(error, results) => {
if (error) {
console.log("There's an error" + error);
} else {
console.log("result of async", results);
}
});
Since this smell heavily like a plssendzecode question, I've removed every console.log and gone ES6 to make sure you will not be able to use it as such and need to read the change I made.
I simplify code a little.
datas and processData aren't good names.
setTimeout(onTimer, 3000);
function onTimer() {
var datas = service_name.map(function(name, i) {
return {
space_guid: space_guid,
name: name,
service_plan_guid: service_plan_guid[i]
}
});
function processData(data, callback) {
var options = {
host: 'api.ng.bluemix.net',
path: '/v2/service_instances?accepts_incomplete=true',
method: 'POST',
headers: {
'Authorization': full_token_new
}
};
var reqst = http.request(options, function(res) {
var msg = '';
res.setEncoding('utf8');
res.on('data', function(chunk) {
msg += chunk;
});
res.on('end', function() {
try {
msg = JSON.parse(msg);
callback(null, msg);
} catch (err) {
callback(err);
}
});
});
reqst.on('error', callback);
reqst.write(JSON.stringify(data));
reqst.end();
}
async.map(datas, processData, function(err, results) {
if (err);
return console.log(err);
// process msg of each request db_insert(...);
});
};
I would use the promises of angularJS to fill data to a grid. I'd like to load data "row by row" as soon as the nodeJS's server, on which use the module "mssql" with the "stream" enabled, back to client every single line from the DB.
On the client side I use these functions:
function asyncGreet() {
var deferred = $q.defer();
var _url = 'http://localhost:1212/test';
$http.get(_url).
then(function(result) {
deferred.resolve(result);
}, function(error) {
deferred.reject(error);
}, function(value) {
deferred.notify(value); //<<-- In "value" I would like to get every single row
});
return deferred.promise;
}
$scope.btnTest = function () {
var promise = asyncGreet();
promise.then(function(res) {
console.log('Success: ' + res.data + "\n");
}, function(reason) {
console.log('Failed: ' + reason);
}, function(update) {
console.log('Got notification: ' + update); //<<--
});
};
On nodeJS server those:
app.get('/test', function (req, res) {
//sql for test
var _query = 'select top 50 * from tb_test';
var sql = require('mssql');
var connection;
var config = {
user: 'testUser',
password: '12345',
server: 'localhost\\test',
database: 'testDB',
stream: true
};
connection = new sql.Connection(config, function (err) {
var request = new sql.Request(connection);
request.query(_query);
request.on('recordset', function(columns) {
// Emitted once for each recordset in a query
//res.send(columns);
});
request.on('row', function(row) {
res.write(JSON.stringify(row)); //<<-- I would like intercept this event on client side
// and get the result in my angularJS function on deferred.notify
});
request.on('error', function(err) {
// May be emitted multiple times
console.error(err)
});
request.on('done', function(returnValue) {
// Always emitted as the last one
res.end('DONE');
});
});
});
Anyone can help me with this?
Thanks!
I'm done it using socket.io :)
On angularJS side:
// count the row for test only
$scope.count = 0;
$scope.prova = function () {
mySocket.emit('getTableByRow', {});
mySocket.on('resRow', function (data) {
if (data.event == 'ROW') {
$scope.count += 1;
}else {
$scope.count += " !!DONE!! ";
}
});
};
On NodeJS side:
[ ... connection with DB ... ]
io.on('connection', function (socket) {
socket.on('getTableByRow', function (data) {
_getTableByRow(socket, data);
});
});
_getTableByRow function:
var _getTableByRow = function (socket, data) {
var _query = 'select top 50 * from tb_test';
request.query(_query);
request.on('row', function(row) {
// return only the ids for test
socket.emit('resRow', {event: 'ROW', data: row.id.toString()});
});
request.on('done', function(returnValue) {
socket.emit('resRow', {event: 'DONE'});
});
request.on('recordset', function(columns) {
console.log(columns);
});
request.on('error', function(err) {
socket.emit('resRow', {event: 'ERROR', data: err});
});
}
In this way, as soon as one row is read from the DB, it is immediately sent to the client :)