how to use request with async module - javascript

One of my controller is doing a crawl of 100s of pages wherein its extracting all the links from the page and when the links after saved to the database, i'm using a afterCreate lifecycle callback (below is the code)
afterCreate: function(createdLink, next) {
var request = require("request");
var currentLink = config.apiUrl + "update/linkstatus?linkPath=" + createdLink.linkUrl;
request(currentLink, function(error, response, body) {
console.log("saved", body);
});
next();
}
this triggers another controller which gets the status of each of those links; below is the code for this controller:
linkstatus: function(req, res) {
var request = require("request");
var currentLink = req.query.linkPath;
request(currentLink, function(error, response, body) {
if(error) {
console.log(error);
}
var thisStatusCode = response.statusCode;
Link.update(
{linkUrl: currentLink}, {statusCode: thisStatusCode}
).exec(function(err, updatedLink) {
res.status(200).send(updatedLink);
});
});
}
The issue is that after some calls, I start getting the following error:
TypeError: Cannot read property 'statusCode' of undefined
at Request._callback (/home/ubuntu/myapp/api/controllers/UpdateController.js:11
2:32)
at self.callback (/home/ubuntu/myapp/node_modules/request/request.js:360:22)
at Request.emit (events.js:107:17)
at Request.onRequestError (/home/ubuntu/myapp/node_modules/request/request.js:1
Assuming that the 'linkstatus' controller action that I've written is not done properly to handle getting status code responses from 1000s of links simultaneously and as per my understanding, I need to use async module and Promises to solve this but i'm having difficulty grabbing the concept of callback and Promises so would be fantastic to get input on this

Your issue is that your not guaranteed a response and in those cases where the requests fails inside of node you to check for that before moving forward.
linkstatus: function(req, res, next) {
var request = require("request");
var currentLink = req.query.linkPath;
request(currentLink, function(error, response, body) {
if(error || !response) {
console.log(error);
}
else {
var thisStatusCode = response.statusCode;
Link.update(
{linkUrl: currentLink}, {statusCode: thisStatusCode}
).exec(function(err, updatedLink) {
res.status(200).send(updatedLink);
});
}
});
}

Related

Getting "Can't set headers after they are sent" error on node js after send a post request in angular

my problem is that I'm trying to send some posts requests in an angular loop.
This is my angular code:
for (var i = 0; i < response.length;i++) {
response[i].pedido_id = $scope.pedidos.pedidoSeleccionado;
PedidoDetalleFactory.save(
{
obraId: obra._id,
pedidoId: response[i].pedido_id
},
response[i],
function (result) {
pedidosTransferidos++;
},
function (result) {
// Handle error
}
);
}
And this is my node.js code
.post(Verify.verifyOrdinaryUser, function (req, res, next) {
Servicios.findById(req.body.servicio_id).exec(function (err, servicio) {
if (err) next(err);
req.body.price = servicio.price;
req.body.pedido_id = req.params.pedidoId;
PedidoDetalles.create(
req.body,
function (err, pedido_detalle) {
if (err) next(err);
res.json(pedido_detalle);
}
);
});
});
Everything looks fine, but I get this error
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:335:11)
Could this be happening beacause angular sends the request too fast to the server?
Please, help
it seem that you set header again after you had response to client. maybe you can try if (err) {return next(err)}, if error, please return, and don't exec the other codes.
I was fetching some results in my server and then I was sending back a json with a attribute "_id" so mongoose was saying "This item already exists", but I couldn't debug the code because I had if (err) next(err); instead of if (err) {return next(err);}

Node.JS with Express async API call

I'm using Node.JS with Express (and ejs for the views) and I want to call two API endpoints to render them in the same page and use the data. I've tried doing it with async but I get
ECONNRESET: Request could not be proxied!
This is my code
app.get('/profile', function(req, res) {
async.parallel([
function(next) {
var query = req.query.search;
var url = '1st url' + query;
var request = require('request');
request(url, function(error, body) {
var data = JSON.parse(body);
next(error, data);
});
},
function(next) {
request('2nd url', function(error, tlist) {
var list = JSON.parse(tlist);
next(error, list);
});
}], function(err, results) {
if (!err && results.statusCode == 200)
var data = results[0];
var list = results[1];
res.render('profile', {data: data, list: list});
});
});
Unsure about Cloud9, but if the issue is around parsing data, there's a couple of things here.
You should handle the error on each request before you attempt to parse; if parse throws an exception, your callback won't be executed:
request(url, function(error, body) {
if (error) return next(error);
var data = JSON.parse(body);
next(null, data);
});
You should probably also have a try/catch around the parse, and execute your callback with an error if there's an exception:
request(url, function(error, body) {
if (error) return next(error);
var data;
try {
data = JSON.parse(body);
} catch (e) {
return next(new Error('Unable to parse body for ' + url));
}
next(null, data);
});
Finally, your current check for results.statusCode will always return false, since it's an array, so you won't reach the end of the request. I'm guessing this is probably where the problem lies. I would also recommend passing any errors from async on to the Express error handler:
function(err, results) {
if (err) {
// pass to Express error handler...
}
var data = results[0];
var list = results[1];
res.render('profile', {data: data, list: list});
});

Node.JS server side script to update JSON file

I have partially written a NODE.JS file to update the JSON file with data received from the client. The post works successfully. The Get command does not. I was wondering if there's a better way to do this? I have about 6 different callback options to write for. All different. I was wondering if there's a node.JS script already done that has all of the things I need. Or if there's a different language that would make it easier.
Here's the NODE:
var http = require('http');
var fs = require('fs');
http.createServer(function (req, res) {
console.log('Request received: ');
if (req.method == 'POST') {
req.on('data', function (chunk) {
fs.writeFile("comments-data.json", chunk, function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
})
});
res.end('{"msg": "success"}');
};
if (req.method == 'GET') {
req.on('data', function (chunk) {
fs.readFile('comments-data.json', 'utf8', function (err, data) {
if (err) throw err;
obj = JSON.parse(data);
return data;
});
});
res.end(data);
};
}).listen(8080, '127.0.0.1');
console.log('Server running at http://127.0.0.1:8080/');
Here's the AJAX call:
postComment: function(commentJSON, success, error) {
$.ajax({
type: 'post',
url: 'http://127.0.0.1:8080',
data: commentJSON,
success: function(comment) {
success(comment)
},
error: error
});
},
But there's an ajax call for all sorts of things with the jquery plugin that i'm using. I need to GET, POST, PUT, DELETE, and sometimes multiple within the call.
Here's a full list of all of the callbacks i'm using:
http://viima.github.io/jquery-comments/#link-3-6
Using express you can do this much easily.
const express = require('express');
const app = express.Router();
//POST Request
app.post('/',(req, res, next)=>{
fs.writeFile("comments-data.json", chunk, function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
res.json({'status': 'Success'})
})
})
//GET Request
app.get('/',(req, res, next)=>{
fs.readFile('comments-data.json', 'utf8', function (err, data) {
if (err) throw err;
obj = JSON.parse(data);
res.json({'status': 'Success', 'data':data})
});
})
As for your question regarding writing it in a different module. That is based on the pattern adopted by you. There are various nodejs patterns available eg. Controller based or classes based. It all depends on what you find comfortable.

How to have express handle and capture my errors

var database = require('database');
var express = require('express');
var app = express();
var cors = require('cors');
app.use(cors());
var bodyParser = require('body-parser');
var urlencodedParser = bodyParser.urlencoded({
extended: false
});
app.post('/dosomething', urlencodedParser, function(req, res) {
if (!req.body.a) {
res.status(500).send(JSON.stringify({
error: 'a not defined'
}));
return;
}
firstAsyncFunction(req.body.a, function(err, result) {
if (err) {
res.status(500).send('firstAsyncFunction was NOT a success!');
} else {
if (result.b) {
secondAsyncFunction(result.b, function(err, data) {
if (err) {
res.status(500).send('secondAsyncFunction was NOT a success!');
return;
}
res.send('EVERYTHING WAS A SUCCESS! ' + data);
});
}
else {
res.status(500).send('result.b is not defined');
}
}
});
});
function firstAsyncFunction(param, callback) {
//Some network call:
// Return either return (callback(null,'success')); or return (callback('error'));
var query = database.createQuery(someOptionsHere);
database.runDatabaseQuery(query, function(err, entities, info) {
if (err) {
return (callback('error'));
}
return (callback(null, 'success'));
});
};
function secondAsyncFunction(param, callback) {
//Some network call:
// Return either return (callback(null,'success')); or return (callback('error'));
var query = database.createQuery(someOptionsHere);
database.runDatabaseQuery(query, function(err, entities, info) {
if (err) {
return (callback('error'));
}
return (callback(null, 'success'));
});
};
var server = app.listen(process.env.PORT || 3000, function() {
var host = server.address().address;
var port = server.address().port;
console.log('App listening at http://%s:%s', host, port);
});
module.exports = app;
I have here a basic express http server. This server has one route, dosomething, which makes two network calls and tells the user if they were a success or not.
This is my entire webserver (this is a bare bones server of my actual server for example purposes). I am now concerned with this server crashing. Reading the docs for express I see there is a default error handler which will catch errors and prevent the server from crashing (http://expressjs.com/en/guide/error-handling.html). I have added the code:
function defaultErrorHandler(err, req, res, next) {
if (res.headersSent) {
return next(err);
}
res.status(500);
res.render('error', { error: err });
}
app.use(defaultErrorHandler);
This still crashes my server though. For example. I had a problem with my database returning an improper JSON response and inside of my firstAsyncFunction (not shown in the code) I tried to parse the JSON and it caused an error telling me it was improper JSON and the server crashed and was unable to take requests anymore until I restarted it. I would like to avoid this and have the default error handler send out a generic response back to the user when this occurs. I thought if I specified the defaultErrorHandler and put it inside of app.use that it would capture and handle all errors, but this does not seem to be the case? Inside of my async function for example you can see I am looking if an error was returned and if it was I send an error back to the user, but what if some other error occurs, how can I get express to capture and handle this error for me?
The defaultErrorHandler cannot handle exceptions that are thrown inside asynchronous tasks, such as callbacks.
If you define a route like:
app.get('/a', function(req, res) {
throw new Error('Test');
});
An error will be thrown, and in this case defaultErrorHandler will successfully catch it.
If the same exception occurs in an async manner, like so:
app.get('/a', function(req, res) {
setTimeout(function () {
throw new Error('Test');
}, 1000);
});
The server will crush, because the callback is actually in another context, and exceptions thrown by it will now be caught by the original catcher. This is a very difficult issue to deal with when it comes to callback.
There is more than one solution though. A possible solution will be to wrap every function that is prone to throw error with a try catch statement. This is a bit excessive though.
For example:
app.get('/a', function(req, res) {
setTimeout(function () {
try {
var x = JSON.parse('{');
}
catch (err) {
res.send(err.message);
}
}, 1000);
});
A nicer solution:
A nicer solution, would be to use promises instead, if it's possible, then for example you can declare a single errorHandler function like so:
function errorHandler(error, res) {
res.send(error.message);
}
Then, let's say you have to following function with fetches stuff from the database (I used setTimeout to simulate async behavior):
function getStuffFromDb() {
return new Promise(function (resolve, reject) {
setTimeout(function () {
resolve("{");
}, 100);
});
}
Notice that this function returns an invalid JSON string. Your route will look something like:
app.get('/a', function(req, res) {
getStuffFromDb()
.then(handleStuffFromDb)
.catch(function (error) { errorHandler(error, res) });
});
function handleStuffFromDb(str) {
return JSON.parse(str);
}
This is a very simplified example, but you can add a lot more functionality to it, and (at least theoretically) have a single catch statement which will prevent your server from crushing.

NodeJS : app.get() does not operate as I exected

I am new to NodeJS and here is my routes/recom.js which is now doing a lot of logics stuffs. I will move the logics to somewhere else later. But now I need to solve the error first.
exports.scrape = function(req, res) {
var APIURI = 'https://www.kimonolabs.com/api/abbxqyg4?apikey=6IrCGNcorodTBSfNawS6sZkHw3LeZVIN';
var receivedJason;
res.type('text/plain');
request({
url: APIURI,
json: true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
console.log('API Retrieved Successfully!');
res.json(body);
receivedJason = JSON.stringify(body);
var node = db.createNode({hello: 'world'}); // instantaneous, but...
node.save(function (err, node) { // ...this is what actually persists.
if (err) {
console.error('Error saving new node to database:', err);
} else {
console.log('Node saved to database with id:', node.id);
}
});
}
});
};
And this is my app.js
var express = require('express');
var recom = require('./server/routes/recom.js');
var app = express();
app.get('/', recom.scrape);
app.listen(process.env.PORT || 2019);
and I got the following error:
Route.get() requires callback functions but got a [object Undefined]
any help would be appreciated.
The error seems to be in the export. In routes/recom.js use:
module.exports.scrape = function(req, res) { //...
The reason you get an [object Undefined] is that exports is not the same object as module.exports, and thus the item looked for does not exist.
In addition, in app.js you may need to use one of:
var recom = require('routes/recom.js');
// or:
var recom = require('server/routes/recom.js');
Depending on you full structure.

Categories

Resources