I am getting inconsistent results from my server. Sometimes the right response is sent and sometimes I get the error
Can't remove headers after they are sent
Using Node.js, Koa.js, and Mongoose
router
.get('/events/', function* getEvent() {
let eventList = [];
yield Event.find({}, (error, events) => {
if (error) {
this.response.body = 'Unable to get events.';
this.status = 404;
return;
}
eventList = events;
eventList.sort((first, second) => {
// sort implementation
});
this.response.body = eventList;
this.status = 200;
});
});
The issue is caused by your callback which introduces a race condition since your yield isn't waiting for it to finish. In Koa v1.x, you generally only use a callback API to return a promise.
Here's how you'd write your example with Koa v1.x:
router
.get('/events', function * () {
let events
try {
events = yield Event.find({})
} catch (err) {
this.status = 503
this.body = 'Unable to get events'
return
}
events = sort(events)
this.body = events // Implicit 200 response
})
Event.find just needs to return something yieldable like a promise. Check to see if the library your using has a promise-returning version.
Though normally you'd just write it like this:
router
.get('/events', function * () {
let events = yield Event.find({})
events = sort(events)
this.body = events
})
Since it's an internal error (500 response) if Event.find is down. Koa will turn uncaught errors into 500 responses.
Basically, after you set this.status to 200, it throws an error because this.response.body is probably undefined. Go ahead and console.log() this.response.body and see if it is defined. If it is undefined, I would guess req.body is not being populated correctly OR it is an asynchronous node problem. Basically, eventList.sort() is asynchronously executing as this.response.body = eventList is set. Thus eventList is not sorted yet when you set it. To fix this put it inside eventList.sort() callback.
EDIT: after seeing your comment, I am pretty sure it is asynchronous problem now. Let me know if putting the last two lines inside the sort call works out for you.
Related
I need to execute unknown number of http requests in a node.js program, and it needs to happen synchronously. only when one get the response the next request will be execute. How can I implement that in JS?
I tried it synchronously with the requset package:
function HttpHandler(url){
request(url, function (error, response, body) {
...
})
}
HttpHandler("address-1")
HttpHandler("address-2")
...
HttpHandler("address-100")
And asynchronously with request-promise:
async function HttpHandler(url){
const res = await request(url)
...
}
HttpHandler("address-1")
HttpHandler("address-2")
...
HttpHandler("address-100")
Non of them work. and as I said I can have unknown number of http request over the program, it depends on the end user.
Any ideas on to handle that?
Use the got() library, not the request() library because the request() library has been deprecated and does not support promises. Then, you can use async/await and a for loop to sequence your calls one after another.
const got = require('got');
let urls = [...]; // some array of urls
async function processUrls(list) {
for (let url of urls) {
await got(url);
}
}
processUrls(urls).then(() => {
console.log("all done");
}).catch(err => {
console.log(err);
});
You are claiming some sort of dynamic list of URLs, but won't show how that works so you'll have to figure out that part of the logic yourself. I'd be happy to show how to solve that part, but you haven't given us any idea how that should work.
If you want a queue that you can regularly add items to, you can do something like this:
class sequencedQueue {
// fn is a function to call on each item in the queue
// if its asynchronous, it should return a promise
constructor(fn) {
this.queue = [];
this.processing = false;
this.fn = fn;
}
add(...items) {
this.queue.push(...items);
return this.run();
}
async run() {
// if not already processing, start processing
// because of await, this is not a blocking while loop
while (!this.processing && this.queue.length) {
try {
this.processing = true;
await this.fn(this.queue.shift());
} catch (e) {
// need to decide what to do upon error
// this is currently coded to just log the error and
// keep processing. To end processing, throw an error here.
console.log(e);
} finally {
this.processing = false;
}
}
}
}
In my controller called MapController I'm doing a function to do a parse of remote json files, and from an if-else structure add some values in an array called "parsewebservice", apparently everything is working fine but console.log ( parsewebservice); is not returning the values that were passed to the array "parsewebservice" in the place where it is returning it empty. But when I put it inside the forEach it returns, but everything cluttered and repeated then is not the right way.
I wanted to know why the values that were passed to the array "parsewebservice" are not going along with the variable after populada and what would be the correct way to do it?
Here is my code below:
/**
* MapController
*
* #description :: Server-side logic for managing Maps
* #help :: See http://sailsjs.org/#!/documentation/concepts/Controllers
*/
module.exports = {
index: function(req, res, next) {
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
datas.forEach(function(data, index) {
var req = require("request");
var url = data.address + "?f=pjson";
req(url, function(err, res, retorno) {
if (err) {
console.log(err);
} else {
var camadas = JSON.parse(retorno);
if (camadas.mapName) {
camadas.layers.forEach(function(campo, i) {
if (campo.subLayerIds != null) {
} else if (campo.subLayerIds == null) {
parsewebservice.push([i, "dynamicMapLayer", campo.name, data.address]);
}
});
} else if (camadas.serviceDataType) {
parsewebservice.push([null, "imageMapLayer", camadas.name, data.address]);
} else if (camadas.type) {
parsewebservice.push([null, "featureLayer", camadas.name, data.address]);
}
}
});
});
console.log(parsewebservice);
});
},
};
My first comment has to be that you should not combine function(req, res) with var req = require('request')... you lose your access to the original req object!
So, you need to run a list of async tasks, and do something when they are all complete. That will never be entirely easy, and no matter what, you will have to get used to the idea that your code does not run from top to bottom as you've written it. Your console.log at the bottom runs before any of the callbacks (functions you pass in) you pass to your external requests.
The right way to do this is to use promises. It looks like you are using this request library, whose returned requests can only accept callbacks, not be returned as promises. You can create your own promise wrapper for them, or use an alternative library (several are recommended on the page).
I don't want to write a whole intro-to-promises right here, so what I will do is give you a less pretty, but maybe more understandable way to run some code at the completion of all your requests.
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
// here we will write some code that we will run once per returned data
var processResponse = function(resp) {
parsewebservice.push(resp);
if(parsewebservice.length >= datas.length) {
// we are done, that was the final request
console.log(parsewebservice);
return res.send({data: parsewebservice)}); // or whatever
}
};
datas.forEach(function(data, index) {
var request = require("request");
var url = data.address + "?f=pjson";
request(url, function(err, res, retorno) {
// do some processing of retorno...
// call our function to handle the result
processResponse(retorno);
});
});
console.log(parsewebservice); // still an empty array here
});
I solved the problem.
the "request" module is asynchronous so we need to wait for it to respond and then send the response to the view.
To do this we created a function called "foo" to contain the foreach and the request, we made a callback of that function and finally we made the response (res.view) within that function, so that the controller response would only be sent after the response of the "foo" function to the callback. So we were able to parse.json the data from the "data" collection using foreach and the "request" module and send the objects to the view.
Many thanks to all who have helped me, my sincere thanks.
Ok, so I've just spent a day figuring out how to use callback-functions within sagas. (Please be nice, I'm just learning this saga-stuff)
My initial problem:
I get an xml-response from the server and before going into my reducer I want to parse it into a js-object. Therefore I use xml2js.
Calling this xml2js library works with a callback:
parseString(xmlInput, (err, jsResult) => {
// here I'd like to put() my success-event, however that's not possible in this scope
})
After reading a lot about eventChannels, I've come up with this solution:
My Channel-Function:
function parseXMLAsyncronously (input) {
return eventChannel(emitter => {
parseString(input, (err, result) => {
emitter(result)
emitter(END)
})
return () => {
emitter(END)
}
})
}
Using it inside the saga:
const parsedJSObject = yield call(parseXMLAsyncronously, xmlFromServer)
const result = yield take(parsedJSObject)
The problem that I'm encountering now is that apparently even while using a callback-structure, the parseString-function is still executed synchronously. So when I get to my yield-line, the parsing has already been done and I can wait forever because nothing will happen anymore.
What's working is to make the parsing asynchronously, by replacing
parseString(input, (err, result) => {...}
with
const parser = new Parser({async: true})
parser.parseString(input, (err, result) => {...}
So basically I'm making an already blocking function unblocking just to block (yield) it again and then wait for it to finish.
My question is now pretty simple: Is there maybe a smarter way?
Why not just use the cps effect?
try {
const result = yield cps(parseString, input)
} catch (err) {
// deal with error
}
I am trying to write a function that:
Takes an array of URLs
Gets files from URLs in parallel (order's irrelevant)
Processes each file
Returns an object with the processed files
Furthermore, I don't need for errors in #2 or #3 to affect the rest of the execution in my application in any way - the app could continue even if all the requests or processing failed.
I know how to fire all the requests in a loop, then once I have all the data, fire the callback to process the files, by using this insertCollection pattern.
However, this is not efficient, as I shouldn't need to wait for ALL files to download before attempting to process them - I would like to process them as each download finishes.
So far I have this code:
const request = require('request');
const urlArray = [urlA, urlB, urlC];
const results = {};
let count = 0;
let processedResult;
const makeRequests = function (urls, callback) {
for (let url of urls) {
request(url, function(error, response, body) {
if (error) {
callback(error);
return;
}
processedResult = callback(null, body)
if (processedResult) {
console.log(processedResult); // prints correctly!
return processedResult;
}
})
}
};
const processResult = function(error, file) {
if (error) {
console.log(error);
results.errors.push(error);
}
const processedFile = file + `<!-- Hello, Dolly! ${count}-->`;
results.processedFiles.push(processedFile);
if (++count === urlArray.length) {
return results;
}
};
const finalResult = makeRequests(urlArray, processResult);
console.log(finalResult); // undefined;
In the last call to processResult I manage to send a return, and makeRequests captures it, but I'm failing to "reign it in" in finalResult after that.
My questions are:
Why is this not working? I can print a well-formed processedResult
on the last iteration of makeRequests, but somehow I cannot return
it back to the caller (finalResult)
How can this be solved, ideally "by hand", without promises or the
help of libraries like async?
The makeRequests function returns undefined to finalResult because that is a synchronous function. Nothing stops the code executing, so it gets to the end of the function and, because there is no defined return statement, it returns undefined as default.
I'm trying to use Node.js to get a response from an API, I want to clean the API response and use the result.
So to access the first API I have the following code.
To store and use the result I believe I need to store the JSON output globally.
However, I can't work out how to do this.
Example -
var request = require('request');
request({url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true}, function(err, res, json) {
if (err) {
throw err;
}
car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
console.log(car2go);
});
This prints the desired result however I know that this is because my variable is defined within the function.
I want to access the variable outside of the function.
To test if I could do this I changed the code to -
var request = require('request');
request({url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true}, function(err, res, json) {
if (err) {
throw err;
}
car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
});
console.log(car2go);
But if I do this I get
ReferenceError: car2go is not defined
I am running Node v0.12.2 on Mac OS Yosemite (10.10.3).
Admittedly I am very new to node and I am more familiar with R, Python and PL SQL.
There is no way to get reference to it outside of the callback function because the console.log line runs before the callback function is invoked. The reason you have to pass a callback function into the request API is because the request library needs to invoke that function when it's done making the request. Meanwhile, your app moves on and does other things (such as running that console.log line) while it waits for the callback function to fire.
That said, there are a number of ways to deal with asynchronous code. My favorite way is with promises. I use a library called bluebird for handling promises.
var request = require('request');
var Promise = require('bluebird');
var requestP = Promise.promisify(request);
The call to Promise.promisify(request) returns a new function that doesn't take a callback function, but instead returns a promise.
requestP({ url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true })
.spread(function(res, json) {
var car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
})
.then(function (car2go) {
console.log(car2go);
})
.catch(function (err) {
console.error(err);
});
Note: .spread is the same as .then except if the resolved value is an array (which it will be because the callback passed to the request library accepts 2 arguments, which bluebird will translate into an array that the promise resolves to) .spread will split up the array back into multiple arguments passed into the function you give to .spread.
Promise.resolve(['hi', 'there']).then(function (result) {
console.log(result); // "['hi', 'there']"
});
Promise.resolve(['hi', 'there']).spread(function (str1, str2) {
console.log(str1); // 'hi'
console.log(str2); // 'there'
});
You're not going to be able to return that value all the way back out to the same context from which you began the asynchronous call, but you can at least write code that looks somewhat synchronous when using promises.
Without promises you'll be forced to call functions from within functions from within functions from within functions ;)
The response is asynchronous. That means the callback function gets called sometime LATER in the future so your console.log(car2go) is executing BEFORE the callback has even been called.
The only place you can reliably use the response is inside the callback or in a function called from the callback. You cannot use it the way you are trying to. Using asynchronous responses in Javascript requires programming in an asynchronous fashion which means processing results and using results IN the asynchronous callbacks only.
Here's where the console.log() should be:
var request = require('request');
request({url: 'https://www.car2go.com/api/v2.1/vehicles?loc=wien&oauth_consumer_key=car2gowebsite&format=json', json: true}, function (err, res, json) {
if (err) {
throw err;
}
car2go = json.placemarks;
for (i = 0; i < car2go.length; i++) {
delete car2go[i].address;
delete car2go[i].charging;
delete car2go[i].exterior;
delete car2go[i].interior;
delete car2go[i].smartPhoneRequired;
delete car2go[i].vin
car2go[i].vendor = 'car2go';
car2go[i].city = 'wien';
car2go[i].carmake = 'Smart';
car2go[i].carmodel = 'Fortwo';
}
// here is where the result is available
console.log(car2go);
});