Develop Reference

How does one access, process and pass along response data outside of the request?

In the snippet below, I want to be able to access locationArray outside of the request function, I understand that in my code below why it will not work, however, I have tried many different methods to access the array. I have tried using promises, callback functions etc, however, none of them seem to be working.
Any other ideas on how to do this? Even open to ways I've tried as at this point everything is worth a try.
request(process.env.RESOURCE_SHEET, (error, response, html) => {
var locationArray = new Array
if(!error && response.statusCode == 200) {
const $ = cheerio.load(html);
$("h3").each((i, lle) => {
const location = $(lle).text();
if(location.includes("Kansas")) return;
if(location.includes("In Stock")) {
var level = location + " ✅";
} else {
var level = location + " ❌";
}
locationArray.push(level);
});
}
console.log(locationArray) // Output 1: [level1,level2,level3,leveletc]
});
console.log(locationArray) // Output 2: []

One not only might consider a Promise based approach, as it got already suggested, but also a code refactoring which separates the different concerns into tasks and implements the latter as functions which can be fed/passed to the promise chain. As an advantage the refactoring pays back with human readable code which also might be easier to maintain ...
function createRequest(src) {
return new Promise((resolve, reject) => {
request(src, (error, response, html) => {
if (!error && response.statusCode === 200) {
resolve(html);
} else {
reject({ error, response });
}
});
};
}
function handleFailedRequest(reason) {
const { error, response } = reason;
// proceed with failure handling based on
// either request data `error` and/or `response`.
}
function createLocationArray(html) {
const locationArray = [];
const $ = cheerio.load(html);
$('h3').each((i, lle) => {
const location = $(lle).text();
if (!location.includes('Kansas')) {
const isInStock = location.includes('In Stock');
locationArray.push(
`${ location } ${ isInStock && '✅' || '❌' }`
);
}
});
return locationArray;
}
function processLocationArray(array) {
console.log('locationArray ... ', array);
}
const promisedResponse = createRequest(process.env.RESOURCE_SHEET);
promisedResponse
.then(createLocationArray)
.then(processLocationArray)
.catch(handleFailedRequest);

#StackSlave was right, it just needed a promise, I believe I messed up the syntaxing when I first tried resolving it using a promise, but this seemed to work.
const promise = new Promise((resolve,reject) => {
request(process.env.RESOURCE_SHEET, (error, response, html) => {
var locationArray = new Array
if(!error && response.statusCode == 200) {
const $ = cheerio.load(html);
$("h3").each((i, lle) => {
const location = $(lle).text();
if(location.includes("Kansas")) return;
if(location.includes("In Stock")) {
var level = location + " ✅";
} else {
var level = location + " ❌";
}
locationArray.push(level);
resolve(locationArray);
});
}
console.log(locationArray) // Output 1: [level1,level2,level3,leveletc]
});
});
promise.then(array => {
console.log(array);
});

Javascript - Chaining 2 (or more) arrays of promises

I have some code that does this: First scrape this array of webpages. After that, scrape another array of webpages.
The following code does what I expect:
let bays=[];
let promises=promisesN=[];
for (let y=2019;y>=2015;y--)
promises.push(new Promise(resolve=>
curl.get(`/*url*/${y}.html`,null, (error,resp,body)=>
resp.statusCode==200? resolve(parse(body)):reject(error)
)));
Promise.all(promises).then(()=>{
bays.forEach(bay=>{
if (bay.no.match(/\d+/)<=103) return;
promisesN.push(new Promise(resolve=>
curl.get(`/*url*/${bay.code}/`,null, (error,resp,body)=>
resp.statusCode==200? resolve(image(bey,body)):reject(error)
)))});
Promise.all(promisesN).then(()=>{
bays.sort((a,b)=>{return parseInt(a.no.match(/\d+/))<parseInt(b.no.match(/\d+/))? -1:1});
console.log(bays);
});
}).catch(error=>console.log(error));`
So I've read you can write a simplier nesting-free syntax:
doSomething()
.then(function(result) {
return doSomethingElse(result);
})
.then(function(newResult) {
return doThirdThing(newResult);
})
.then(function(finalResult) {
console.log('Got the final result: ' + finalResult);
})
.catch(failureCallback);
How to apply this to the code above?

correctness
let promises=promisesN=[];
This is really incorrect. It makes both variables reference the same array, and makes promisesN an implicit global. The fact that it appears to work means you aren’t in strict mode. Always use strict mode. The correct version of what you intended is:
let promises = [];
let promisesN = [];
cleanliness
new Promise(resolve=>
curl.get(`/*url*/${y}.html`,null, (error,resp,body)=>
resp.statusCode==200? resolve(parse(body)):reject(error)
))
You’re repeating this pattern, so make it into a function, or use a package that does the job for you, like request-promise[-native] or axios. (Also, please show your real code. reject isn’t defined here.)
const getAsync = url => new Promise((resolve, reject) => {
curl.get(url, null, (error, resp, body) => {
if (resp.statusCode === 200) {
resolve(body);
} else {
reject(error);
}
});
});
Notice how you’re free to make the function more readable when it isn’t repeated, and to extend it later.
let promises = [];
let promisesN = [];
for (let y = 2019; y >= 2015; y--) {
promises.push(getAsync(`/*url*/${y}.html`).then(parse));
}
Promise.all(promises).then(bays => {
bays.forEach(bay => {
if (bay.no.match(/\d+/) <= 103) return;
promisesN.push(getAsync(`/*url*/${bay.code}/`).then(body => image(bay, body)));
});
Promise.all(promisesN).then(() => {
bays.sort((a, b) => {return parseInt(a.no.match(/\d+/)) < parseInt(b.no.match(/\d+/)) ? -1 : 1;});
console.log(bays);
});
}).catch(error => console.log(error));
I had to take a few guesses at what your real code looks like again, because you’re surely doing something with the resolved value of Promise.all(promises). It doesn’t have any easily-accessible side-effects. bey also seemed likely enough to be bay.
Now you can give promisesN a more appropriate scope:
let promises = [];
for (let y = 2019; y >= 2015; y--) {
promises.push(getAsync(`/*url*/${y}.html`).then(parse));
}
Promise.all(promises).then(bays => {
let promisesN = bays
.filter(bay => bay.no.match(/\d+/) > 103)
.map(bay => getAsync(`/*url*/${bay.code}/`).then(body => image(bay, body)));
Promise.all(promisesN).then(() => {
bays.sort((a, b) => {return parseInt(a.no.match(/\d+/)) < parseInt(b.no.match(/\d+/)) ? -1 : 1;});
console.log(bays);
});
}).catch(error => console.log(error));
and use an expression-bodied arrow function where appropriate, since you’re already using them whenever they aren’t appropriate:
bays.sort((a, b) => parseInt(a.no.match(/\d+/)) < parseInt(b.no.match(/\d+/)) ? -1 : 1);
Now, if my guess about bays is right, then you can’t unnest. If it comes from somewhere else then you can. Normally I would leave a comment about that but I already wrote all this, so… please clarify that for further cleanup.

If you're looking to simplify your code, you might consider the use of async/await instead of promises.
The async/await syntax will greatly simplify the presentation and ease comprehension of the code, especially given that your logic relies on asynchronous iteration of arrays.
Consider the following code revision of your code:
/* Define local helper that wraps curl() in async function declaration */
function async doRequest(url) {
return (await new Promise(resolve=> curl.get(url, null, (error,resp,body) =>
resp.statusCode==200 ? resolve(res) : reject(error))))
}
/* Re-define simplified scrape logic using await/async */
function async doScrape() {
try {
var bays = []
/* Iterate date range asynchronously */
for (let y=2019; y>=2015; y--) {
/* Use doRequest helper function to fetch html */
const response = await doRequest(`/*url*/${y}.html`)
const bay = parse(response)
bays.push(bay)
}
/* Iterate bays array that was obtained */
for(const bay of bays) {
/* Use doRequest helper again to fetch data */
const response = await doRequest(`/*url*/${bay.code}/`)
/* Await may not be needed here */
await image(bay, response)
}
/* Perform your sort (which is non asynchronous) */
bays.sort((a,b)=> parseInt(a.no.match(/\d+/))<parseInt(b.no.match(/\d+/))? -1:1);
console.log("Result", bays);
}
catch(err) {
/* If something goes wrong we arrive here - this is
essentially equivalent to your catch() block */
console.error('Scrape failed', err);
}
}
/* Usage */
doScrape()
Hope that helps!

Not entirely sure if this is what you want, but I've separated your code out a bit because I found it easier for me to read.
let bays = [];
let promises = [];
let promisesN = [];
for (let y = 2019; y >= 2015; y--) {
const promiseOne = new Promise((resolve, reject) => {
return curl.get(`/*url*/${y}.html`, null, (error, resp, body) => {
resp.statusCode === 200 ? resolve(parse(body)) : reject(error);
});
});
promises.push(promiseOne);
}
Promise.all(promises)
.then(() => {
bays.forEach((bay) => {
if (bay.no.match(/\d+/) <= 103) {
return;
}
const promiseTwo = new Promise((resolve, reject) => {
return curl.get(`/*url*/${bay.code}/`, null, (error, resp, body) => {
resp.statusCode === 200 ? resolve(image(bay, body)) : reject(error);
});
});
promisesN.push(promiseTwo);
});
return Promise.all(promisesN);
})
.then(() => {
bays.sort((a, b) => {
return parseInt(a.no.match(/\d+/), 10) < parseInt(b.no.match(/\d+/), 10) ? -1 : 1;
});
console.log(bays);
})
.catch((error) => {
console.log(error);
});
I am wondering though, you are firing the promises instantly on each iteration of your for loop. This might be intentional, but it means if those promises resolve before the code gets to execute Promise.all you may run into issues. I personally would do something like, e.g. const promiseOne = () => somePromise, that way you can create a bunch of promises, and then once they're all created, map over that array and fire them at once. Same thing goes for the second promises.
Not sure if this is helpful, let me know if it is. Feel free to ask more questions too.

Retry nodejs http.request (post,put,delete)

What is the correct way to implement a retry on error/condition without using any third party modules in nodejs, please?
I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
Do I need to destroy/end the sockets?
I've tried looking for examples but have only found reference to third party modules and http.get samples which don't seem to work. How does one test this?
I have attempted the below without success:
async pingApi(cb) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, /ect do I reference this path?
}
};
let request = await http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', function () {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.result != 'OK') {
setTimeout(pingApi, 1000); //cant pass callback
} else {
cb(null, json.result) //works
}
}
});
})
request.end(); //does the socket need to be closed if retry is this function?
}
Any help, pointing in the right direction or criticism will be greatly appreciated as I think this is a very important learning curve for me.
Thank you in advance,

I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
I don't know for sure that everything else in your function is correct, but you can fix the recursion that you're asking about by changing this:
setTimeout(pingApi, 1000); //cant pass callback
to this:
setTimeout(() => {
this.pingApi(cb);
}, 1000);
You aren't showing the whole context here, but if pingApi() is a method, then you also need to keep track of the this value to you can call this.pingApi(db). You can preserve the value of this by using arrow function callbacks like this:
response.on('end', () => { ... });
Other things I notice that look off here:
There's no reason to use await http.request(). http.request() does not return a promise so using await with it does not do anything useful.
Without the await, there's then no reason for your function to be declared async since nobody is using a returned promise from it.
It's not clear what if (this.complete) is meant to do. Since this is inside a regular function callback, the value of this won't be your pingApi object. You should either save this higher in the scope typically with const self = this or all callbacks internally need to be arrow functions so the value of this is preserved.
You should probably put try/catch around JSON.parse() because it can throw if the input is not perfect JSON.
You should probably not retry forever. Servers really hate clients that retry forever because if something goes wrong, the client may just be bashing the server every second indefinitely. I'd suggest a certain number of max retries and then give up with an error.
Do I need to destroy/end the sockets?
No, that will happen automatically after the request ends.
How does one test this?
You have to create a test route in your server that returns the error condition for the first few requests and then returns a successful response and see if your code works with that.
Here's an attempt at a code fixup (untested):
const maxRetries = 10;
pingApi(cb, cnt = 0) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, // ect do I reference this path?
};
let request = http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', () => {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
try {
let json = JSON.parse(decoded);
if (json.result != 'OK') {
if (cnt < maxRetries)
setTimeout(() => {
this.pingApi(cb, ++cnt);
}, 1000);
} else {
cb(new Error("Exceeded maxRetries with error on pingApi()"));
}
} else {
cb(null, json.result) //works
}
} catch(e) {
// illegal JSON encountered
cb(e);
}
}
});
})
request.end();
}
Remaining open questions about this code:
What is this.complete doing and what this should it be referencing?
Why is there no request.write() to send the body of the POST request?
I know you ask for no external modules, but my preferred way of doing this would be to use promises and to use the request-promise wrapper around http.request() because it handles a lot of this code for you (checks response.status for you, parses JSON for you, uses promise interface, etc...). You can see how much cleaner the code is:
const rp = require('request-promise');
const maxRetries = 5;
pingApi(cnt = 0) {
let options = {
method: "post",
url: `http://somedomain.com/API/pingAPI?${this.auth}`,
json: true
};
return rp(options).then(result => {
if (result.result === "OK") {
return result;
} else {
throw "try again"; // advance to .catch handler
}
}).catch(err => {
if (cnt < maxRetries) {
return pingApi(++cnt);
} else {
throw new Error("pingApi failed after maxRetries")
}
});
}
And, then sample usage:
pingApi().then(result => {
console.log(result);
}).catch(err => {
console.log(err);
})

your use of async/await with core node server intrigued me and I've tried to use much as possible of this new async features.
This is what I end up with: https://runkit.com/marzelin/pified-ping
const pify = require("util").promisify;
const http = require("http");
const hostname = "jsonplaceholder.typicode.com";
const failEndpoint = "/todos/2";
const goodEndpoint = "/todos/4";
let options = {
method: "get",
path: `${failEndpoint}`,
hostname
};
async function ping(tries = 0) {
return new Promise((res) => {
const req = http.request(options, async (response) => {
let body = new Buffer(0);
response.on("data", (chunk) => {
body = Buffer.concat([body, chunk]);
})
const on = pify(response.on.bind(response));
await on("end");
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.completed) {
return res("all good");
}
if (tries < 3) {
console.log(`retrying ${tries + 1} time`);
return res(ping(tries + 1));
}
return res("failed");
})
req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});
// write data to request body
req.end();
})
}
const status = await ping();
"status: " + status

How do you get javascript promises to work with node + express and display on an EJS file?

Before saying to look at the docs, I have and they were not helpful in the slightest.
I have an web page with node as the backbone. On one page I need to request the past 10 images from NASA's Astronomy Picture of the Day (APOD) API and then after that, I need to request the next 5 upcoming launches from the Launch Library API (https://launchlibrary.net/docs/1.3/api.html). My problem is that not all the APODs will load (which I understand because of the nature of asynchronous requests).
Here is my concise app.js file for the Node backend:
app.get("/index", function(req, res) {
/**Requesting NASA's Astronomy Picture of the Day**/
var apod_url = "https://api.nasa.gov/planetary/apod?api_key=[My Key]"
var apod_img_urls = [];
var curr_moment = moment();
for(var i = 0; i < 10; i++) {
var appended_url = apod_url + "&date=" + curr_moment.subtract(i, "days").format("YYYY-MM-DD");
request(appended_url, function(error, reponse, body) {
if(!error && reponse.statusCode == 200) {
var img_json = JSON.parse(body);
if(img_json.media_type == "image") {
var apod_promise = new Promise(function(resolve, reject){
resolve(img_json.hdurl);
});
apod_img_urls.push(apod_promise);
}
} else {
console.log(error);
}
});
}
/**************************************************/
var url = "https://launchlibrary.net/1.3/launch?next=20&mode=verbose";
request(url, function(error, response, body) {
if(!error && response.statusCode == 200) {
var data = JSON.parse(body);
res.render("index", {data: data, apod_img_urls: apod_img_urls});
} else {
console.log(error);
}
});
});
Here is an EJS snippet
<% apod_img_urls.forEach(function(promise, index) { %>
<div class="carousel-item <%= (index == 0 ? 'active' : '') %>">
<div class="w-100 home-image" style="background-image: url('<%= promise.then(function(url) {return url}); %>')"></div>
</div>
<% }); %>
When I check in the source it shows that the background image urls for the divs are [object Promise]. The way I have it, no images show up. Also the number of divs displayed (i.e. the number of images I should have) is variable; sometimes it's 5, sometimes it's 3, and other times it's none. Could my problem be that I'm rendering the page inside of another request? Also how can I get the actual image URL to show up in the EJS file?

You are creating the promise too late, inside the asynchronous callback to reequest - fairly simple reorganisation of code required
Once all the promises are in an array, you then need to wait for them to complete, using Promise.all
app.get("/index", function(req, res) {
/**Requesting NASA's Astronomy Picture of the Day**/
var apod_url = "https://api.nasa.gov/planetary/apod?api_key=[My Key]"
var promises = [];
var curr_moment = moment();
for(var i = 0; i < 10; i++) {
var appended_url = apod_url + "&date=" + curr_moment.subtract(i, "days").format("YYYY-MM-DD");
promises.push(new Promise((resolve, reject) => {
request(appended_url, function(error, reponse, body) {
if(!error && reponse.statusCode == 200) {
var img_json = JSON.parse(body);
resolve(img_json.hdurl);
} else {
reject(error);
console.log(error);
}
});
}));
}
Promise.all(promises).then(apod_img_urls => {
var url = "https://launchlibrary.net/1.3/launch?next=20&mode=verbose";
request(url, function(error, response, body) {
if(!error && response.statusCode == 200) {
var data = JSON.parse(body);
res.render("index", {data, apod_img_urls});
} else {
console.log(error);
}
});
});
});
Rather than using request-promise and all the cruft it requires, you can always make your own promisified request function, and in this case, it rejects if status is anything other than 200
const requestP = url => new Promise((resolve, reject) => request(url, (error, response, body) => {
if (error) {
reject({error, response, body});
} else if (resonse.statusCode != 200) {
reject(resonse.statusCode);
} else {
resolve({response, body});
}
}));
now, your code can be written as:
app.get("/index", function(req, res) {
/**Requesting NASA's Astronomy Picture of the Day**/
const apod_url = "https://api.nasa.gov/planetary/apod?api_key=[My Key]"
const curr_moment = moment();
Promise.all(Array.from({length:10}, (_, i) => {
const appended_url = apod_url + "&date=" + curr_moment.subtract(i, "days").format("YYYY-MM-DD");
return requestP(appended_url).then(({response, body}) => JSON.parse(body).hdurl);
})).then(apod_img_urls => {
const url = "https://launchlibrary.net/1.3/launch?next=20&mode=verbose";
return requestP(url).then(({response, body}) => {
const data = JSON.parse(body);
return res.render("index", {data, apod_img_urls});
});
});
});
Note: there's a lot of ES2015+ going on in there

If you use a promise-returning request library like request-promise then you can do something like:
app.get("/index", function(req, res) {
var apod_url = "https://api.nasa.gov/planetary/apod?api_key=[My Key]"
var curr_moment = moment();
var urls = [];
for(var i = 0; i < 10; i++) {
urls[i] = apod_url + "&date=" + curr_moment.subtract(i, "days").format("YYYY-MM-DD");
}
Promise.all(urls.map(url => rp({ url, json: true})).then((results) => {
// here you have all results with JSON already parsed for you
// ...
}).catch((err) => {
// handle error
// make sure to return response to client
// ...
});
// ...
});
If you want to work with promises then use a promise-returning module like request-promise or axios instead of the standard request, see:
https://www.npmjs.com/package/request-promise
https://www.npmjs.com/package/axios
Or, alternatively, use bluebird.promisify or built-in util.primisify (since 8.0) to promisify the request module, see:
http://bluebirdjs.com/docs/api/promise.promisify.html
https://nodejs.org/dist/latest-v8.x/docs/api/util.html
For more options of promisifying request see:
https://www.npmjs.com/package/request#promises--asyncawait
Then, when you have a promise-returning request library, make an array of URLs however you want, then map that array with the request functions, like this:
let rp = require('request-promise');
let urls = ['http://...', 'http://...'];
let promises = urls.map(rp);
and then use Promise.all to wait for all of them to finish, while being done concurrently:
Promise.all(promises)
.then(...)
.catch(...);
or, if you're using async/await, then:
try {
let x = await Promise.all(promises);
...
} catch (err) {
...
}
Many of modules like axios or request-promise-json or request-promise will parse JSON for you if you run it correctly, see:
https://github.com/request/request-promise#get-something-from-a-json-rest-api
Avoid parsing JSON yourself but if you do then always put JSON.parse() inside of try/catch (or use my little tryjson module) - see those answers to know why:
Calling a JSON API with Node.js
Reading requests body Nodejs
Node JS ignores undefined check

Error can't set headers after they are sent. (res.send())

I am struggle with a error, and I would like a help if it is possible. I need to send an array from my Router: pronunciation / route: /words to a javaScript file which sent to that route ("/words") a array with words like (ask).
So in that route("/words") I am requesting to a API the word (ask), then I can grab all its values from the database.
Then I send an array back to the file where everything started by "return res.send(array)". All right is does work. But my problem starts when a have two words in that array which is sent to the router/route, even though the request works well and I am able to grab information of those words for example (ask, a) from the API, when it runs the line "return res.send(array)" I have a message error saying "Error can't set headers after they are sent."
Is that explained good enough to be understood?
Thanks in advance!
router.get("/words", function(req, res) {
var wordsToBeSearched = req.query.str;
var dictionary = [];
for(var i = 0; i < wordsToBeSearched.length; i++) {
var url = "http://www.dictionaryfake.com/api/xml/" + wordsToBeSearched[i];
request(url, function(error, response, body) {
if(!error && response.statusCode == 200) {
parseString(body, function (err, result) {
var word = (JSON.stringify(result["entry_list"]["entry"][0]["ew"][0]));
var sound = (JSON.stringify(result["entry_list"]["entry"][0]["sound"][0]["wav"][0]));
var symbol = (JSON.stringify(result["entry_list"]["entry"][0]["pr"][0]));
var singleWord = {word: word.replace(/"/g,''), symbol: symbol.replace(/"/g,''), sound: sound.replace(/"/g,'')};
dictionary.push(singleWord);
if(i === wordsToBeSearched.length) {
return res.send(dictionary);
};
};
};
};
};
};
enter image description here

You want to wait for all of your requests to finish and then return you dictionary.
This can be nicely done using (async) promises. You can execute multiple promises and wait for all of them to resolve before res.sending the results.
router.get("/words", function(req, res) {
var wordsToBeSearched = req.query.str;
var dictionary = [];
var requests = [];
for(var i = 0; i < wordsToBeSearched.length; i++) {
var req = new Promise((resolve, reject) => {
var url = "http://www.dictionaryfake.com/api/xml/" + wordsToBeSearched[i];
request(url, function(error, response, body) {
if (error || !response.statusCode === 200) {
reject(error);
} else {
parseString(body, function (err, result) {
var word = (JSON.stringify(result["entry_list"]["entry"][0]["ew"][0]));
var sound = (JSON.stringify(result["entry_list"]["entry"][0]["sound"][0]["wav"][0]));
var symbol = (JSON.stringify(result["entry_list"]["entry"][0]["pr"][0]));
var singleWord = {word: word.replace(/"/g,''), symbol: symbol.replace(/"/g,''), sound: sound.replace(/"/g,'')};
dictionary.push(singleWord);
resolve();
});
}
});
});
requests.push(req);
}
Promise.all(requests)
.then(() => {
res.send(dictionary);
})
.catch(err => console.error(err));
});