I have an array of APIs to query. For example var urls = ['example.org?x=1', 'example.org?x=2,3'] How can I have ReactJS wait for all the responses (which are JSON) to come back before responding to the client?
I've tried a modification #Shanoor's answer from Promises with http.get node.js but the result to the client is empty ("").
var http = require('http');
var urls = ['example.org', 'example2.org', '...'];
var done = 0;
var result = [];
function callback(index, data) {
result[index] = data;
done++;
if (done == urls.length) {
result.forEach(console.log);
//I tried using the resp object here to return result[] but the data seemed corrupt
}
}
function processUrl(url, index) {
var finalData = '';
http.get(url, function(response) {
response.setEncoding('utf8');
response.on('data', function(data) {
finalData += data;
});
response.on('error', console.error);
response.on('end', function() {
callback(index, finalData);
})
});
}
app.get("/api/blah", (resp) => {
urls.forEach(processUrl);
//resp(result); this would result in an empty response
}
You can use Promise to do that, should be something like
function processUrl(url, index) {
var finalData = '';
return new Promise((resolve, reject) => {
http.get(url, function(response) {
response.setEncoding('utf8');
response.on('data', function(data) {
finalData += data;
});
response.on('error', reject);
response.on('end', function() {
resolve(finalData);
})
});
})
}
app.get("/api/blah", (resp) => {
Promise.all(urls.map(processUrl)).then(function(results) {
console.log(results);
});
}
Note that I didn't checked if the code is working, so please read more about Promises and the Promise.all() method
Related
I'm trying to send a GET request, parse its response and return it to another method. Apparently I have problems handling the asynchronous response.
I want to use Node.js' standard modules, so no Axios.
// class 1: Calling API, processing and returning the response
export async function getData() {
let str = '';
const options = {
hostname: 'jsonplaceholder.typicode.com',
path: '/posts/',
method: 'GET',
json: true,
};
https
.get(options, response => {
response.on('data', chunk => {
str += chunk;
});
response.on('end', () => {
return parseJson(str);
});
})
.on('error', error => {
console.log(error);
});
}
async function parseJson(str) {
const json = JSON.parse(str);
var text;
try {
json.forEach(element => {
text += element.body;
});
// console.log(text); // I'm getting the expected output
return text;
} catch (error) {
console.log('error');
}
}
// class 2: Calling the 2 methods above
getData().then(function (value) {
console.log('DATA: ' + value); // this is called first
});
Unfortunately as output I get an undefined. Despite using async and then:
DATA: undefined
Change getData as follows
export function getData() {
return new Promise((resolve, reject) => {
let str = '';
const options = {
hostname: 'jsonplaceholder.typicode.com',
path: '/posts/',
method: 'GET',
json: true,
};
https.get(options, response => {
response.on('data', chunk => {
str += chunk;
});
response.on('end', () => {
try {
const result = parseJson(str);
resolve(result);
} catch (error) {
reject(error);
}
});
response.on('error', reject);
})
.on('error', reject);
});
}
Now it returns a Promise, which resolves to the result of parseJson(str) or rejects with the error at on('error'
And parseJson as follows - doesn't need to be async since there's nothing asynchronous about the code inside it
Also, removing the try/catch in parseJson and using try/catch in .on("end" means that you can reject the promise returned by getData if there's an error in the parseJson call
function parseJson(str) {
const json = JSON.parse(str);
let text = '';
json.forEach(element => {
text += element.body;
});
return text;
}
alternatively (IMHO better)
function parseJson(str) {
const data = JSON.parse(str); // call it data, not json
return data.map(({body}) => body).join('');
}
or even
const parseJson = (str) => JSON.parse(str).map(({body}) => body).join('');
But that's not important :p
Task is to return data from getData function to main function.
function getData(){
const https = require('https')
const url = "https://...../api/movies";
https.get(url, res => {
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
data = JSON.parse(data);
console.log(data);
//How can I return this data to main function?
})
}).on('error', err => {
console.log(err.message);
})
}
function main(){
console.log(getData());
}
I am not able to access data or print data in main function
I think you already have the answer in your comment '//How can I return this data to main function?'.
...
res.on('end', () => {
data = JSON.parse(data);
console.log(data);
return data;
})
...
So the return-value of your getData-function should now be the parsed json-data and be accessible in the main-function.
I would create a variable to save value which you want to return
const https = require('https')
function getData() {
let ret = null;
const url = "https://...../api/movies";
https.get(url, res => {
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
ret = JSON.parse(data);
})
}).on('error', err => {
console.log(err.message);
})
return ret;
}
function main() {
console.log(getData());
}
What you could do is to wrap your request into a Promise. You would want to return this promise and and wait for it to be fullfilled:
function getData() {
const url = "https://...../api/movies";
return new Promise((resolve, reject) => {
const req = https.get(url, (res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
resolve(JSON.parse(data));
});
});
req.on('error', (err) => {
reject(err);
});
req.end();
});
}
function main() {
getData().then(data => {
console.log("Response:", data);
}, error => {
console.error(error);
});
}
Moreover you can make your main function async and use await to get the response:
async function main() {
const data = await getData();
console.log(data);
}
// Start an IIFE to use `await` at the top level. See: https://stackoverflow.com/a/14220323/7661119
(async () => {
await main()
})();
I have a problem to send string that is from s3 object body to a client
I'm using aws sdk for node, and apollo server(express), express, react
I did get object from s3 and create readable stream. and then i listen data event so I might send string to a client
let data = '';
s3.getObject(params).createReadStream().on('data', function(chunk) {
data += chunk;
});
return { data }
I thought data is not a empty string but it is empty string
what can I do to solve the problem?
Edit:
let data = '';
function promiseBasedRequest (params) {
return new Promise((resolve, reject) => {
s3.getObject(params).createReadStream()
.on('data', function (chunk) {
data += chunk;
})
.on('end', function () {
resolve(data);
})
.on('error', function (err) {
reject(err);
});
});
}
await promiseBasedRequest(params);
This works as I intended.
You are not waiting for the writing to end. First the function need to be asynchronous a promise or callback.
function getData(params) {
let data = ''
return new Promise((res, rej) => {
let data = '';
s3.getObject(params).createReadStream()
.on('data', function (chunk) {
data += chunk;
})
.on('end', function(){
res(data);
})
.on('error', function(){
rej()
})
})
}
You can use the function by:
(async(){
const data = await getData()
})();
Or getData().then(..)
EDIT: Also, getObject has one promise method as well.
s3.getObject(params).promise().then(...).catch(...)
I need to save data from GET requests to a variable and then save it in a file. However, in some cases GET request does not save data to global variables.
var fs = require("fs");
var http = require("http");
var request = require('request');
var tmp_json = {};
var g_last = 0;
var data = {};
//request 1
http.get('server:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server1 = {};
tmp_json.server1 = JSON.parse(data);
g_last = tmp_json.height; // 100500
console.log(g_last); // 100500
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
//request 2
http.get('server2:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server2 = {};
tmp_json.server2 = JSON.parse(data);
g_last = tmp_json.height; // 256
console.log(g_last); // 256
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
console.log(g_last); // 0
data = JSON.stringify(tmp_json);
fs.writeFile('data.json', data, 'utf8'); // empty file
Also I was trying to do it with fs.createWriteStream, but again I can save one request to file, but if there more then one I catch only buffer data.
Your problem is that request1 and request2 are happening while you are writing the file. This is because of the async nature of node. The execution order looks something like this:
Declare empty variables
Request1 goes out
Request2 goes out
Write empty variables to file
Request1 comes back and writes to variables
Request2 comes back and writes to variables
One way to fix this would be with Promises. The following allows for the function in then to be executed after the promises in Promise.all([ ... ]) have resolved:
var fs = require("fs");
var http = require("http");
var tmp_json = {};
var g_last = 0;
var data = {};
//request 1
var req1 = new Promise((resolve, reject) => {
http.get('server:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server1 = {};
tmp_json.server1 = JSON.parse(data);
g_last = tmp_json.height; // 100500
console.log(g_last); // 100500
resolve()
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(error)
});
});
//request 2
var req2 = new Promise((resolve, reject) => {
http.get('server2:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server2 = {};
tmp_json.server2 = JSON.parse(data);
g_last = tmp_json.height; // 256
console.log(g_last); // 256
resolve()
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(error)
});
});
Promise.all([ req1, req2 ]).then(() => {
console.log(g_last);
data = JSON.stringify(tmp_json);
fs.writeFile('data.json', data, 'utf8');
})
Edit:
function handleGet (url) {
return new Promise((resolve, reject) => {
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server1 = {};
tmp_json.server1 = JSON.parse(data);
g_last = tmp_json.height; // 100500
console.log(g_last); // 100500
resolve()
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(error)
});
})
}
// Then use it
Promise.all([
handleGet('http://google.ca'),
handleGet('http://somesite.com')
])
So I have a method, which I want to call multiple times in a loop. This is the function:
function PageSpeedCall(callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[0]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
As you see this is an async function that calls the PageSpeed API. It then gets the response thanks to the callback and renders it in the view. Now how do I get this to be work in a for/while loop? For example
function PageSpeedCall(websites, i, callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[i]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
var websites = ['google.com','facebook.com','stackoverflow.com'];
for (let i = 0; i < websites.length; i++) {
PageSpeedCall(websites, i);
}
I want to get a raport for each of these sites. The length of the array will change depending on what the user does.
I am using async.parallel to call the functions like this:
let freeReportCalls = [PageSpeedCall, MozCall, AlexaCall];
async.parallel(freeReportCalls, (err, results) => {
if (err) {
console.log(err);
} else {
res.render('reports/report', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}
});
I tried to use promise chaining, but for some reason I cannot put it together in my head. This is my attempt.
return Promise.all([PageSpeedCall,MozCall,AlexaCall]).then(([ps,mz,al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
I tried doing this:
return Promise.all([for(let i = 0;i < websites.length;i++){PageSpeedCall(websites, i)}, MozCall, AlexaCall]).
then(([ps, mz, al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
But node just said it's stupid.
And this would work if I didn't want to pass the websites and the iterator into the functions. Any idea how to solve this?
To recap. So far the functions work for single websites. I'd like them to work for an array of websites.
I'm basically not sure how to call them, and how to return the responses.
It's much easier if you use fetch and async/await
const fetch = require('node-fetch');
async function PageSpeedCall(website) {
const pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
const result = await fetch(pagespeeddCall);
return await result.json();
}
async function callAllSites (websites) {
const results = [];
for (const website of websites) {
results.push(await PageSpeedCall(website));
}
return results;
}
callAllSites(['google.com','facebook.com','stackoverflow.com'])
.then(results => console.log(results))
.error(error => console.error(error));
Which is better with a Promise.all
async function callAllSites (websites) {
return await Promise.all(websites.map(website => PageSpeedCall(website));
}
Starting on Node 7.5.0 you can use native async/await:
async function PageSpeedCall(website) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
return await promisify(pagespeedCall);
}
async function getResults(){
const websites = ['google.com','facebook.com','stackoverflow.com'];
return websites.map(website => {
try {
return await PageSpeedCall(website);
}
catch (ex) {
// handle exception
}
})
}
Node http "callback" to promise function:
function promisify(url) {
// return new pending promise
return new Promise((resolve, reject) => {
// select http or https module, depending on reqested url
const lib = url.startsWith('https') ? require('https') : require('http');
const request = lib.get(url, (response) => {
// handle http errors
if (response.statusCode < 200 || response.statusCode > 299) {
reject(new Error('Failed to load page, status code: ' + response.statusCode));
}
// temporary data holder
const body = [];
// on every content chunk, push it to the data array
response.on('data', (chunk) => body.push(chunk));
// we are done, resolve promise with those joined chunks
response.on('end', () => resolve(body.join('')));
});
// handle connection errors of the request
request.on('error', (err) => reject(err))
})
}
Make PageSpeedCall a promise and push that promise to an array as many times as you need, e.g. myArray.push(PageSpeedCall(foo)) then myArray.push(PageSpeedCall(foo2)) and so on. Then you Promise.all the array.
If subsequent asynch calls require the result of a prior asynch call, that is what .then is for.
Promise.all()
Promise.all([promise1, promise2, promise3]).then(function(values) {
console.log(values);
});