I'm trying to do a GET request in node.js. I would like to access the result of the request but there is a problem. Here is the code :
const https = require('https');
var data;
https.get('https://www.example.com', (resp) => {
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
console.log("Inside : "+data+"\n");
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
console.log("Outside : "+data+"\n");
The outside message appears before the inside one, and only the latter displays the request. That is due to asynchronous process I guess. But if I want to use my request outside, it's a problem because of this delay.
How may I do?
Your code execute asyncronymos. http.get(...) execute, but programm don't wait result of your get. So when you do "console.log("Outside : "+data+"\n");" - your data is empty. You will never get data to outside message, without js constructions like async/await.
This is what you need:
(async function () {
const https = require('https');
var data;
await new Promise((resolve, reject) => {https.get('https://www.example.com', (resp) => {
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
console.log("\n\n===>Inside : "+data+"\n");
resolve();
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject();
});
});
console.log("\n\n===>Outside : "+data+"\n");
})();
Thanks for answering. After searches, I made this, which work, but I don't know if it's the best way to do this.
const https = require('https');
var data;
(async function () {
var p = new Promise((resolve) => {https.get('https://www.example.com', (resp) => {
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
console.log("\n\n===>Inside : "+data+"\n");
resolve();
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
});
await p;
console.log("\n\n===>Outside : "+data+"\n");
})();
Related
I am fetching data from an API in order to show sales and finance reports, but I receive a type gzip file which I managed to convert into a Uint8Array. I'd like to somehow parse-decode this into a JSON file that I can use to access data and create charts in my frontend with.
I was trying with different libraries (pako and cborg seemed to be the ones with the closest use cases), but I ultimately get an error Error: CBOR decode error: unexpected character at position 0
This is the code as I have it so far:
let req = https.request(options, function (res) {
console.log("Header: " + JSON.stringify(res.headers));
res.setEncoding("utf8");
res.on("data", function (body) {
const deflatedBody = pako.deflate(body);
console.log("DEFLATED DATA -----> ", typeof deflatedBody, deflatedBody);
console.log(decode(deflatedBody));
});
res.on("error", function (error) {
console.log("connection could not be made " + error.message);
});
});
req.end();
};
I hope anyone has stumbled upon this already and has some idea.
Thanks a lot!
Please visit this answer https://stackoverflow.com/a/12776856/16315663 to retrieve GZIP data from the response.
Assuming, You have already retrieved full data as UInt8Array.
You just need the UInt8Array as String
const jsonString = Buffer.from(dataAsU8Array).toString('utf8')
const parsedData = JSON.parse(jsonString)
console.log(parsedData)
Edit
Here is what worked for me
const {request} = require("https")
const zlib = require("zlib")
const parseGzip = (gzipBuffer) => new Promise((resolve, reject) =>{
zlib.gunzip(gzipBuffer, (err, buffer) => {
if (err) {
reject(err)
return
}
resolve(buffer)
})
})
const fetchJson = (url) => new Promise((resolve, reject) => {
const r = request(url)
r.on("response", (response) => {
if (response.statusCode !== 200) {
reject(new Error(`${response.statusCode} ${response.statusMessage}`))
return
}
const responseBufferChunks = []
response.on("data", (data) => {
console.log(data.length);
responseBufferChunks.push(data)
})
response.on("end", async () => {
const responseBuffer = Buffer.concat(responseBufferChunks)
const unzippedBuffer = await parseGzip(responseBuffer)
resolve(JSON.parse(unzippedBuffer.toString()))
})
})
r.end()
})
fetchJson("https://wiki.mozilla.org/images/f/ff/Example.json.gz")
.then((result) => {
console.log(result)
})
.catch((e) => {
console.log(e)
})
Thank you, I actually just tried this approach and I get the following error:
SyntaxError: JSON Parse error: Unexpected identifier "x"
But I managed to print the data in text format using the below function:
getFinancialReports = (options, callback) => {
// buffer to store the streamed decompression
var buffer = [];
https
.get(options, function (res) {
// pipe the response into the gunzip to decompress
var gunzip = zlib.createGunzip();
res.pipe(gunzip);
gunzip
.on("data", function (data) {
// decompression chunk ready, add it to the buffer
buffer.push(data.toString());
})
.on("end", function () {
// response and decompression complete, join the buffer and return
callback(null, buffer.join(""));
})
.on("error", function (e) {
callback(e);
});
})
.on("error", function (e) {
callback(e);
});
};
Now I would need to pass this into a JSON object.
I have a node js with express application. I need to expose a rest endpoint that will return the response of a http call. Whatever I try it returns before the http request. Can you please help
app.all('/query', function(req, res){
// here i need to make a http call
let urlCall = new Promise((resolve, reject) => {
http.get('http://test.com', (response) => {
let sdata = '';
response.on('data', (fragments) => {
sdata += fragments;
});
response.on('end', () => {
let response_body = sdata;
resolve(response_body.toString());
});
response.on('error', (error) => {
// promise rejected on error
reject(error);
});
});
});
urlCall.then((response) => {
var responseData=response;
res.json(responseData);
res.end();
}).catch((error) => {
console.log(error);
res.end();
});
}
Your code should work, but I suspect a request error not being handled (the error event handler being missing on your request)
You can try moving the error handler from the response to the request
app.all("/query", function (req, res) {
// here i need to make a http call
let urlCall = new Promise((resolve, reject) => {
http
.get("http://test.com", (response) => {
let sdata = "";
response.on("data", (fragments) => {
sdata += fragments;
});
response.on("end", () => {
let response_body = sdata;
resolve(response_body.toString());
});
})
.on("error", (error) => { // handling request errors
console.error("error");
// promise rejected on error
reject(error);
});
});
urlCall.then(
(response) => {
res.json(response);
res.end();
},
(error) => {
console.log(error);
res.end();
}
);
});
I have the following code
const https = require("https");
it("wait for some result", function (done) {
this.timeout(15000);
const options = {
hostname: "httpbin.org",
path: "/get",
headers: {
Authorization: "bearer ",
},
};
https.get(options, (resp) => {
let data = "";
// A chunk of data has been recieved.
resp.on("data", (chunk) => {
data += chunk;
console.log(data);
});
// The whole response has been received. Print out the result.
resp.on("end", () => {
console.log(JSON.parse(data).explanation);
});
resp.on("error", (err) => {
console.log("Error: " + err.message);
});
done();
});
});
this returns:
should respond with redirect on post:
Error: timeout of 15000ms exceeded. Ensure the done() callback is being called in this test.
I am trying to make this asynchronous because of mocha, which is why I am using callback. I am even waiting 15000 instead of the default 2000ms.
this code normally works, only fails with mocha.
Not sure how to fix this issue. Any help is appreciated.
Thanks in advance.
try to add .end() on your request object
const req = https.get(options, (resp) => {
let data = "";
// A chunk of data has been recieved.
resp.on("data", (chunk) => {
data += chunk;
console.log(data);
});
// The whole response has been received. Print out the result.
resp.on("end", () => {
console.log(JSON.parse(data).explanation);
done();
});
resp.on("error", (err) => {
console.log("Error: " + err.message);
});
});
req.end()
I am trying to invoke a rest API inside an API but it is not returning anything. So I am making a simple lambda which returns a JSON but getting a null value as a response.
var https = require('https');
var dt;
exports.handler = async (event, context) => {
var data = '';
return new Promise((resolve, reject) => {
var params = {
host: "cvwtzygw4a.execute-api.ap-south-1.amazonaws.com",
path: "/test/first"
};
const req = https.request(params, (res) => {
console.log('STATUS: ' + res.statusCode);
res.setEncoding('utf8');
res.o n('data', function(chunk) {
data += chunk;
});
res.on('end', function() {
console.log("DONE");
console.log(data);
dt = JSON.parse(data);
console.log(dt);
});
resolve(dt);
});
req.on('error', (e) => {
reject(e.message);
});
// send the request
req.write('');
req.end();
});
};
You should go through this article to understand how to use NodeJs promises in AWS Lambda. In this, the second solution addresses your use case.
To be specific to your code, I modified to make it very simple using the async/await syntax and the request-promise library.
const request = require('request-promise');
exports.handler = async (event, context) => {
var data = '';
try {
data = await request.get('https://cvwtzygw4a.execute-api.ap-south-1.amazonaws.com/test/first');
console.log('response received', res);
} catch (error) {
console.log('Error', error);
}
return data;
};
Following was the output:
START RequestId: 80d75f93-5fa6-1354-c22c-0597beb075e7 Version: $LATEST
2020-01-03T17:51:19.987Z 80d75f93-5fa6-1354-c22c-0597beb075e7 response received {
"basic" : {"name":"John","age":31,"city":"New York"}
}
END RequestId: 80d75f93-5fa6-1354-c22c-0597beb075e7
REPORT RequestId: 80d75f93-5fa6-1354-c22c-0597beb075e7 Init Duration: 907.81 ms Duration: 1258.54 ms Billed Duration: 1300 ms Memory Size: 128 MB Max Memory Used: 55 MB
"{\n\"basic\" : {\"name\":\"John\",\"age\":31,\"city\":\"New York\"}\n}"
My promise-then chain does not appear to wait for each previous return statement.
new Promise(function (resolve, reject) {
console.log("1");
const http = require('http');
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
var info;
// process data
resolve(info);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}).then(function (info) {
console.log("2");
if (info.item) {
console.log("item exists, don't retry");
return (info);
}
const http = require('http');
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
var info;
// process data
return(info);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}).then(function (info) {
console.log("3");
const http = require('http');
http.get('otherurl' + info.item, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
console.log("processed");
return (info);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}).then(function (info) {
console.log("4 " + info);
});
I want my output to be, eg.:
1
2
3
processed
4 [someinfo]
Here is what I get:
1
2
3
4 undefined
processed
It appears that only the first promise-then happen asynchronously. Why aren't the second and third then statements waiting for the prior return?
The code currently is:
new Promise(function (resolve, reject) {
console.log("1");
return(http.ClientRequest)
}).then(function (info) {
console.log("2");
return(http.ClientRequest)
}).then(function (info) {
console.log("3");
resolve(http.ClientRequest)
}).then(function (info) {
console.log("4 " + info);
});
To work a Promise chain needs to return a promise from the then part. But anything you return from then is treated as promise. But in your case
You are not returning anything.
If you are returning, it is from the callback, so basically it doesn't go out of the function. if you do return http.get(...) you will get http.ClientRequest object in your next then chain. Not the actual data.
So in your case a crude way to do this will be: Promisifying each http call.
new Promise(function (resolve, reject) {
console.log("1");
const http = require('http');
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
var info;
// process data
resolve(info);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}).then(function (info) {
console.log("2");
if (info.item) {
console.log("item exists, don't retry");
return (info);
}
return new Promise(function (resolve, reject) {
const http = require('http');
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
var info;
// process data
resolve(info);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
})
}).then(function (info) {
console.log("3");
return new Promise(function (resolve, reject) {
const http = require('http');
http.get('otherurl' + info.item, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
console.log("processed");
resolve(info);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
})
}).then(function (info) {
console.log("4 " + info);
});
Note: As I said it is a very non-elegant way of doing things, I would suggest using a promise based library like axios or maybe use async library instead of promises. Also you can use async-await. Each of them are better approach.
You are only resolving your first promise.
When you return a value inside a callback you are not resolving the promise.
You need to use the same strategy you use for the first one, wrapping the callback on a promise. So in your steps 2 and 3 you should return a new Promise and resolve it in the callback.
(...)
.then(function(result){
return new Promise(function(resolve, reject){
someThingWithCallback(result, function(err, data){
resolve(data)
})
})
})
.then(function(data){
...
})
You should try to avoid using a module that uses callbacks if you want to work with promises. You can use something like request-promise or axios.