Develop Reference

Getting 'undefined' from asynchronous response despite 'await' and 'then'

I'm trying to send a GET request, parse its response and return it to another method. Apparently I have problems handling the asynchronous response.
I want to use Node.js' standard modules, so no Axios.
// class 1: Calling API, processing and returning the response
export async function getData() {
let str = '';
const options = {
hostname: 'jsonplaceholder.typicode.com',
path: '/posts/',
method: 'GET',
json: true,
};
https
.get(options, response => {
response.on('data', chunk => {
str += chunk;
});
response.on('end', () => {
return parseJson(str);
});
})
.on('error', error => {
console.log(error);
});
}
async function parseJson(str) {
const json = JSON.parse(str);
var text;
try {
json.forEach(element => {
text += element.body;
});
// console.log(text); // I'm getting the expected output
return text;
} catch (error) {
console.log('error');
}
}
// class 2: Calling the 2 methods above
getData().then(function (value) {
console.log('DATA: ' + value); // this is called first
});
Unfortunately as output I get an undefined. Despite using async and then:
DATA: undefined

Change getData as follows
export function getData() {
return new Promise((resolve, reject) => {
let str = '';
const options = {
hostname: 'jsonplaceholder.typicode.com',
path: '/posts/',
method: 'GET',
json: true,
};
https.get(options, response => {
response.on('data', chunk => {
str += chunk;
});
response.on('end', () => {
try {
const result = parseJson(str);
resolve(result);
} catch (error) {
reject(error);
}
});
response.on('error', reject);
})
.on('error', reject);
});
}
Now it returns a Promise, which resolves to the result of parseJson(str) or rejects with the error at on('error'
And parseJson as follows - doesn't need to be async since there's nothing asynchronous about the code inside it
Also, removing the try/catch in parseJson and using try/catch in .on("end" means that you can reject the promise returned by getData if there's an error in the parseJson call
function parseJson(str) {
const json = JSON.parse(str);
let text = '';
json.forEach(element => {
text += element.body;
});
return text;
}
alternatively (IMHO better)
function parseJson(str) {
const data = JSON.parse(str); // call it data, not json
return data.map(({body}) => body).join('');
}
or even
const parseJson = (str) => JSON.parse(str).map(({body}) => body).join('');
But that's not important :p

How to return data from http.get to parent function in nodejs

Task is to return data from getData function to main function.
function getData(){
const https = require('https')
const url = "https://...../api/movies";
https.get(url, res => {
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
data = JSON.parse(data);
console.log(data);
//How can I return this data to main function?
})
}).on('error', err => {
console.log(err.message);
})
}
function main(){
console.log(getData());
}
I am not able to access data or print data in main function

I think you already have the answer in your comment '//How can I return this data to main function?'.
...
res.on('end', () => {
data = JSON.parse(data);
console.log(data);
return data;
})
...
So the return-value of your getData-function should now be the parsed json-data and be accessible in the main-function.

I would create a variable to save value which you want to return
const https = require('https')
function getData() {
let ret = null;
const url = "https://...../api/movies";
https.get(url, res => {
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
ret = JSON.parse(data);
})
}).on('error', err => {
console.log(err.message);
})
return ret;
}
function main() {
console.log(getData());
}

What you could do is to wrap your request into a Promise. You would want to return this promise and and wait for it to be fullfilled:
function getData() {
const url = "https://...../api/movies";
return new Promise((resolve, reject) => {
const req = https.get(url, (res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
resolve(JSON.parse(data));
});
});
req.on('error', (err) => {
reject(err);
});
req.end();
});
}
function main() {
getData().then(data => {
console.log("Response:", data);
}, error => {
console.error(error);
});
}
Moreover you can make your main function async and use await to get the response:
async function main() {
const data = await getData();
console.log(data);
}
// Start an IIFE to use `await` at the top level. See: https://stackoverflow.com/a/14220323/7661119
(async () => {
await main()
})();

Can't get an object from a get request

I can console log userData.name and userData.summonerLevel just fine, but why can't I return userData as an object?
I am trying to assign the returned object to a variable, but I really don't understand why it doesn't work.
function getStats() {
https.get(https://euw1.api.riotgames.com/lol/summoner/v3/summoners/by-name/yojimbozx?api_key=${API_KEY},
(res) => {
let userData = ''
res.on('data', (chunk) => {
userData += chunk
})
res.on('end', () => {
userData = JSON.parse(userData)
if(!userData.hasOwnProperty('status')) {
console.log(userData.name, userData.summonerLevel)
return userData
}
return console.log("Summoner not found")
})
}).on("error", (err) => {
return console.log("Error: " + err.message)
})
}

You can, but since you are doing some async stuff you should have your function getStats () return the http call and it should work. Currently you are returning the outcome of the call to the function, but the function is not returning anything

s3.getObject().createReadStream() : How to catch the error?

I am trying to write a program to get a zip file from s3, unzip it, then upload it to S3.
But I found two exceptions that I can not catch.
1. StreamContentLengthMismatch: Stream content length mismatch. Received 980323883 of 5770104761 bytes. This occurs irregularly.
2. NoSuchKey: The specified key does not exist. This happens when I input the wrong key.
When these two exceptions occur, this program crashes.
I'd like to catch and handle these two exceptions correctly.
I want to prevent a crash.
const unzipUpload = () => {
return new Promise((resolve, reject) => {
let rStream = s3.getObject({Bucket: 'bucket', Key: 'hoge/hoge.zip'})
.createReadStream()
.pipe(unzip.Parse())
.on('entry', function (entry) {
if(entry.path.match(/__MACOSX/) == null){
// pause
if(currentFileCount - uploadedFileCount > 10) rStream.pause()
currentFileCount += 1
var fileName = entry.path;
let up = entry.pipe(uploadFromStream(s3,fileName))
up.on('uploaded', e => {
uploadedFileCount += 1
console.log(currentFileCount, uploadedFileCount)
//resume
if(currentFileCount - uploadedFileCount <= 10) rStream.resume()
if(uploadedFileCount === allFileCount) resolve()
entry.autodrain()
}).on('error', e => {
reject()
})
}
}).on('error', e => {
console.log("unzip error")
reject()
}).on('finish', e => {
allFileCount = currentFileCount
})
rStream.on('error', e=> {
console.log(e)
reject(e)
})
})
}
function uploadFromStream(s3,fileName) {
var pass = new stream.PassThrough();
var params = {Bucket: "bucket", Key: "hoge/unzip/" + fileName, Body: pass};
let request = s3.upload(params, function(err, data) {
if(err) pass.emit('error')
if(!err) pass.emit('uploaded')
})
request.on('httpUploadProgress', progress => {
console.log(progress)
})
return pass
}
This is the library I use when unzipping.
https://github.com/mhr3/unzip-stream
Help me!!

If you'd like to catch the NoSuchKey error thrown by createReadStream you have 2 options:
Check if key exists before reading it.
Catch error from stream
First:
s3.getObjectMetadata(key)
.promise()
.then(() => {
// This will not throw error anymore
s3.getObject().createReadStream();
})
.catch(error => {
if (error.statusCode === 404) {
// Catching NoSuchKey
}
});
The only case when you won't catch error if file was deleted in a split second, between parsing response from getObjectMetadata and running createReadStream
Second:
s3.getObject().createReadStream().on('error', error => {
// Catching NoSuchKey & StreamContentLengthMismatch
});
This is a more generic approach and will catch all other errors, like network problems.

You need to listen for the emitted error earlier. Your error handler is only looking for errors during the unzip part.
A simplified version of your script.
s3.getObject(params)
.createReadStream()
.on('error', (e) => {
// handle aws s3 error from createReadStream
})
.pipe(unzip)
.on('data', (data) => {
// retrieve data
})
.on('end', () => {
// stream has ended
})
.on('error', (e) => {
// handle error from unzip
});
This way, you do not need to make an additional call to AWS to find out if out if it exists.

You can listen to events (like error, data, finish) in the stream you are receiving back. Read more on events
function getObjectStream (filePath) {
return s3.getObject({
Bucket: bucket,
Key: filePath
}).createReadStream()
}
let readStream = getObjectStream('/path/to/file.zip')
readStream.on('error', function (error) {
// Handle your error here.
})
Tested for "No Key" error.
it('should not be able to get stream of unavailable object', function (done) {
let filePath = 'file_not_available.zip'
let readStream = s3.getObjectStream(filePath)
readStream.on('error', function (error) {
expect(error instanceof Error).to.equal(true)
expect(error.message).to.equal('The specified key does not exist.')
done()
})
})
Tested for success.
it('should be able to get stream of available object', function (done) {
let filePath = 'test.zip'
let receivedBytes = 0
let readStream = s3.getObjectStream(filePath)
readStream.on('error', function (error) {
expect(error).to.equal(undefined)
})
readStream.on('data', function (data) {
receivedBytes += data.length
})
readStream.on('finish', function () {
expect(receivedBytes).to.equal(3774)
done()
})
})

To prevent a crash, you need to asynchronously listen to the object's head metadata, where it does not return the whole object, which will take less time. Try this one!
isObjectErrorExists = async functions () => {
try {
const s3bucket = {
secret key: '',
client id: ''
}
const params = {
Bucket: 'your bucket name',
Key: 'path to object'
};
await s3bucket.headObject(params).promise(); // adding promise will let you add await to listen to process untill it completes.
return true;
} catch (err) {
return false; // headObject threw error.
}
throw new Error(err.message);
}
}
public yourFunction = async() => {
if (await this.isObjectErrorExists()) {
s3Bucket.getObject().createReadStream(); // works smoothly
}
}

Node JS for loop and array push

I have 1,211,434 IP addresses that needed to be converted into geolocations. I found an api that answers this question by using GET request. But the thing is, the when using a for loop, I can not send the ip address and receive the description correctly.
Majorly I have two questions:
I just can not output the ip_and_info array, and can't find the reason. Can anybody tell me what went wrong?
Now, the code I wrote can retrieve all the information that I need, there are around 200 ip addresses in the test_ip.txt. Would there be a potential problem if I try to send all those 1M IP addresses?
Is there anyone can give me some advice?
Much Appreciated.
My code is as below:
fs = require('fs')
async = require("async")
http = require('http')
ip_and_info = []
// getIPInfo("1.171.58.24")
fs.readFile("../test_ips.txt", "utf-8", (err, content, printArr) => {
content = content.split("\n")
async.each(content, (ip) => {
content = getIPInfo(ip)
// console.log(ip)
}, (err) => {
if (err) {
console.log(err)
} else {
console.log(ip_and_info)
}
})
// for (i in content) {
// ((ip) => {
// getIPInfo(ip)
// })(content[i])
// }
});
function getIPInfo(ipAddress) {
options = {
host: 'freegeoip.net',
path: '/csv/' + ipAddress
}
request = http.get(options, function(response) {
// console.log('STATUS: ' + response.statusCode)
// console.log('HEADERS: ' + JSON.stringify(response.headers))
// Buffer the body entirely for processing as a whole.
bodyChunks = []
response.on('data', function(chunk) {
bodyChunks.push(chunk)
}).on('end', function() {
body = Buffer.concat(bodyChunks)
content = body.toString('ascii')
ip_and_info.push(content)
console.log(content)
return content
})
})
request.on('error', function(e) {
console.log('ERROR: ' + e.message)
})
}
Much Appreciated!

The problem lies in this line
content = getIPInfo(ip)
getIPInfo should be an async function. One way of doing it would be to send a callback to the function and in the function return the output in the callback.
async.each(content, getIPInfo, (err) => {
if (err) {
console.log(err)
} else {
console.log(ip_and_info)
}
})
And in the getIPInfo function
function getIPInfo(ipAddress, callback) {
.....
.....
ip_and_info.push(content)
callback();
}
Also, instead of using async.each use async.eachSeries or async.eachLimit else it will try to send request for all 1,211,434 ips .

Use Promise.
Use the let and const keywords. Seriously, implicit global aren't fun.
Decide whether to use ' or " and stick with it, it is way more readable.
With Promise, no need for async or your ip_and_info variable.
'use strict';
const fs = require('fs'),
http = require('http');
fs.readFile('../test_ips.txt', 'utf-8', (err, content) => {
content = content.split('\n');
Promise.resolve().then(() => {
return getAllIPInfo(content);
}).then((ipsInfos) => {
console.log('Info:' + ipsInfos);
}).catch((error) => {
console.error('Error: ' + error);
});
});
function getAllIPInfo(ipsAddress) {
return new Promise((resolve, reject) => {
let ipsInfo = [];
ipsAddress.reduce((previous, current, index, ips) => {
return previous.then(() => {
return getIPInfo(ips[index]).then((content) => {
ipsInfo.push(content);
return Promise.resolve();
});
});
}, Promise.resolve()).then(() => {
resolve(ipsInfo);
}).catch((error) => {
reject(error);
});
});
}
function getIPInfo(ipAddress) {
return new Promise((resolve, reject) => {
let options = {
host: 'freegeoip.net',
path: '/csv/' + ipAddress
};
http.get(options, function(response) {
// console.log('STATUS: ' + response.statusCode)
// console.log('HEADERS: ' + JSON.stringify(response.headers))
// Buffer the body entirely for processing as a whole.
let bodyChunks = [];
response.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
let body = Buffer.concat(bodyChunks),
content = body.toString('ascii');
resolve(content);
});
}).on('error', function(e) {
console.log('ERROR: ' + e.message);
reject(e);
});
});
}

I think your problem might be that you are re-declaring the 'content' variable each loop you make.
So perhaps change the loop to this so you don't reset the variable each time the loop executes. I hope that fixes your issue:
IPList = content.split("\n")
async.each(IPList, (ip) => {
IPGeoLocation = getIPInfo(ip)
console.log(IPGeoLocation)
}, (err) => {
As for doing this with a million IPs, I cant see major problem as long as you have a decent amount of memory on your computer. You might like to add a 'wait' call so you don't hammer the server so consistently. They might block you!
I would wait 1 second between each call by adding
sleep(1000);
after getting the IP.