I have 1,211,434 IP addresses that needed to be converted into geolocations. I found an api that answers this question by using GET request. But the thing is, the when using a for loop, I can not send the ip address and receive the description correctly.
Majorly I have two questions:
I just can not output the ip_and_info array, and can't find the reason. Can anybody tell me what went wrong?
Now, the code I wrote can retrieve all the information that I need, there are around 200 ip addresses in the test_ip.txt. Would there be a potential problem if I try to send all those 1M IP addresses?
Is there anyone can give me some advice?
Much Appreciated.
My code is as below:
fs = require('fs')
async = require("async")
http = require('http')
ip_and_info = []
// getIPInfo("1.171.58.24")
fs.readFile("../test_ips.txt", "utf-8", (err, content, printArr) => {
content = content.split("\n")
async.each(content, (ip) => {
content = getIPInfo(ip)
// console.log(ip)
}, (err) => {
if (err) {
console.log(err)
} else {
console.log(ip_and_info)
}
})
// for (i in content) {
// ((ip) => {
// getIPInfo(ip)
// })(content[i])
// }
});
function getIPInfo(ipAddress) {
options = {
host: 'freegeoip.net',
path: '/csv/' + ipAddress
}
request = http.get(options, function(response) {
// console.log('STATUS: ' + response.statusCode)
// console.log('HEADERS: ' + JSON.stringify(response.headers))
// Buffer the body entirely for processing as a whole.
bodyChunks = []
response.on('data', function(chunk) {
bodyChunks.push(chunk)
}).on('end', function() {
body = Buffer.concat(bodyChunks)
content = body.toString('ascii')
ip_and_info.push(content)
console.log(content)
return content
})
})
request.on('error', function(e) {
console.log('ERROR: ' + e.message)
})
}
Much Appreciated!
The problem lies in this line
content = getIPInfo(ip)
getIPInfo should be an async function. One way of doing it would be to send a callback to the function and in the function return the output in the callback.
async.each(content, getIPInfo, (err) => {
if (err) {
console.log(err)
} else {
console.log(ip_and_info)
}
})
And in the getIPInfo function
function getIPInfo(ipAddress, callback) {
.....
.....
ip_and_info.push(content)
callback();
}
Also, instead of using async.each use async.eachSeries or async.eachLimit else it will try to send request for all 1,211,434 ips .
Use Promise.
Use the let and const keywords. Seriously, implicit global aren't fun.
Decide whether to use ' or " and stick with it, it is way more readable.
With Promise, no need for async or your ip_and_info variable.
'use strict';
const fs = require('fs'),
http = require('http');
fs.readFile('../test_ips.txt', 'utf-8', (err, content) => {
content = content.split('\n');
Promise.resolve().then(() => {
return getAllIPInfo(content);
}).then((ipsInfos) => {
console.log('Info:' + ipsInfos);
}).catch((error) => {
console.error('Error: ' + error);
});
});
function getAllIPInfo(ipsAddress) {
return new Promise((resolve, reject) => {
let ipsInfo = [];
ipsAddress.reduce((previous, current, index, ips) => {
return previous.then(() => {
return getIPInfo(ips[index]).then((content) => {
ipsInfo.push(content);
return Promise.resolve();
});
});
}, Promise.resolve()).then(() => {
resolve(ipsInfo);
}).catch((error) => {
reject(error);
});
});
}
function getIPInfo(ipAddress) {
return new Promise((resolve, reject) => {
let options = {
host: 'freegeoip.net',
path: '/csv/' + ipAddress
};
http.get(options, function(response) {
// console.log('STATUS: ' + response.statusCode)
// console.log('HEADERS: ' + JSON.stringify(response.headers))
// Buffer the body entirely for processing as a whole.
let bodyChunks = [];
response.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
let body = Buffer.concat(bodyChunks),
content = body.toString('ascii');
resolve(content);
});
}).on('error', function(e) {
console.log('ERROR: ' + e.message);
reject(e);
});
});
}
I think your problem might be that you are re-declaring the 'content' variable each loop you make.
So perhaps change the loop to this so you don't reset the variable each time the loop executes. I hope that fixes your issue:
IPList = content.split("\n")
async.each(IPList, (ip) => {
IPGeoLocation = getIPInfo(ip)
console.log(IPGeoLocation)
}, (err) => {
As for doing this with a million IPs, I cant see major problem as long as you have a decent amount of memory on your computer. You might like to add a 'wait' call so you don't hammer the server so consistently. They might block you!
I would wait 1 second between each call by adding
sleep(1000);
after getting the IP.
Related
I am new to node js. I have used the python call where data comes from. Since the process is a little complex so it taking time to execute and node js to receive a response. The code I mentioned here tried to wait for the function to complete, but the function is repeating the call from the start and goes repeating until the python function finished and to get a response.
So after completion of function
console.log("Image Conversation::")
await bot.say(dumy_msg);
await bot.beginDialog('classfication_done_');
These statements are repeating for two times.
How to resolve the issue?
convo1.ask('Post Your message', async(answer, convo1, bot,message) => {
if(message.type=='file_share') {
console.log("Image Conversation::")
const botToken = bot.api.token;
const destination_path = '/_classification/upload/' + message.files[0].name;
console.log("URl :::",message.files[0].url_private)
console.log('Testing file share' + message.files[0].name);
const url = message.files[0].url_private;
const opts = {
method: 'GET',
url: url,
headers: {
Authorization: 'Bearer ' + botToken,
}
};
request(opts, function (err, res, body) {
console.log('FILE RETRIEVE STATUS', res.statusCode);
}).pipe(fs.createWriteStream(destination_path));
const img = '/_classification/upload/' + message.files[0].name;
spawn = require("child_process").spawn;
const getResponse = () => {
return new Promise((resolve, reject) => {
const process = spawn('python',["/_classification/tensorflow-image-detection-master/classify.py",img] );
process.stdout.on('data', data => {
resolve(data);
});
});
}
const response = await getResponse();
var dumy_msg = `${response}`;
await bot.say(dumy_msg);
await bot.beginDialog('classfication_done_');
}
else{
await bot.say(message,"Please provide jpeg file format, other files are not allowed");
await bot.beginDialog('classfication_done_');
}
});
I think the callback function(async () => {}) itself has been called twice.
I don't know exactly what the role convo1.ask do, but make sure it's not in a sitution call a callback twice.
I am calling an API where I can only fetch 1000 records per request,
I was able to achieve this using recursion.
I am now trying to achieve the same using promises, I am fairly new to Node.js and JavaScript too.
I tried adding the recursion code in an if else block but failed
var requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
}
const callback = (body) => {
// some code
.
.
.//saving records to file
.
//some code
if (totlExtractedRecords < total) {
requestP(option, callback).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
}
}
requestP(option).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
I want to execute the method using promise and in a synchronous way,
i.e. I want to wait until the records are all exported to a file and continue with my code
I think its better to create your own promise and simply resolve it when your done with your recursion. Here's a simply example just for you to understand the approach
async function myRecursiveLogic(resolveMethod, ctr = 0) {
// This is where you do the logic
await new Promise((res) => setTimeout(res, 1000)); // wait - just for example
ctr++;
console.log('counter:', ctr);
if (ctr === 5) {
resolveMethod(); // Work done, resolve the promise
} else {
await myRecursiveLogic(resolveMethod, ctr); // recursion - continue work
}
}
// Run the method with a single promise
new Promise((res) => myRecursiveLogic(res)).then(r => console.log('done'));
Here's a clean and nice solution using the latest NodeJS features.
The recursive function will continue executing until a specific condition is met (in this example asynchronously getting some data).
const sleep = require('util').promisify(setTimeout)
const recursive = async () => {
await sleep(1000)
const data = await getDataViaPromise() // you can replace this with request-promise
if (!data) {
return recursive() // call the function again
}
return data // job done, return the data
}
The recursive function can be used as follows:
const main = async () => {
const data = await recursive()
// do something here with the data
}
Using your code, I'd refactored it as shown below. I hope it helps.
const requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
};
/*
NOTE: Add async to the function so you can udse await inside the function
*/
const callback = async (body) => {
// some code
//saving records to file
//some code
try {
const result = await requestP(option, callback).auth('api-reader', token, true);
if (totlExtractedRecords < total) {
return callback(result);
}
return result;
} catch (error) {
console.log('Error Observed ' + err);
return error;
}
}
Created this code using feed back from Amir Popovich
const rp = require('Request-Promise')
const fs = require('fs')
const pageSize = 200
const options = {
url: 'https://jira.xyz.com/rest/api/2/search',
json: true,
qs: {
jql: "project in (PROJECT_XYZ)",
maxResults: pageSize,
startAt: 0,
fields: '*all'
},
auth: {
user: 'api-reader',
pass: '<token>',
sendImmediately: true
}
}
const updateCSV = (elment) => {
//fs.writeFileSync('issuedata.json', JSON.stringify(elment.body, undefined, 4))
}
async function getPageinatedData(resolve, reject, ctr = 0) {
var total = 0
await rp(options).then((body) => {
let a = body.issues
console.log(a)
a.forEach(element => {
console.log(element)
//updateCSV(element)
});
total = body.total
}).catch((error) => {
reject(error)
return
})
ctr = ctr + pageSize
options.qs.startAt = ctr
if (ctr >= total) {
resolve();
} else {
await getPageinatedData(resolve, reject, ctr);
}
}
new Promise((resolve, reject) => getPageinatedData(resolve, reject))
.then(() => console.log('DONE'))
.catch((error) => console.log('Error observed - ' + error.name + '\n' + 'Error Code - ' + error.statusCode));
I've been looking around for a while now and this is the only resource I've found on the internet related to my problem. I'm trying to download files from one ftp server then upload them to another ftp server, one by one using promises and without having to save the files locally during the process.
First I'm calling client.List() recursively from ftp module to get an array of file paths I'll need to download from the source ftp server. This works fine.
getRecursively(client, path) {
var _this = this;
let downloadList = [];
let paths = [];
let promise = new Promise((resolve, reject) => {
client.list(path, function(err, list) {
async function loop() {
for (var i = 0; i < list.length; i++) {
if (list[i].type == 'd') {
let _list = await _this.getRecursively(client, path + '/' + list[i].name)
downloadList = downloadList.concat(_list);
} else {
if ( list[i].name.match(/\.(jpg|jpeg)$/i) ) {
downloadList.push({path: path, name: list[i].name});
}
}
}
console.log("One complete");
resolve(downloadList);
}
loop();
})
})
return promise;
}
Next, I'm looping through the list of file paths and sending off promises which are throttled using es6-promise-pool module, so right now its concurrency limit is set to 10.
This is what each promise looks like:
getAndInsert(file) {
let _this = this;
let promise = new Promise((resolve, reject) => {
let c = new Client();
c.on('ready', () => {
let d = new Client();
d.on('ready', () => {
c.get(file.path + '/' + file.name, function(err, stream) {
if (err) {console.log(err); console.log("FILE NAME: " + file.name)}
d.put(stream.pipe(passThrough()), '/images/' + file.name, function() {
_this.uploadCount += 1;
_this.uploadedImages.push(file.name)
console.log(_this.uploadCount + '/' + _this._list.length + " uploaded.")
c.end();
d.end();
resolve(true);
});
});
})
d.on('error', (err) => {
if (err) console.log(err);
_this.onCompleteCallback();
})
d.connect(destinationFTP);
})
c.on('error', (err) => {
if (err) console.log(err);
_this.onCompleteCallback();
})
c.connect(sourceFTP);
})
return promise;
}
Each promise makes its own connection to the source and destination ftp server. I'm using the stream module's Transform object as well when I call d.put(stream.pipe(passThrough()). Here is that function.
const passThrough = () => {
var passthrough = new Transform();
passthrough._transform = function(data, encoding, done) {
this.push(data);
done();
};
return passthrough;
}
Finally, here is the main code that fires off the promises.
*buildPromises(list) {
for (let i = 0; i < list.length; i++) {
yield this.getAndInsert(list[i]);
}
}
let iterator = _this.buildPromises(list);
var pool = new PromisePool(iterator, 10);
pool.start()
.then(function(){
console.log("Finished")
}).catch((err) => {
console.log(err);
console.log("error processing pool promise");
})
This will go through and build the list just fine, however when I send the promises off I'm getting the following error:
Error: write after end
at writeAfterEnd (_stream_writable.js:236:12)
at Transform.Writable.write (_stream_writable.js:287:5)
at Socket.ondata (_stream_readable.js:639:20)
at emitOne (events.js:116:13)
at Socket.emit (events.js:211:7)
at Socket.Readable.read (_stream_readable.js:475:10)
at flow (_stream_readable.js:846:34)
at Transform.<anonymous> (_stream_readable.js:707:7)
at emitNone (events.js:106:13)
It might make it through like 5 and then error out, and sometimes more, but it seems to be pretty consistent. I've also noticed sometimes I'll get similar errors saying that 'The file is already in use', but each file im uploading has a unique name. Any help is appreciated, and if you need any more info I'll do my best to provide more information. Thank you.
So I found a solution. In my getAndInsert() function was doing:
c.get(file.path + '/' + file.name, function(err, stream) {
if (err) {console.log(err); console.log("FILE NAME: " + file.name)}
d.put(stream.pipe(passThrough()), '/images/' + file.name, function() {
_this.uploadCount += 1;
_this.uploadedImages.push(file.name)
console.log(_this.uploadCount + '/' + _this._list.length + " uploaded.")
c.end();
d.end();
resolve(true);
});
});
The problem resided in stream.pipe(passThrough()). It seemed as if I was trying to write after the stream had already ended. This is what solved my problem:
let chunks = [];
stream.on('data', (chunk) => {
chunks.push(chunk);
})
stream.on('end', () => {
d.put(Buffer.concat(chunks), '/images/' + file.name, function() {
_this.uploadCount += 1;
_this.uploadedImages.push(file.name)
console.log(_this.uploadCount + '/' + _this._list.length + " uploaded.")
c.end();
d.end();
resolve(true);
});
})
When new data is available from the stream push to an array called chunks. When the stream is finished, call .put and pass in Buffer.concat(chunks)
Hope this helps anyone going through a similar problem.
I'm trying to call a 3rd party API in my Alexa Skill, and I'm getting a "Session ended with reason: ERROR" in CloudWatch log. The issue appears to be in my NumberIntentHandler or my httpGet function, but I'm not sure where.
UPDATED CODE
-- Handler that's getting fired --
const NumberIntentHandler = {
canHandle(handlerInput) {
return handlerInput.requestEnvelope.request.type === 'IntentRequest'
&& handlerInput.requestEnvelope.request.intent.name === 'NumberIntent';
},
handle(handlerInput) {
let slotNum = handlerInput.requestEnvelope.request.intent.slots.number.value;
//var myRequest = parseInt(slotNum);
const myRequest = parseInt(slotNum);
console.log('NumberIntentHandler myRequest: ', myRequest);
var options = `http://numbersapi.com/${myRequest}`;
console.log('NumberIntentHandler options: ', options);
// Use the async function
const myResult = httpGet(options);
console.log("sent : " + options);
console.log("received : " + myResult);
const speechText = myResult;
console.log('speechText: ', speechText); // Print the speechText */
return handlerInput.responseBuilder
.speak(speechText)
.withSimpleCard('Here is your fact: ', speechText)
.getResponse();
},
};
-- Function that's getting called from the Handler --
async function httpGet(options) {
// return new pending promise
console.log(`~~~~~~~~~ httpGet ~~~~~~~~~`);
console.log(`~~~~~${JSON.stringify(options)}~~~~~`);
return new Promise((resolve, reject) => {
const request = http.get(options, (response) => {
// handle http errors
if (response < 200 || response > 299) {
reject(new Error('Failed to load page, status code: ' + response));
}// temporary data holder
const body = [];
// on every content chunk, push it to the data array
response.on('data', (chunk) => body.push(chunk));
// we are done, resolve promise with those joined chunks
response.on('end', () => resolve(body.join('')));
console.log('body: ', body[0]);
});
// handle connection errors of the request
request.on('error', (err) => reject(err));
request.end();
});
}
Updated Code - Eliminated async/await/promise
-- Handler --
const NumberIntentHandler = {
canHandle(handlerInput) {
return handlerInput.requestEnvelope.request.type === 'IntentRequest'
&& handlerInput.requestEnvelope.request.intent.name === 'NumberIntent';
},
handle(handlerInput) {
let slotNum = handlerInput.requestEnvelope.request.intent.slots.number.value;
//var myRequest = parseInt(slotNum);
const myRequest = parseInt(slotNum);
console.log('NumberIntentHandler myRequest: ', myRequest);
var options = `http://numbersapi.com/${myRequest}`;
console.log('NumberIntentHandler options: ', options);
// Use the async function
//const myResult = httpGet(options);
const myResult = httpGet(options, res => {
console.log("sent : " + options);
console.log("received : " + myResult);
const speechText = myResult;
console.log('speechText: ', speechText); // Print the speechText */
return handlerInput.responseBuilder
.speak(speechText)
.withSimpleCard('Here is your fact: ', speechText)
.getResponse();
});
},
};
-- Function --
function httpGet(options, cb) {
http.get(options, res => {
console.log(`~~~~~${JSON.stringify(options)}~~~~~`);
// simplified version without error handling
let output = [];
res.on('data', d => output.push(d)); // or concat to a string instead?
res.on('end', () => cb(output));
console.log('output: ', output[0]);
});
}
I believe you will need to call resolve with your response in httpGet.
As a side note (unrelated to your problem) - I can recommend using request-promise, it implements a very nice promise api around http and would simplify your code in this case. (I know I know, async/await are new and fun tools but in this case I would go with "simpler" :) ).
Also, if I remember correctly, the callback for http.get is being invoked with only one argument.
edit, after changes:
you could get rid of the promise and async to simplify your code.
Just a note on async/await - if the awaited expression isn't a promise then it is cast into one automatically. In your current code you either need to use it like a promise (chain a .then() for example) or await it.
Anyways, here is an example that is just using a callback:
function httpGet(options, cb) {
http.get(options, res => {
// simplified version without error handling
let output = [];
res.on('data', d => output.push(d)); // or concat to a string instead?
res.on('end', () => cb(output));
});
}
httpGet(options, res => {
// building the alexa response, all your intent handler code that needs the response from your request
})
So I have a method, which I want to call multiple times in a loop. This is the function:
function PageSpeedCall(callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[0]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
As you see this is an async function that calls the PageSpeed API. It then gets the response thanks to the callback and renders it in the view. Now how do I get this to be work in a for/while loop? For example
function PageSpeedCall(websites, i, callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[i]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
var websites = ['google.com','facebook.com','stackoverflow.com'];
for (let i = 0; i < websites.length; i++) {
PageSpeedCall(websites, i);
}
I want to get a raport for each of these sites. The length of the array will change depending on what the user does.
I am using async.parallel to call the functions like this:
let freeReportCalls = [PageSpeedCall, MozCall, AlexaCall];
async.parallel(freeReportCalls, (err, results) => {
if (err) {
console.log(err);
} else {
res.render('reports/report', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}
});
I tried to use promise chaining, but for some reason I cannot put it together in my head. This is my attempt.
return Promise.all([PageSpeedCall,MozCall,AlexaCall]).then(([ps,mz,al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
I tried doing this:
return Promise.all([for(let i = 0;i < websites.length;i++){PageSpeedCall(websites, i)}, MozCall, AlexaCall]).
then(([ps, mz, al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
But node just said it's stupid.
And this would work if I didn't want to pass the websites and the iterator into the functions. Any idea how to solve this?
To recap. So far the functions work for single websites. I'd like them to work for an array of websites.
I'm basically not sure how to call them, and how to return the responses.
It's much easier if you use fetch and async/await
const fetch = require('node-fetch');
async function PageSpeedCall(website) {
const pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
const result = await fetch(pagespeeddCall);
return await result.json();
}
async function callAllSites (websites) {
const results = [];
for (const website of websites) {
results.push(await PageSpeedCall(website));
}
return results;
}
callAllSites(['google.com','facebook.com','stackoverflow.com'])
.then(results => console.log(results))
.error(error => console.error(error));
Which is better with a Promise.all
async function callAllSites (websites) {
return await Promise.all(websites.map(website => PageSpeedCall(website));
}
Starting on Node 7.5.0 you can use native async/await:
async function PageSpeedCall(website) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
return await promisify(pagespeedCall);
}
async function getResults(){
const websites = ['google.com','facebook.com','stackoverflow.com'];
return websites.map(website => {
try {
return await PageSpeedCall(website);
}
catch (ex) {
// handle exception
}
})
}
Node http "callback" to promise function:
function promisify(url) {
// return new pending promise
return new Promise((resolve, reject) => {
// select http or https module, depending on reqested url
const lib = url.startsWith('https') ? require('https') : require('http');
const request = lib.get(url, (response) => {
// handle http errors
if (response.statusCode < 200 || response.statusCode > 299) {
reject(new Error('Failed to load page, status code: ' + response.statusCode));
}
// temporary data holder
const body = [];
// on every content chunk, push it to the data array
response.on('data', (chunk) => body.push(chunk));
// we are done, resolve promise with those joined chunks
response.on('end', () => resolve(body.join('')));
});
// handle connection errors of the request
request.on('error', (err) => reject(err))
})
}
Make PageSpeedCall a promise and push that promise to an array as many times as you need, e.g. myArray.push(PageSpeedCall(foo)) then myArray.push(PageSpeedCall(foo2)) and so on. Then you Promise.all the array.
If subsequent asynch calls require the result of a prior asynch call, that is what .then is for.
Promise.all()
Promise.all([promise1, promise2, promise3]).then(function(values) {
console.log(values);
});