I have a list of proxies which I want to test if they are still active or not, after creating a promise list I run through the results, but after the last promise result the process doesnt stop and the programm freezes even though it should finish because there isnt any code after that.
The only fix I found was to put a process.exit() after running through all results, but this seems really odd to me. I want to automate the process but because the result function never ends its really difficult to workaround it.
const axios = require('axios')
const HttpsProxyAgent = require('https-proxy-agent');
const fs = require('fs');
const data = fs.readFileSync("./proxies.txt", "utf-8");
const proxyList = data.split("\n")
const axiosGet = (url, proxy) => {
const abort = axios.CancelToken.source()
const id = setTimeout(() => abort.cancel(`Timeout of 60000ms.`), 60000)
return axios.get(url, { cancelToken: abort.token, httpsAgent: new HttpsProxyAgent(`http://${proxy.trim()}`), proxy: false })
.then(response => { clearTimeout(id); return true })
.catch(error => { clearTimeout(id); return false })
}
const promises = proxyList.map((proxy, index) => {
return axiosGet(`https://api.ipify.org`, proxy, index)
}).map(p => p.catch(e => { e; return false }))
Promise.all(promises)
.then((results) => {
results.forEach((result, index) => console.log(index + 1, result))
//not exiting after running through all promises, only process.exit() works
})
.catch(err => console.log(err));
Related
I have the following code that is used to get JSON data from an Amazon Web Server API.
var json1 = new Promise((resolve, reject) => {
fetch(url[0])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
I have this repeating 14 times using different urls and json vars and have it return the promises at the end using.
return Promise.all([json1,json2,json3,json4,json5,json6,json7,json8,json9,json10,json11,json12,json13,json14]).then(function(values) {
return values;
});
This works, but it takes up 150+ lines. I want to make a for loop that runs through the same code using a for loop. I created this...
for(var jsonCount = 0;jsonCount<url.length-1;jsonCount++){
jsonArr[jsonCount] = new Promise((resolve, reject) => {
fetch(url[jsonCount])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
}
This doesn't work because the promise functions come back as undefined even though it is called by an await function.
const data = await fetchURL(urlToQuery())
Does anyone have suggestions to make this work? There is JSON being returned.
Thanks for your help.
Edit: Here is a larger chunk of the code.
function fetchURL(urls) {
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
return arr;
});
(async function() {
const data = await fetchURL(urlToQuery())
console.log(data);
for(var r=0;r<numStations;r++){
if (data[r] == ""){
onlineArr[r] = false;
wdDataArr[r].push(cardinalToDeg(stationHistAvgArr[r]));
wsDataArr[r].push(0);
You can use .map for the loop. But don't use new Promise. You don't need a new promise when fetch already provides you with one.
Also, call your array urls instead of url. A plural will be a good indication for the reader of your code that indeed it is a collection of URLs.
Here is how it could look:
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
// process your data
for (let obj of arr) {
console.log(obj);
}
});
I think this example can helps you:
// Mock async function
const getDataAsync = callback => {
setTimeout(
() => callback(Math.ceil(Math.random() * 100)),
Math.random() * 1000 + 2000
)
}
// Create the promise
const getDataWithPromise = () => {
return new Promise((resolve, reject) => {
try {
getDataAsync(resolve);
} catch(e) {
reject(e);
}
});
}
// Using the promise one time
getDataWithPromise()
.then(data => console.log("Simple promise:",data))
.catch(error => console.error(`Error catched ${error}`));
// Promises compound: Promise.all
const promise1 = getDataWithPromise();
promise1.then(data => console.log("promise1 ends:",data));
const promise2 = getDataWithPromise();
promise2.then(data => console.log("promise2 ends:",data));
const promise3 = getDataWithPromise();
promise3.then(data => console.log("promise3 ends:",data));
const promise4 = getDataWithPromise();
promise4.then(data => console.log("promise4 ends:",data));
const promise5 = getDataWithPromise();
promise5.then(data => console.log("promise5 ends:",data));
Promise.all([promise1,promise2,promise3,promise4,promise5])
.then(data => console.log("Promise all ends !!",data));
Hope this helps
you will have issues with closure and var variable capture.
You may want to change var to let to capture the right value in the closure so that url[jsonCount] is actually what you want.
also I think it would be much easier to do something like that in one line :)
let results = [];
for(let i = 0; i < urls.length; ++i) results.push(await (await fetch[urls[i]]).json());
This is a good use for map, mapping urls to promises...
function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
return Promise.all(promises).then(results => {
return results.map(result => result.json())
})
}}
// url is your array of urls (which would be better named as a plural)
fetchUrls(url).then(results => {
// results will be the fetched json
})
Using the async/await syntax (equivalent meaning)
// this can be called with await from within another async function
async function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
let results = await Promise.all(promises)
return results.map(result => result.json())
}
I have to do a functionality to test if 3 APIs are running.
Thus, the user will click on the Test APIs button and it will return the status of each API (status: 200, 500, 404 etc). If an API return an error, I should show the error stack.
Screen example:
API Status Detail
url1.com 200 -
url2.com 200 -
url3.com 500 internal server error
My question is, how can I call the 3 requests in parallel and return the async result, I mean how can I update the screen of API request status without having to wait for the result of all requests
I was basing on that How do I call three requests in order?, but it returns the result synchronously.
*******EDIT*****
Thats my current code
app.get('/testDependencies', function (req, res, next) {
let objTestsResul = {}
var urls = ['url1', 'url2', 'url3'];
let index = 0
while(urls.length > 0) {
let url = urls.shift();
objTestsResult[index++] = testURL(url)
}
res.send(objTestsResult)
});
This function is the same for each URL:
function testURL(URL){
fetch(URL, {
method: 'GET'
})
.then(res => {
res.json()
})
.then(json => {
console.log(json)
return json
})
.catch(error => {
return error
})
}
Promises (mdn) seem to be what you're looking for. They're essentially a more readable version of callbacks, which allow you to execute code when something else occurs rather than having to wait for that trigger to occur before resuming execution.
let endpoint1 = () => new Promise(resolve => setTimeout(() => resolve('200'), 1000));
let endpoint2 = () => new Promise(resolve => setTimeout(() => resolve('201'), 2000));
let endpoint3 = () => new Promise(resolve => setTimeout(() => resolve('500'), 1500));
document.getElementById('test').addEventListener('click', () => {
document.getElementById('status').textContent = 'test running...';
Promise.all([
endpoint1().then(a => document.getElementById('result1').textContent = a),
endpoint2().then(a => document.getElementById('result2').textContent = a),
endpoint3().then(a => document.getElementById('result3').textContent = a),
]).then(() => document.getElementById('status').textContent = 'test complete');
});
<button id="test">test</button>
<div>status: <span id="status">not running</span></div>
<div>endpoint 1: <span id="result1"></span></div>
<div>endpoint 2: <span id="result2"></span></div>
<div>endpoint 3: <span id="result3"></span></div>
This is actually pretty straightforward if you can use Bluebird:
const { Promise } = require('bluebird');
app.get('/testDependencies', function (req, res, next) {
Promise.map(['url1', 'url2', 'url3'], url => testURL(url)).then(results => {
res.send(results);
});
});
You'll just need to ensure your promise function actually returns a promise:
function testURL(URL) {
let start_time = new Date().getTime();
return fetch(URL, {
method: 'GET'
}).then(res => {
res.json()
}).then(json => {
console.log(json)
return json
}).catch(error => {
return error
})
}
Promises can't be dependency chained unless you explicitly return them from the function that's involved in chaining.
If you're able to use async and await, I'd also recommend doing that as well as that can vastly simplify otherwise complex code.
Express can't send multiple responses. You will have to finish all calls or use WebSockets to stream data.
function testURL(URL) {
return new Promise((resolve, reject) => {
if (URL === 'url2') {
reject(new Error('Internal Server Error'));
return;
}
resolve({ status: 200 });
});
}
const main = async () => {
const urls = ['url1', 'url2', 'url3'];
// return resolved and rejected Promises because if one fails in Promise.all
// the function will throw and we won't have any access to any resolved Promises.
const results = await Promise.all(urls
.map(url => testURL(url).then(response => response).catch(error => error)));
// every error have a stack property, Set the status to whatever you want
// based on the error and store the stack and the message
const objTestsResul = results.reduce((result, cur, i) => {
result[urls[i]] = cur.stack
? { status: 500, message: cur.message, stack: cur.stack }
: cur;
return result;
}, {});
console.log(objTestsResul);
};
main();
I'm importing a json file and loop through the object.
Each object has an array with eventIDs, which I loop through as well.
In this forEach I push a wrapper function that returns a Promise into an array promises
After looping through all objects I perform a promise.all() on the promises array. But strangely anything I do after this loop is not executed.
const fs = require('fs')
const promisify = require('util').promisify
const rp = require('request-promise')
const readFile = promisify(fs.readFile)
const filePath = 'json/results.json'
let promises = []
function init() {
readFile(filePath, 'utf8')
.then(file => {
const json = JSON.parse(file)
for (const country of json) {
if (country.eventIDs.length === 0) return
country.eventIDs.forEach(id => promises.push(getEventData(country, id)))
}
// nothing here is executed,
console.log('this is not fired')
Promise.all(promises)
.then(results => writeFile(results))
.catch( err => console.log(err))
})
.catch(err => console.log(err))
}
getEventData = (country, id) => new Promise((resolve, reject) => {
setTimeout(() => {
rp(url)
.then((results) => {
resolve ({
...results
})
})
.catch((err) => reject(`getEventData Error:\n ${err}`))
}, 2000)
})
writeFile = (results) => {
const json = JSON.stringify(results)
// const date = new Date()
// const filename = `${date.getTime()}-all-event-ids.json`
const filename = `results-allevents.json`
fs.writeFile(`json/${filename}`, json, 'utf8', () => console.log(`Succesfully written: ${filename}`))
}
init()
After the investigation, the line:
if (country.eventIDs.length === 0) return
was the main issue (and many others solved through comments).
The issue, to explain that, is that the return is not skipping the looped item (as perhaps expected), but rather returning void in the then callback, so skipping further executions of that block.
In order to "skip" the item if that condition is true, just do this instead
for (const country of json) {
if (country.eventIDs.length > 0) {
country.eventIDs.forEach(id => promises.push(getEventData(country, id)))
}
}
better is to capture each result from each promise. Because if you want array of responses of the .All . If one of those promises reject you will get no results.
const results = [];
country.eventIDs.forEach(id => promises.push(getEventData(country, id)
.then(res => results.push(res));
Promise.all(promises).then(()=> writeFile(results))
Stackblitz Example
I'm trying to remove the event node once it expires and remove all it's children
image of the event node before getting removed
the problem is when the time passes and i delete the event node all it's children get removed except this child
image of the event node after getting removed
source code
exports.removeOldEvents = functions.https.onRequest((req, res) => {
const eventsRef = admin.database().ref('events')
eventsRef.once('value', (snapshot) => {
snapshot.forEach((child) => {
child.forEach((child) => {
if (1000*Number(child.val()['endDate']) <= new Date().getTime()) {
child.ref.set(null)
}
})
})
})
return res.status(200).end()
})
Since you call several time the set() method, which returns a promise, you should use Promise.all() in order to wait all the promises resolve before sending back the response.
The following adaptation of your code should work (not tested however):
exports.removeOldEvents = functions.https.onRequest((req, res) => {
const eventsRef = admin.database().ref('events')
eventsRef.once('value')
.then((snapshot) => {
const promises = [];
snapshot.forEach((child) => {
child.forEach((child) => {
if (1000*Number(child.val().endDate) <= new Date().getTime()) {
promises.push(child.ref.set(null));
}
});
});
return Promise.all(promises);
})
.then(results => {
const responseObj = {response: 'success'};
res.send(responseObj);
})
.catch(err => {
res.status(500).send(err);
})
});
I kind of got the same question as has been asked here.
I've made a function:
const rewardStayingViewersOrNewcomers = () => {
fetch('https://tmi.twitch.tv/group/user/instak/chatters')
.then(parseJSON)
.then(r => {
let chatters = r.chatters;
viewerKeys = Object.keys(chatters); // [mods, viewers,...]
let ChattersPerRole = viewerKeys.map(role => {
return chatters[role].map(username => ({
username, role
}));
});
return flattenDeep(ChattersPerRole);
}).catch(err => {
console.log(`Error in fetch: ${err}`);
});
};
Why can't I assign that return value to my variable? The log of that variable returns undefined...
let viewersPerRole = rewardStayingViewersOrNewcomers();
setTimeout(() => console.log(viewersPerRole), 7000);
Bonus question, how could I easily wait for viewersPerRole to be filled with the data I'm waiting for because of the fetch? (so I don't have to use setTimeout())?
First of all, try returning something from the main function. There's no return in front on the fetch(...). Code should look like:
const rewardStayingViewersOrNewcomers = () => {
return fetch('https://tmi.twitch.tv/group/user/instak/chatters')
.then(parseJSON)
.then(r => {
let chatters = r.chatters;
viewerKeys = Object.keys(chatters); // [mods, viewers,...]
let ChattersPerRole = viewerKeys.map(role => {
return chatters[role].map(username => ({
username, role
}));
});
return Promise.resolve(flattenDeep(ChattersPerRole));
}).catch(err => {
console.log(`Error in fetch: ${err}`);
});
};
// now that you're returning a promise
rewardStayingViewersOrNewcomers()
.then(viewersPerRole => console.log(viewersPerRole))
If you're using Babeljs to transpile with stage3 enabled, have a look at async / await.