I have to do a functionality to test if 3 APIs are running.
Thus, the user will click on the Test APIs button and it will return the status of each API (status: 200, 500, 404 etc). If an API return an error, I should show the error stack.
Screen example:
API Status Detail
url1.com 200 -
url2.com 200 -
url3.com 500 internal server error
My question is, how can I call the 3 requests in parallel and return the async result, I mean how can I update the screen of API request status without having to wait for the result of all requests
I was basing on that How do I call three requests in order?, but it returns the result synchronously.
*******EDIT*****
Thats my current code
app.get('/testDependencies', function (req, res, next) {
let objTestsResul = {}
var urls = ['url1', 'url2', 'url3'];
let index = 0
while(urls.length > 0) {
let url = urls.shift();
objTestsResult[index++] = testURL(url)
}
res.send(objTestsResult)
});
This function is the same for each URL:
function testURL(URL){
fetch(URL, {
method: 'GET'
})
.then(res => {
res.json()
})
.then(json => {
console.log(json)
return json
})
.catch(error => {
return error
})
}
Promises (mdn) seem to be what you're looking for. They're essentially a more readable version of callbacks, which allow you to execute code when something else occurs rather than having to wait for that trigger to occur before resuming execution.
let endpoint1 = () => new Promise(resolve => setTimeout(() => resolve('200'), 1000));
let endpoint2 = () => new Promise(resolve => setTimeout(() => resolve('201'), 2000));
let endpoint3 = () => new Promise(resolve => setTimeout(() => resolve('500'), 1500));
document.getElementById('test').addEventListener('click', () => {
document.getElementById('status').textContent = 'test running...';
Promise.all([
endpoint1().then(a => document.getElementById('result1').textContent = a),
endpoint2().then(a => document.getElementById('result2').textContent = a),
endpoint3().then(a => document.getElementById('result3').textContent = a),
]).then(() => document.getElementById('status').textContent = 'test complete');
});
<button id="test">test</button>
<div>status: <span id="status">not running</span></div>
<div>endpoint 1: <span id="result1"></span></div>
<div>endpoint 2: <span id="result2"></span></div>
<div>endpoint 3: <span id="result3"></span></div>
This is actually pretty straightforward if you can use Bluebird:
const { Promise } = require('bluebird');
app.get('/testDependencies', function (req, res, next) {
Promise.map(['url1', 'url2', 'url3'], url => testURL(url)).then(results => {
res.send(results);
});
});
You'll just need to ensure your promise function actually returns a promise:
function testURL(URL) {
let start_time = new Date().getTime();
return fetch(URL, {
method: 'GET'
}).then(res => {
res.json()
}).then(json => {
console.log(json)
return json
}).catch(error => {
return error
})
}
Promises can't be dependency chained unless you explicitly return them from the function that's involved in chaining.
If you're able to use async and await, I'd also recommend doing that as well as that can vastly simplify otherwise complex code.
Express can't send multiple responses. You will have to finish all calls or use WebSockets to stream data.
function testURL(URL) {
return new Promise((resolve, reject) => {
if (URL === 'url2') {
reject(new Error('Internal Server Error'));
return;
}
resolve({ status: 200 });
});
}
const main = async () => {
const urls = ['url1', 'url2', 'url3'];
// return resolved and rejected Promises because if one fails in Promise.all
// the function will throw and we won't have any access to any resolved Promises.
const results = await Promise.all(urls
.map(url => testURL(url).then(response => response).catch(error => error)));
// every error have a stack property, Set the status to whatever you want
// based on the error and store the stack and the message
const objTestsResul = results.reduce((result, cur, i) => {
result[urls[i]] = cur.stack
? { status: 500, message: cur.message, stack: cur.stack }
: cur;
return result;
}, {});
console.log(objTestsResul);
};
main();
Related
I have the following structure of code:
var cdict = new Map();
fetch("randomurl")
.then(res => res.json())
.then(data => {
for (const obj of data.result) {
// insert stuff into Map cdict
}
counter(0).then(res => {
console.log(cdict);
})
// ^ here after calling of counter i need to do stuff
});
const cbegin = 0;
const ccount = 10;
function counter(cnt) {
if (cnt < ccount) {
setTimeout( () => {
cnt++;
fetch(someurl)
.then(res => res.json())
.then(data => {
for (const obj of data.result) {
// do stuff
}
})
.catch(err => {
console.log(err);
});
counter(cnt);
}, 1000);
}
}
Here after execution of counter(0) call and all its fetch requests, I wish to execute a line console.log(cdict);
How can this be achieved? And is this proper way to call fetch requests with delay of 1 second?
Don't mix setTimeout event queue callbacks with promise-based code -
const sleep = time =>
new Promise(resolve => setTimeout(resolve, time))
async function main()
{ console.log("please wait 5 seconds...")
await sleep(5000)
console.log("thanks for waiting")
return "done"
}
main().then(console.log, console.error)
Don't write .then(res => res.json()) every time you need some JSON. Write it once and reuse it -
const fetchJSON(url, options = {}) =>
fetch(url, options).then(res => res.json())
async function main()
{ const data = await fetchJSON("path/to/data.json")
console.log("data received", data)
return ...
}
main().then(console.log, console.error)
Don't attempt to declare variables outside of Promises and modify them later. You cannot return the result of an asynchronous call. Asynchronous data needs to stay contained within the promise or you will be chasing hard-to-find bugs in your code -
async function main(urls)
{ const result = []
for (const u of urls) // for each url,
{ result.push(await getJSON(u)) // await fetch and append to result
sleep(1000) // wait 1 second
}
return result
}
const myUrls =
[ "foo/path/data.json"
, "another/something.json"
, "and/more/here.json"
]
main(urls)
.then(result => /* counter logic */)
.then(console.log, console.error)
Continue abstracting as you see fit -
// give reusable functions a name; use parameters for configurable behaviour
async function fetchAll(urls, delay = 100)
{ const result = []
for (const u of urls)
{ result.push(await getJSON(u))
sleep(delay)
}
return result
}
async function counter(urls)
{ const results = await fetchAll(urls) // use our generic fetchAll
const cdict = /* counter logic... */
return cdict
}
const myUrls =
[ "foo/path/data.json"
, "another/something.json"
, "and/more/here.json"
]
counter(urls).then(console.log, console.error)
As you can see async and await prevent nesting that occurs with the use of setTimeout or .then callbacks. If you use them correctly, your code remains flat and you can think about your code in a synchronous way.
I have a list of proxies which I want to test if they are still active or not, after creating a promise list I run through the results, but after the last promise result the process doesnt stop and the programm freezes even though it should finish because there isnt any code after that.
The only fix I found was to put a process.exit() after running through all results, but this seems really odd to me. I want to automate the process but because the result function never ends its really difficult to workaround it.
const axios = require('axios')
const HttpsProxyAgent = require('https-proxy-agent');
const fs = require('fs');
const data = fs.readFileSync("./proxies.txt", "utf-8");
const proxyList = data.split("\n")
const axiosGet = (url, proxy) => {
const abort = axios.CancelToken.source()
const id = setTimeout(() => abort.cancel(`Timeout of 60000ms.`), 60000)
return axios.get(url, { cancelToken: abort.token, httpsAgent: new HttpsProxyAgent(`http://${proxy.trim()}`), proxy: false })
.then(response => { clearTimeout(id); return true })
.catch(error => { clearTimeout(id); return false })
}
const promises = proxyList.map((proxy, index) => {
return axiosGet(`https://api.ipify.org`, proxy, index)
}).map(p => p.catch(e => { e; return false }))
Promise.all(promises)
.then((results) => {
results.forEach((result, index) => console.log(index + 1, result))
//not exiting after running through all promises, only process.exit() works
})
.catch(err => console.log(err));
I have the following code that is used to get JSON data from an Amazon Web Server API.
var json1 = new Promise((resolve, reject) => {
fetch(url[0])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
I have this repeating 14 times using different urls and json vars and have it return the promises at the end using.
return Promise.all([json1,json2,json3,json4,json5,json6,json7,json8,json9,json10,json11,json12,json13,json14]).then(function(values) {
return values;
});
This works, but it takes up 150+ lines. I want to make a for loop that runs through the same code using a for loop. I created this...
for(var jsonCount = 0;jsonCount<url.length-1;jsonCount++){
jsonArr[jsonCount] = new Promise((resolve, reject) => {
fetch(url[jsonCount])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
}
This doesn't work because the promise functions come back as undefined even though it is called by an await function.
const data = await fetchURL(urlToQuery())
Does anyone have suggestions to make this work? There is JSON being returned.
Thanks for your help.
Edit: Here is a larger chunk of the code.
function fetchURL(urls) {
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
return arr;
});
(async function() {
const data = await fetchURL(urlToQuery())
console.log(data);
for(var r=0;r<numStations;r++){
if (data[r] == ""){
onlineArr[r] = false;
wdDataArr[r].push(cardinalToDeg(stationHistAvgArr[r]));
wsDataArr[r].push(0);
You can use .map for the loop. But don't use new Promise. You don't need a new promise when fetch already provides you with one.
Also, call your array urls instead of url. A plural will be a good indication for the reader of your code that indeed it is a collection of URLs.
Here is how it could look:
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
// process your data
for (let obj of arr) {
console.log(obj);
}
});
I think this example can helps you:
// Mock async function
const getDataAsync = callback => {
setTimeout(
() => callback(Math.ceil(Math.random() * 100)),
Math.random() * 1000 + 2000
)
}
// Create the promise
const getDataWithPromise = () => {
return new Promise((resolve, reject) => {
try {
getDataAsync(resolve);
} catch(e) {
reject(e);
}
});
}
// Using the promise one time
getDataWithPromise()
.then(data => console.log("Simple promise:",data))
.catch(error => console.error(`Error catched ${error}`));
// Promises compound: Promise.all
const promise1 = getDataWithPromise();
promise1.then(data => console.log("promise1 ends:",data));
const promise2 = getDataWithPromise();
promise2.then(data => console.log("promise2 ends:",data));
const promise3 = getDataWithPromise();
promise3.then(data => console.log("promise3 ends:",data));
const promise4 = getDataWithPromise();
promise4.then(data => console.log("promise4 ends:",data));
const promise5 = getDataWithPromise();
promise5.then(data => console.log("promise5 ends:",data));
Promise.all([promise1,promise2,promise3,promise4,promise5])
.then(data => console.log("Promise all ends !!",data));
Hope this helps
you will have issues with closure and var variable capture.
You may want to change var to let to capture the right value in the closure so that url[jsonCount] is actually what you want.
also I think it would be much easier to do something like that in one line :)
let results = [];
for(let i = 0; i < urls.length; ++i) results.push(await (await fetch[urls[i]]).json());
This is a good use for map, mapping urls to promises...
function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
return Promise.all(promises).then(results => {
return results.map(result => result.json())
})
}}
// url is your array of urls (which would be better named as a plural)
fetchUrls(url).then(results => {
// results will be the fetched json
})
Using the async/await syntax (equivalent meaning)
// this can be called with await from within another async function
async function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
let results = await Promise.all(promises)
return results.map(result => result.json())
}
I have a lambda function written in javascript (node) that takes a large payload, breaks it down, hits multiple APIs, builds the response, and returns.
The gist of the code looks like so:
'use strict';
let API_KEY = '';
exports.createObjects = (event, context, callback) => {
let promises = createObject(event);
Promise.all(promises).then(results => {
let res = [];
results.forEach((r) => {
res.push(r.obj);
});
callback(null, {"message" : res});
})
.catch((e) =>{
console.log('Error Creating obj ', e);
});
};
const createObject = (req) => {
let objects = req.body.obj;
let promiseReturn = [];
if(req.headers.Authorization === undefined)
return promiseReturn;
API_KEY = req.headers.Authorization;
objects.forEach((s) => {
let newPromise = create(s);
promiseReturn.push(newPromise);
});
return promiseReturn;
};
const create = (object) => {
return new Promise((resolve, reject) => {
let chain = createObject(s);
chain
.then((res)=>{
return updateObject(s);
})
.then((res)=>{
return createAttribute1(object.x, object.y);
})
.then((res)=>{
return createAttribute2(object.z, object.q);
})
.then((res)=>{
return createAttribute3(object.s, object.t);
})
.then((res)=>{
return createAttribute4(object.v, object.c);
})
.then((finalResponse)=>{
resolve(finalResponse);
})
.catch((e)=>{
return rollBack(object);
})
});
};
I'm having problems dealing with concurrent request. For example if I hit the lambda function with a 10 mb payload it processes just fine, but If 3 users hit the lambda function with 10mb payloads at the same time i see the error "Endpoint request timed out".
What is the best way to "parallelize" this and move it out of the web server with lambda? I'm not sure how to approach this... I have increased the response time from 3 seconds to the maximum of 30 but this only buys a little time and doesn't fix the problem.
Please forgive the fairly case-specific question, though I think the general end goal could be of use to other people.
Goal: Populate a MongoDB with data requested from multiple JSON API URLs.
Short question: So far I've had some success with request-promise, which uses Bluebird:
var rp = require('request-promise');
var options = {
uri: 'http://www.bbc.co.uk/programmes/b006qsq5.json',
headers: {
'User-Agent': 'Request-Promise'
},
json: true
};
rp(options)
.then(function (body) {
// Mongoose allows us query db for existing PID and upsert
var query = {pid: body.programme.pid},
update = {
name: body.programme.title,
pid: body.programme.pid,
desc: body.programme.short_synopsis
},
options = { upsert: true, new: true };
// Find the document
Programme.findOneAndUpdate(query, update, options, function(err, result) {
if (err) return res.send(500, { error: err });
return res.send("succesfully saved");
});
})
.catch(function (err) {
return res.send(err);
})
But how do I loop over an array of URLs, without the program failing if any of the promises are rejected?
Something like this for example, using Bluebird, fails if any of the URLs errors.
const urls = ['http://google.be', 'http://google.uk']
Promise.map(urls, rp)
.map((htmlOnePage, index) => {
return htmlOnePage;
})
.then(console.log)
.catch((e) => console.log('We encountered an error' + e));
As I want to write to the DB with successful requests, and ignore those that might not be responding right then, I need something that skips over rejected promises, which .all does not do.
Long question:
I've been reading up about promises all day and it's making my head hurt! But I've found some good resources, such as https://pouchdb.com/2015/05/18/we-have-a-problem-with-promises.html, which mentions the use of a Promise factory. Would this work for my case? I initially thought I should make each request, process the result and add it to the DB, then move on to the next request; but having seen .all I thought I should do all the requests, save the results in an array and loop over that with my DB saving function.
Should I even be using Promises for this? Maybe I should just make use of something like async.js and run my requests in series.
Thanks very much for any help or ideas.
But how do I loop over an array of URLs, without the program failing if any of the promises are rejected?
if you return a value from .catch other than a rejected promise, you will return a resolved promise
So, your .then for each individual request could return an object like
{
success: true,
result: whateverTheResultIs
}
and your catch returns
{
success: false,
error: whateverTheErrorIs
}
Really you don't NEED the success property, it's a convenience though
So the code would be - assuming process(url) returns a Promise
Promise.map(urls, url =>
process(url)
.then(result => ({result, success:true}))
.catch(error => ({error, success:false}))
)
.then(results => {
let succeeded = results.filter(result => result.success).map(result => result.result);
let failed = results.filter(result => !result.success).map(result => result.error);
});
Or, in ES5
Promise.map(urls, function (url) {
return process(url).then(function (result) {
return { result: result, success: true };
}).catch(function (error) {
return { error: error, success: false };
});
}).then(function (results) {
var succeeded = results.filter(function (result) {
return result.success;
}).map(function (result) {
return result.result;
});
var failed = results.filter(function (result) {
return !result.success;
}).map(function (result) {
return result.error;
});
});
I don't know if this fit your case, but I think You can use a counter to check when all promises has returned, regardless of the fact that each one has been resolved or rejected
var heroes = [
'Superman',
'Batman',
'Spiderman',
'Capitan America',
'Ironman',
];
function getHero(hero) {
return new Promise((resolve, reject) => {
setTimeout(() => {
return Math.round(Math.random()) ? resolve(hero + ' lives') : reject(hero + ' dead');
}, Math.random() * 3000)
})
}
function checkHeroes() {
var checked = heroes.length;
heroes.forEach((hero) => {
getHero(hero)
.then((res) => {
checked --;
console.log(res);
if (!checked) done();
})
.catch((err) => {
checked --;
console.log(err);
if (!checked) done();
});
})
}
function done() {
console.log('All heroes checked');
}
checkHeroes();
I think your issue is less about the bluebird api than structuring your promise chain.
const reducePropsToRequests = (props) => Promise.resolve(Object
.keys(props)
.reduce((acc, key) => {
acc[key] = request(sources[key]);
return acc;
}, {}));
const hashToCollection = (hash) => Promise.resolve(Object
.keys(hash)
.reduce((acc, k) => {
return [...acc, {source: k, data: hash[k]}];
}, []));
const fetchFromSources = (sources) => Promise.props(sources);
const findSeveralAndUpdate = (results) => Promise
.each(results.map(obj => {
// you have access to original {a: 'site.com'}
// here, so use that 'a' prop to your advantage by abstracting out
// your db config somewhere outside your service
return Programme.findOneAndUpdate(someConfig[obj.source], obj.data);
}))
const requestFromSeveralAndUpdate = (sources) => reducePropsToRequests(sources)
.then(fetchFromSources)
.then(hashToCollection)
.then(findSeveralAndUpdate)
.catch(/* some err handler */);
requestFromSeveralAndUpdate({ a: 'site.com', b: 'site.net' });
I'd just use request and write my own promise with try catch inside that only resolves. Pseudo example below
var request = require('request')
var urls = ['http://sample1.com/json', 'http://sample2.com/json']
var processUrl = (url) => {
return new Promise((resolve,reject)=> {
var result;
try {
var myRequest = {
uri: url,
method: 'GET',
header: {...}
};
request(option, (res,body,err)=> {
if(err) {
result = err;
return;
}
result = body;
})
}
catch(e) {
result = e;
}
finally {
resolve(result)
}
})
}