I'm using promises to fetch large albums of images and when pull random samples from that album. I have managed to request all the albums and then push the links to images to an array of objects.
Now I want to print out that array but only after I've actually filled it. Whenever I add a .then() on the end it prints out only the initialized empty array.
What can I do to force async and only print the array once it's filled. (I'm printing it out at the bottom)
let findImagesCatalyst = new Promise(function(resolve, reject) {
//url options
const options = {
url: 'https://api.imgur.com/3/gallery/hot/time/',
headers: {
"Authorization": "Client-ID xxxx"
}
};
//inital request
request(options, function(err, res, body) {
//parse the response
body = JSON.parse(body)
//access the data in the response
const responseData = body.data;
//filter only those with image counts great than 50
const largeAlbums = responseData.filter(findDumps)
//test to see if a dump is present
if (largeAlbums.length > 0) {
largeAlbums.forEach(function(i) {})
resolve(largeAlbums)
} else {
reject()
}
})
})
//if successful in finding a dump, then go through them and find their albumIds
.then(function(largeAlbums) {
let dumpIds = largeAlbums.map(index => index.id)
return dumpIds;
})
//with the album/dump ids, get each of them with a new request
.then(function(dumpIds) {
//for each of the dumpIds create the needed url using ES6 and then request it.
dumpIds.forEach(function(i) {
const albumUrlOptions = {
url: `https://api.imgur.com/3/album/${i}/images`,
headers: {
"Authorization": "Client-ID xxxx"
}
}
//make a request to each of the albums/dumps
request(albumUrlOptions, function(err, res, body) {
body = JSON.parse(body)
const responseData = body.data
//pick one sample image from the album/dump
let sampleImage = responseData[randomSelector(responseData.length)].link;
dumps.push({
"dump": i,
'sample': sampleImage
})
})
})
return dumps;
})
.then(function(dumps) {
console.log(dumps)
})
You're second .then should return Promise.all of the (promisified) requests
.then(function(dumpIds) {
//for each of the dumpIds create the needed url using ES6 and then request it.
return Promise.all(dumpIds.map(function(i) {
const albumUrlOptions = {
url: `https://api.imgur.com/3/album/${i}/images`,
headers: {
"Authorization": "Client-ID xxxx"
}
};
return new Promise((resolve, reject) => {
//make a request to each of the albums/dumps
request(albumUrlOptions, function(err, res, body) {
body = JSON.parse(body)
const responseData = body.data
//pick one sample image from the album/dump
let sampleImage = responseData[randomSelector(responseData.length)].link;
resolve({
"dump": i,
'sample': sampleImage
});
});
});
}))
})
As you are using node.js, which has very good ES2015+ implementation, you can simplify (in my opinion) your code by, firstly, creating a "promisified version of request
let requestP = (options) => new Promise((resolve, reject) => {
request(options, (err, res, body) => {
if (err) {
return reject(err);
}
resolve({res, body});
});
});
The rest of the code could be then re-written as follows
const options = {
url: 'https://api.imgur.com/3/gallery/hot/time/',
headers: {
"Authorization": "Client-ID xxxx"
}
};
//inital request
let findImagesCatalyst = requestP(options)
.then(({res, body}) => {
//parse the response
body = JSON.parse(body)
//access the data in the response
const responseData = body.data;
//filter only those with image counts great than 50
const largeAlbums = responseData.filter(findDumps)
//test to see if a dump is present
if (largeAlbums.length > 0) {
largeAlbums.forEach(function(i) {})
return(largeAlbums)
} else {
return Promise.reject();
}
})
//if successful in finding a dump, then go through them and find their albumIds
.then((largeAlbums) => largeAlbums.map(index => index.id))
//with the album/dump ids, get each of them with a new request
.then((dumpIds) =>
//for each of the dumpIds create the needed url using ES6 and then request it.
Promise.all(dumpIds.map((i) => {
const albumUrlOptions = {
url: `https://api.imgur.com/3/album/${i}/images`,
headers: {
"Authorization": "Client-ID xxxx"
}
};
return requestP(albumUrlOptions)
.then(({res, body}) => {
body = JSON.parse(body)
const responseData = body.data
//pick one sample image from the album/dump
let sampleImage = responseData[randomSelector(responseData.length)].link;
return({
"dump": i,
'sample': sampleImage
});
});
}))
)
.then(function(dumps) {
console.log(dumps)
});
So, you have a few building blocks here:
Request for imgur albums reflected into options object.
findDumps — a simple function that you filter the list of albums against.
A function that applies the preceding two and returns an array of large albums. It's an asynchronous function, so it likely employs Promise.
A function that takes every item of the array of large albums and receives a single image. It's an asynchronous function, so, again, a Promise.
You want to wait until all the single images have been received.
Finally, you expect an array of objects of two properties: "dump" and "sample".
Let's try to contruct an example.
const findImagesCatalyst = new Promise((resolveImagesCatalyst, rejectImagesCatalyst) => {
const options = {
url: 'https://api.imgur.com/3/gallery/hot/time/',
headers: {
Authorization: 'Client-ID xxxx'
}
};
request(options, (err, res, body) => {
//access the data in the response
const responseData = JSON.parse(body).data;
//filter only those with image counts great than 50
const largeAlbums = responseData.filter(findDumps);
//test to see if a dump is present
if (largeAlbums.length > 0) {
// /!\ The trickiest part here: we won't resolve this promise until an "inner Promise" has been resolved.
// Note that next line declares a new function to resolve inner Promise, resolveLargeAlbum. Now we have two functions:
// - resolveImagesCatalyst - to resolve the main Promise, and
// - resolveLargeAlbum — to resolve every image request, and there can be many of them.
const imagesPromises = largeAlbums.map(largeAlbum => new Promise((resolveLargeAlbum, rejectLargeAlbun) => {
// take id from every large album
const dumpId = largeAlbum.id;
// declare options for inner request
const options = {
url: `https://api.imgur.com/3/album/${i}/images`,
headers: {
"Authorization": "Client-ID xxxx"
}
};
request(albumUrlOptions, (err, res, body) => {
const responseData = JSON.parse(body).data;
//pick one sample image from the album/dump
const sampleImage = responseData[randomSelector(responseData.length)].link;
if (sampleImage) {
// A-HA!
// It's inner Promise's resolve function. For N albums, there will be N resolveLargeAlbum calls. Just a few lines below, we're waiting for all of them to get resolved.
resolveLargeAlbum({
dump: dumpId,
sample: sampleImage
});
} else {
rejectLargeAlbun('Sorry, could not receive sample image:', dumpId, responseData);
}
});
}));
// Now we have imagePromises, an array of Promises. When you have an array of Promises, you can use Promise.all to wait until all of them are resolved:
Promise.all(imagesPromises).then(responses => {
// Take a look at responses: it has to be an array of objects of two properties: dump and sample.
// Also, note that we finally use outer Promise's resolve function, resolveImagesCatalyst.
resolveImagesCatalyst(responses);
}).catch(errors => {
rejectImagesCatalyst(errors);
});
} else {
rejectImagesCatalyst('Sorry, nope.');
}
});
});
That's a huge one. What you really need to see is that
With Promise.all, you can wait for a collection of Promises to get resolved, and the "then" part won't get executed until all of them have been resolved.
You can put a Promise into a Promise, and resolve outer Promise when inner Promise gets resolved.
The code is really hard to read, because the order of execution is not top-to-bottom. If you use Webpack with Babel, you might want to take a look at async/await. With async/await, the code looks synchronous: you read it from top to bottom and that's exactly the order results of its execution appear, but under the hood, it's all asynchronous. Pretty neat ES6 feature, imho.
Make sure there is no existing Node module that is handles your imgur searching business. Search on npms.io.
If there is no existing module, find one that is close and expand it for your use case (hot images).
If you really can't find a module for imgur to expand, then make your own. All of the imgur request stuff goes in its own module (and own file).
Make sure that module supports promises.
Your code should look something like this:
import {getHotAlbums, getAlbumImages, config} from 'imgur';
config({clientID: 'BLAHXXXX'});
async function getHotImages() {
let hotAlbums = await getHotAlbums();
hotAlbums = hotAlbums.filter(a => a.imageCount > 50);
const sampleImages = [];
let albumIDs = hotAlbums.map(a => a.id);
for (let albumID of albumIDs) {
const images = await getAlbumImages(albumID);
const randomImageNum = Math.round(Math.random()*images.length)+1;
sampleImages.push(images[randomImageNum].link);
}
return sampleImages;
}
Related
I'm trying to limit the number of requests I send to an API.
I'm using Limiter and it's working just like I need, the only issue is that I can't find a way to use it with await (I need all the responses before rendering my page)
Can someone give me a hand with it?
Btw the Log returns a boolean.
const RateLimiter = require('limiter').RateLimiter;
const limiter = new RateLimiter(50, 5000)
for (let i = 0; i < arrayOfOrders.length; i++) {
const response = limiter.removeTokens(1, async (err, remainingRequests) => {
console.log('request')
return await CoreServices.load('updateOrder', {
"OrderNumber": arrayOfOrders[i],
"WorkFlowID": status
})
})
console.log('response', response)
}
console.log('needs to log after all the request');
this is loggin:
response true
response true
response false
needs to log after all the request
request
request
request
...
Promisifying .removeTokens will help, see if this code works
const RateLimiter = require('limiter').RateLimiter;
const limiter = new RateLimiter(50, 5000);
const tokenPromise = n => new Promise((resolve, reject) => {
limiter.removeTokens(n, (err, remainingRequests) => {
if (err) {
reject(err);
} else {
resolve(remainingRequests);
}
});
});
(async() => { // this line required only if this code is top level, otherwise use in an `async function`
const results = await Promise.all(arrayOfOrders.map(async (order) => {
await tokenPromise(1);
console.log('request');
return CoreServices.load('updateOrder', {
"OrderNumber": order,
"WorkFlowID": status
});
}));
console.log('needs to log after all the request');
})(); // this line required only if this code is top level, otherwise use in an `async function`
explanation
Firstly:
const tokenPromise = n => new Promise((resolve, reject) => {
limiter.removeTokens(n, (err, remainingRequests) => {
if (err) {
reject(err);
} else {
resolve(remainingRequests);
}
});
});
promisifies the limiter.removeTokens to use in async/await - in nodejs you could use the built in promisifier, however lately I've had too many instances where that fails - so a manual promisification (I'm making up a lot of words here!) works just as well
Now the code is easy - you can use arrayOfOrders.map rather than a for loop to create an array of promises that all run parallel as much as the rate limiting allows, (the rate limiting is done inside the callback)
await Promise.all(... will wait until all the CoreServices.load have completed (or one has failed - you could use await Promise.allSettled(... instead if you want)
The code in the map callback is tagged async so:
await tokenPromise(1);
will wait until the removeTokens callback is called - and then the request
return CoreServices.load
is made
Note, this was originally return await CoreServices.load but the await is redundant, as return await somepromise in an async function is just the same as return somepromise - so, adjust your code too
I am trying to query the Twitter API for users' followers to compare the two. I am using a cursor to retrieve the list of users. I set it up using promises, where the first get request gets the users followers to a limit, then looks for the cursor passes the data into an array and then resolves with the value of the cursor into a the same recurring function until the cursor returns 0, when that happens I want to send it all in a res. When I set up my initial promises and use promise.all to resolve them and attach a .then, the promise. all resolves first. I've tried resolving the promise from the recurring function and then returning another promise to the first function but I can't seem to get it to work. Any ideas?
const express = require("express");
const router = express.Router();
const twitter = require("../controllers/twitter");
let object1 = {
params: {
screen_name: "xxxx",
count: "40"
}
};
let object = {
params: {
screen_name: "xxxxx",
count: "40"
}
};
let data = [];
router.get("", (req, res, next) => {
let one = getUserFollowers("/1.1/followers/ids.json", object);
let two = getUserFollowers("/1.1/followers/ids.json", object1);
Promise.all([one, two]).then(console.log);
});
function getUserFollowers(uri, parameters) {
twitter
.get(uri, parameters)
.then(resp => {
let ids = resp.data.ids;
data.push(...ids);
return recurr(resp, uri, parameters);
})
.then(data => {
//if instead I re console.log the data I can get it, but I need to resolve it back in the router so I can send it as a res.
return new Promise((resolve, reject) => {
resolve(data);
reject(error);
});
});
}
let recurr = (response, uri, params) => {
if (response.data.next_cursor > 0) {
params.params.cursor = response.data.next_cursor;
return getUserFollowers(uri, params);
} else {
return Promise.resolve(data);
}
};
module.exports = router;
The mistakes in your code:
getUserFollowers doesn't return anything - so, lets add a return before twitter.get
.then(data => {
//if instead I re console.log the data I can get it, but I need to resolve it back in the router so I can send it as a res.
return new Promise((resolve, reject) => {
resolve(data);
reject(error);
});
});
This makes no sense for several reasons.
you can only "complete" a Promise once, so having resolve followed by reject means only resolve is executed ...
.then always returns a promise, so there's usually no need for a Promise constructor when the .then contains only synchronous code; and
what you're actually doing is a no-op ... you've received data in .then, done nothing to it, and in a complicated way, returned data out the other end
Since recurr is only ever called inside a .then you don't need to guarantee you're returning a Promise using Promise.resolve - however, I'd lose the recurr function altogether, it just complicates things in such simple code
he fact that you have one global data variable that has data pushed by both getUserFollowers will probably cause you problems too
Rewriting to fix all of the above, you get:
const express = require("express");
const router = express.Router();
const twitter = require("../controllers/twitter");
let object1 = {
params: {
screen_name: "xxxx",
count: "40"
}
};
let object = {
params: {
screen_name: "xxxxx",
count: "40"
}
};
router.get("", (req, res, next) => {
let one = getUserFollowers("/1.1/followers/ids.json", object, []);
let two = getUserFollowers("/1.1/followers/ids.json", object1, []);
Promise.all([one, two]).then(console.log);
});
function getUserFollowers(uri, parameters, data) {
return twitter
.get(uri, parameters)
.then(resp => {
let ids = resp.data.ids;
data.push(...ids);
if (resp.data.next_cursor > 0) {
params.params.cursor = response.data.next_cursor;
return getUserFollowers(uri, params, data);
}
return data;
});
}
module.exports = router;
I am making call to the bitbucket API to get all the files that are in a repo. I have reached to a point where I can get the list of all the folders in the repo and make the first API call to all the root folders in the repo in parallel and get the the list of first 1000 files for all folders.
But the problem is bitbucket api can give me only 1000 files per folder at a time.
I need to append a query param &start =nextPageStart and make the call again, until it is null and isLastPage is true per API. How can I achieve that with below code??
I get the nextPageStart from first call to the api. See the API response below.
Below is the code that I have so far.
Any help or guidance is appreciated.
Response from individual API thats called per folder.
{
"values": [
"/src/js/abc.js",
"/src/js/efg.js",
"/src/js/ffg.js",
...
],
"size": 1000,
"isLastPage": false,
"start": 0,
"limit": 1000,
"nextPageStart": 1000
}
function where i made asynchronous calls to get the list of files
export function getFilesList() {
const foldersURL: any[] = [];
getFoldersFromRepo().then((response) => {
const values = response.values;
values.forEach((value: any) => {
//creating API URL for each folder in the repo
const URL = 'https://bitbucket.abc.com/stash/rest/api/latest/projects/'
+ value.project.key + '/repos/' + value.slug + '/files?limit=1000';
foldersURL.push(URL);
});
return foldersURL;
}).then((res) => {
// console.log('Calling all the URLS in parallel');
async.map(res, (link, callback) => {
const options = {
url: link,
auth: {
password: 'password',
username: 'username',
},
};
request(options, (error, response, body) => {
// TODO: How do I make the get call again so that i can paginate and append the response to the body till the last page.
callback(error, body);
});
}, (err, results) => {
console.log('In err, results function');
if (err) {
return console.log(err);
}
//Consolidated results after all API calls.
console.log('results', results);
});
})
.catch((error) => error);
}
I was able to get it working be creating a function with callback.
export function getFilesList() {
const foldersURL: any[] = [];
getFoldersFromRepo().then((response) => {
const values = response.values;
values.forEach((value: any) => {
//creating API URL for each folder in the repo
const URL = 'https://bitbucket.abc.com/stash/rest/api/latest/projects/'
+ value.project.key + '/repos/' + value.slug + '/files?limit=1000';
foldersURL.push(URL);
});
return foldersURL;
}).then((res) => {
// console.log('Calling all the URLS in parallel');
async.map(res, (link, callback) => {
const options = {
url: link,
auth: {
password: 'password',
username: 'username',
},
};
const myarray = [];
// This function will consolidate response till the last Page per API.
consolidatePaginatedResponse(options, link, myarray, callback);
}, (err, results) => {
console.log('In err, results function');
if (err) {
return console.log(err);
}
//Consolidated results after all API calls.
console.log('results', results);
});
})
.catch((error) => error);
}
function consolidatePaginatedResponse(options, link, myarray, callback) {
request(options, (error, response, body) => {
const content = JSON.parse(body);
content.link = options.url;
myarray.push(content);
if (content.isLastPage === false) {
options.url = link + '&start=' + content.nextPageStart;
consolidatePaginatedResponse(options, link, myarray, callback);
} else {
// Final response after consolidation per API
callback(error, JSON.stringify(myarray));
}
});
}
I think the best way is to wrap it in a old school for loop (forEach doesn't work with async, since it's synchronous and it will cause all the requests to be spawn at the same time).
What I understood is that you do some sort of booting query where you get the values array and then you should iterate among the pages. Here some code, I didn't fully grasp the APIs so I'll give a simplified (and hopefully readable) answer, you should be able to adapt it:
export async function getFilesList() {
logger.info(`Fetching all the available values ...`);
await getFoldersFromRepo().then( async values => {
logger.info("... Folders values fetched.");
for (let i = 0; ; i++ ) {
logger.info( `Working on page ${i}`);
try {
// if you are using TypeScript, the result is not the promise but the succeeded value already
const pageResult: PageResult = await yourPagePromise(i);
if (pageResult.isLastPage) {
break;
}
} catch(err) {
console.err(`Error on page ${i}`, err);
break;
}
}
logger.info("Done.");
});
logger.info(`All finished!`);
}
The logic behind is that first getFoldersFromRepo() returns a promise which returns the values, and then I sequentially iterate on all available pages through the yourPagePromise function (which returns a promise). The async/await construct allows to write more readable code, rather then having a waterfall of then().
I'm not sure it respects your APIs specs, but it's the logic you can use as foundation! ^^
I get an array of args as an argument, and then I make lots of server calls based on the algo below.
Post to endpoint /abc with args array as data.
Iterate over args array,
a. Pull 3 at a time and send 3 Get calls to endpoint /pqr
b. Once 3 calls in step '2.a' succeeds send 3 Post calls to endpoint /def
c. Collect responses from step '2.a' server call and push it in an array.
d. Repeat step a,b,c till args length.
Code Snippet for the entire process is given below, execution starts at function execute(args).
import Promise from 'bluebird';
import request from 'superagent';
// sends a post request to server
const servercall2 = (args, response) => {
const req = request
.post(`${baseUrl}/def`)
.send(args, response)
.setAuthHeaders();
return req.endAsync();
};
// sends a post request to server
const servercall1 = (args) => {
const req = request
.post(`${baseUrl}/abc`)
.send(args)
.setAuthHeaders();
return req.endAsync()
.then((res) => resolve({res}))
.catch((err) => reject(err));
};
async function makeServerCalls(args, length) {
// convert args to two dimensional array, chunks of given length [[1,2,3], [4,5,6,], [7,8]]
const batchedArgs = args.reduce((rows, key, index) => (index % length === 0 ? rows.push([key])
: rows[rows.length - 1].push(key)) && rows, []);
const responses = [];
for (const batchArgs of batchedArgs) {
responses.push(
// wait for a chunk to complete, before firing the next chunk of calls
await Promise.all(
***// Error, expected to return a value in arrow function???***
batchArgs.map((args) => {
const req = request
.get(`${baseUrl}/pqr`)
.query(args)
// I want to collect response from above req at the end of all calls.
return req.endAsync()
.then((response) =>servercall2(args,response));
})
)
);
}
// wait for all calls to finish
return Promise.all(responses);
}
export function execute(args) {
return (dispatch) => {
servercall1(args)
.then(makeServerCalls(args, 3))
.then((responses) => {
const serverresponses = [].concat(...responses);
console.log(serverresponses);
});
};
}
I am facing couple of issues
2.c seems not to be working fine "Collect responses from step '2.a' server call and push it in an array.". Error: expected to return a value in arrow function. What am I doing wrong here? Please note that at the end I care about the response from step 2.a only.
Is this a right chaining or it can be optimized, based on the requirements mentioned above?
Is there any other failure handling I have to do?
You have a brick wall of text so its becoming a little hard to decipher what you're actually trying to achieve, but I will give my two cents on the code given.
//Both server calls can be simplified.. no need to
//wrap in another promise if one is being returned
const servercall2 = (args, response) => {
const req = request
.post(`${baseUrl}/def`)
.send(args, response)
.setAuthHeaders();
return req.endAsync();
};
//Here... you return no value in the function passed to map, thus an
//error is being thrown. You need to return a Promise from here so that
//it can be passed into Promise.all
const allFinished = await Promise.all(
batchArgs.map((args) => {
const req = request
.get(`${baseUrl}/pqr`)
.query(args)
// I want to collect response from above req at the end of all calls.
return req.endAsync()
})
);
allFinished.then(function(results){
});
It might be this -- each item in batchArgs.map should be a Promise I think? Then Promise.all will wait for each to finish:
batchArgs.map((args) => {
const req = request
.get(`${baseUrl}/pqr`)
.query(args)
// Return promise here
return req.endAsync()
.then((response) =>servercall2(args,response))
.then((res) => res);
})
I am trying to create an async queue for an array of get requests to an api, i am just unsure how to combine and use the responses. Maybe my implementation is wrong since i am using async.queue inside a promise then function ?
Ultimately i would like to get results from first promise ->
use results of that first promise to create an array of get requests for the async.queue ->
then combine the results of all the get responses. I need to throttle the amount of requests that go out at a time due to API rate limit.
const rp = require("request-promise");
app.get("/", (req,res) => {
let arr = []
rp.get(url)
.then((response) => {
let arrayID = response
let q = async.queue((task, callback) => {
request({
method: "GET",
url: url,
qs: {
id: task.id
}
}, (error, response, body) => {
arr.push(body)
console.log(arr.length)
// successfully gives me the response i want. im trying to push into an array with all of my responses,
// but when i go to next then chain it is gone or if i try to return arr i get an empty []
})
callback()
}, 3)
for(var i = 0; i < arrayID.length; i++){
q.push({ id : arrayID[i]} );
}
q.drain = function() {
console.log('all items have been processed');
}
return arr
})
.then((responseArray) => {
//empty array even though the length inside the queue said other wise, i know its a problem with async and sync actions but is there a way to make the promise chain and async queue play nice?
res.json(responseArray)
})
})
Figured it out, ended up having to wrap it in a promise and resolve the final array in q.drain()
const rp = require("request-promise");
app.get("/", (req,res) => {
rp.get(url)
.then((response) => {
let arrayID = response
return new Promise((resolve, reject) => {
var q = async.queue((task, callback) => {
request({
method: "GET",
url: url,
qs: {
id:task.id,
},
}, (error, response, body) => {
arr.push(body)
callback();
})
}, 2);
q.drain = () => resolve(arr);
q.push(arrayID);
})
})
.then((response) => res.json(response))
.catch((error) => res.json(error))
}
To launch multiple async calls in parallel you can use Promise.all()
To launch multiple async calls sequentially (i.e they depend on each other) you can return each promise and use its result inside a then() function
Code below:
app.get("/", (req,res)
.then(function(firstResult)) {
//You can use result of first promise here
return Promise.all([
//Create array of get request here
//To also return firstResult just add it in the Promise.All array
]);
})
.then(function(allResults){
//You can use results of all the get requests created in the previous then()
})
.catch(function(error){
//Deal with any error that happened
});