I'm consuming an API with rate-limit, every time a hit my rate limit it returns header retry-after specifying the amount of seconds to wait for rate limit reset.
I need to:
Send 100 calls with Promise.allSettled([...]);
Some requests will succeed then process it;
Retry rejected requests after specified seconds.
My solution so far:
async *indicators(items: string[]): AsyncIterableIterator<any[]> {
const res = await Promise.allSettled(items.map((item) => this.makeRequest(item)))
const fulfilledRequests = res.filter((r) => r.status === 'fulfilled') as PromiseFulfilledResult<any>[]
for (const { value } of fulfilledRequests) {
console.log('Yielding')
yield value
console.log('Yielded')
}
const rejectedRequest = res.find((r) => r.status === 'rejected') as any
const failedItems = res.filter((p) => p.status === 'rejected').map(({ reason }: any) => reason.item)
if (failedItems.length === 0 || !failedItems?.reason?.retryAfter)
return Logger.log(`No more items to check`)
setTimeout(this.indicators(failedItems).next.bind(this), rejectedRequest.reason.retryAfter)
}
async makeRequest(item: string): Promise<Indicator[]> {
try {
const { data: { data } } = await firstValueFrom(this.httpService.post('https://api.io', { item }))
return data
} catch (error) {
throw { retryAfter: error.response.headers['retry-after'] * 1000, symbol }
}
}
main() {
for await (const item of this.indicators(['', ''])) {
console.log(item)
}
}
First iterations runs fine, from 100 items it fetches 30 and yields as expected;
Then setTimeout is working as expected;
Indicators functions runs for the second time;
The request works;
The first Yielding log is shown and then it stops.
I'm using NestJS with Typescript on Node v16.
I'd suggest retry every request independently until succeed or abandond.
The code below is NOT TESTED
async function main() {
const promises = ["", ""].map(item=>makeRequestUntilDone(item));
const results = await Promise.allSettled(promises);
for(const r of results) {
if(r.status === 'fulfilled') {
// process result
console.log(r.value);
} else {
// process error if you sometimes
// throw in `makeRequestUntilDone`
console.log(r.reason);
}
}
}
async function makeRequestUntilDone(item: string) {
while(true){
try {
const { data: { data } } = await firstValueFrom(this.httpService.post('https://api.io', { item }))
return data
} catch (error) {
// `throw error` if you don't wanna retry
// anymore or it is not a retryable error,
// otherwise delay and continue
const retryAfter = error.response.headers['retry-after'] * 1000;
await delay(retryAfter);
}
}
}
function delay(ms: number) {
return new Promise(function(resolve) {
setTimeout(resolve, ms);
});
}
then you can
I made it working by using a do while loop awaiting for retry after time.
async *listMeetings(meetingIds: string[]): AsyncIterableIterator<MeetingResponse> {
let hasMore = false
do {
const res = await Promise.allSettled(meetingIds.map((id) => this.makeRequest(id)))
const fulfilledRequests = res.filter((r) => r.status === 'fulfilled')
for (const { value } of fulfilledRequests) {
yield value
}
const rejectedRequest = res.find((r) => r.status === 'rejected')
const failedMeetings = res.filter((p) => p.status === 'rejected').map(({ reason }: any) => reason.meetingId)
if (failedMeetings.length === 0 || !rejectedRequest?.reason?.retryAfter) {
hasMore = false
} else {
await new Promise<void>((resolve) => setTimeout(() => resolve(), rejectedRequest.reason.retryAfter))
yield* this.listMeetings(failedMeetings)
}
} while (hasMore)
}
main() {
for await (const meeting of this.client.listMeetings([])) {
console.log(meeting)
}
}
Related
I'm attempting to setup an async function so that my next step will not start until the function finishes.
I coded one module to connect to mongodb server, and then check to see if it's connected. These two functions work well together.
const mongoose = require('mongoose');
const mongoServer = `mongodb://127.0.0.1/my_database`;
const consoleColor = { green: '\x1b[42m%s\x1b[0m', yellow: '\x1b[43m%s\x1b[0m', red: '\x1b[41m%s\x1b[0m' }
exports.connectMongoose = () => {
mongoose.connect(mongoServer, { useNewUrlParser: true });
}
exports.checkState = () => {
const mongooseState = mongoose.STATES[mongoose.connection.readyState];
return new Promise((resolve) => {
if (mongooseState === 'connected') {
console.log(consoleColor.green, `Mongoose is ${mongooseState}.`);
resolve();
} else if (mongooseState === 'connecting') {
console.log(`Mongoose is ${mongooseState}.`);
setTimeout(() => {
this.checkState();
}, 1000);
} else {
console.log(consoleColor.red, `Mongoose is ${mongooseState}.`);
}
});
}
The next thing I tried to do was connect to the mongo db using my connectMongoose function, and then call a second function that will run my checkState function, and only perform the next function if it resolves (the if statement for the "connected" state.
const dbconfig = require('./dbconfig')
dbconfig.connectMongoose()
const testAwait = async () => {
await dbconfig.checkState();
console.log("Do this next");
}
testAwait()
The testAwait function runs, but it does not get to the console.log function which leads me to believe I'm doing something wrong when passing the resolve.
setTimeout(() => {
this.checkState();
}, 1000);
When this block is hit, the promise is never resolved. The original promise needs to resolve (as your code is currently, if the status is connecting, a new promise is created, but nothing waits for it, and the original promise never resolves). You could go with a pattern like this:
let attempts = 0;
const isConnected = async () => {
console.log("checking connection state...");
attempts++;
if (attempts >= 5) {
return true;
}
return false;
}
const wait = ms => new Promise(res => setTimeout(res, ms));
const checkState = async () => {
while (!(await isConnected())) {
await wait(1000);
}
return;
};
checkState().then(() => console.log("done"));
But to keep it more in line with what you've written, you could do:
const checkState = () => {
const mongooseState = Math.random() > 0.2 ? "connecting" : "connected";
return new Promise((resolve) => {
if (mongooseState === 'connected') {
console.log(`Mongoose is ${mongooseState}.`);
resolve();
} else if (mongooseState === 'connecting') {
console.log(`Mongoose is ${mongooseState}.`);
setTimeout(() => {
checkState().then(resolve);
}, 1000);
}
});
}
checkState().then(() => console.log("done"));
I think the issue here in the above code, you are only resolving your promise once. There is no rejection either. Thus, your code is blocked inside the promise. See the below example. You should exit the promise in any case resolve or reject.
const random = parseInt(Math.random());
const testAwait =
async() => {
await new Promise((resolve, reject) => {
if (random === 0) {
resolve(random);
} else {
reject(random);
}
});
console.log("Do this next");
}
testAwait()
I have a button when user click on it I will send a request and receive answer. If user click 100 times on this button I want to send 100 requests to server and each request send after previous. because I need previous response in next request.
example:
<button #click="sendRequest">send</button>
methods:{
sendRequest:function(){
axios.post('https:/url/store-project-item', {
'id': this.project.id,
"items": this.lists,
'labels': this.labels,
'last_update_key': this.lastUpdateKey,
'debug': 'hYjis6kwW',
}).then((r) => {
if (r.data.status) {
this.change = false
this.lastUpdateKey = r.data.lastUpdateKey;
this.showAlert('success')
} else {
if (r.data.state == "refresh") {
this.showAlert('error')
this.getProject()
} else {
this.showAlert('error')
}
}
}).catch(() => {
this.showAlert('error')
})
}}
I keep a higher-order function (i.e. a function that returns a function) withMaxDOP (DOP = degrees-of-parallelism) handy for this kind of thing:
const withMaxDOP = (f, maxDop) => {
const [push, pop] = createAsyncStack();
for (let x = 0; x < maxDop; ++x) {
push({});
}
return async(...args) => {
const token = await pop();
try {
return await f(...args);
} finally {
push(token);
}
};
};
The function makes use of an async stack data structure (implementation is in the attached demo), where the pop function is async and will only resolve when an item is available to be consumed. maxDop tokens are placed in the stack. Before invoking the supplied function, a token is popped from the stack, sometimes waiting if no token is immediately available. When the supplied completes, the token is returned to the stack. This has the effect of limiting concurrent calls to the supplied function to the number of tokens that are placed in the stack.
You can use the function to wrap a promise-returning (i.e. async) function and use it to limit re-entrancy into that function.
In your case, it could be used as follows:
sendRequest: withMaxDOP(async function(){ /*await axios.post...*/ }, 1)
to ensure that no call to this function ever overlaps another.
Demo:
const createAsyncStack = () => {
const stack = [];
const waitingConsumers = [];
const push = (v) => {
if (waitingConsumers.length > 0) {
const resolver = waitingConsumers.shift();
if (resolver) {
resolver(v);
}
} else {
stack.push(v);
}
};
const pop = () => {
if (stack.length > 0) {
const queueItem = stack.pop();
return typeof queueItem !== 'undefined' ?
Promise.resolve(queueItem) :
Promise.reject(Error('unexpected'));
} else {
return new Promise((resolve) => waitingConsumers.push(resolve));
}
};
return [push, pop];
};
const withMaxDOP = (f, maxDop) => {
const [push, pop] = createAsyncStack();
for (let x = 0; x < maxDop; ++x) {
push({});
}
return async(...args) => {
const token = await pop();
try {
return await f(...args);
} finally {
push(token);
}
};
};
// example usage
const delay = (duration) => {
return new Promise((resolve) => setTimeout(() => resolve(), duration));
};
async function doSomething(name) {
console.log("starting");
// simulate async IO
await delay(1000);
const ret = `hello ${name}`;
console.log(`returning: ${ret}`);
return ret;
}
const limitedDoSomething = withMaxDOP(doSomething, 1);
//call limitedDoSomething 5 times
const promises = [...new Array(5)].map((_, i) => limitedDoSomething(`person${i}`));
//collect the resolved values and log
Promise.all(promises).then(v => console.log(v));
I am writing a node js program as follows. The purpose of this code is to parse from multiple pages of an API (variable number of pages thus scraping the first page to see how many pages are to be scraped) followed by uploading all the pages to MongoDB and then "analysing the pages" with a function in another file (manipulate keyword):
const MongoClient = require('mongodb').MongoClient
const fetch = require('node-fetch')
const config = require('./config.json')
const manipulate = require('./manipulateDB')
async function startAHLoop() {
async function getAuctionPage(page = 0) {
return fetch(`https://website.net/page/${page}`).then(res => {
return res.json()
}).catch (error => console.error("Faced an error: " + error))
}
async function getFullAH() {
try {
let ah = []
let completedPages = 0
let firstPage = await getAuctionPage(0)
for (let i = 1; i <= firstPage.totalPages; i++) {
getAuctionPage(i).then((page) => {
if (completedPages !== firstPage.totalPages - 1) {
completedPages++
}
if (page.success) {
for (auction of page.auctions) {
ah.push(auction)
if (completedPages == firstPage.totalPages - 1) {
completedPages++
}
}
} else if (completedPages == firstPage.totalPages - 1) {
completedPages++
}
})
}
// Wait for the whole ah to download
while (completedPages !== firstPage.totalPages)
await new Promise((resolve) => setTimeout(resolve, 10))
return ah
} catch (e) {
console.log('Failed to update auctions', e)
return
}
}
async function main() {
let startTime = Date.now()
if (!db.isConnected()) await connectToDB()
let auctionCollection = data.collection('auctions')
let ah = await getFullAH()
let timeTaken = Date.now() - startTime
if (typeof ah.ok == 'undefined') {
auctionCollection.drop()
auctionCollection.insertMany(ah)
console.log(`Auction update complete in ${timeTaken} ms ${Date().toLocaleString()}`)
console.log("Starting analysis")
await auctionCollection.insertMany(ah)
manipulate.printAHInfos()
} else {
console.log(`Auction update failed in ${timeTaken} ms ${Date().toLocaleString()}`)
}
// This essentially is the delay instead of every 60000 ms
setTimeout(main, 60000 - timeTaken)
}
main()
}
async function connectToDB(isFirstConnect) {
console.log('Connecting to db...')
MongoClient.connect(
config.mongoSRV,
{ useNewUrlParser: true, useUnifiedTopology: true },
(err, DB) => {
if (err) return connectToDB()
db = DB
skyblock = DB.db('skyblock')
}
)
while (typeof db == 'undefined') {
await new Promise((resolve) => setTimeout(resolve, 10))
}
if (!db.isConnected()) {
console.log('Something weird happened... re-starting db connection')
return connectToDB()
}
console.log('Successful connection to database')
if (isFirstConnect) startAHLoop()
return db
}
connectToDB(true)
I am looking for a way to wait until collection.insertMany(ah) has finished before doing manipulate.AHdata
The issue I get is that manipulate.AHdata is invoked before collection.insertMany(ah) is finished. Resulting as follows when manipulate.AHdata outputs "Invoked":
Invoked
Connecting to db...
I tried using the following:
collection.insertMany(ah)
await collection.insertMany(ah)
manipulate.AHdata
But it doesn't work tho...
Any idea of what I could do?
Thanks for the help and have a great day!
Following up with all my comments and points, here's what I believe is a better (but obviously untested) version of the code :
const MongoClient = require('mongodb').MongoClient
const fetch = require('node-fetch')
const config = require('./config.json')
const manipulate = require('./manipulateDB')
let auctionCollection
async function getAuctionPage(page = 0) {
try {
const response = await fetch(`https://website.net/page/${page}`)
const data = await response.json()
return data
} catch (err) {
console.error("Faced an error: " + err)
}
}
async function getFullAH() {
try {
let ah = []
let firstPage = await getAuctionPage(0)
for (let i = 1; i <= firstPage.totalPages; i++) {
const page = await getAuctionPage(i);
for (let auction of page.auctions) ah.push(auction);
}
return ah
} catch (e) {
console.log('Failed to update auctions', e)
return
}
}
async function main() {
let startTime = Date.now()
let ah = await getFullAH()
let timeTaken = Date.now() - startTime
auctionCollection.drop()
auctionCollection.insertMany(ah)
console.log(`Auction update complete in ${timeTaken} ms ${Date().toLocaleString()}`)
console.log("Starting analysis")
await auctionCollection.insertMany(ah)
manipulate.printAHInfos()
// This essentially is the delay instead of every 60000 ms
setTimeout(main, 60000 - timeTaken)
}
async function connectToDB() {
console.log('Connecting to db...')
let db
try {
db = await MongoClient.connect(
config.mongoSRV,
{ useNewUrlParser: true, useUnifiedTopology: true });
} catch (err) {
return connectToDB()
}
auctionCollection = db.collection('auctions');
console.log('Successful connection to database')
main() // You don't need hacks and verifications to check if the DB is connected. If there was a problem, it got caught in the catch()
}
connectToDB()
I have an async function that checks for the status of an order (checkOrderStatus()). I would like to repeat this function until it returns either "FILLED" or "CANCELED", then use this return value in another function to decide to continue or stop the code. Every order goes through different status before being "FILLED" or "CANCELED", therefore the need to repeat the checkOrderStatus() function (it is an API call).
What I have now is this, to repeat the checkOrderStatus() function:
const watch = filter => {
return new Promise(callback => {
const interval = setInterval(async () => {
if (!(await filter())) return;
clearInterval(interval);
callback();
}, 1000);
});
};
const watchFill = (asset, orderId) => {
return watch(async () => {
const { status } = await checkOrderStatus(asset, orderId);
console.log(`Order status: ${status}`);
if (status === 'CANCELED') return false;
return status === 'FILLED';
});
};
I then call watchFill() from another function, where I would like to check its return value (true or false) and continue the code if true or stop it if false:
const sellOrder = async (asset, orderId) => {
try {
const orderIsFilled = await watchFill(asset, orderId);
if (orderIsFilled) {
//… Continue the code (status === 'FILLED'), calling other async functions …
}
else {
//… Stop the code
return false;
}
}
catch (err) {
console.error('Err sellIfFilled() :', err);
}
};
However, this does not work. I can see the status being updated in the terminal via the console.log in watchFill(), but it never stops and most importantly, the value in the orderIsFilled variable in sellOrder() does not get updated, whatever the value returned by watchFill() becomes.
How can I achieve the desired behavior?
watch never calls resolve (in the original code, this is misleadingly named callback()) with any value, so there's no way const orderIsFilled = await watchFill(asset, orderId); will populate orderIsFilled with anything but undefined.
If you save the result of await filter() in a variable and pass it to
callback as callback(result), your code seems like it should work.
That said, the code can be simplified by using a loop and writing a simple wait function. This way, you can return a value (more natural than figuring out how/when to call resolve), keep the new Promise pattern away from the logic and avoid dealing with setInterval and the bookkeeping that goes with that.
const wait = ms =>
new Promise(resolve => setTimeout(resolve, ms))
;
const watch = async (predicate, ms) => {
for (;; await wait(ms)) {
const result = await predicate();
if (result) {
return result;
}
}
};
/* mock the API for demonstration purposes */
const checkOrderStatus = (() => {
let calls = 0;
return async () => ({
status: ++calls === 3 ? "FILLED" : false
});
})();
const watchFill = (asset, orderId) =>
watch(async () => {
const {status} = await checkOrderStatus();
console.log(`Order status: ${status}`);
return status === "CANCELLED" ? false : status === "FILLED";
}, 1000)
;
const sellOrder = async () => {
try {
const orderIsFilled = await watchFill();
console.log("orderIsFilled:", orderIsFilled);
}
catch (err) {
console.error('Err sellIfFilled() :', err);
}
};
sellOrder();
You can use recursive functionality like this:
const checkOrderStatus = async () => {
// ... function does some work ...
await someOtherFunction() // you can use here the other async function as well
// ... function does some more work after returning from await ...
if(/* if status is FILLED or CANCELED */) {
// return true or false or some info about response for your needs
} else {
checkOrderStatus();
}
}
// this will response back when status will be FILLED or CANCELED
await checkOrderStatus();
The watch function clears the interval timer after the first call if filter resolves with false. setInterval doesn't wait for an async function to finish executing either so you'll have to create a loop yourself. Try this:
const delay = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));
const watch = async check => {
while (true) {
if (await check()) {
return;
}
await delay(1000);
}
};
Because watch only resolves when check succeeds, it is not possible to fail so you don't need to check for it (this might be a bug in your code):
const sellOrder = async (asset, orderId) => {
try {
await watchFill(asset, orderId);
//… Continue the code (status === 'FILLED'), calling other async functions …
}
catch (err) {
console.error('Err sellIfFilled() :', err);
}
};
p-wait-for contains an excellent implementation of this. You can use it like so:
import pWaitFor from 'p-wait-for';
const watchFill = (asset, orderId) => pWaitFor(async () => {
const { status } = await checkOrderStatus(asset, orderId);
console.log(`Order status: ${status}`);
if (status === 'CANCELED') return false;
return status === 'FILLED';
}, {
interval: 1000,
leadingCheck: false
});
I have a function that is called that must return a response to a server. Inside this function are two await function calls that are nested. To track error handling, I added try/catch blocks. Is there a way to avoid having nested try catch blocks to track all cases where the function might fail so I can send back an error server response?
Here's my function, it queries for a user's unique device id's and sends a push notification to each one. If a token becomes invalid, then I delete it from my database:
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).pipe(
filter((userDevices) => userDevices && userDevices.length > 0),
take(1)
).subscribe( async (devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens !== undefined && userDeviceTokens.length != 0) {
try {
message['tokens'] = userDeviceTokens;
const pushResponse = await admin.messsaging().sendMulticast(message);
if (pushResponse.failureCount > 0) {
const failedTokens = [];
pushResponse.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(userDeviceTokens[idx]);
}
});
failedTokens.forEach( async (token) => {
var tokenInstanceID = token.split(':')[0];
try {
await deleteOldToken(tokenInstanceID);
console.log(`Token ${tokenInstanceID} deleted`)
} catch {
return res.status(500).send("err");
}
})
return res.status(200).send("ok");
} else {
return res.status(200).send("ok");
}
} catch {
return res.status(500).send("err");
}
} else {
return res.status(200).send("ok");
}
})
}
It just feels a bit excessive with all the returns I must have. Where can I improve?
EDIT, broke apart code into three blocks to prevent arrow coding
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).pipe(
filter((userDevices) => userDevices && userDevices.length > 0),
take(1)
).subscribe(async (devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens !== undefined && userDeviceTokens.length != 0) {
try {
message['tokens'] = userDeviceTokens;
const response = await admin.messaging().sendMulticast(message);
const oldTokensArray = checkOldTokens(response, userDeviceTokens);
if (oldTokensArray.length > 0) {
await deleteOldTokens(oldTokensArray);
return res.status(200).send("ok");
} else {
return res.status(200).send("ok");
}
} catch (err) {
return res.status(500).send(err);
}
} else {
return res.status(200).send("ok");
}
})
}
function checkOldTokens(response, userDeviceTokens) {
if (response.failureCount > 0) {
const failedTokens = [];
response.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(userDeviceTokens[idx]);
}
});
return failedTokens;
} else {
return [];
}
}
async function deleteOldTokens(tokenArray) {
for (const token of tokenArray) {
await fb.firestore().collection('devices').doc(token).delete();
}
}