Develop Reference

nodejs/javascript on stream data post loop delay

I am trying to use a twitter npm to search for tweets in realtime and like them. It streams the tweets data and then uses .post to create the likes.
Currently works but I keep running into 429 too many request errors because of the api rate limit. Ive been trying to get it to pause after each like, however nothing I've tried seems to work. At most it effects the loop before or after but never in between the post/like action.
Any ideas how to get it to delay after each post(like)? I've commented out some of the things I've already tried.
// Returns a Promise that resolves after "ms" Milliseconds
const timer = ms => new Promise(res => setTimeout(res, ms))
const wait = (duration, ...args) => new Promise(resolve => {
setTimeout(resolve, duration, ...args);
});
function LikeTweets() {
client.stream('statuses/filter', { track: terms }, function (stream) {
stream.on('data', async function (tweet) {
// try {
// for (var i = 0; i < 3;) {
v1Client.post('favorites/create', { id: tweet.id_str })
.then(async (result) => {
console.log(result.text);
i++;
console.log(i);
await timer(10000);
}).then(async (newresult) => {
console.log(newresult);
await timer(10000);
}).catch(error => {
console.log(error);
return;
});
//await timer(3000); // then the created Promise can be awaited
// }
// } catch(err) {
// console.log("or catching here?");
// setTimeout(function() {
// LikeTweets();
// }, 15000);
// }
});
});
}
setTimeout(function() {
LikeTweets();
}, 15000);

You make one request per invocation of the stream.on("data", ...) event handler, therefore if 100 data events arrive within a minute, you will make 100 requests within that minute. This exceeds the rate limit.
You must ensure that the sequence of requests made is slower than the sequence of incoming events. The following code illustrates how this decoupling of sequences can be achieved:
/* Make one request every 20 seconds. */
var requestQueue = [];
function processQueue() {
var r = requestQueue.shift();
if (r) v1Client.post("favorites/create", r.payload).then(r.resolve, r.reject);
setTimeout(processQueue, 20000);
}
processQueue();
/* Use this function to schedule another request. */
function makeRequest(payload) {
var r = {payload};
requestQueue.push(r);
return new Promise(function(resolve, reject) {
r.resolve = resolve;
r.reject = reject;
});
}
stream.on("data", function(tweet) {
makeRequest({id: tweet.id_str}).then(async (result) => {...});
});
The promise returned by makeRequest can take a while until it resolves, therefore the code {...} may be executed only after several seconds or even minutes. In other words: The code uses the power of promises to keep within the strictures of the API rate limit.
This works only if, in the long run average, the number of incoming data events does not exceed the possible rate of outgoing requests, which is 1 every 20 seconds. This is nothing you can get around without a mass-update API (which would not be in the interest of the Twitter community, I assume).

Why does setInterval never run, in my NodeJs code that streams an generator to file?

I have this situation in my NodeJs code, which calculates permutations (code from here), but no matter what I don't get any output from setInterval.
const { Readable } = require('stream');
const { intervalToDuration, formatDuration, format } = require('date-fns');
const { subsetPerm } = require('./permutation');
function formatLogs(counter, permStart) {
const newLocal = new Date();
const streamTime = formatDuration(intervalToDuration({
end: newLocal.getTime(),
start: permStart.getTime()
}));
const formattedLogs = `wrote ${counter.toLocaleString()} patterns, after ${streamTime}`;
return formattedLogs;
}
const ONE_MINUTES_IN_MS = 1 * 60 * 1000;
let progress = 0;
let timerCallCount = 1;
let start = new Date();
const interval = setInterval(() => {
console.log(formatLogs(progress, start));
}, ONE_MINUTES_IN_MS);
const iterStream = Readable.from(subsetPerm(Object.keys(Array.from({ length: 200 })), 5));
console.log(`Stream started on: ${format(start, 'PPPPpppp')}`)
iterStream.on('data', () => {
progress++;
if (new Date().getTime() - start.getTime() >= (ONE_MINUTES_IN_MS * timerCallCount)) {
console.log(`manual timer: ${formatLogs(progress, start)}`)
timerCallCount++;
if (timerCallCount >= 3) iterStream.destroy();
}
});
iterStream.on('error', err => {
console.log(err);
clearInterval(interval);
});
iterStream.on('close', () => {
console.log(`closed: ${formatLogs(progress, start)}`);
clearInterval(interval);
})
console.log('done!');
But what I find is that it prints 'done!' (expected) and then the script seems to end, even though if I put a console.log in my on('data') callback I get data printed to the terminal. But even hours later the console.log in the setInterval never runs, as nothing ends up on file, besides the output from the on('close',...).
The output log looks like:
> node demo.js
Stream started on: Sunday, January 30th, 2022 at 5:40:50 PM GMT+00:00
done!
manual timer: wrote 24,722,912 patterns, after 1 minute
manual timer: wrote 49,503,623 patterns, after 2 minutes
closed: wrote 49,503,624 patterns, after 2 minutes
The timers in node guide has a section called 'leaving timeouts behind' which looked relevant. But where I though using interval.ref(); told the script to not garbage collect the object until .unref() is called on the same timeout object, on second reading that's not quite right, and doesn't make a difference.
I'm running this using npm like so npm run noodle which just points to the file.

The generator is synchronous and blocks the event loop
Readable.from processes the whole generator in one go, so if the generator is synchronous and long running it blocks the event loop.
Here is the annotated code that it runs:
async function next() {
for (;;) {
try {
const { value, done } = isAsync ?
await iterator.next() : // our generator is not asynchronous
iterator.next();
if (done) {
readable.push(null); // generator not done
} else {
const res = (value &&
typeof value.then === 'function') ?
await value :
value; // not a thenable
if (res === null) {
reading = false;
throw new ERR_STREAM_NULL_VALUES();
} else if (readable.push(res)) { // readable.push returns false if it's been paused, or some other irrelevant cases.
continue; // we continue to the next item in the iterator
} else {
reading = false;
}
}
} catch (err) {
readable.destroy(err);
}
break;
}
}
Here is the api for readable.push, which explains how this keeps the generator running:
Returns: true if additional chunks of data may continue to be pushed; false otherwise.
Nothing has told NodeJs not to continue pushing data, so it carries on.
Between each run of the event loop, Node.js checks if it is waiting for any asynchronous I/O or timers and shuts down cleanly if there are not any.
I raised this as a NodeJs Github Issue and ended up workshopping this solution:
cosnt yieldEvery = 1e5;
function setImmediatePromise() {
return new Promise(resolve => setImmediate(resolve));
}
const iterStream = Readable.from(async function* () {
let i = 0
for await (const item of baseGenerator) {
yield item;
i++;
if (i % yieldEvery === 0) await setImmediatePromise();
}
}());
This is partly inspired by this snyk.io blog, which goes into more detail on this issue.

Make page display 10 seconds in a loop

I'm building a guessing game with Node JS. After collecting some data on the back-end, I send it to the front-end and the game starts. The data contains all 10 levels, so the game can run on a single page. Each level runs for 10 seconds. After the time is up, the user selection is sent to the server, and a result comes back. The answer is displayed, and then the content is changed to the "next level" (using the content in the big data object, therefore no refresh is needed).
I'm having some issues with having 10 levels run for 10 seconds each (or ~12 seconds with a delay for displaying the results).
This can't be done in some type of loop, since all awaits for each level will run at once. For instance:
function timeout(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
for (let i = 0; i < 10; i++) {
displayPage(i);
await timeout(10000);
const result = await $.post(...) // gets results
displayResults(result);
await timeout(2000);
}
all the timeouts will run at once, and it won't work.
I thought of using a setInterval, but I'm not sure how to.. since I want to wait 10 seconds until checking the input, and then display the results for 2 seconds, and then move on.
For now the result I came up with is:
displayPage(level1);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
displayPage(level2);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
displayPage(level3);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
displayPage(level4);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
...
This does not seem efficient and I think there's a better way to do this, but I'm not sure how.
Any suggestions would be appreciated. Thanks!

I think this is what you are looking for:
const pages = [1, 2, 3, 4, 5];
run();
async function run() {
for (let i = 0; i < pages.length; i++) {
await displayPage(i);
const result = 'Some result';
await displayResult(result);
}
}
function displayPage(number) {
text.innerText = 'Page ' + number;
return new Promise(res => {
setTimeout(res, 10000);
});
}
function displayResult(result) {
text.innerText = 'Result: ' + result;
return new Promise(res => {
setTimeout(res, 2000);
});
}
<div id="text"><div>
Another solution, without promises and loops:
const pages = [1, 2, 3, 4, 5];
let currentPageIndex = 0;
displayPage();
function displayPage() {
const index = currentPageIndex++;
if (pages[index] === undefined) return;
const pageNumber = pages[index];
text.innerText = 'Page ' + pageNumber;
const result = 'Some result';
setTimeout(() => {
displayResult(result);
}, 10000);
}
function displayResult(result) {
text.innerText = 'Result: ' + result;
setTimeout(() => {
displayPage();
}, 2000);
}
<div id="text"></div>

Your first option seems to work assuming it is wrapped within an async function:
function timeout(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
const testFunc = async () =>{
for (let i = 0; i < 10; i++) {
console.log('page' , i+1 , '- question')
await timeout(3000);
console.log('page' , i+1 , '- answer')
await timeout(1000);
}
}
testFunc()

use setInterval on about 1000ms to create worker and add a state-machine that toggles the playing(10s) and the waiting(2s). You need a procedure that does the post call to the server and an object to keep the data(levels and stuff).
for example:
setInterval(funcWorker,1000,s_gameObj);
function funcWorker(f_context){
var l_dateNow = new Date();
if(f_context.is_playing){
var l_elapsed = l_dateNow.getTime() - f_context.dateLevelStart.getTime();
if(l_elapsed.totalSeconds() >= f_context.LEVEL_TIME){
f_context.end_level();
}
}else if(f_context.is_waiting_user){
//same check as above but on the waiting start
....
f_context.next_level();
}else if(f_context.is_waiting_server){
//do whatever
}
}
the end_level() should set the state flag in the context(game) object, to sending and send to the server. As the server returns in the response function set the state to waiting user and init the corresponding time variable to now(). The next_level() should set the state to playing and init the corresponding time variable to now() so that the timer can count. Consider the code above as a reference and not as a copy-paste resource.

call an async function out of another async function to run parallel

I have a working async function as main code (async function (){...});, working with some inputs coming from reed contacts posting them into a cloud. All good so far.
Now I want to add the feature analyzing if a reed is open for more then x seconds. If so, I want to do something additionally but still listening to other inputs while that. So what I try is:
var threshold;
async function time(threshold){
var sec = 0;
var start = new Date(); //set time stamp
while (sec < threshold){
// take actual time
var akt = new Date();
// calc difference
var diff = akt.getTime() - start.getTime();
// calc secs from mills
sec = Math.floor(diff / 1000);
}
post threshold data to cloud;
return "Threshold reached";
}
(async function () {
reading reed permanent {
if (reed === 1){
post data to cloud;
var call = time(20);
console.log(call);
}
}
})();
I wan't the main function still listeing to new reed changes while the time loop should wait for the threshold and do it's job parallel.
But my code waits for the threshold to be reached before continuing.
How can I parallel them?
EDIT after Gibors help:
Now I stuck at the point verifying if the reed is still open or was closed meanwhile. I always get a diff > threshold even if the reed is closed in the meanwhile so that the timestamp should be newer...
var id = [];
if (value['contact'] === 1) {
var bool = "false";
id[device['manufacturer']] = new Date();
} else if(value['contact'] === 0) {
var bool = "true";
id[device['manufacturer']] = new Date();
setTimeout( () => {
var akt = new Date();
var diff = akt.getTime() - id[device['manufacturer']].getTime();
if (diff > threshold) {
console.log("now: " + diff + "=" + akt.getTime() + "-" + id[device['manufacturer']].getTime());
}
}, threshold);
/*var call = time (50);
console.log(call);*/
}
```

First i'll explain what wrong with your code, and then i'll give you something I believe is a better and simpler solution:
Your time function is marked as async, but it actually does nothing asynch-ish. This effectively means that when the function "returns"- it actually returns a resolved promise. But the promise is created only at the end and resolves immediately, so it doesn't do its job to be async.
I believe something like this should work:
async function time(threshold){
return new Promise( (resolve, reject) => {
var sec = 0;
var start = new Date(); //set time stamp
while (sec < threshold){
// take actual time
var akt = new Date();
// calc difference
var diff = akt.getTime() - start.getTime();
// calc secs from mills
sec = Math.floor(diff / 1000);
}
post threshold data to cloud;
resolve("Threshold reached)";
}
}
Now this will run in parallel, and call variable will only get the string "threshold reached" when promise is resolved- which means in your current code, the log you'll get is something like Promise<pending>.
To log only when its done (and do your other stuff), use the .then on the promise held by call.
One thing you should notice tho, is that you'll have to somehow sync between the reed status (which I didn't really get what it actually is but I think its irrelevant) and the promise status, because you want to do your post-timeout code when the 20 sec are over AND reed is still open, so you'll have to check it in your .then clause, or pass it to time function which will reject the promise if the reed status changes before time runs out, etc.
Now- for the simple solution: it seems to me you're way better off using the setTimeout function:
The setTimeout() method sets a timer which executes a function or specified piece of code once the timer expires.
So you can simply do something like this:
(async function () {
reading reed permanent {
if (reed === 1){
post data to cloud;
setTimeout( () => { if(reed still open) {/*do what you want after timeout*/} }, 20 );
}
}
})();

Calculate total elapsed time of Promises till reject?

I want to test how much requests i can do and get their total time elapsed. My Promise function
async execQuery(response, query) {
let request = new SQL.Request();
return new Promise((resolve, reject) => {
request.query(query, (error, result) => {
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
}
And my api
app.get('/api/bookings/:uid', (req, res) => {
let st = new stopwatch();
let id = req.params.uid;
let query = `SELECT * FROM booking.TransactionDetails WHERE UID='${id}'`;
for (let i = 0; i < 10000; i++) {
st.start();
db.execQuery(res, query);
}
});
I can't stop the for loop since its async but I also don't know how can I stop executing other calls after the one which first rejects so i can get the counter and the elapsed time of all successful promises. How can i achieve that?

You can easily create a composable wrapper for this, or a subclass:
Inheritance:
class TimedPromise extends Promise {
constructor(executor) {
this.startTime = performance.now(); // or Date.now
super(executor);
let end = () => this.endTime = performance.now();
this.then(end, end); // replace with finally when available
}
get time() {
return this.startTime - this.endTime; // time in milliseconds it took
}
}
Then you can use methods like:
TimedPromise.all(promises);
TimedPromise.race(promises);
var foo = new TimedPromise(resolve => setTimeout(resolve, 100);
let res = await foo;
console.log(foo.time); // how long foo took
Plus then chaining would work, async functions won't (since they always return native promises).
Composition:
function time(promise) {
var startTime = performance.now(), endTime;
let end = () => endTime = performance.now();
promise.then(end, end); // replace with finally when appropriate.
return () => startTime - endTime;
}
Then usage is:
var foo = new Promise(resolve => setTimeout(resolve, 100);
var timed = time(foo);
await foo;
console.log(timed()); // how long foo took
This has the advantage of working everywhere, but the disadvantage of manually having to time every promise. I prefer this approach for its explicitness and arguably nicer design.
As a caveat, since a rejection handler is attached, you have to be 100% sure you're adding your own .catch or then handler since otherwise the error will not log to the console.

Wouldn't this work in your promise ?
new Promise((resolve, reject) => {
var time = Date.now();
request.query(query, (error, result) => {
if (error) {
reject(error);
} else {
resolve(result);
}
});
}).then(function(r){
//code
}).catch(function(e){
console.log('it took : ', Date.now() - time);
});
Or put the .then and .catch after your db.execQuery() call

You made 2 comments that would indicate you want to stop all on going queries when a promise fails but fail to mention what SQL is and if request.query is something that you can cancel.
In your for loop you already ran all the request.query statements, if you want to run only one query and then the other you have to do request.query(query).then(-=>request.query(query)).then... but it'll take longer because you don't start them all at once.
Here is code that would tell you how long all the queries took but I think you should tell us what SQL is so we could figure out how to set connection pooling and caching (probably the biggest performance gainer).
//removed the async, this function does not await anything
// so there is no need for async
//removed initializing request, you can re use the one created in
// the run function, that may shave some time off total runtime
// but not sure if request can share connections (in that case)
// it's better to create a couple and pass them along as their
// connection becomes available (connection pooling)
const execQuery = (response, query, request) =>
new Promise(
(resolve, reject) =>
request.query(
query
,(error, result) =>
(error)
? reject(error)
: resolve(result)
)
);
// save failed queries and resolve them with Fail object
const Fail = function(detail){this.detail=detail;};
// let request = new SQL.Request();
const run = (numberOfTimes) => {
const start = new Date().getTime();
const request = new SQL.Request();
Promise.all(
(x=>{
for (let i = 0; i < numberOfTimes; i++) {
let query = `SELECT * FROM booking.TransactionDetails WHERE UID='${i}'`;
db.execQuery(res, query, request)
.then(
x=>[x,query]
,err=>[err,query]
)
}
})()//IIFE creating array of promises
)
.then(
results => {
const totalRuntime = new Date().getTime()-start;
const failed = results.filter(r=>(r&&r.constructor)===Fail);
console.log(`Total runtime in ms:${totalRuntime}
Failed:${failed.length}
Succeeded:${results.length-failed.length}`);
}
)
};
//start the whole thing with:
run(10000);