I have an external service I am making HTTP requests to from Node.js. The service has current limitations that only 10 requests per second can be made. I have a naive rate limiter I wrote that I am trying to test, but falling down on timing of it. I know that Javascript times are not very accurate, but I'm getting wildly different swings, in the range of up to 50 milliseconds different.
Here's the gist of what I'm doing:
var RATE_LIMIT_MS = 100 // No more than 10 requests per second
var NEXT_WAIT_MS = 0
function goGetRequest(...) {
return new Promise(function(resolve, reject) {
setTimeout(function() {
function complete() {
// Rollback next time after this request finishes
NEXT_WAIT_MS = Math.max(0, NEXT_WAIT_MS - RATE_LIMIT_MS)
}
// ... requests are fired off here asynchronously...
http.get(...).then(complete)
}, NEXT_WAIT_MS)
// Push time back on the queue
NEXT_WAIT_MS += RATE_LIMIT_MS
})
}
var chai = require('chai')
var expect = chai.expect
it('should rate limit requests properly', function() {
var iterations = [0, 1, 2, 3, 4]
var lastResponseMs = 0
var promises = iterations.map(function(i) {
return goGetRequest(...).
then(function(result) {
// Diff the times
var thisResponseMs = Date.now()
var thisDiffMs = Math.abs(thisResponseMs - lastResponseMs)
expect(wrapper.results).to.not.be.empty
expect(thisDiffMs, 'failed on pass ' + i).to.be.at.least(RATE_LIMIT_MS)
// Move last response time forward
lastResponseMs = thisResponseMs
})
})
return Promise.all(promises)
})
What happens next is that the tests will fail at random passes. A time diff on 92 milliseconds on pass 2, a time diff of 68 milliseconds on pass 4.... what am I missing here? Thanks!
Javascript setTimeout and setInterval (as with any non-realtime code) is never precise. If you're using NodeJS however, you can try using Nanotimer:
https://www.npmjs.com/package/nanotimer
var NanoTimer = require('nanotimer');
var timerA = new NanoTimer();
timerA.setTimeout(yourFunction, '', NEXT_WAIT_MS);
Alternatively, I suggest simply testing that rate limit occurs and not to worry too much about exact precision. If you spam it with 11 requests consecutively (which should hopefully take less than one second), and it gets blocked, and one second later it is fine, then your test can be considered passing.
One final solution is to use exec() and call the OS sleep command, which is significantly more precise.
Related
I have a REST API coded with Node.js and there is one route that applies NTFS Rights to folders. The request can take 1 second but can also last several minutes (it depends on the size of the folder).
To summarize, if the rights take less than 5 seconds to be applied, I want to return the value with the code 200.
If the rights have not finished being applied I would like to return the value with the code 202
...
inProgress = true;
ApplyRights()
.then(() => {
inProgress = false
}
// Here I want to return as fast inProgress = false otherwise wait a bit but max 5s
return ...;
I tried to wait a bit with setTimeout like this:
const sleep = s => new Promise(resolve => {
setTimeout(resolve, s * 1000)
});
let nbCheck = 0;
while (inProgress && nbCheck < 5){
await sleep(1)
nbCheck++;
}
But setTimeout is not called before the end of my previous promise (ApplyRights).
I read here that promise are executed before setTimeout
So, i tried to find a solution without setTimeout and I tried this: (I know it's not very elegant)
let dateStop = Date.now() + 5 * 1000;
while (dateStop = Date.now() && inProgress){}
return ...;
But in this case the .then() of ApplyRights is only reached at the end of the 5s.
Is there a way to let my promise ApplyRights do its job. And if the job take time, wait maximum 5 seconds before returning the response. If the job is quick, I want to return the responses without waiting.
You can use Promise.race:
Promise.race([ApplyRights(), sleep(5)]).then(result => {
if (result === undefined) { // Timeout of 5 seconds occurred
// Some message to the user?
} else { // Got reply within 5 seconds
// Do something with the result
}
});
Do realise however, that this does not interrupt the work of ApplyRights(), which eventually may still do the job.
Essentially, how do you pause javascript execution without having to waste computation on something like a while loop?
For example, say I want to only perform a function next after 10 seconds and not interrupt other processes or waste computation? setTimeout won't work because I want the processes to actually pause / not continue any operations during that time.
const next = () => console.log("next");
/** pause for 10 seconds */
next()
Also what if I want to only run a method conditionally, where every couple seconds or something I can either abort the operation or continue. Notably I don't mean to use setInterval because in that case it's not actually conditionally pausing the actual javascript execution.
const next = () => console.log("next");
const start = new Date();
const startSeconds = now.getSeconds();
const checkEvery = 1; // seconds
const requiredDiff = 10; // seconds
const checker = () => {
const now = new Date();
let secondsNow = now.getSeconds();
secondsNow < startSeconds ? secondsNow += 60 : null;
const diff = secondsNow - startSeconds;
if (diff < 10) {
return false
}
return true;
}
/** something that runs checker every 1 seconds and pauses computation until checker() returns true and then runs next() */
A good way to do this is by creating a block in the pipe. i.e.
process -> pause -> next so that you can know that the flow of the program won't continue until a certain criteria is met
You can do this by pausing things inline
...
await pause();
...
where pause is something that does a timed-out promise or something along those lines. I don't know too much about this method, but it seems relatively complicated.
Another way to do this is by stopping the execution inline with a while loop
...
// pause for 10 seconds
let result;
while (!result) { result = checker() };
...
but as you alluded to, this wastes a lot of operations and can interfere with other actions in the background from running properly. Another thing is you can't only check checker every 1 second.
I suggest you do the following instead:
const max = 20; // max number of recursive calls (i.e. timeout after 20 seconds)
// checker is a function that returns true or false and is agnostic to this implementation
// timeout is the time (in milliseconds) to wait before running the checker again
// next is the next step in your pipeline that you want to prevent from e
const pause = async (checker, timeout, next, calls = 0) => {
if (calls > max) return; // prevents stack overflow
const result = await checker(); // just in case your checker is async
// if the condition was met then continue on to the next stage in your pipeline
// if the condition was not met then run this method again to re-check in timeout
result ? next() : setTimeout(() => pause(checker, timeout, next, calls + 1), timeout)
}
// with the functions you provided...
pause(checker, 1000, next)
pause will only execute operations when the timeout is met, and it won't allow the program to continue to the next stage until the checker is met.
Is it possible to cancel a regex.match operation if takes more than 10 seconds to complete?
I'm using an huge regex to match a specific text, and sometimes may work, and sometimes can fail...
regex: MINISTÉRIO(?:[^P]*(?:P(?!ÁG\s:\s\d+\/\d+)[^P]*)(?:[\s\S]*?))PÁG\s:\s+\d+\/(\d+)\b(?:\D*(?:(?!\1\/\1)\d\D*)*)\1\/\1(?:[^Z]*(?:Z(?!6:\s\d+)[^Z]*)(?:[\s\S]*?))Z6:\s+\d+
Working example: https://regex101.com/r/kU6rS5/1
So.. i want cancel the operation if takes more than 10 seconds. Is it possible? I'm not finding anything related in sof
Thanks.
You could spawn a child process that does the regex matching and kill it off if it hasn't completed in 10 seconds. Might be a bit overkill, but it should work.
fork is probably what you should use, if you go down this road.
If you'll forgive my non-pure functions, this code would demonstrate the gist of how you could communicate back and forth between the forked child process and your main process:
index.js
const { fork } = require('child_process');
const processPath = __dirname + '/regex-process.js';
const regexProcess = fork(processPath);
let received = null;
regexProcess.on('message', function(data) {
console.log('received message from child:', data);
clearTimeout(timeout);
received = data;
regexProcess.kill(); // or however you want to end it. just as an example.
// you have access to the regex data here.
// send to a callback, or resolve a promise with the value,
// so the original calling code can access it as well.
});
const timeoutInMs = 10000;
let timeout = setTimeout(() => {
if (!received) {
console.error('regexProcess is still running!');
regexProcess.kill(); // or however you want to shut it down.
}
}, timeoutInMs);
regexProcess.send('message to match against');
regex-process.js
function respond(data) {
process.send(data);
}
function handleMessage(data) {
console.log('handing message:', data);
// run your regex calculations in here
// then respond with the data when it's done.
// the following is just to emulate
// a synchronous computational delay
for (let i = 0; i < 500000000; i++) {
// spin!
}
respond('return regex process data in here');
}
process.on('message', handleMessage);
This might just end up masking the real problem, though. You may want to consider reworking your regex like other posters have suggested.
Another solution I found here:
https://www.josephkirwin.com/2016/03/12/nodejs_redos_mitigation/
Based on the use of VM, no process fork.
That's pretty.
const util = require('util');
const vm = require('vm');
var sandbox = {
regex:/^(A+)*B/,
string:"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC",
result: null
};
var context = vm.createContext(sandbox);
console.log('Sandbox initialized: ' + vm.isContext(sandbox));
var script = new vm.Script('result = regex.test(string);');
try{
// One could argue if a RegExp hasn't processed in a given time.
// then, its likely it will take exponential time.
script.runInContext(context, { timeout: 1000 }); // milliseconds
} catch(e){
console.log('ReDos occurred',e); // Take some remedial action here...
}
console.log(util.inspect(sandbox)); // Check the results
I'm working with firebase cloud function.
My function is triggered every 5-minutes.
When my function is triggered, my function send HTTP request to external websites.
If number of websites is less than 10, everything is ok.
But, when number of websites is greater than 10, timeout error occurs.
Can you explain why? any solution? see below code.
const functions = require('firebase-functions');
admin.initializeApp();
const rp = require('request-promise');
exports.scheduledFunction = functions.pubsub.schedule('*/5 * * * *')
.timeZone("Asia/Tokyo").onRun((context) => {
// URLs
const urlArray = ['http://a.com', /* ..... */, 'http://z.com']; // total 25 urls.
function getDataFromUrl(){
const promises = []
var N = 10 // if N<=10 working well, but if N>10 not working
for(var i=0; i<N; i++){
promises.push(rp.get(urlArray[i]))
}
return Promise.all(promises)
}
return getDataFromUrl().then(results =>{
results.forEach(result=>{
// do something
})
})
}
Since we can't benchmark your code for you, or observe how it's behaving, the only logical answer is that at least one of the ten is taking longer than your configured timeout, which makes the entire set of promises fail when sent through Promise.all(). Try increasing the timeout for that function.
Is it possible to cancel a regex.match operation if takes more than 10 seconds to complete?
I'm using an huge regex to match a specific text, and sometimes may work, and sometimes can fail...
regex: MINISTÉRIO(?:[^P]*(?:P(?!ÁG\s:\s\d+\/\d+)[^P]*)(?:[\s\S]*?))PÁG\s:\s+\d+\/(\d+)\b(?:\D*(?:(?!\1\/\1)\d\D*)*)\1\/\1(?:[^Z]*(?:Z(?!6:\s\d+)[^Z]*)(?:[\s\S]*?))Z6:\s+\d+
Working example: https://regex101.com/r/kU6rS5/1
So.. i want cancel the operation if takes more than 10 seconds. Is it possible? I'm not finding anything related in sof
Thanks.
You could spawn a child process that does the regex matching and kill it off if it hasn't completed in 10 seconds. Might be a bit overkill, but it should work.
fork is probably what you should use, if you go down this road.
If you'll forgive my non-pure functions, this code would demonstrate the gist of how you could communicate back and forth between the forked child process and your main process:
index.js
const { fork } = require('child_process');
const processPath = __dirname + '/regex-process.js';
const regexProcess = fork(processPath);
let received = null;
regexProcess.on('message', function(data) {
console.log('received message from child:', data);
clearTimeout(timeout);
received = data;
regexProcess.kill(); // or however you want to end it. just as an example.
// you have access to the regex data here.
// send to a callback, or resolve a promise with the value,
// so the original calling code can access it as well.
});
const timeoutInMs = 10000;
let timeout = setTimeout(() => {
if (!received) {
console.error('regexProcess is still running!');
regexProcess.kill(); // or however you want to shut it down.
}
}, timeoutInMs);
regexProcess.send('message to match against');
regex-process.js
function respond(data) {
process.send(data);
}
function handleMessage(data) {
console.log('handing message:', data);
// run your regex calculations in here
// then respond with the data when it's done.
// the following is just to emulate
// a synchronous computational delay
for (let i = 0; i < 500000000; i++) {
// spin!
}
respond('return regex process data in here');
}
process.on('message', handleMessage);
This might just end up masking the real problem, though. You may want to consider reworking your regex like other posters have suggested.
Another solution I found here:
https://www.josephkirwin.com/2016/03/12/nodejs_redos_mitigation/
Based on the use of VM, no process fork.
That's pretty.
const util = require('util');
const vm = require('vm');
var sandbox = {
regex:/^(A+)*B/,
string:"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC",
result: null
};
var context = vm.createContext(sandbox);
console.log('Sandbox initialized: ' + vm.isContext(sandbox));
var script = new vm.Script('result = regex.test(string);');
try{
// One could argue if a RegExp hasn't processed in a given time.
// then, its likely it will take exponential time.
script.runInContext(context, { timeout: 1000 }); // milliseconds
} catch(e){
console.log('ReDos occurred',e); // Take some remedial action here...
}
console.log(util.inspect(sandbox)); // Check the results