I have a node.js program that runs around 50 different python script instances. I would like to be able to throttle the phase - such that at any one time, only 4 processes will run parallel.
I tried a simple loop that calls a function that runs 4 instances of the python script. I used 60 seconds delay as this is the average time takes the script to run.
function startPythonScraping(){
for(var i = 0; i < finalList.length; i++){
setTimeout(function(){
runFourProccesses(finalList[i]);
}, i*60*1000);
}
}
function runFourProccesses(stockSymbol){
console.log("working on " + stockSymbol);
for(var j = 0; j < 4; j++){
if(j == 0){
trueOrFalse = "y"
} else {
trueOrFalse = "n"
}
let secondPythonProcess = spawn('python', ["/Users/nybgwrn/Desktop/AlphaSecWebsite/getAllData.py", stockSymbol, (j*10).toString(), "10", trueOrFalse]);
secondPythonProcess.stdout.on('data', (data) => {
let messegeFromPython = JSON.stringify(data.toString('utf8')).replace("\\n", "");
console.log(messegeFromPython + " with stock " + stockSymbol);
if(messegeFromPython != "something went wrong"){
//console.log("created file for " + symbol);
} else {
//console.log("couldn't create file for " + symbol + "_" + (j*10).toString() + "-" + (j*10 + 10).toString());
}
});
}
}
It doesn't work because somehow the index i begin with 50 instead of 0, and also I want a better solution as I want to be SURE that only 4 instances are running.
This is usually done using semaphores. A semaphore is basically a pool of locks, so that multiple locks can be held at the same time.
In your case you could use this package and wrap each python process spawning with sem.take and sem.leave.
Related
I have for loop here for calling API calls from ticketmaster website and import in google sheets. The problem is I can't make many API calls at once I have to wait about 1 second.
So when I make the loop wait 1 second for example if the array has 40 element I wait 40 sec.
No problem with that due to the massive data but when the fucntion wait it gives an error of execution time limit, I need to set the execution time unlimited.
for (var Veunue_id1 = 0; Veunue_id1 < Venue_Id_List.length; Veunue_id1++) {
var Venue_API_Url = "https://app.ticketmaster.com/discovery/v2/venues?apikey=" + API_key + "&keyword=" + Venue_Id_List[Veunue_id1] + "&locale=*";
// ImportJSON(url, "/","noInherit,noTruncate,rawHeaders");
// console.log(ImportJSON(url, "/", "noInherit,noTruncate,rawHeaders"));
// console.log("Veuneid" + Veunue_id + Venue_Id_List.length);
results = results.concat(ImportJSON(Venue_API_Url, "/_embedded/venues/id", "noInherit,noTruncate,rawHeaders"));
console.log(results);
// wait1 second
Utilities.sleep(1000);
}
This a common issue, especially when working with heavy data API. General approach is:
Run as much as you can, until timeout(6 min) is close
Save loaded data
Store some indicator on there to resume on next run
On next run, repeat from position saved in 3)
Put this on trigger.
Example code based on your situation (not tested)
function SimpleTimer(timeout){
var start = Date.now();
this.getElapsed = () => Date.now() - start;
this.isTimeout = () => this.getElapsed() > timeout;
}
function resetProgress(){
PropertiesService.getScriptProperties().setProperty('last', '-1')
}
function test_SimpleTimer() {
var timer = new SimpleTimer(4*60*1000) // 4 min;
var start = PropertiesService.getScriptProperties().getProperty('last') || '-1';
for (var Veunue_id1 = parseInt(start) + 1; Veunue_id1 < Venue_Id_List.length && ! timer.isTimeout(); Veunue_id1++) {
var Venue_API_Url = "https://app.ticketmaster.com/discovery/v2/venues?apikey=" + API_key + "&keyword=" + Venue_Id_List[Veunue_id1] + "&locale=*";
results = results.concat(ImportJSON(Venue_API_Url, "/_embedded/venues/id", "noInherit,noTruncate,rawHeaders"));
Utilities.sleep(1000);
PropertiesService.getScriptProperties().setProperty('last', '' + Veunue_id1)
}
// SAVE YOUR DATA HERE, NEXT CALL WILL PROCEED FROM LAST STEP
}
So Ive been fighting with this for loop for a day and a half now, when I get it to actually print it goes infinitely, even when the if statement (logCount === 10) is satisfied...
Not too sure what to try anymore, really feel like its far simpler than what I'm trying....
Any attempts at a solution are appreciated..
var timers = [];
var log = (function(outputFunc) {
var counter = 0;
var callerLog = [];
var dateTime = [];
//assigning current Date to a variable to print for logs
let logDate = new Date();
return function(timer) {
for (var logCount = 0; logCount <= 10; logCount++) {
//printing data without using printFunc, as specified
document.getElementById("output").innerHTML += logCount + " " + timer + " " + logDate + "<br>";
//TODO: add after for loop is resolved.
if (logCount >= 10) {
clearInterval(timer1);
clearInterval(timer2);
clearInterval(timer3);
document.getElementById("output").innerHTML += "<br><br/> Logging stopped.";
}
}
}
})(printFunc);
function printFunc(output) {
document.write(output + "<br>");
}
function startMeUp() {
// add each of the timer references to the timers array
// as part of invoking the log function following each interval
timers.push(setInterval("log('Timer1')", 1000));
timers.push(setInterval("log('Timer2')", 1200));
timers.push(setInterval("log('Timer3')", 1700));
}
I'm guessing this is what you're trying to achieve:
function printFunc(output) {
// Replaced with console.log for my own convenience
console.log(output);
}
// Not really a factory, just a curry of the outputFunc
function loggerFactory(outputFunc) {
return function startLogger(name, interval) {
// Variables to keep track of the iterations and a reference to the interval to cancel
let logCount = 0;
let intervalRef;
function tick() {
// On each tick, check if we're done
// If yes, clear the interval and do the last output
// If no, do some output and increment the iterator
// Once the next tick passes we'll check again
// If you were using setTimeout instead you would have to requeue the tick here
if (logCount >= 10) {
clearInterval(intervalRef);
outputFunc('Done ' + name);
} else {
outputFunc(logCount + " " + name);
logCount += 1;
}
}
// Start it of
intervalRef = setInterval(tick, interval);
}
}
const logger = loggerFactory(printFunc);
function startMeUp() {
console.log('Starting');
logger('Log1', 1000);
logger('Log2', 1200);
logger('Log3', 1700);
}
startMeUp();
Some notes:
You could push the intervalRefs into an array but I find it nicer to encapsulate that work within the same logger, since it should only clean up itself anyway.
When working with intervals (or asynchronous code in general) for loops are usually not what you're looking for. For loops are inherently synchronous, all iterations will be run directly after each other, without space for anything else. What you were looking for was a way to run multiple asynchronous "tracks" at the same time. You could use a for loop to start these "tracks", such as:
for () {
logger('Log' + i, 1000 * i);
}
But the key lies in that the logger quickly sets up the interval function and then returns. That way your for loop can quickly "schedule" the tasks but the logger runs the iterations internally asynchronously by using setInterval or setTimeout.
Attached is the problem and the solution code I wrote. Basically callback should console log the output in ascending order not the order in which it received.
function processRequest(i, callback) {
setTimeout(function() {
callback('Action processed ' + i);
}, Math.random() * 1000);
}
callAction(6);
callAction(count) {
//write code here
}
expected output>>
Action processed 1
Action processed 2
Action processed 3
Action processed 4
Action processed 5
Action processed 6
I know this is not the best way to solve this. Can someone rate my solution and possibly suggest a better way? Something Async I am guessing?
Much appreciated! Thanks.
function processRequest(i, callback) {
setTimeout(function() {
callback('Action processed ' + i);
}, Math.random() * 1000);
}
callAction(6);
function callAction(count) {
const arr = [];
let counter = 0;
for (let i = 1; i <= count; i++) {
arr[i - 1] = i;
}
for (let i = 1; i <= count; i++) {
processRequest(i, function(str) {
counter++;
let currentActionNum = parseInt(str.match(/\d+/g));
let message = str.substr(0, str.indexOf(currentActionNum));
if (currentActionNum === arr[0]) {
console.log(message + currentActionNum);
arr.shift();
}
if (counter === count) {
for (let i = arr[0]; i <= count; i++) {
console.log(message + i);
}
}
});
}
}
Your task is not to use super-complicated processing to rewrite the outputs to appear in the correct order, but to ensure that the 6 processes run sequentially. You can do that with a simple recursive approach:
function callAction(count) {
if (count > 1)
processRequest(7 - count, result => {
// ^^^^^^^^^ ok, that part is a bit weird
console.log(result);
callAction(count - 1);
});
}
I think what the interviewer want you to do is to make sure these processes would run in the right order (the 'real-life' process would be different, not just print some string). So, here is a better solution:
function callAction(count){
i = 1;
cb = function(str){
console.log(str)
count-i++ && processRequest(i, cb) //'same' as: if (count-i++>0) then ...
}
processRequest(1, cb)
}
Just out of curiosity, may I ask if it is a face to face interview? Do you have access to a computer? How long do you have to solve this? If I dont have a computer with me to try and test then I might do worst than that...
function Test(time) {
i = 1;
fun1 = setInterval(function () {
if (i >= 1 && i <= 10) {
$('.class.' + i).children('.BUTTON').click();
}
}, time * i);
fun2 = setInterval(function () {
if (result > 0 && i < 10) {
console.log('The number is ' + i);
i++;
}
else if (result < 0) {
console.log('Sorry at ' + i + ' point you fell in the hole');
clearInterval(fun2);
stats.loss++;
$('.loss').text(stats.loss);
console.log('Number of fell are ' + stats.loss);
} else {
console.log('Exited from maze');
clearInterval(fun);
clearInterval(fun2);
stats.wins++;
$('.wins').text(stats.wins);
console.log('Number of wins are ' + stats.wins);
$('#START').click();
}
}, time * 1.3);
}
Last time I asked about making the if's in a for holding the i, loved it, then, I had the idea of checking the result of the path, showed in this case as points. And this way works great, problem is, what if I want to create an "infinite loop" to check if in the long run my character would be a winner or loser?
Solutions I tried: for,while,do while, they ends up requestin thousands of requests to the server.
tried with A main set interval, but somehow it just creates parallel requests and even If I do clearInterval(masterloop); it just activate the test again and again, speeding up and clicks only the start/reset level button.
I have a page that is using jQuery to load an XML file, which I'm then outputting the contents of to the page.
Recently I added a sorting function to the output which is causing a 1+ or 2+ minute hang on Safari on an iPod Touch (depending upon how many fields I sort by) and a less than 1 minute hang on an iPad. The same sort returns within a few seconds on Firefox 4.0.1.
I'm afraid it's just a limitation of the iOS, but before I removed the sort, perhaps there's an optimization that could be made.
Before the filter there's 357 items in the XML. After the filter there's 199 items that are sorted through.
var videoGames = $($.parseXML(videoGameXml)).find("game");
videoGames = videoGames.filter(function (a) {
return ($(this).attr('addOn') != "true" && $(this).find('own').text() == "yes");
});
videoGames.sort(function (a, b) {
var firstTitle = $(a).find('title').text().toLowerCase();
var secondTitle = $(b).find('title').text().toLowerCase();
var firstSystem = ($(a).find("console").text() + " " + $(a).find("version").text()).toLowerCase();
var secondSystem = ($(b).find("console").text() + " " + $(b).find("version").text()).toLowerCase();
if (firstSystem != secondSystem) {
if (firstSystem > secondSystem) {
return 1;
} else {
return -1;
}
} else {
if (firstTitle > secondTitle) {
return 1;
} else if (secondTitle < firstTitle) {
return -1;
}
}
return 0;
});
videoGames.each(function () {
// runs quickly, so removed
});
Note that if I remove the system check as an initial 'optimization' that cuts the time about in half on the iPod Touch, but still results in the 1+ minute hang mentioned above.
So, is it an iOS device limitation, or can I optimize my sort?
Every time you do $(a) it will perform a very complex set of operations, so you better cache it. Also, you don't need the Title if System is different. This version should speed it up a bit:
videoGames.sort(function (a, b) {
var first = $(a);
var second = $(b);
var firstSystem = (first.find("console").text() + " " + first.find("version").text()).toLowerCase();
var secondSystem = (second.find("console").text() + " " + second.find("version").text()).toLowerCase();
if (firstSystem != secondSystem) {
if (firstSystem > secondSystem) {
return 1;
} else {
return -1;
}
} else {
var firstTitle = first.find('title').text().toLowerCase();
var secondTitle = second.find('title').text().toLowerCase();
if (firstTitle > secondTitle) {
return 1;
} else if (secondTitle < firstTitle) {
return -1;
}
}
return 0;
});
You could also cache the values in the object
Then, instead of:
var firstSystem = (first.find("console").text() + " " + first.find("version").text()).toLowerCase();
Do:
var firstSystem = first.data('system');
if (!firstSystem) {
firstSystem = (first.find("console").text() + " " + first.find("version").text()).toLowerCase();
first.data('system') = firstSystem;
}
You should move any selectors calls like this:
var firstTitle = $(a).find('title').text().toLowerCase();
out from the comparator function. The comparator function is supposed to be lightweight.
Either use children(), next() and the like or
scan you set once and create an array of keys upfront and then sort it using those keys.
The comparator function will be called 2n * ln(n) times (depends on algorithm used) where n is a number of elements in a set. So your code does the same expensive calculations twice at least.