Question: How can I make my files.map(...) pause every 50 iterations?
Problem: the gm().size() is a very expensive function. which completely shits the bed after about 300 iterations. I have a theory that this will be remedied if I let the function keep up.
//interaction happens that will traverse a bunch of folder and create an array of files paths
glob(filePath + '/**/*{.png,.jpg,.gif}', function (er, files) {
var chunksize = 50; // sets the iteration size
if (er) return er;
service.stuff[name] = files.map(function (entry, i) {
return {
identity: getIdentity() //returns the identity(size) of images
};
function getIdentity() {
if(i % chunksize == 0) { // if the 50th iteration
(function(chunksize, i){
setTimeout(function () {
var entrySize = gm(entry) //graphics magic will return size of images based on path.
.size(function (err, size) {
return size;
});
}, 2000); //pause for 2 seconds.
}());
} else {
var entrySize = gm(entry)
.size(function (err, size) {
return size;
});
}
return entrySize.data; //returns identity data.
}
});
});
Alternatively, implement your own batch processor. This alternative to map will only process options.batchSize items at a time, then takes an options.timeoutMs break to give the application time to do other things.
function batchMap(array, fn, options, callback) {
var batchSize = options.batchSize || 100,
timeoutMs = options.timeoutMs || 0;
function map(done, todo) {
if(todo.length > 0) {
setTimeout(function() {
var mapped = todo.slice(0, batchSize).map(fn);
map(done.concat(mapped), todo.slice(batchSize));
}, timeoutMs);
} else {
callback(null, done);
}
}
map([], array);
}
You can use async.mapSeries for this. It waits for each iteration to finish before continuing to the next.
npm install async
var async = require("async");
var noFile = 0;
var done = function (err, result) {
console.log(result);
}
var yourLogic = function(file){
}
var processFile = function(file, callback) {
if(noFile > 50) {
setTimeout(function() {
noFile++;
callback(null, yourLogic(file));
}, 1000);
} else {
noFile++;
callback(null, yourLogic(file));
}
}
async.mapSeries(files, processFile, done);
Sorry, I just feel like shamelessly plugging my new library so I will. It goes something like this:
var CL = require('coastline');
CL.run(function* () {
var files = yield glob(filePath + '/**/*{.png,.jpg,.gif}', CL.cb());
var chunksize = 50;
service.stuff[name] = yield CL.map(files, function* (entry, i) {
if (i && i % chunksize == 0) yield CL.sleep(2000);
var entrySize = yield gm(entry).size(CL.cb());
return {
identity: entrySize.data
};
});
});
Edit: checked and it works, except there is no .data in entrySize, just .width and .height?
Edit: removed var i since we can use array index.
Related
var message = "I like apple and sunny and crazy bannaas ... and lovey cats";
var getMsgOneCharATime = function(cb, idx) {
// Fetches a character from the message. Simulates an API call by adding a delay to the request.
const apiDelay = 10;
setTimeout(function() {
cb(message[idx]);
}, apiDelay);
};
//// only after following line I can edit!
var countDistinctAsyncWords = function() {
const messageArray = [];
let currentIndex = 0;
function getPromisedCounter(value) {
return new Promise((resolve, reject) => {
if (!value) {
resolve({
value,
index
});
} else {
reject('there is a error');
}
});
}
var counter = setInterval(() => {
getMsgOneCharATime(getPromisedCounter, currentIndex);
currentIndex ++;
});
function saveWord(word) {
if (!word) clearInterval(counter);
console.log(word);
messageArray.push(word);
}
console.log(messageArray);
return messageArray.join();
}
console.log('---');
console.log(countDistinctAsyncWords());
console.log('---end-');
The purpose is to use countDistinctAsyncWords to print out the message that got fetched by the fake timeout api getMsgOneCharATime calling. I still not able to come out an idea how should I intercept each character correctly, assuming that the message can be infinity and I do not know the length of it. I am also open to other solutions.
Thanks!
I am having some problem using the settimeout() in my function. I am new to async. No matter how much I try I just can't make the timeout work. My code works perfect so that is not the problem. I need the request to execute every 10 seconds. Thanks for the help.
function getContent() {
function getPelicula(pelicula, donePelicula) {
var peli = pelicula.title;
//request id
request({
url: "http://api.themoviedb.org/3/search/movie?query=" + peli + "&api_key=3e2709c4c051b07326f1080b90e283b4&language=en=ES&page=1&include_adult=false",
method: "GET",
json: true,
}, function(error, res, body) {
if (error) {
console.error('Error getPelicula: ', error);
return;
}
var control = body.results.length;
if (control > 0) {
var year_base = pelicula.launch_year;
var id = body.results[0].id;
var year = body.results[0].release_date;
var d = new Date(year);
var year_solo = d.getFullYear();
if (year_base == year_solo) {
pelicula.id = id;
pelicula.year_pagina = year_solo;
}
} else {
pelicula.id = null;
pelicula.year_pagina = null;
}
donePelicula();
});
}
}
To do something in a loop, use setInterval.
UPD:
In general, there're two ways of executing some code in loop
1 setTimeout :
var someTimer = setTimeout(function sayHello(){
console.log("hello!");
someTimer = setTimeout(sayHello, 2000);
}, 2000);
Notice that someTimer variable is needed to stop the looping process if you need: clearTimeout(someTimer)
2 setInterval:
var someIntervalTimer = setInterval(function(){
console.log("I'm triggered by setInterval function!");
}, 2000);
Invoke clearInterval(someIntervalTimer) to stop the looping
Both functions are treated as properties of the global Window variable. By default, the following code works:
var window = this;
console.log("type of setTimeout: " + typeof window.setTimeout);
console.log("type of setInterval: " + typeof window.setInterval);
Try putting it in another function so:
domore(pelicula,donePelicula);
function domore(pelicula,donePelicula) {
// 1 second
var timeout = 1000;
for (var i = 1; i < pelicula.length; i++) {
createData(pelicula[i],donePelicula,timeout);
timeout = timeout + 800;
}
}
function createData(peli,donePelicula,timeout) {
setTimeout(function() { getData(peli,donePelicula); }, timeout);
}
function getData(peli,donePelicula) {
var txtFile = new XMLHttpRequest();
txtFile.open("GET", "http://api.themoviedb.org/3/search/movie?query=" + peli + "&api_key=3e2709c4c051b07326f1080b90e283b4&language=en=ES&page=1&include_adult=false", true);
txtFile.onreadystatechange = function() {
if (txtFile.readyState === 4) { // Makes sure the document is ready to parse.
if (txtFile.status === 200) { // Makes sure it's found the file.
allText = txtFile.responseText;
domore(allText,donePelicula);
}
}
}
txtFile.send(null);
}
I have a function say myMainFunction that is called from a client, that in turn calls mypromisified function.
Scenario:
mypromisified function can fail intermittently and I need to call this function with a delay (at an exponential increase) until success or until max no of tries reached.
What I have so far
The following code illustrates my scenario and repeats itself until success, but it tries indefinitely and not until certain count is reached
// called once from the client
myMainFuntion();
function rejectDelay(delay, reason) {
// call main function at a delayed interval until success
// but would want to call this only a limited no of times
setTimeout(() => {
myMainFuntion(); // calling main function again here but with a delay
}, delay);
}
function myMainFuntion() {
var delay = 100;
var tries = 3;
tryAsync().catch(rejectDelay.bind(null, delay));
}
function tryAsync() {
return new Promise(function(resolve, reject) {
var rand = Math.random();
console.log(rand);
if (rand < 0.8) {
reject(rand);
} else {
resolve();
}
});
}
while loop inside the rejectDelay would certainly not work as the counter would increment even before the actual function inside setInterval is executed, so am unsure as to how to go about this? so...
I tried promisifying the setInterval something like this knowing it will fail :( as it doesnt decrement the counter, but not sure how to get it right either .
function rejectDelay(delay, maximumTries, reason) {
return new Promise(function (resolve, reject) {
console.log(tries + ' remaining');
if (--maximumTries > 0) {
setTimeout(function() {
foo();
}, 500);
}
});
}
function myMainFunction() {
var delay = 100;
var maximumTries = 3;
tryAsync().catch(rejectDelay.bind(null, delay, maximumTries));
}
Using a couple of helper functions I've used a lot, this becomes very easy
The "helpers"
Promise.wait = (time) => new Promise(resolve => setTimeout(resolve, time || 0));
Promise.retry = (cont, fn, delay) => fn().catch(err => cont > 0 ? Promise.wait(delay).then(() => Promise.retry(cont - 1, fn, delay)) : Promise.reject('failed'));
The code:
function myMainFuntion() {
var delay = 100;
var tries = 3;
Promise.retry(tries, tryAsync, delay);
}
ES5 versions of the helpers
Promise.wait = function (time) {
return new Promise(function (resolve) {
return setTimeout(resolve, time || 0);
});
};
Promise.retry = function (cont, fn, delay) {
return fn().catch(function (err) {
return cont > 0 ? Promise.wait(delay).then(function () {
return Promise.retry(cont - 1, fn, delay);
}) : Promise.reject('failed');
});
};
A slightly different approach that uses "asynchronous recursion" to call a nested function within a function that returns a promise:
function retry( func, maxTries, delay) {
var reTry = 0;
return new Promise( function(resolve, reject) {
function callFunc() {
try
{
func().then(resolve, function( reason) {
if( ++reTry >= maxTries) {
reject( reason);
}
else {
setTimeout( callFunc,
typeof delay=="function" ? delay( retry) : delay );
}
});
}
catch(e) {
reject(e);
}
}
callFunc();
});
}
// ******* run snippet to test ********
var retryCount = 0;
function getDelay( n) {
// return 100 * n*n + 500; // for example
++ retryCount;
return 100; // for testing
}
function testFunc() {
return Math.random() < 0.8 ? Promise.reject("too many tries") : Promise.resolve( "success");
}
retry( testFunc, 5, getDelay).then(
function(data) { console.log("data: %s, retryCount %s", data, retryCount);},
function(reason){console.log("reason: %s, retryCount %s", reason, retryCount);}
)
Since the support for arrow functions in JS is still quite low, I'd like to replace them from the following code snippet:
Promise.all([f1(), f2()])
.then(values => { f3(values); })
.catch(values => { f4(values); });
Any idea how can this be achieved?
Should be straight forward, just type "function" instead
Promise.all([f1(), f2()]).then(function(values) {
f3(values);
}).catch(function(values) {
f4(values);
});
Why are you using function expressions at all in there? Do you want to avoid the return values? If not, just go for
Promise.all([f1(), f2()]).then(f3).catch(f4);
This is a solution I wrote to allow existing promise-based code work on promise-less browsers like IE11. Not perfect, but works well.
CanPromise = !!window.Promise;
PromiseResolve = [];
PromiseReject = [];
if (!CanPromise)
{
window.Promise = function (f)
{
var p = {};
p.resolve = function (result)
{
PromiseResolve.push(result);
}
p.reject = function (result)
{
PromiseReject.push(result);
}
return f(p.resolve, p.reject);
};
}
//...........
PromiseResolve.length = 0;
PromiseReject.length = 0;
F1(args);
F2(args);
var AjaxTimeout = 10000;
var period = 100;
var cnt = 0;
var cntMax = parseInt(AjaxTimeout / period);
var t = setInterval(function ()
{
if (PromiseReject.length > 0)
{
clearInterval(t);
Error(PromiseReject[0]);
}
else if (PromiseResolve.length >= 2)
{
clearInterval(t);
Success(PromiseResolve);
}
else if (cnt >= cntMax)
{
clearInterval(t);
}
cnt++;
}, period);
I have a very expensive task running on a Worker, similar as this
for(var i = 0; i < 1000000000; i++)
//operations using i...
How can I make is so that, in that loop, I can check if a message was received from the Worker owner asking it to stop? I would like to have something like this
for(var i = 0; i < 1000000000; i++)
if(windowDidNotAskToStop())
//operations using i...
Right now I have a onmessage function registered so I can start listen to message coming from the owner, but it is blocked while my loop is running (obviously).
I imagine that the postMessage calls from the owner are queued somewhere, so I would simply have to access that in order to process the calls from inside my loop.
You’ll have to handle the events as usual and set a flag, but make sure to leave time for the event to be received in your loop, probably using setTimeout:
var exitLoop = false;
(function loop(i) {
if (exitLoop || i >= 1000000000) {
return;
}
// …
setTimeout(function () {
loop(i + 1);
}, 0);
})(0);
onmessage = function (e) {
if (e.data === 'stop') {
exitLoop = true;
}
};
Or, as a general utility function:
function asyncIterateRange(start, end, callback) {
var exitLoop = false;
var doneCallbacks = [];
(function loop(i) {
if (exitLoop) {
return;
}
if (i >= end) {
doneCallbacks.forEach(function (callback) {
callback();
});
return;
}
callback(function () {
setTimeout(function () {
loop(i + 1);
}, 0);
});
})(start);
return {
abort: function abort() {
exitLoop = true;
},
done: function addDoneCallback(callback) {
doneCallbacks.push(callback);
}
};
}
Use it like so:
var innerIterator;
var outerIterator = asyncIterateRange(0, 1000000, function outerLoop(next) {
innerIterator = asyncIterateRange(0, 1000, function innerLoop(next) {
// …
next();
}).done(next);
});
// To stop:
if (innerIterator) {
innerIterator.abort();
}
outerIterator.abort();