I am new to the async programming and currently stuck with this simple task. I have a Mysql table which has 20k records and need to process all the records. The current approach is looping in batch of records.
let limit = 100;
let offset = 0;
sequelize.query(`Select * from abc limit ${limit} offset ${offset}`, {type: sequelize.QueryTypes.SELECT}).then((records) => {
// process
return processResult;
});
Therefore, i need to increase the offset in the while loop.
while (/*....*/) {
let p = sequelize.query(`Select * from abc limit ${limit} offset ${offset}`, {type: sequelize.QueryTypes.SELECT}).then((records) => {
// process records
return processResult;
});
offset += limit;
}
My question is what is the condition in the while loop in order to stop it ? Since p is always a Promise, thus I cannot use it as the stop condition
You can use the .each() of bluebird Promise.
var Promise = require('bluebird');
let limit = 100;
let offset = 0;
let queryStrings = [];
let max = 20000;
while (offset < max) {
queryStrings.push(`Select * from abc limit ${limit} offset ${offset}`);
offset += limit;
}
let p = Promise.each(queryStrings, (queryString)=> {
return sequelize.query(queryString, {type: sequelize.QueryTypes.SELECT}).then((records) => {
// process records
if(/* need to break */){
return Promise.reject(new Error('...'));
}
return processResult;
});
});
update
If you don't know the maximum, you can do some recursion:
let _iterate = function (limit, offset) {
return sequelize.query(`Select * from abc limit ${limit} offset ${offset}`, {type: sequelize.QueryTypes.SELECT}).then((records) => {
// process records
if (/* need to break */) {
return Promise.reject(new Error(''));
}
offset += limit;
return _iterate(limit, offset);
});
};
let p = _iterate(100, 0);
Related
how to get data from an object Store from index number n to k.
For example if we have 100 records in an store (ProductStore), an I need to get from index 11 to 20 or 50 to 60. I need to use this in pagination. In mysql we can use LIMIT and OFFSET clouse,is there any equivalent to OFFSET in Indexeddb.
You can use cursor.advance to skip past some records. There is no limit with cursors, you simply stop iterating by counting how many times you iterated.
Something like this:
function query(db, criteria, offset, limit) {
return new Promise((resolve, reject) => {
const results = [];
const transaction = db.transaction('store');
transaction.oncomplete = event => resolve(results);
transaction.onerror = event => reject(event.target);
const store = transaction.objectStore('store');
const index = store.index('index');
const request = index.openCursor(criteria);
let advanced = offset === 0;
let counter = 0;
request.onsuccess = event => {
const cursor = event.target.result;
if (!cursor) {
return;
}
if (!advanced) {
advanced = true;
cursor.advance(offset);
}
counter++;
results.push(cursor.value);
if (counter >= limit) {
return;
}
cursor.continue();
};
});
}
I'm trying to sum up the array of elements by every seconds. My array has 20 elements, and it should be summed up in 20seconds.
I'm using setTimeout in for loop but its not working , the loop finish before the first second. Anyway to achieve ?
for (var o = 0; o < 20; o++) {
setTimeout(function () {
tempPsum += array[o];
}, 1000);
}
Right, the loop finishes before the first setTimeout callback occurs. Your code schedules 20 callbacks that all occur, one after another, a second later.
You have two choices:
Schedule each one a second later than the previous one, or
Don't schedule the next one until the previous one finishes
#1 is simpler, so let's do that:
for (let o = 0; o < 20; o++) {
// ^^^−−−−−−−−−−−−−−−−−−−−−−−−−−−−−−−
setTimeout(function () {
tempPsum += array[o];
}, 1000 * o);
// ^^^^−−−−−−−−−−−−−−−−−−−−−−−−−−
}
By multiplying the delay time by o, we get 0 for the first timer, 1000 for the second, 3000 for the third...
Note the change from var to let. That way, there's a separate o for the callback to close over for each loop iteration. (See answers here, or...Chapter 2 of my new book. :-) )
If you can't use ES2015+ features in your environment, you can use the extra arguments to setTimeout instead:
for (var o = 0; o < 20; o++) {
setTimeout(function (index) {
// ^^^^^−−−−−−−−−−−
tempPsum += array[index];
// ^^^^^−−−−−−−−−−−
}, 1000 * o, o);
// ^^^^−−^−−−−−−−−−−−−−−−−−−−−−−−
}
That tells setTimeout to pass the third argument you give it to the callback as its first argument.
Ideal use case of setTimeout() is when you have to perform an operation once after a specific time frame. The scenario mentioned above requires same operation that is adding array element after a fixed time frame, so I believe setInterval() would be a better option.
var array = [1,2,3,4,5];
var tempSum = 0, i = 0;
var intrvl = setInterval( ()=> {
tempSum += array[i++];
console.log(tempSum)
if(i === array.length)
clearInterval(intrvl)
}, 1000)
This answer doesn't explain what the issue is, but offers an alternative to the other solutions. Instead of modifying the timeout using the index, you could add a sleep function that resolves a promise in a given amount of time.
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
Then loop through your array using for...of and sleep each iteration for a given amount of time.
let sum = 0;
for (const number of numbers) {
await sleep(1000);
sum += number;
}
This does require the code to be placed in an async function.
const randomInt = (min, max) => Math.floor(Math.random() * (max - min + 1)) + min;
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
const numbers = Array.from({length: 20}, () => randomInt(10, 99));
console.log("numbers =", ...numbers);
(async () => {
let sum = 0;
for (const number of numbers) {
await sleep(1000);
console.log(sum, "+", number, "=", sum += number);
}
})();
#TJCrowder has said why your version doesn't work.
I will suggest another solution:
Another way: use setInterval
let numbers = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
let tempPsum = 0
let i = 0
let intervalId = setInterval(function () {
tempPsum += numbers[i];
i = i + 1;
console.log(tempPsum);
if (i == 20) {clearInterval(intervalId)}
}, 1000);
I think you can use this approach:
var i = 0;
var arr = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20];
var sum = 0;
function myLoop() {
setTimeout(() => {
sum = sum + arr[i];
console.log(sum);
i++;
if (i < 20) {
myLoop();
}
}, 1000)
}
myLoop();
so I was given the task to make a unique code by adding numbers after the code.
and I have tried and succeeded but there is one thing that blocks what if the number has reached its limit? 9999 value
how to reset it to 00001. Lift digit increased by 1
here's my snippet code
function getNewCode(value, callback){
let newCode = _.upperCase(value) + "-";
let lastCode = newCode + "0001"
Transaction.findOne({tr_number: new RegExp(newCode, 'i')}, (err, doc) => {
if(err) callback(err)
if (!_.isNil(doc)){
let arr = doc.tr_number.split("-");
// in this part, i want got some conditional to set a new value
//when it got max length = 9999 for an example
let inc = parseInt(arr[1]) + 1;
lastCode = newCode + ("0000"+inc).slice(-4);
console.log('lastCode', ciduk, lastCode);
return callback(lastCode);
}else{
return callback(lastCode);
}
}).sort({tr_number: -1})
};
sorry for my bad language :) grats.
You can get number size by converting it toString() and get it length.
function getNext(prevStr) {
let padSize = prevStr.length;
let next = parseInt(prevStr, 10) + 1;
if (next >= 10 ** padSize) {
padSize++;
next = 1;
}
return next.toString().padStart(padSize, '0');
}
console.log(getNext('0099')); // '0100'
console.log(getNext('9999')); // '00001'
I am trying to generate fixed length nonce (length 9).
But my code is printing sometimes nonce of 8 length and sometime 9 length.
this is what I am trying to do but with different approach (I have modified it for fixed nonce length)
I am not able to understand why it is printing nonce of length 8 when i am passing length as 9 as argument??
It would be great if someone can tell why this is happening.
Below is complete Nodejs code
var last_nonce = null;
var nonce_incr = null;
// if you call new Date to fast it will generate
// the same ms, helper to make sure the nonce is
// truly unique (supports up to 999 calls per ms).
module.exports = {
getNonce: function(length) {
if (length === undefined || !length) {
length = 8;
}
var MOD = Math.pow(10, length);
var now = (+new Date());
if (now !== last_nonce) {
nonce_incr = -1;
}
nonce_incr++;
last_nonce = now;
var nonce_multiplier = ((nonce_incr < 10) ? 10 : ((nonce_incr < 100) ? 100 : 1000));
var s = (((now % MOD) * nonce_multiplier) + nonce_incr) % MOD;
return s;
}
}
//test code
if(require.main === module) {
console.time("run time");
//importing async module
var async = require('async');
var arr = [];
//generating 1000 length array to use it in making 1000 async calls
//to getNonce function
for(var i=0; i<1000; i++) arr.push(i);
//this will call getNonce function 1000 time parallely
async.eachLimit(arr, 1000, function(item, cb) {
console.log(module.exports.getNonce(9));
cb();
}, function(err) {console.timeEnd("run time");});
}
Sample output:
708201864 --> nonce length 9
708201865
708201866
70820190 --> nonce length 8 (why it is coming 8?? when passed length is 9)
70820191
70820192
70820193
70820194
70820195
70820196
70820197
70820198
70820199
708201910
708201911
708201912
708201913
708201914
708201915
708201916
708201917
708201918
In case someone needs it, here is a nonce generator free from convoluted logic, allowing you to control both character sample and nonce size:
const generateNonce = (options) => {
const {
length = 32,
sample = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
} = options || {};
const getRand = () => Math.floor(Math.random() * sample.length);
return Array.from({ length }, () => sample.charAt(getRand())).join('');
};
If you prefer Typescript:
const generateNonce = (options?: { sample?: string, length?: number }) => {
const {
length = 32,
sample = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
} = options || {};
const getRand = () => Math.floor(Math.random() * sample.length);
return Array.from({ length }, () => sample.charAt(getRand())).join('');
};
I have a long-running function that does a huge calculation: all the possible permutations of x n-sided dice and the probability of those outcomes. For small x and n, the calculation is quick. For large values (n = 100, x > 3), the calculation takes tens of seconds if not longer; meanwhile, the browser stalls.
Here's a snippet of my code:
let dist = [];
// min and max are the minimum and maximum possible values
// (if dice are all 1's or all their maximum values)
for (let i = min; i <= max; i++) {
// initialize possible values of our distribution to 0
dist.push([ i, 0 ]);
}
// total is the total outcome value so far
// dIndex is the index into the dice-array (diceList) for the die we're
// currently applying to our total--the die we're "rolling"
function applyDie(total, dIndex) {
if (dIndex === diceList.length) {
dist[total - min][1]++;
permutationsCount++;
return;
}
// diceList is an array of integers representing the number of sides
// for each die (one array element = one die of element-value sides)
for (let i = 1; i <= diceList[dIndex]; i++) {
applyDie(total + i, dIndex + 1);
}
}
// kick off recursive call
applyDie(0, 0);
I want to add two functionalities:
Cancellation
Progress reporting
Cancellation will be easy (I can do it myself) once I have the async pattern in place, so I really only need help with progress reporting, or, more accurately, simply breaking the recursive pattern into chunks based on the permutationsCount variable. i.e.
/* ... */
permutationsCount++;
if (permutationsCount % chunkSize === 0)
/* end this chunk and start a new one */
I would prefer to use Javasciprt Promises, but I'm open to other suggestions.
Ideas?
Here's a function I wrote to do something similar. It's for a calculation done entirely in javascript... I couldn't tell from your question whether you were working entirely on the client side or what.
// Break the list up into equal-sized chunks, applying f() to each item of
// the list, writing a %-complete value to the progress span after each
// chunk. Then execute a callback with the resulting data.
var chunkedLoop = function (list, f, progressSpan, callback) {
var numChunks = 10,
chunkSize = Math.round(list.length / numChunks),
chunkedList = [], // will be a list of lists
// Concatenated results of all chunks, passed to the callback.
resultList = [],
x, // just a loop variable
chunkIndex = 0; // tracks the current chunk across timeouts
progressSpan.html(0);
// Splice of chunks of equal size, but allow the last one to be of an
// arbitrary size, in case numChunks doesn't divide the length of the
// list evenly.
for (x = 0; x < numChunks - 1; x += 1) {
chunkedList.push(list.splice(0, chunkSize));
}
chunkedList.push(list);
// Schedule a series of timeouts, one for each chunk. The browser
// stops blocking for a moment between each chunk, allowing the screen
// to update. This is the only way to have progress values reported to
// the view mid-loop. If it was done in a single loop, execution would
// block all the way to the end, and the screen would only update once
// at 100%.
var chunkFunction = function () {
setTimeout(function () {
// Run f on the chunk.
var chunk = chunkedList[chunkIndex];
var r = forEach(chunk, f);
resultList = resultList.concat(r);
chunkIndex += 1;
// Update progress on the screen.
progressSpan.html(Math.round(chunkIndex / numChunks * 100));
// Schedule the next run, if this isn't the last chunk. If it
// is the last chunk, execute the callback with the results.
if (chunkIndex < chunkedList.length) {
chunkFunction();
} else if (callback instanceof Function) {
callback.call(undefined, resultList);
}
// There's no reason to delay more than the minimum one
// millisecond, since the point is just to break up javascript's
// single-threaded blocking.
}, 1);
};
chunkFunction();
};
For reporting status you can pass callback function into your recursuve function and do there whatever you like (increase counter, push status into page etc).
Also think about rewriting recursive into iterative algorithm because it will have lesser memory overhead and it will be much easier to put there some other logic (like cancellations you mentioned)
You could use setTimeout to let JavaScript do other things and unstuck the event loop. This way even infinite loop would nonblocking. Here is a quick dirty example.
http://jsfiddle.net/xx5adut6/
function isPrime(n) {
// If n is less than 2 or not an integer then by definition cannot be prime.
if (n < 2) {
return false
}
if (n != Math.round(n)) {
return false
}
var isPrime = true;
for (var i = 2; i <= Math.sqrt(n); i++) {
if (n % i == 0) {
isPrime = false
}
}
// Finally return whether n is prime or not.
return isPrime;
}
var cancel = false;
var i = 0;
var primesFound = 0;
var status = $('.status');
var timeout;
function loop4Primes() {
if (cancel) {
clearTimeout(timeout);
return;
}
if (isPrime(i++)) primesFound++;
timeout = setTimeout(loop4Primes, 1);
}
function updateText() {
status.text(primesFound);
}
var interval = setInterval(updateText, 1000);
$('#start').click(function () {
loop4Primes();
$(this).off('click');
});
$('#stop').click(function () {
clearInterval(interval);
updateText();
cancel = true;
});