Develop Reference

How do I run code after all threads have finished running?

I have a multithreaded web crawler that downloads a website and stores it in a database (which takes around 4 minutes). To make the crawling faster, I used node.js cluster module, but I have a problem, I want to iterate over to the next segment of the while loop, after all the threads have done their processes, not as soon as they start. How do I make sure all my threads are concluded and then move on?
Here is the relevant code in the main while loop:
while (indexSize !== indexSizeLimit) {
const queueLength = queue.length;
const numberOfThreads = Math.min(numberOfCPUs, queueLength);
const threadAllocations = Array(numberOfThreads).fill(0);
let queuesAllocated = 0;
const queueChunks = [];
function fillQueueChunks() {
loop: while (true) {
for (let i = 0; i < numberOfThreads; i++) {
threadAllocations[i] += 1;
queuesAllocated += 1;
if (queuesAllocated === queueLength) {
break loop;
};
};
};
let start = 0;
for (let threadAllocation of threadAllocations) {
const end = start + threadAllocation;
queueChunks.push(queue.slice(start, end));
start = end;
};
};
fillQueueChunks();
// Find out how to make multithreading finish, and then move on with the loop.
if (cluster.isMaster) {
for (let i = 0; i < numberOfThreads; i++) {
cluster.fork();
};
} else {
const chunk = queueChunks[cluster.worker.id - 1];
await Promise.all(chunk.map(function (url) {
return new Promise(async function (resolve) {
const webcode = await request(url);
if (webcode !== "Failure") {
indexSize += 1;
const document = new Document(url, webcode);
const hrefs = document.hrefs();
const hrefsQuery = Query(hrefs);
// Also make sure it is not included in indexed webpages.
const hrefIndividualized = hrefsQuery.individualize();
hrefIndividualized;
// Do something with hrefIndividualized in regards to maintaining a queue in the database.
// And in adding a nextQueue which to replace the queue in code with.
await document.save();
};
resolve("Written");
});
}));
process.exit(0);
};
};

Wrap the threading in a promise. You can check in the parent thread if there is a disconnect event, and if the amount of disconnects is equal to the number of threads, then you can resolve the promise.
Here is what I have
while (indexSize !== indexSizeLimit) {
let nextQueue = [];
const queueLength = queue.length;
const numberOfThreads = Math.min(numberOfCPUs, queueLength);
const threadAllocations = Array(numberOfThreads).fill(0);
let queuesAllocated = 0;
// queueChunks: [[{_id: ..., ...}], [...], ...]
const queueChunks = [];
function fillQueueChunks() {
loop: while (true) {
for (let i = 0; i < numberOfThreads; i++) {
threadAllocations[i] += 1;
queuesAllocated += 1;
if (queuesAllocated === queueLength) {
break loop;
};
};
};
let start = 0;
for (let threadAllocation of threadAllocations) {
const end = start + threadAllocation;
queueChunks.push(queue.slice(start, end));
start = end;
};
};
fillQueueChunks();
await new Promise(async function (resolve) {
if (cluster.isMaster) {
let threadsDone = 0;
for (let i = 0; i < numberOfThreads; i++) {
cluster.fork();
};
cluster.on("disconnect", function (_) {
threadsDone += 1;
if (threadsDone === numberOfThreads) {
resolve("Queue Processed");
};
});
} else {
const queueJob = queueChunks[cluster.id - 1];
await Promise.all(queueJob.map(function (queueItem) {
return new Promise(async function (resolve) {
const url = queueItem._id;
const webcode = await request(url);
if (webcode !== "Failure") {
const document = Document(url, webcode);
let hrefs = document.hrefs();
const hrefsQuery = Query(hrefs);
await document.save();
indexSize += 1;
hrefs = hrefsQuery.individualize();
const hrefIncidences = Promise.all(hrefs.map(function (href) {
return new Promise(async function (resolve) {
const incidences = await Site.countDocuments({
url: href
});
resolve(incidences);
});
}));
hrefs = hrefs.filter(function (_, i) {
return hrefIncidences[i] === 0;
}).map(function (href) {
return {
_id: href
};
});
await Queue.insertMany(hrefs);
nextQueue = nextQueue.concat(hrefs);
};
await Queue.deleteOne({
_id: url
});
resolve("Success");
});
}));
process.exit(0);
};
});
queue = nextQueue;
};

Custom function not running asynchronously in Promise.all()

I am trying to run a function 4 times (with different parameters) asynchronously with a Promise.all. This snippet runs, however it will only run each 'loopGridValidate' function in order synchronously. I have replaced the loopGridValidate function with the logic from this https://www.javascripttutorial.net/es6/javascript-promise-all/ tutorial and the Promise.all returns what is expected. I'm not sure what I am missing... The loop function is async, everything is set up correctly in the promise all (I have also split out the new promise's into 'const p1 = new Promise' and have the same effect as the code snippet below)
function validateCells() {
var grid = $("#NonStartUpGrid").data("kendoGrid");
var rows = grid.tbody.find("tr"); //rows from excel sheet
var valid = true;
var errorCount = 0;
var rowSec1 = Math.floor(rows.length * .25)
var rowSec2 = Math.floor(rows.length * .5)
var rowSec3 = Math.floor(rows.length * .75)
Promise.all([new Promise((resolve, reject) => { resolve(loopGridValidate(rows, grid, errorCount, 0, rowSec1, valid)); })
, new Promise((resolve, reject) => { resolve(loopGridValidate(rows, grid, errorCount, rowSec1 + 1, rowSec2, valid)); })
, new Promise((resolve, reject) => { resolve(loopGridValidate(rows, grid, errorCount, rowSec2 + 1, rowSec3, valid)); })
, new Promise((resolve, reject) => { resolve(loopGridValidate(rows, grid, errorCount, rowSec3 + 1, rows.length - 1, valid)); })
]).then(results => { const total = results.reduce((a, b) => a + b, 0) });
return errorCount;
};
async function loopGridValidate(rows, grid, errorCount, begin, end, valid) {
console.log(begin);
for (var i = begin; i <= end; i++) {
var rowModel = grid.dataItem(rows[i]);
if (rowModel) {
console.log(i);
var colCells = $(rows[i]).find("td[role=gridcell]");
for (var j = 0; j < colCells.length; j++) {
//custom logic
}
}
}
scrollToTop(grid)
if (valid) {
$("#SubmitGrid").prop("disabled", "");
}
$("#loading").prop("hidden", "hidden");
return errorCount;
}
Here is a fiddle to show a basic example of what I am talking about that is happening. If you click the button you will see in the console, the functions are ran synchronously, instead of asynchronously(the i values would be mixed up instead of in order) https://jsfiddle.net/tap16fbo/

Promise body is synchronous, try using setTimeout. For more information
function loop(begin, end, results) {
setTimeout(() => {
for (var i = begin; i <= end; i++) {
console.log(i);
}
results();
}, Math.random() * 1000)
}
function BeginPromise() {
const p1 = new Promise((resolve, reject) => {
loop(0, 100, resolve)
});
const p2 = new Promise((resolve, reject) => {
loop(110, 200, resolve)
});
const p3 = new Promise((resolve, reject) => {
loop(210, 300, resolve)
});
return Promise.all([p1, p2, p3]).then(() => console.log('done'))
}

Javascript - Axios in for loop. Using the response of current fetch for the next iteration of fetching data

I use the the variable previous_epoch to fetch reddit posts before today in the first iteration. The next iteration previous_epoch must contain the date from the response data. Oddly enough, the date doesn't change after the first two loops
const getposts = async(user) => {
var dataArray = [];
var utc = new Date()
.toJSON()
.slice(0, 10)
.replace(/-/g, "/");
var previous_epoch;
for(let i=0; i <=5; i++) {
if(i == 0) {
console.log('i is 0');
previous_epoch = new Date(utc).getTime() / 1000;
console.log('p',previous_epoch); <--keeps doesn't work after second loop
}
else {
// console.log('else',dataArray);
previous_epoch = dataArray[dataArray.length-1].created_utc
console.log('else',previous_epoch);
}
await getdat(user,previous_epoch).then((resp) => {
let jsondat = resp.data.data;
dataArray = jsondat.concat(dataArray);
})
}
// console.log( dataArray);
var result = _.map( dataArray, function( o ) {
return o.id;
});
/* console.log("%j",result); */
}
Function getdat()
const getdat = (user,epoch) => {
const sub_url = `https://api.pushshift.io/reddit/search/submission/?subreddit=${user}&limit=300&sort=desc&before=${epoch}`;
console.log('elZK',sub_url);
return new Promise((resolve, reject) => {
axios.get(sub_url)
.then(response => {
return resolve(response)
})
.catch(error => {
return reject(error.message)
})
})
}
here is the fiddle
EDIT 2:
I've changed the logic so that I get the next date from getdat function itself
const getdat = (user,epoch) => {
let res= {};
const sub_url = `https://api.pushshift.io/reddit/search/submission/?subreddit=${user}&limit=300&sort=desc&before=${epoch}`;
console.log('elZK',sub_url);
return new Promise((resolve, reject) => {
axios.get(sub_url)
.then(response => {
let d = response.data.data;
let e = d[d.length - 1].created_utc;
console.log('e',e);
res.data = d;
res.epoch = e;
return resolve(res)
})
.catch(error => {
return reject(error.message)
})
})
}
export const getUserPosts = async(user) => {
var dataArray = [];
var utc = new Date()
.toJSON()
.slice(0, 10)
.replace(/-/g, "/");
var previous_epoch;
var epoch;
for(let i=0; i <=5; i++) {
if(dataArray.length === 0) {
console.log('i is 0');
previous_epoch = new Date(utc).getTime() / 1000;
console.log('p',previous_epoch);
}
else {
// console.log('else',dataArray);
previous_epoch = epoch;
console.log('else',previous_epoch);
}
const resp = await getdat(user,previous_epoch);
const jsondat = resp.data;
dataArray = jsondat.concat(dataArray);
epoch = resp.epoch;
}
// console.log( dataArray);
var result = _.map( dataArray, function( o ) {
return o.id;
});
console.log("%j",result);
}

On first iteration you fetch posts before today, so last post in result will have created_utc before or equal today. Next iterations just repeat you first request.
Try this code
for(let i=0; i <=5; i++) {
if(dataArray.length === 0) {
console.log('i is 0');
previous_epoch = new Date(utc).getTime() / 1000;
console.log('p',previous_epoch);
}
else {
// console.log('else',dataArray);
previous_epoch = dataArray[0].created_utc
console.log('else',previous_epoch);
}
const resp = await getdat(user,previous_epoch);
const jsondat = resp.data.data;
dataArray = jsondat.concat(dataArray);
}

Try this code
const getdat = (user,epoch) => {
const sub_url = `https://api.pushshift.io/reddit/search/submission/?subreddit=${user}&limit=10&sort=desc&before=${epoch}`;
console.log('elZK',sub_url);
return axios.get(sub_url);
};
const getposts = async(user) => {
let dataArray = [];
let previous_epoch;
for(let i=0; i <=5; i++) {
previous_epoch = dataArray.length>0 ? dataArray[dataArray.length-1].created_utc-1 : Math.floor(Date.now()/1000);
const result = await getdat(user,previous_epoch);
dataArray = dataArray.concat(result.data.data);
}
console.log(dataArray.map(e => e.created_utc));
};
getposts('data');

NodeJS Async - Use it like for-cycle

How to use NodeJS Async (https://caolan.github.io/async/) as normal for-cycle?
for(var i = 0; i < 100; i++){
doSomething();
}
I need serial flow but normal for-cycle not waiting until action is finished.
I am open to ES6 solutions, if any.

You can try to use Async/Await from ES6 which is cleaner and common shared in many standards. And in addition you don't need any dependencies from third party
const forLoopiNSeria = async () => {
for(var i = 0; i < 10; i++){
console.log(await doSomething(i))
}
}
function doSomething(index) {
return new Promise( (resolve, reject) => {
setInterval(() => resolve(index), 500)
})
}
forLoopiNSeria()

I believe you could always do something like this:
function asyncWhile(condition, action, ctx) {
const whilst = function(data) {
return condition.call(ctx, data) ?
Promise.resolve(action.call(ctx, data)).then(whilst) :
data;
}
return whilst();
}
let i = 1
asyncWhile(
() => {
if (i <= 100) {
i += 1
return true
}
return false
},
() => {
console.log(`iteration ${i}`)
},
)

I found another way with usage of until function from async:
var i = 0;
var max = 100;
async.until(() => {
i++;
return i === max-1;
}, function (callback) {
//doMagic()
callback();
}, function () {
console.log("Finished");
});

Javascript Coding Challenge with setTimeout/Asynchronous Output

I am trying solve the following challenge where I have to write a function triggerActions that passes a callback into the processAction, and produces the output:
"Process Action 1"
"Process Action 2"
...
"Process Action n"
Here is the provided function:
function processAction(i, callback) {
setTimeout(function() {
callback("Processed Action " + i);
}, Math.random()*1000);
}
Function to code:
function triggerActions(count) {
}
Note that the code for processAction cannot be altered. I was thinking of using a Promise but I'm not sure how. I believe the setTimeout is actually synchronous so I don't know if async/await would work.
My attempt:
triggerActions = count => {
let promises = [];
for(let i=1; i<=count; i++) {
promises.push(new Promise( (resolve, reject) => processAction(i, str => resolve(str))));
}
let results = []
promises.forEach( promise => Promise.resolve(promise).then( async res => results.push(await res)));
return results;
}

I kind of like short and sweet:
var n = 5
var stop = 1
triggerActions = function(text) {
if (text) console.log(text)
if (stop <= n){
processAction(stop++, triggerActions)
}
}
triggerActions()
P.S
It occurred to me that perhaps you are only allowed to provide a function which means the stop variable declaration outside the function is a problem. It makes it a little more verbose, but you can wrap it all inside the function like this:
function triggerActions(stop) {
var rFn = (text) => {
if (text) console.log(text)
if (stop <= n){
processAction(stop++, rFn)
}
}
rFn()
}
triggerActions(1)

There you go:
// Your unaltered function
function processAction(i, callback) {
setTimeout(function() {
callback("Processed Action " + i);
}, Math.random()*1000);
}
// The function you want to implement
function triggerActions(count) {
var triggerAction = function (i) { // Local function to process the given action number:
if (i <= count) { // More actions to execute?
processAction(i, function (text) {// Process current action number and pass a callback in parameter
console.log(text); // Write the result of processAction
triggerAction(i + 1); // Trigger the next action
}); //
} //
}
triggerAction(1); // First things first: start at action one
}
// Call the function
triggerActions(10);

The original poster's instinct to use promises was correct.
The two solutions above may work but because each call to triggerActions() has to wait for the delay to elapse before the next call can be made, this is considerably slow.
Maybe this is what you want but here's an optimized solution using promises and Promise.all():
const processAction = (i, callback) => {
setTimeout(function() {
callback("Processed Action " + i);
}, Math.random()*1000);
}
const triggerActions = (n) => {
const promises = [];
const generatePromise = (i) => {
return new Promise((resolve, reject) => {
processAction(i, resolve);
});
}
for (let i = 1; i <= n; i += 1) {
promises.push(generatePromise(i));
}
Promise.all(promises)
.then((strings) => strings.forEach((string) => console.log(string)));
}
triggerActions(10);
To compare the performance differences, try running the two approaches side by side.

Here's my solution:
function processAction(i, callback) {
setTimeout(function() {
callback("Processed Action " + i);
}, Math.random()*1000);
}
// Function to code:
function triggerActions(count) {
const asyncArr = [];
for (let i = 1; i <= count; i++) {
asyncArr.push(new Promise(resolve => processAction(i, resolve)));
}
Promise.all(asyncArr).then((vals) => {
vals.forEach((val) => console.log(val))
});
}
triggerActions(5);

Here is my solution using Promise.all:
function triggerActions(count) {
const promises = range(count).map(
i => new Promise(resolve => processAction(i, resolve))
);
Promise.all(promises).then(results => {
results.forEach(result => console.log(result));
});
}
// Generates an array from 1...n
function range(n) {
return Array.from({ length: n }, (_, i) => i + 1);
}

The requirements are that the function ‘processAction’ should remain unchanged and invoked in a batch.
For this I have used the util.promisify function that takes a function and converts it into a promise. A promise can be invoked in a batch with Promise.all.
Another requirement is that the callback should output “Processed Action i” where i is a number. The anonymous function ‘func’ has been defined to do this.
The triggerActions function takes a number, x, creates an array of numbers containing indices from 0 to x and then invokes a count of x asynchronous functions simultaneously.
const {promisify} = require('util');
function processAction(i, callback) {
setTimeout(function() {
callback("Processed Action " + i);
}, Math.random()*1000);
}
const func = (param1) => console.log(param1);
const promisifyedProcessAction = promisify(processAction);
async function triggerActions(count) {
const arr = [];
for(let i = 0; i < count;)
arr.push(++i);
await Promise.all(
arr.map((value) => promisifyedProcessAction(value,func)));
}
triggerActions(5);

Here's an overview of all the possible approaches:
Callback-based:
Sequential:
function triggerActions(count) {
;(function recur(i = 0) {
processAction(i, (data) => {
console.log(data)
if (i < count) {
recur(i + 1)
}
})
})()
}
Concurrent
function triggerActions(count) {
const data = Array.from({ length: count })
for (let i = 0; i < count; i++) {
processAction(i, (result) => {
data[i] = result
count--
if (count == 0) {
for (const x of data) {
console.log(x)
}
}
})
}
}
Promise-based:
We can use this function to make processAction async:
function processActionP(i) {
return new Promise((res) => processAction(i, res))
}
Sequential:
async function triggerActions(count) {
for (let i = 0; i < count; i++) {
const data = await processActionP(i)
console.log(data)
}
}
Concurrent:
async function triggerActions(count) {
const data = await Promise.all(
Array.from({ length: count }, (_, i) => processActionP(i)),
)
for (const x of data) {
console.log(x)
}
}
Concurrent, using lodash/fp
const _ = require('lodash/fp')
const triggerActions = _.pipe(
_.range(0),
_.map(processActionP),
Promise.all.bind(Promise),
data => data.then(
_.each(console.log)
),
)