Develop Reference

How do I run code after all threads have finished running?

I have a multithreaded web crawler that downloads a website and stores it in a database (which takes around 4 minutes). To make the crawling faster, I used node.js cluster module, but I have a problem, I want to iterate over to the next segment of the while loop, after all the threads have done their processes, not as soon as they start. How do I make sure all my threads are concluded and then move on?
Here is the relevant code in the main while loop:
while (indexSize !== indexSizeLimit) {
const queueLength = queue.length;
const numberOfThreads = Math.min(numberOfCPUs, queueLength);
const threadAllocations = Array(numberOfThreads).fill(0);
let queuesAllocated = 0;
const queueChunks = [];
function fillQueueChunks() {
loop: while (true) {
for (let i = 0; i < numberOfThreads; i++) {
threadAllocations[i] += 1;
queuesAllocated += 1;
if (queuesAllocated === queueLength) {
break loop;
};
};
};
let start = 0;
for (let threadAllocation of threadAllocations) {
const end = start + threadAllocation;
queueChunks.push(queue.slice(start, end));
start = end;
};
};
fillQueueChunks();
// Find out how to make multithreading finish, and then move on with the loop.
if (cluster.isMaster) {
for (let i = 0; i < numberOfThreads; i++) {
cluster.fork();
};
} else {
const chunk = queueChunks[cluster.worker.id - 1];
await Promise.all(chunk.map(function (url) {
return new Promise(async function (resolve) {
const webcode = await request(url);
if (webcode !== "Failure") {
indexSize += 1;
const document = new Document(url, webcode);
const hrefs = document.hrefs();
const hrefsQuery = Query(hrefs);
// Also make sure it is not included in indexed webpages.
const hrefIndividualized = hrefsQuery.individualize();
hrefIndividualized;
// Do something with hrefIndividualized in regards to maintaining a queue in the database.
// And in adding a nextQueue which to replace the queue in code with.
await document.save();
};
resolve("Written");
});
}));
process.exit(0);
};
};

Wrap the threading in a promise. You can check in the parent thread if there is a disconnect event, and if the amount of disconnects is equal to the number of threads, then you can resolve the promise.
Here is what I have
while (indexSize !== indexSizeLimit) {
let nextQueue = [];
const queueLength = queue.length;
const numberOfThreads = Math.min(numberOfCPUs, queueLength);
const threadAllocations = Array(numberOfThreads).fill(0);
let queuesAllocated = 0;
// queueChunks: [[{_id: ..., ...}], [...], ...]
const queueChunks = [];
function fillQueueChunks() {
loop: while (true) {
for (let i = 0; i < numberOfThreads; i++) {
threadAllocations[i] += 1;
queuesAllocated += 1;
if (queuesAllocated === queueLength) {
break loop;
};
};
};
let start = 0;
for (let threadAllocation of threadAllocations) {
const end = start + threadAllocation;
queueChunks.push(queue.slice(start, end));
start = end;
};
};
fillQueueChunks();
await new Promise(async function (resolve) {
if (cluster.isMaster) {
let threadsDone = 0;
for (let i = 0; i < numberOfThreads; i++) {
cluster.fork();
};
cluster.on("disconnect", function (_) {
threadsDone += 1;
if (threadsDone === numberOfThreads) {
resolve("Queue Processed");
};
});
} else {
const queueJob = queueChunks[cluster.id - 1];
await Promise.all(queueJob.map(function (queueItem) {
return new Promise(async function (resolve) {
const url = queueItem._id;
const webcode = await request(url);
if (webcode !== "Failure") {
const document = Document(url, webcode);
let hrefs = document.hrefs();
const hrefsQuery = Query(hrefs);
await document.save();
indexSize += 1;
hrefs = hrefsQuery.individualize();
const hrefIncidences = Promise.all(hrefs.map(function (href) {
return new Promise(async function (resolve) {
const incidences = await Site.countDocuments({
url: href
});
resolve(incidences);
});
}));
hrefs = hrefs.filter(function (_, i) {
return hrefIncidences[i] === 0;
}).map(function (href) {
return {
_id: href
};
});
await Queue.insertMany(hrefs);
nextQueue = nextQueue.concat(hrefs);
};
await Queue.deleteOne({
_id: url
});
resolve("Success");
});
}));
process.exit(0);
};
});
queue = nextQueue;
};

javascript multiple function callback syntax

I can't wrap my head around callback syntax, can you please help me re-write my code so that it executes in this order:
MenuBuilder.load()
MenuBuilder.draw()
Translator.load()
(in my case it executes in this order MenuBuilder.load(), Translator.load(), MenuBuilder.draw() so it doesn't do what I want)
onload.js
import MenuBuilder from "./menu-builder.js";
import Translator from "./translator.js";
var menuBuilder = new MenuBuilder();
var translator = new Translator();
menuBuilder.load();
translator.load();
menu-builder.js
"use strict"
class MenuBuilder {
constructor() {
this._nav = document.getElementsByTagName("nav")[0];
this._url = window.location.href;
}
load() {
console.log("MenuBuilder.load() start");
fetch(`/json/menu.json`)
.then((res) => res.json())
.then((jsonMenu) => {
this.draw(jsonMenu);
})
/*.catch(() => {
console.error(`Could not load ${this._lang}.json.`);
});*/
console.log("MenuBuilder.load() end");
}
draw(jsonMenu) {
console.log("MenuBuilder.draw(jsonMenu) start");
var htmlMenu = `<div id="siteTitleDiv"><p id="siteTitle" data-i18n="general.title"></p><p id="siteTitleShadow" data-i18n="general.title-shadow"></p><p id="siteSubtitle"data-i18n="general.subtitle"></p></div><ul>`;
for(var i = 0; i < jsonMenu.length; i++) {
var menuItem = jsonMenu[i];
var regexp = /http:\/\/cypher-f\.com\/(([a-z\-]*\/)?([a-z\-]*\/))?/g;
var fullPage = "something format_abc";
var match = regexp.exec(this._url);
var level_1 = match[1];
var level_2 = match[3];
var parent = match[2];
var full_suffix = match[0];
if ((parent == null) || (menuItem.parent === parent)) {
var material_icon = menuItem["material-icon"];
var href = menuItem["href"];
var i18n = menuItem["data-i18n"];
htmlMenu += `<li><i class="material-icons">${material_icon}</i></li>`;
}
}
htmlMenu += `</ul>`;
this._nav.innerHTML = htmlMenu;
console.log("MenuBuilder: nav.innerHTML");
console.log(this._nav.innerHTML);
console.log("MenuBuilder: document.elements");
console.log(document.querySelectorAll("[data-i18n]"));
console.log("MenuBuilder.draw(jsonMenu) end");
}
}
export default MenuBuilder;
translator.js
"use strict"
class Translator {
constructor() {
this._lang = this.getLanguage();
this._elements = document.querySelectorAll("[data-i18n]");
}
getLanguage() {
var lang = navigator.languages ? navigator.languages[0] : navigator.language;
return lang.substr(0, 2);
}
load(lang = null) {
console.log("Translator.load() start");
console.log("this._elements");
console.log(this._elements);
if (lang) {
this._lang = lang;
}
else {
var re = new RegExp("lang=([^;]+)");
var value = re.exec(document.cookie);
var cookieLang = (value != null) ? unescape(value[1]) : null;
if (cookieLang) {
this._lang = cookieLang;
}
}
fetch(`/i18n/${this._lang}.json`)
.then((res) => res.json())
.then((translation) => {
this.translate(translation);
})
.then(this.toggleLangTag())
.then(document.cookie = `lang=${this._lang};path=/`)
/*.catch(() => {
console.error(`Could not load ${this._lang}.json.`);
});*/
console.log("Translator.load() end");
}
translate(translation) {
console.log("Translator.load(translation) start");
this._elements.forEach((element) => {
var keys = element.dataset.i18n.split(".");
var text = keys.reduce((obj, i) => obj[i], translation);
if (text) {
element.innerHTML = text;
}
else {
element.innerHTML = `key ${keys} not found for ${this._lang}!`
}
});
console.log("Translator.load(translation) end");
}
toggleLangTag() {
if (document.documentElement.lang !== this._lang) {
document.documentElement.lang = this._lang;
}
}
switchLanguage(translator) {
var availableLang = ["en", "fr"];
var currentLangIndex = availableLang.indexOf(translator._lang);
var nextLang = availableLang[(currentLangIndex + 1)%availableLang.length];
translator.load(nextLang);
}
}
export default Translator;
I'm sorry I know this is kind of a newbie question but I haven't programmed in three years.

You're working with Promises here, so you want to stick with that paradigm. Return the promise that is returned from the fetch call, then "chain" off of that promise to call the translator.
load() {
console.log("MenuBuilder.load() start");
// The return here gives control of the promise to the caller...
return fetch(`/json/menu.json`)
.then((res) => res.json())
.then((jsonMenu) => {
this.draw(jsonMenu);
})
/*.catch(() => {
console.error(`Could not load ${this._lang}.json.`);
});*/
console.log("MenuBuilder.load() end");
}
So back in onload.js you can use the promise returned from menuBuilder.load() to call translator.load() after menuBuilder.load() is done.
import MenuBuilder from "./menu-builder.js";
import Translator from "./translator.js";
var menuBuilder = new MenuBuilder();
var translator = new Translator();
menuBuilder.load().then(() => translator.load());

How do I assign a variable defined in a synchronous scope in an async function?

I am trying to write a GUI frontend that uses a service to get data about the system. I am using a net.Socket for the client end of this. I want to be able to access certain variables assigned in the data event handlers in other modules but the assignment does not stay after that callback function finishes.
Problematic code:
client.on('data', (data) => {
var array = [...data];
array.splice(0,2);
for (var i=0;i<array.length;i++) {
dataInBuffer = dataInBuffer + String.fromCharCode(array[i]);
}
console.log(dataInBuffer);
if (dataInBuffer.startsWith('batStat')) {
let lastBatteryJSON = JSON.parse(dataInBuffer.split(';')[1]);
module.exports.hasBattery = lastBatteryJSON.hasBattery == 'true';
module.exports.isCharging = lastBatteryJSON.isCharging == 'true';
module.exports.lastBatteryReading = parseFloat(lastBatteryJSON.batteryLife);
}
dataInBuffer = '';
});
Those three exported variable assignments don't actually work, the variables always either stay undefined or their default values outside of the function. I tried using a Promise to solve this problem but got the same result. I'm at a loss and I can't find any other questions or forum posts that solve this problem.
EDIT
I do not have the option of moving the code that depends on those variables into the callback. In order to do that I would have to wait for the data every frame and flood the server as a result.

As apple commented; you can export an object and mutate it every time you receive data:
const data = {};
client.on('data', (data) => {
var array = [...data];
array.splice(0, 2);
for (var i = 0; i < array.length; i++) {
dataInBuffer = dataInBuffer + String.fromCharCode(array[i]);
}
console.log(dataInBuffer);
if (dataInBuffer.startsWith('batStat')) {
let lastBatteryJSON = JSON.parse(dataInBuffer.split(';')[1]);
//mutate the data object
data.hasBattery = lastBatteryJSON.hasBattery == 'true';
data.isCharging = lastBatteryJSON.isCharging == 'true';
data.lastBatteryReading = parseFloat(lastBatteryJSON.batteryLife);
}
dataInBuffer = '';
});
//export the data object
module.exports.batteryData = data;
Or as CertainPerformance answered you can have the caller decide when to ask for the information and provide a promise.
Here is an extended version of CertainPerformance answer that listens to error as well so a promise can be rejected and cleans up the event listeners when promise is resolved or rejected:
//wrapper for client.on to add and remove event listeners
const listeners = (function(){
var listenerCounter = -1;
const listeners = [];
const triggerEvent = event => data =>{
listeners.filter(
listener=>listener[2] === event
).forEach(
listener=>listener[1](data)
);
};
client.on('data', triggerEvent("data"));
client.on('error', triggerEvent("error"));//assuming you have an error event
return {
add:(event,fn)=>{
listenerCounter = listenerCounter + 1;
if(listenerCounter>1000000){
listenerCounter=0;
}
listeners.push([listenerCounter,fn,event]);
return listenerCounter;
},
remove:num=>{
listeners = listeners.filter(
listener=>{
num !== listener[0];
}
)
}
}
}());
//convert data to object or false
const getObjectFromData = data => {
var array = [...data];
var dataInBuffer="";
array.splice(0,2);
for (var i=0;i<array.length;i++) {
dataInBuffer = dataInBuffer + String.fromCharCode(array[i]);
}
console.log(dataInBuffer);
if (dataInBuffer.startsWith('batStat')) {
let lastBatteryJSON = JSON.parse(dataInBuffer.split(';')[1]);
return {
hasBattery : lastBatteryJSON.hasBattery == 'true',
isCharging : lastBatteryJSON.isCharging == 'true',
lastBatteryReading : parseFloat(lastBatteryJSON.batteryLife)
};
}
return false;
}
//export this function
const getBatteryData = () =>
new Promise((resolve,reject) => {
const removeListeners = ()=>{
listeners.remove(okId);
listeners.remove(errorId);
}
const okId = listeners.add(
"data",
data=>{
const resultObject = getObjectFromData(data);
if(resultObject){
resolve(data);
removeListeners();//clean up listeners
}else{
//not sure of on data is triggered multiple times by client.on.data
// if it is then at what point do we need to reject the returned promise?
}
}
)
const errorId = listeners.add(
"error",
error=>{
reject(error);
removeListeners();//clean up listeners
}
)
});
//you can call getBatteryData like so:
//getBatteryData()
// .then(batteryData=>console.log(batteryData))
// .catch(error=>console.warn("an error getting battery data:",error))

Your module should export a function that returns a promise that returns the desired values. Also, use const and not var when possible:
let resolveObj;
const haveData = new Promise((resolve) => {
let resolved = false;
client.on('data', (data) => {
const array = [...data];
array.splice(0, 2);
for (let i = 0; i < array.length; i++) {
dataInBuffer = dataInBuffer + String.fromCharCode(array[i]);
}
console.log(dataInBuffer);
if (dataInBuffer.startsWith('batStat')) {
const {
hasBattery,
isCharging,
batteryLife,
} = JSON.parse(dataInBuffer.split(';')[1]);
resolveObj = {
hasBattery: hasBattery === 'true',
isCharging: isCharging === 'true',
lastBatteryReading: Number(batteryLife),
};
if (!resolved) resolve();
resolved = true;
}
dataInBuffer = '';
});
});
const getData = () => haveData.then(() => resolveObj);
module.exports = getData;
Then consume with
moduleFunction().then(({ hasBattery, isCharging, lastBatteryReading }) => {
// do something with results
});
If called before resolveObj is populated, the promise will wait until the first client.on('data' to resolve. After that, the function will return a promise that resolves immediately to the current value of resolveObj (which will be properly updated on client.on('data')

Node js return function does not return data

Hi so i just made this function and I'm trying to make return the data but for some reason it does not return the data but the function is called as i tried console logging it when the function is executed it works fine but it just does return the data when i want to use the function
As you can see I'm using the function trying to put the returned data in a array
Thank you all.
a.push(methods.UserInfo(id));
Here's all the code I'm using
methods.UserDatas = ((client, json)=> {
let a = [];
if (json.Params.length > 0) {
json.Params.forEach((id)=>{
if (id) {
a.push(methods.UserInfo(id));
}
});
let cmd = {
'Cmd': 'GetUserInfo',
'userinfo': a
};
packet.send(client, JSON.stringify(cmd));
}
});
methods.UserInfo = ((id)=> {
sql.query('SELECT * FROM users WHERE ?',[{id:id}], (err, row)=>{
if (err) {
throw err;
} else {
if (row.length > 0) {
let playerData = {};
playerData.userid = row[0].id;
playerData.strGender = row[0].Gender;
playerData.Username = row[0].Username;
let items = {};
sql.query('SELECT * FROM users_items WHERE ? AND equipped=1',[{userid:id}],(err,rows) => {
if (err) {
throw err;
} else {
if (rows.length > 0) {
for (var i =0; i< rows.length; i++) {
let item = DBData.items.get(parseInt(rows[i].itemid));
items[rows[i].equipment] = {
'ItemID': item.id,
'File':item.File,
};
}
}
playerData["equipment"] = items;
return playerData;
}
});
}
}
});
});

An async function (like for example
sql.query('SQL', [{PARAM:VALUE}], (ERR, RESULT) => { CODE }
) does not return as a regular function: you could use promises, for example (see here).
And note, too, that a sync function calling an async function, does not synchronously return values coming from that async function...

How do I iterate through all the id's?

Check out the api --> https://api.icndb.com/jokes/random/10
Everytime when the user clicks on a specific joke, it will be added to the favorite list.
To keep the code concise I will only show the function itself:
(function() {
"use strict";
const getJokesButton = document.getElementById('getData');
getJokesButton.addEventListener('click', getData);
loadLocalStorage();
function loadLocalStorage() {
let storage = JSON.parse(localStorage.getItem('favoList')) || [];
let listOfFavorites = document.getElementById("favorites");
let emptyArray = '';
if(storage.length > 0) {
for(let i = 0; i < storage.length; i++) {
let idNumberJoke = storage[i].id;
emptyArray +=
`<li><input type="checkbox" id='${idNumberJoke}'/> User title: ${storage[i].joke}</li>`;
listOfFavorites.innerHTML = emptyArray;
}
} else {
return false;
}
}
// fetch data from api
function getData() {
let listOfJokes = document.getElementById("list-of-jokes");
fetch('https://api.icndb.com/jokes/random/10')
.then(function(res) {
return res.json();
}).then(function(data) {
// variable is undefined because it is not initialized. Therefore at some empty single quotes
let result = '';
console.log(data.value);
data.value.forEach((joke) => {
result +=
`<li><input type="checkbox" class='inputCheckbox' id='${joke.id}'/> User title : ${joke.joke}</li>`;
listOfJokes.innerHTML = result;
});
bindCheckbox();
}).catch(function(err) {
console.log(err);
});
}
function clickedButton() {
getJokesButton.setAttribute('disabled', 'disabled');
getJokesButton.classList.add('opacity');
}
function bindCheckbox() {
let inputCheckbox = document.querySelectorAll('input[type=checkbox]');
let elems = document.getElementById('list-of-jokes').childNodes;
let favoriteList = document.getElementById('favorites');
let fav = JSON.parse(localStorage.getItem('favoList'))|| [];
if(elems.length > 0) {
inputCheckbox.forEach(function(element, index) {
inputCheckbox[index].addEventListener('change', function() {
let joke = this;
if(joke.checked && joke.parentNode.parentNode.id === 'list-of-jokes') {
joke.checked = false;
favoriteList.appendChild(joke.parentNode);
addFavorite(joke.id, joke.parentNode.innerText, fav);
}
if(joke.checked && joke.parentNode.parentNode.id === 'favorites') {
joke.checked = false;
removeFavorite(joke, index);
}
});
});
}
clickedButton();
}
function removeFavorite(favorite, index) {
let favoriteCheckBox = favorite;
let i = index;
// convert iterable object to an array, otherwise splice method would give an error.
let favoriteListItem = Array.from(favoriteCheckBox.parentNode);
favoriteListItem.splice(i, 1);
document.getElementById('list-of-jokes').appendChild(favorite.parentNode);
localStorage.setItem('favoList', JSON.stringify(favoriteListItem));
}
// store favorites in localStorage
function addFavorite(jokeId, jokeText, fav) {
let norrisJoke = {
id: jokeId,
joke: jokeText
};
let favorites = fav;
for (let i = 0; i < favorites.length; i++) {
if(favorites[i].id !== norrisJoke.id) {
favorites.push(norrisJoke);
}
}
// favorites[i].id !== norrisJoke.id
// always get the object before the push method and pass it into stringify
localStorage.setItem('favoList', JSON.stringify(favorites));
}
// function which will randomly add one joke to favorite list every 5 seconds
// function need a button which allows you to turn on and off this auto add function
})();
<div class="inner-body">
<button id="getData">GET Jokes</button>
<div class='inner-block'>
<h2>Chuck Norris Jokes</h2>
<ul class='unordered-list' id="list-of-jokes">
</ul>
</div>
<div class='inner-block'>
<h2>Favorites</h2>
<ul class='unordered-list' id="favorites">
</ul>
</div>
</div>
The keys and values would not be pushed into localStorage, the only thing I see is an empty [] in localStorage. The norrisJoke object literal will be dynamically changed. So how could I make this function works?
Too complex, but click on the link below and scroll down to the bottom:
https://codepen.io/chichichi/pen/Gyzzvb

You are trying to run through an empty list here
for (let i = 0; i < favorites.length; i++) {
if(favorites[i].id !== norrisJoke.id) {
favorites.push(norrisJoke);
}
}
This means that nothing will ever be pushed. You can reduce your list to an array of id, then check if the joke exists in the list.
const favIds = favorites.reduce((sum, element) => {
return sum.concat(element.id);
},
[]);
Now you can check if the joke doesn't exists in favorites
if(!favIds.includes(jokeId)){
favorites.push(norrisJoke);
}

The problem is the for loop, the first time it's executed favorites will be an empty array so it's length will be 0, so it will never enter the loop
Something like this should work:
favorites = favorites.filter(joke => joke.id !== norrisJoke.id).concat(norrisJoke);

let favorites = JSON.parse(localStorage.getItem('favoList'))|| {};
favorites[norrisJoke.id] =norrisJoke.joke
Why don't you use a map in place of an array?
Also as #fl9 points out your for loop will never start off! because favorites.length is 0 to begin with
But I want to check duplicates before the joke will be pushed into favorite list
By definition a hash will not allow duplicate entries, so no need to worry about duplications
Run localStorage.getItem('favoList') in the console of this fiddle :
(function() {
"use strict";
const getJokesButton = document.getElementById('getData');
getJokesButton.addEventListener('click', getData);
loadLocalStorage();
function loadLocalStorage() {
let storage = JSON.parse(localStorage.getItem('favoList')) || [];
let listOfFavorites = document.getElementById("favorites");
let emptyArray = '';
if(storage.length > 0) {
for(let i = 0; i < storage.length; i++) {
let idNumberJoke = storage[i].id;
emptyArray +=
`<li><input type="checkbox" id='${idNumberJoke}'/> User title: ${storage[i].joke}</li>`;
listOfFavorites.innerHTML = emptyArray;
}
} else {
return false;
}
}
// fetch data from api
function getData() {
let listOfJokes = document.getElementById("list-of-jokes");
fetch('https://api.icndb.com/jokes/random/10')
.then(function(res) {
return res.json();
}).then(function(data) {
// variable is undefined because it is not initialized. Therefore at some empty single quotes
let result = '';
console.log(data.value);
data.value.forEach((joke) => {
result +=
`<li><input type="checkbox" class='inputCheckbox' id='${joke.id}'/> User title : ${joke.joke}</li>`;
listOfJokes.innerHTML = result;
});
bindCheckbox();
}).catch(function(err) {
console.log(err);
});
}
function clickedButton() {
getJokesButton.setAttribute('disabled', 'disabled');
getJokesButton.classList.add('opacity');
}
function bindCheckbox() {
let inputCheckbox = document.querySelectorAll('input[type=checkbox]');
let elems = document.getElementById('list-of-jokes').childNodes;
let favoriteList = document.getElementById('favorites');
let fav = JSON.parse(localStorage.getItem('favoList'))|| [];
if(elems.length > 0) {
inputCheckbox.forEach(function(element, index) {
inputCheckbox[index].addEventListener('change', function() {
let joke = this;
if(joke.checked && joke.parentNode.parentNode.id === 'list-of-jokes') {
joke.checked = false;
favoriteList.appendChild(joke.parentNode);
addFavorite(joke.id, joke.parentNode.innerText, fav);
}
if(joke.checked && joke.parentNode.parentNode.id === 'favorites') {
joke.checked = false;
removeFavorite(joke, index);
}
});
});
}
clickedButton();
}
function removeFavorite(favorite, index) {
let favoriteCheckBox = favorite;
let i = index;
// convert iterable object to an array, otherwise splice method would give an error.
let favoriteListItem = Array.from(favoriteCheckBox.parentNode);
favoriteListItem.splice(i, 1);
document.getElementById('list-of-jokes').appendChild(favorite.parentNode);
localStorage.setItem('favoList', JSON.stringify(favoriteListItem));
}
// store favorites in localStorage
function addFavorite(jokeId, jokeText, fav) {
let norrisJoke = {
id: jokeId,
joke: jokeText
};
let favorites = fav;
for (let i = 0; i < favorites.length; i++) {
if(favorites[i].id !== norrisJoke.id) {
favorites.push(norrisJoke);
}
}
// favorites[i].id !== norrisJoke.id
// always get the object before the push method and pass it into stringify
localStorage.setItem('favoList', JSON.stringify(favorites));
}
// function which will randomly add one joke to favorite list every 5 seconds
// function need a button which allows you to turn on and off this auto add function
})();