Promise and work thread chunking not working - javascript

I have this in my code where i am using promise to limit at max MAX_WORKERS workgroup at a time.
const MAX_WORKERS = 3;
let promises = [];
let chunk = 3;
let totalNodes = keyCountMap.length / 2;
let doneCount = 0;
for (let m = 0; m < keyCountMap.length; ) {
let remaining = totalNodes - doneCount;
let numbWorker = Math.min(chunk, remaining);
for (let i = 0; i < numbWorker; i++) {
promises.push(createWorker(keyCountMap[m], keyCountMap[m + 1]));
doneCount++;
m += 2;
console.log(doneCount);
if (doneCount % MAX_WORKERS == 0) {
Promise.all(promises).then((response) => {
console.log("one chunk finish");
});
}
}
}
and createWorker function is:
function createWorker(data1, data2) {
return new Promise((resolve) => {
let worker = new Worker();
worker.onmessage = (event) => {
postMessageRes = event.data;
if (postMessageRes == 200) {
worker.postMessage([
nodePagesString,
pagesString,
copcString,
data1,
data2,
]);
} else {
workerCount += 1;
let position = postMessageRes[0];
let color = postMessageRes[1];
for (let i = 0; i < position.length; i++) {
positions.push(position[i]);
colors.push(colors[i]);
}
if (workerCount == MAX_WORKERS) {
worker.terminate();
workerCount = 0;
promises = [];
}
resolve(true);
}
};
});
}
but the problem i have here is that it is not working like i wanted "3 work thread at a time and when it is done, spawn more thread/workers"
This is my console
and
which mean it is called at once and not after 3 threads which is count of chunks
Can anyone help me please on this
This is my worker code:
import { Copc, Key } from "copc";
import * as THREE from "three";
const color = new THREE.Color();
const colors = [];
let firstTime = true;
var nodePages, pages, receivedData, copc;
let x_min, y_min, z_min, x_max, y_max, z_max, width;
let positions = [];
let filename = "https://s3.amazonaws.com/data.entwine.io/millsite.copc.laz";
const readPoints = (id, getters) => {
let returnPoint = getXyzi(id, getters);
positions.push(
returnPoint[0] - x_min - 0.5 * width,
returnPoint[1] - y_min - 0.5 * width,
returnPoint[2] - z_min - 0.5 * width
);
const vx = (returnPoint[3] / 65535) * 255;
color.setRGB(vx, vx, vx);
colors.push(color.r, color.g, color.b);
firstTime = false;
};
function getXyzi(index, getters) {
return getters.map((get) => get(index));
}
async function load() {
// copc = await Copc.create(filename);
// let scale = copc.header.scale[0];
// [x_min, y_min, z_min, x_max, y_max, z_max] = copc.info.cube;
// width = Math.abs(x_max - x_min);
// // let center_x = (x_min + x_max) / 2;
// // let center_y = (y_min + y_max) / 2;
// // let center_z = (z_min + z_max) / 2;
// receivedData = await Copc.loadHierarchyPage(
// filename,
// copc.info.rootHierarchyPage
// );
// nodePages = receivedData.nodes;
// pages = receivedData.pages;
postMessage(200);
}
async function loadData(nodes, pages, copc, myRoot, pointCount) {
// console.log(copc, myRoot);
const view = await Copc.loadPointDataView(filename, copc, myRoot);
let getters = ["X", "Y", "Z", "Intensity"].map(view.getter);
for (let j = 0; j < pointCount; j += 1) {
readPoints(j, getters);
}
postMessage([positions, colors]);
}
load();
onmessage = function (message) {
let nodePages = message.data[0];
let nodes = JSON.parse(nodePages);
let pagesStr = message.data[1];
let pages = JSON.parse(pagesStr);
let copcStr = message.data[2];
let copc = JSON.parse(copcStr);
let mapIndex = message.data[3];
let pointCount = message.data[4];
let myRoot = nodes[mapIndex];
// console.log(mapIndex);
loadData(nodes, pages, copc, myRoot, pointCount);
};

What's creating the confusion here is the mix of synchronous and asynchronous processes. In your first code block there are two for loops that iterate over the keyCountMap.length and numbWorker, respectively. The inner for loop sets up an asynchronous callback once the array of promises are resolved. However, it does not wait for the result as it is asynchronous.
So, the behavior you're seeing is the looping over all values in the for loops and queueing of multiple async calls. You can read documentation here about how the event loop works.
The logging for the doneCount values happens first and then your async callbacks happen afterward. You should be able to verify this by changing the console.log within the Promise callback by adding the doneCount to it:
/*
The use of JSON.parse and JSON.stringify is to preserve the initial value.
Otherwise it will show the current value of doneCount when the callback finalizes.
*/
console.log(`one chunk finish: ${JSON.parse(JSON.stringify(doneCount))}`);
Edit Added recommended adjustment.
const chunkWorkers = async () => {
const MAX_WORKERS = 3;
let promises = [];
let chunk = 3;
let totalNodes = keyCountMap.length / 2;
let doneCount = 0;
for (let m = 0; m < keyCountMap.length; ) {
let remaining = totalNodes - doneCount;
let numbWorker = Math.min(chunk, remaining);
for (let i = 0; i < numbWorker; i++) {
promises.push(createWorker(keyCountMap[m], keyCountMap[m + 1]));
doneCount++;
m += 2;
console.log(doneCount);
if (doneCount % MAX_WORKERS == 0) {
await Promise.all(promises).then((response) => {
console.log(`one chunk finish: ${doneCount}`);
});
}
}
}
};

Related

How to improve this nested for loop using GAS?

I'd like to learn an alternative way to run this for loop, because currently, it sets the value through each iteration and as this can get large, it better be more performant.
function calculate() {
const ss = SpreadsheetApp.getActiveSpreadsheet();
const settingSheet = ss.getSheetByName('settings');
const dataSheet= ss.getSheetByName('Data');
let settingData = settingSheet.getRange(1, 1, settingSheet.getLastRow(), settingSheet.getLastColumn()).getValues();
let spcData = dataSheet.getRange(1, 1, dataSheet.getLastRow(), dataSheet.getLastColumn()).getValues();
let recordTypesAccepted = ['strategy1', 'goal2'];
var settingPar1 = settingData.filter(e => e[0] == 'default').map(e => e[1]);
var settingPar2 = settingData.filter(e => e[0] == 'default').map(e => e[2]);
for (let a = 0; a < spcData.length; a++) {
for (let r = 0; r < settingData .length; r++) {
let settingSearchTerm = settingData[r][4];
let spcSearchTerm = spcData[a][10];
let settingMatchType = settingData[r][5];
let spcMatchType = spcData[a][9];
let spcRecordType = spcData[a][8];
if (recordTypesAccepted.indexOf(spcRecordType) > -1 && settingMatchType === spcMatchType) {
if (settingSearchTerm == spcSearchTerm) {
settingPar1 = settingData[r][7];
settingPar2 = settingData[r][6];
}
let spcClicks = spcData[a][10];
let spcOrders = spcData[a][11];
let row = a + 1;
if (spcClicks >= settingsPar1 && spcOrders >= settingsPar2) {
let newValue = 10;
spcSheet.getRange(row, 20).setValue(newBid).setBackground("#D8E271");
}
}
}
}
}
Thank you!!!
Description
Provided using setValues() doesn't overwrite any formulas you could use getValues() to get an array of the old values, replace any that need to be updated and simply put back the array of values using setValues(). The same for back grounds and number formats.
Note it is assumed from the OP that spcSheet is defined elsewhere.
Script
function calculate() {
const ss = SpreadsheetApp.getActiveSpreadsheet();
const settingSheet = ss.getSheetByName('settings');
const dataSheet= ss.getSheetByName('Data');
let settingData = settingSheet.getRange(1, 1, settingSheet.getLastRow(), settingSheet.getLastColumn()).getValues();
let spcData = dataSheet.getRange(1, 1, dataSheet.getLastRow(), dataSheet.getLastColumn()).getValues();
let recordTypesAccepted = ['strategy1', 'goal2'];
var settingPar1 = settingData.filter(e => e[0] == 'default').map(e => e[1]);
var settingPar2 = settingData.filter(e => e[0] == 'default').map(e => e[2]);
let spcRange = spcSheet.getRange(1,20,spcSheet.getLastRow(),1)
var spcValues = spcRange.getValues();
var backgrounds = spcRange.getBackgrounds();
var numberformats = spcRange.getNumberFormats();
for (let a = 0; a < spcData.length; a++) {
for (let r = 0; r < settingData .length; r++) {
let settingSearchTerm = settingData[r][4];
let spcSearchTerm = spcData[a][10];
let settingMatchType = settingData[r][5];
let spcMatchType = spcData[a][9];
let spcRecordType = spcData[a][8];
if (recordTypesAccepted.indexOf(spcRecordType) > -1 && settingMatchType === spcMatchType) {
if (settingSearchTerm == spcSearchTerm) {
settingPar1 = settingData[r][7];
settingPar2 = settingData[r][6];
}
let spcClicks = spcData[a][10];
let spcOrders = spcData[a][11];
if (spcClicks >= settingsPar1 && spcOrders >= settingsPar2) {
let newValue = 10;
spcValues[a][0] = newBid;
backgrounds[a][0] = "#D8E271";
numberformats[a][0] = "€##0.00";
}
}
}
}
spcRange.setValues(spcValues);
spcRange.setBackgrounds(backgrounds);
spcRange.setNumberFormats(numberformats);
}
Reference
Range.getBackgrounds()
Range.setBackgrounds()
Range.getNumberFormats()
Range.setNumberFormats()

Javascript cancel async for loop

I found this code in a project:
const fn = async () => {
let x = 0;
for(let i = 0; i < 50; i++){
const res = await api.call(i);
if(res.someProp) x++;
}
return x;
}
I want to be able to stop it mid way, so that if I call it again, it will start from scratch and discard the previous call results. To avoid making two sets of requests at the same time.
This should do:
let token;
const fn = async () => {
const my = token = Symbol();
let x = 0;
for(let i = 0; i < 50 && my == token; i++){
const res = await api.call(i);
if(res.someProp) x++;
}
return x;
}
While there still can be some overlap between the calls, any previous loops will break their iteration as soon as the next fn() call is started.
You can use any technique of using an external flag variable to break the loop.
As a workaround you can try to use a custom Promise class (Live demo):
import CPromise from "c-promise2";
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
async function api(i) {
console.log(`Async API call [${i}]`);
await delay(100);
return {};
}
const fn = () =>
CPromise.from(function* () {
let x = 0;
for (let i = 0; i < 50; i++) {
const res = yield api.call(i);
if (res.someProp) x++;
}
return x;
});
const cancelablePromise = fn().then(
() => console.log("Done"),
(err) => console.log(`Fail: ${err}`) // Fail: CanceledError: canceled
);
setTimeout(() => {
cancelablePromise.cancel(); // abort the async sequence (loop) after 3500ms
}, 3500);

Recursive function to ensure the return length is 5

I'm fetching some data from an xml source, but I need to check that the length of the array (return value) is 5, sometimes the response serves data with less than 5 elements (it's random).
If the return value (colorArray) is 5, the promise resolves with the correct array. Otherwise, if the function re-runs the promise resolves with undefined.
Appreciate any help in understanding why I'm getting undefined when colorArray.length is less than 5, or if anyone has any better suggestions about how I should run the code.
Thanks.
const runAxios = async () => {
console.log("function");
const res = await axios.get("/api/palettes/random");
let parser = new DOMParser();
let xml = parser.parseFromString(res.data, "application/xml");
let colors = xml.getElementsByTagName("hex");
const colorArray = [];
for (let i = 0; i < colors.length; i++) {
let colorList = colors[i].firstChild.nodeValue;
colorArray.push(colorList);
}
if (colorArray.length === 5) return colorArray;
else runAxios();
};
const result = runAxios();
result.then(e => {
console.log(e);
});
The problem is that you never returned runAxios:
const runAxios = async () => {
console.log("function");
const res = await axios.get("/api/palettes/random");
let parser = new DOMParser();
let xml = parser.parseFromString(res.data, "application/xml");
let colors = xml.getElementsByTagName("hex");
const colorArray = [];
for (let i = 0; i < colors.length; i++) {
let colorList = colors[i].firstChild.nodeValue;
colorArray.push(colorList);
}
if (colorArray.length === 5) return colorArray;
else return runAxios(); // <----------------------------------This
};
const result = runAxios();
result.then(e => {
console.log(e);
});
Also, depending on your requirements, I would suggest a do-while loop:
const runAxios = async () => {
do {
console.log("function");
const res = await axios.get("/api/palettes/random");
let parser = new DOMParser();
let xml = parser.parseFromString(res.data, "application/xml");
let colors = xml.getElementsByTagName("hex");
const colorArray = [];
for (let i = 0; i < colors.length; i++) {
let colorList = colors[i].firstChild.nodeValue;
colorArray.push(colorList);
}
} while(colorArray.length != 5);
return colorArray;
};
const result = runAxios();
result.then(e => {
console.log(e);
});

How do I go to the API's second page when all 1000 arrays had been searched

I have this code, It searches a site for a specific item_name. But it only searches the first page of the site. So I made some changes, I added k and kRequired. But the problem is that I got an error code that says that getItemFind is not defined. I cannot spam requests to the API either because if I reach a certain amount of requests per second, my API key is going to get disabled.
if (command === 'item') {
var k = 0
var kRequired = 0
var itemFinder = message.content.substring(6)
for (var j = 0; j <= 1000; j++) {
console.log("searching")
if (j >= 950) {
console.log('next page!')
kRequired = ++k
j = 0
}
if (k === kRequired) {
console.log("searching next page!")
let getItemFind = async () => {
let response = await axios.get(`https://api.hypixel.net/skyblock/auctions?key=<API-KEY>&page=${k}`);
let itemFind = response.data;
k = ++k
j = 0
return itemFind;
}
}
let itemFindValue = await getItemFind();
var searchItem = itemFindValue.auctions[j].item_name
var itemFound = searchItem.toUpperCase().includes(itemFinder.toUpperCase())
if (itemFound === true) {
var itemNameDescription = itemFindValue.auctions[j].item_name
var itemDescription = itemFindValue.auctions[j].item_lore
var itemDescription = itemDescription.split(/§./).join("");
const itemEmbed = {
color: 0x0099ff,
title: `${itemNameDescription}`,
fields: [{
title: `\u200b`,
value: `${itemDescription}`,
inline: false,
}],
timestamp: new Date(),
footer: {
text: '?help [command | category]',
},
};
message.channel.send({
embed: itemEmbed
})
return j
}
}
}
The keywords let and const behave differently from var. If you define a variable with let or const inside of a code block {...} then the variables you created are only available inside that block.
// begin code block
if (k === kRequired) {
console.log("searching next page!")
// this is only available in this block of code
let getItemFind = async () => {
let response = await axios.get(`https://api.hypixel.net/skyblock/auctions?key=<API-KEY>&page=${k}`);
let itemFind = response.data;
k = ++k
j = 0
return itemFind;
}
}
// end code block
let itemFindValue = await getItemFind();
In your case, you have defined the function getItemFind inside of an if block. There are many ways to fix this. One of the simplest is to just move the variable declaration to the top of the function scope.
Here is an example to show you what I mean
const a = 1;
const b = 2;
if (a < b) {
let c = 3;
console.log('c inside code block', c)
}
console.log('c outside of code block', c)

Why do the last two functions work, but not the first?

I have three different functions that should do the same thing, populate an array with resolved Promises, but it's not working for the first example.
Here's my code:
(async() => {
const items = [];
const someFn = async() => {
const v = await Promise.resolve(10);
items.push(Math.random());
return Promise.resolve(v * 10);
}
const arr = [];
for (let i = 0; i < 10; i++) {
arr.push(someFn);
}
await Promise.all(arr);
console.log("item 1", items);
})();
(async() => {
const items = [];
const someFn = async() => {
const v = await Promise.resolve(10);
items.push(Math.random());
return Promise.resolve(v * 10);
}
const arr = [...Array(10).keys()].map(someFn)
await Promise.all(arr);
console.log("items 2", items);
})();
(async() => {
const items = [];
const someFn = async() => {
const v = await Promise.resolve(10);
items.push(Math.random());
return Promise.resolve(v * 10);
}
for (let i = 0; i < 10; i++) {
await someFn();
}
console.log("items 3", items);
})()
This is the output:
item 1 []
items 2 [ 0.7450904427103939,
0.37106667256699555,
0.12035280341441346,
0.265221052932904,
0.7775494303685422,
0.4872532010723445,
0.6497680191919464,
0.2570485072009576,
0.5613137531648884,
0.95109416178435 ]
items 3 [ 0.25328649499657585,
0.5452758396760038,
0.7274346878509064,
0.9306670111476503,
0.22942578229725785,
0.32547900377461625,
0.9722902638678983,
0.9964743517593542,
0.2828162584401659,
0.7672256760378469 ]
Notice how item 1 is an empty array.
That's because in the first example, someFn is never executed:
for (let i = 0; i < 10; i++) {
arr.push(someFn);
}
await Promise.all(arr);
This part just pushes functions into the arr variable, it doesn't run them, thus not creating Promises and never filling the items array.
On the other hand, the other examples run the function someFn:
const arr = [...Array(10).keys()].map(someFn)
This fills the arr array with 10 executions of someFn (map executes them with the current value (0-9), the index (also 0-9) and the array itself).
for (let i = 0; i < 10; i++) {
await someFn();
}
And this obviously runs someFn in a loop.
To make the first example work, push the result of the function into the array:
(async () => {
const items = [];
const someFn = async () => {
const v = await Promise.resolve(10);
items.push(Math.random());
return Promise.resolve(v * 10);
}
const arr = [];
for (let i = 0; i < 10; i++) {
arr.push(someFn()); // <-- () added
}
await Promise.all(arr);
console.log("item 1", items);
})();
you’re pushing someFn but you want someFn(). note we’re calling the function.

Categories

Resources