Develop Reference

Promise and work thread chunking not working

I have this in my code where i am using promise to limit at max MAX_WORKERS workgroup at a time.
const MAX_WORKERS = 3;
let promises = [];
let chunk = 3;
let totalNodes = keyCountMap.length / 2;
let doneCount = 0;
for (let m = 0; m < keyCountMap.length; ) {
let remaining = totalNodes - doneCount;
let numbWorker = Math.min(chunk, remaining);
for (let i = 0; i < numbWorker; i++) {
promises.push(createWorker(keyCountMap[m], keyCountMap[m + 1]));
doneCount++;
m += 2;
console.log(doneCount);
if (doneCount % MAX_WORKERS == 0) {
Promise.all(promises).then((response) => {
console.log("one chunk finish");
});
}
}
}
and createWorker function is:
function createWorker(data1, data2) {
return new Promise((resolve) => {
let worker = new Worker();
worker.onmessage = (event) => {
postMessageRes = event.data;
if (postMessageRes == 200) {
worker.postMessage([
nodePagesString,
pagesString,
copcString,
data1,
data2,
]);
} else {
workerCount += 1;
let position = postMessageRes[0];
let color = postMessageRes[1];
for (let i = 0; i < position.length; i++) {
positions.push(position[i]);
colors.push(colors[i]);
}
if (workerCount == MAX_WORKERS) {
worker.terminate();
workerCount = 0;
promises = [];
}
resolve(true);
}
};
});
}
but the problem i have here is that it is not working like i wanted "3 work thread at a time and when it is done, spawn more thread/workers"
This is my console
and
which mean it is called at once and not after 3 threads which is count of chunks
Can anyone help me please on this
This is my worker code:
import { Copc, Key } from "copc";
import * as THREE from "three";
const color = new THREE.Color();
const colors = [];
let firstTime = true;
var nodePages, pages, receivedData, copc;
let x_min, y_min, z_min, x_max, y_max, z_max, width;
let positions = [];
let filename = "https://s3.amazonaws.com/data.entwine.io/millsite.copc.laz";
const readPoints = (id, getters) => {
let returnPoint = getXyzi(id, getters);
positions.push(
returnPoint[0] - x_min - 0.5 * width,
returnPoint[1] - y_min - 0.5 * width,
returnPoint[2] - z_min - 0.5 * width
);
const vx = (returnPoint[3] / 65535) * 255;
color.setRGB(vx, vx, vx);
colors.push(color.r, color.g, color.b);
firstTime = false;
};
function getXyzi(index, getters) {
return getters.map((get) => get(index));
}
async function load() {
// copc = await Copc.create(filename);
// let scale = copc.header.scale[0];
// [x_min, y_min, z_min, x_max, y_max, z_max] = copc.info.cube;
// width = Math.abs(x_max - x_min);
// // let center_x = (x_min + x_max) / 2;
// // let center_y = (y_min + y_max) / 2;
// // let center_z = (z_min + z_max) / 2;
// receivedData = await Copc.loadHierarchyPage(
// filename,
// copc.info.rootHierarchyPage
// );
// nodePages = receivedData.nodes;
// pages = receivedData.pages;
postMessage(200);
}
async function loadData(nodes, pages, copc, myRoot, pointCount) {
// console.log(copc, myRoot);
const view = await Copc.loadPointDataView(filename, copc, myRoot);
let getters = ["X", "Y", "Z", "Intensity"].map(view.getter);
for (let j = 0; j < pointCount; j += 1) {
readPoints(j, getters);
}
postMessage([positions, colors]);
}
load();
onmessage = function (message) {
let nodePages = message.data[0];
let nodes = JSON.parse(nodePages);
let pagesStr = message.data[1];
let pages = JSON.parse(pagesStr);
let copcStr = message.data[2];
let copc = JSON.parse(copcStr);
let mapIndex = message.data[3];
let pointCount = message.data[4];
let myRoot = nodes[mapIndex];
// console.log(mapIndex);
loadData(nodes, pages, copc, myRoot, pointCount);
};

What's creating the confusion here is the mix of synchronous and asynchronous processes. In your first code block there are two for loops that iterate over the keyCountMap.length and numbWorker, respectively. The inner for loop sets up an asynchronous callback once the array of promises are resolved. However, it does not wait for the result as it is asynchronous.
So, the behavior you're seeing is the looping over all values in the for loops and queueing of multiple async calls. You can read documentation here about how the event loop works.
The logging for the doneCount values happens first and then your async callbacks happen afterward. You should be able to verify this by changing the console.log within the Promise callback by adding the doneCount to it:
/*
The use of JSON.parse and JSON.stringify is to preserve the initial value.
Otherwise it will show the current value of doneCount when the callback finalizes.
*/
console.log(`one chunk finish: ${JSON.parse(JSON.stringify(doneCount))}`);
Edit Added recommended adjustment.
const chunkWorkers = async () => {
const MAX_WORKERS = 3;
let promises = [];
let chunk = 3;
let totalNodes = keyCountMap.length / 2;
let doneCount = 0;
for (let m = 0; m < keyCountMap.length; ) {
let remaining = totalNodes - doneCount;
let numbWorker = Math.min(chunk, remaining);
for (let i = 0; i < numbWorker; i++) {
promises.push(createWorker(keyCountMap[m], keyCountMap[m + 1]));
doneCount++;
m += 2;
console.log(doneCount);
if (doneCount % MAX_WORKERS == 0) {
await Promise.all(promises).then((response) => {
console.log(`one chunk finish: ${doneCount}`);
});
}
}
}
};

(Javascript) Check if 2 Values Overlap and give percentage value

I have the following code adapted to check if a range overlaps:
function isOverlapping(prevHighLow, currentHighLow) {
const a = prevHighLow[0];
const b = prevHighLow[1];
const c = currentHighLow[0];
const d = currentHighLow[1];
if (a < d && b > c) {
return true;
}
return false;
}
var prevHighLow = [12350, 12900]
var currentHighLow = [12100, 12800]
console.log(isOverlapping(prevHighLow, currentHighLow)) //returns true
It works 100%, however I would like to also return a percentage (from 0 to 100%) of how much they overlap?
Thank you!

The only percentage that might make sense is percentage of common area vs total area.
So
// function from https://www.tutorialspoint.com/get-intersection-between-two-ranges-in-javascript
const findRangeIntersection = (arr1, arr2) => {
const [el11, el12] = arr1;
const [el21, el22] = arr2;
const leftLimit = Math.max(el11, el21);
const rightLimit = Math.min(el12, el22);
return [leftLimit, rightLimit];
};
var arr1 = [0, 12];
var arr2 = [6, 14];
var intersection = findRangeIntersection(arr1, arr2)
console.log("" + intersection)
function size(arr1) {
return arr1[1] - arr1[0];
}
var percent = size(intersection) / (size(arr1) + size(arr2)) * 100
console.log(percent, '%');

function isOverlapping(prevHighLow, currentHighLow) {
const a = prevHighLow[0];
const b = prevHighLow[1];
for (const interval of currentHighLow) {
const c = currentHighLow[0];
const d = currentHighLow[1];
if (a < d && b > c) {
let smallerRange = 0;
let largerRange = 0;
if (b-a < d-c){
smallerRange = b-a;
largerRange = d-c;
} else {
smallerRange = d-c;
largerRange = b-a;
}
return [true, Math.round((smallerRange/largerRange)*100) + "%"];
}
}

How to store Array data into csv format file?

I have an array and I want to convert that array into a csv file here is my code and how i store my arrays :
const words = ['item1','item2','item3']
const txtNumber = 2;
let newArr = []
for(let t= 1 ; t <= txtNumber ; t++) {
const data = "item1 , item2 , item3 ,item4";
const findWord = (word) => data.split(",").filter(x => x.includes(word))
const array = [];
for(let i = 0; i < words.length ; i++) {
const x = findWord(words[i]);
array.push(x[0])
}
newArr = array.map(x => x === undefined ? '' : x.trim());
console.log(newArr)
}
how can i store newArr values into items.csv into my pc ?

Im a little confused about what the desired output is, so no idea if this helps.
const fs = require('fs')
let writeStream = fs.createWriteStream('output.csv')
const words = ['item1','item2','item3']
const txtNumber = 2;
let newArr = []
for(let t= 1 ; t <= txtNumber ; t++) {
const data = "item1 , item2 , item3 ,item4";
const findWord = (word) => data.split(",").filter(x => x.includes(word))
const array = [];
for(let i = 0; i < words.length ; i++) {
const x = findWord(words[i]);
array.push(x[0])
}
newArr = array.map(x => x === undefined ? '' : x.trim());
//console.log(newArr)
writeStream.write(newArr.map(x=>`"${x}"`).join(",")+"\n")
}
writeStream.end()
The output of this should be (although I have not tested it)
"item1","item2","item3"
"item1","item2","item3"

Store newArr values into items.csv
const fs = require('fs');
const words = ['item1','item2','item3']
const txtNumber = 2;
let newArr = []
for(let t= 1 ; t <= txtNumber ; t++) {
const data = "item1 , item2 , item3 ,item4";
const findWord = (word) => data.split(",").filter(x => x.includes(word))
const array = [];
for(let i = 0; i < words.length ; i++) {
const x = findWord(words[i]);
array.push(x[0])
}
newArr = array.map(x => x === undefined ? '' : x.trim());
console.log(newArr)}
const writeStream = fs.createWriteStream('items.csv');
writeStream.write(`item \n`);
writeStream.write('[ "' + newArr.join('","') + '" ]\n');

Super lazy solution would be, for example, this NPM package found at:
https://www.npmjs.com/package/convert-array-to-csv
Disadvantage would be have an additional dependency to your project which you have no (or little) control over.

Recursive function to ensure the return length is 5

I'm fetching some data from an xml source, but I need to check that the length of the array (return value) is 5, sometimes the response serves data with less than 5 elements (it's random).
If the return value (colorArray) is 5, the promise resolves with the correct array. Otherwise, if the function re-runs the promise resolves with undefined.
Appreciate any help in understanding why I'm getting undefined when colorArray.length is less than 5, or if anyone has any better suggestions about how I should run the code.
Thanks.
const runAxios = async () => {
console.log("function");
const res = await axios.get("/api/palettes/random");
let parser = new DOMParser();
let xml = parser.parseFromString(res.data, "application/xml");
let colors = xml.getElementsByTagName("hex");
const colorArray = [];
for (let i = 0; i < colors.length; i++) {
let colorList = colors[i].firstChild.nodeValue;
colorArray.push(colorList);
}
if (colorArray.length === 5) return colorArray;
else runAxios();
};
const result = runAxios();
result.then(e => {
console.log(e);
});

The problem is that you never returned runAxios:
const runAxios = async () => {
console.log("function");
const res = await axios.get("/api/palettes/random");
let parser = new DOMParser();
let xml = parser.parseFromString(res.data, "application/xml");
let colors = xml.getElementsByTagName("hex");
const colorArray = [];
for (let i = 0; i < colors.length; i++) {
let colorList = colors[i].firstChild.nodeValue;
colorArray.push(colorList);
}
if (colorArray.length === 5) return colorArray;
else return runAxios(); // <----------------------------------This
};
const result = runAxios();
result.then(e => {
console.log(e);
});
Also, depending on your requirements, I would suggest a do-while loop:
const runAxios = async () => {
do {
console.log("function");
const res = await axios.get("/api/palettes/random");
let parser = new DOMParser();
let xml = parser.parseFromString(res.data, "application/xml");
let colors = xml.getElementsByTagName("hex");
const colorArray = [];
for (let i = 0; i < colors.length; i++) {
let colorList = colors[i].firstChild.nodeValue;
colorArray.push(colorList);
}
} while(colorArray.length != 5);
return colorArray;
};
const result = runAxios();
result.then(e => {
console.log(e);
});

Get months and weeks from range with momentjs

If I have something like:
let start = moment('2019-01-27');
let end = moment('2019-02-28');
How can I get real months start and end and weeks start and end so I can count some data within those weeks and months.
I have this for weekdays:
getWeekdays(data, labels) {
let start = moment(this.$store.state.labels[0]);
let end = moment(this.$store.state.labels[this.$store.state.labels.length - 1]);
let new_labels = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'];
let store_labels = this.$store.state.labels;
let store_data = this.$store.state.data;
let new_data = [];
for(let i=1; i<8; i++) {
var arr = [];
let tmp = start.clone().day(i);
if( tmp.isAfter(start, 'd') ){
arr.push(tmp.format('YYYY-MM-DD'));
}
while( tmp.isBefore(end) ){
tmp.add(7, 'days');
arr.push(tmp.format('YYYY-MM-DD'));
}
arr.pop();
let count = 0;
_.forEach(arr, function(val) {
let key = store_labels.findIndex(function(i){return i === val});
count = count + store_data[key];
});
new_data.push(count);
}
console.log(new_data);
return {data: new_data, labels: new_labels};
}
Where data = [1,2,5,3,8] and labels = ['2019-01-27','2019-01-28','2019-01-29','2019-01-30','2019-01-31'] and this works, but I am not sure how to do this for months and weeks.

This is answer for months just if someone needs :D
let new_range = {};
_.forEach(labels, function(value, key) {
new_range[moment(value).format('YYYY MM')] = [];
});
_.forEach(labels, function(value, key) {
new_range[moment(value).format('YYYY MM')].push(data[key]);
});
let new_labels = Object.keys(new_range);
let new_data = [];
_.forEach(new_range, function(value, key) {
new_data.push(_.sumBy(value, function(n) {return n;}));
});
return {data: new_data, labels: new_labels};
And after whole day here is one for real weeks:
const range = ['2019-01-22', '2019-01-23', '2019-01-24', '2019-01-25', '2019-01-26', '2019-01-27', '2019-01-28', '2019-01-29', '2019-01-30', '2019-01-31', '2019-02-01', '2019-02-02', '2019-02-03', '2019-02-04', '2019-02-05', '2019-02-06', '2019-02-07', '2019-02-08', '2019-02-09', '2019-02-10', '2019-02-11', '2019-02-12', '2019-02-13', '2019-02-14', '2019-02-15', '2019-02-16', '2019-02-17', '2019-02-18', '2019-02-19', '2019-02-20', '2019-02-21', '2019-02-22', '2019-02-23', '2019-02-24', '2019-02-25', '2019-02-26', '2019-02-27', '2019-02-28', '2019-03-01'];
const backup = ['2019-01-22', '2019-01-23', '2019-01-24', '2019-01-25', '2019-01-26', '2019-01-27', '2019-01-28', '2019-01-29', '2019-01-30', '2019-01-31', '2019-02-01', '2019-02-02', '2019-02-03', '2019-02-04', '2019-02-05', '2019-02-06', '2019-02-07', '2019-02-08', '2019-02-09', '2019-02-10', '2019-02-11', '2019-02-12', '2019-02-13', '2019-02-14', '2019-02-15', '2019-02-16', '2019-02-17', '2019-02-18', '2019-02-19', '2019-02-20', '2019-02-21', '2019-02-22', '2019-02-23', '2019-02-24', '2019-02-25', '2019-02-26', '2019-02-27', '2019-02-28', '2019-03-01'];
const data = [5,2,3,1,5,5,2,3,1,5,5,2,3,1,5,5,2,3,1,5,5,2,3,1,5,5,2,3,1,5,5,2,3,1,5,5,2,3,1];
console.log(bindData(data, backup, getWeeks(range)));
function bindData(data, range, labels) {
let new_data = {};
Object.keys(labels).forEach(function(val) {
let count = 0;
for(let j = 0; j<labels[val].length; j++) {
for(let i = 0; i<range.length; i++) {
if(labels[val][j] == range[i]) {
count = count + data[i];
}
}
}
new_data[val] = count;
});
return new_data;
}
function getWeeks(labels) {
let start = moment(labels[0]);
let new_range = {};
var arr = [];
let tmp = start.clone().day(7);
let sunday = tmp.format('YYYY-MM-DD');
let sunday_index = labels.indexOf(sunday);
if(sunday_index === -1) {
new_range[sunday] = labels;
}else{
let first_week = labels.slice(0, sunday_index + 1);
new_range[first_week[first_week.length - 1]] = first_week;
labels.splice(0, sunday_index + 1);
let last_days = labels.length % 7;
if(last_days != 0) {
let last_week = labels.splice(labels.length - last_days, last_days);
new_range[last_week[last_week.length - 1]] = last_week;
}
for(let i = 0; i<labels.length; i++) {
let a = 0;
new_range[labels[6]] = labels.splice(a, 7);
}
}
return new_range;
}
Now for some reason I need to have double labels array, if I used one from them I would get empty array in bindData function. If you know why. Edit it.