Develop Reference

How do I run code after all threads have finished running?

I have a multithreaded web crawler that downloads a website and stores it in a database (which takes around 4 minutes). To make the crawling faster, I used node.js cluster module, but I have a problem, I want to iterate over to the next segment of the while loop, after all the threads have done their processes, not as soon as they start. How do I make sure all my threads are concluded and then move on?
Here is the relevant code in the main while loop:
while (indexSize !== indexSizeLimit) {
const queueLength = queue.length;
const numberOfThreads = Math.min(numberOfCPUs, queueLength);
const threadAllocations = Array(numberOfThreads).fill(0);
let queuesAllocated = 0;
const queueChunks = [];
function fillQueueChunks() {
loop: while (true) {
for (let i = 0; i < numberOfThreads; i++) {
threadAllocations[i] += 1;
queuesAllocated += 1;
if (queuesAllocated === queueLength) {
break loop;
};
};
};
let start = 0;
for (let threadAllocation of threadAllocations) {
const end = start + threadAllocation;
queueChunks.push(queue.slice(start, end));
start = end;
};
};
fillQueueChunks();
// Find out how to make multithreading finish, and then move on with the loop.
if (cluster.isMaster) {
for (let i = 0; i < numberOfThreads; i++) {
cluster.fork();
};
} else {
const chunk = queueChunks[cluster.worker.id - 1];
await Promise.all(chunk.map(function (url) {
return new Promise(async function (resolve) {
const webcode = await request(url);
if (webcode !== "Failure") {
indexSize += 1;
const document = new Document(url, webcode);
const hrefs = document.hrefs();
const hrefsQuery = Query(hrefs);
// Also make sure it is not included in indexed webpages.
const hrefIndividualized = hrefsQuery.individualize();
hrefIndividualized;
// Do something with hrefIndividualized in regards to maintaining a queue in the database.
// And in adding a nextQueue which to replace the queue in code with.
await document.save();
};
resolve("Written");
});
}));
process.exit(0);
};
};

Wrap the threading in a promise. You can check in the parent thread if there is a disconnect event, and if the amount of disconnects is equal to the number of threads, then you can resolve the promise.
Here is what I have
while (indexSize !== indexSizeLimit) {
let nextQueue = [];
const queueLength = queue.length;
const numberOfThreads = Math.min(numberOfCPUs, queueLength);
const threadAllocations = Array(numberOfThreads).fill(0);
let queuesAllocated = 0;
// queueChunks: [[{_id: ..., ...}], [...], ...]
const queueChunks = [];
function fillQueueChunks() {
loop: while (true) {
for (let i = 0; i < numberOfThreads; i++) {
threadAllocations[i] += 1;
queuesAllocated += 1;
if (queuesAllocated === queueLength) {
break loop;
};
};
};
let start = 0;
for (let threadAllocation of threadAllocations) {
const end = start + threadAllocation;
queueChunks.push(queue.slice(start, end));
start = end;
};
};
fillQueueChunks();
await new Promise(async function (resolve) {
if (cluster.isMaster) {
let threadsDone = 0;
for (let i = 0; i < numberOfThreads; i++) {
cluster.fork();
};
cluster.on("disconnect", function (_) {
threadsDone += 1;
if (threadsDone === numberOfThreads) {
resolve("Queue Processed");
};
});
} else {
const queueJob = queueChunks[cluster.id - 1];
await Promise.all(queueJob.map(function (queueItem) {
return new Promise(async function (resolve) {
const url = queueItem._id;
const webcode = await request(url);
if (webcode !== "Failure") {
const document = Document(url, webcode);
let hrefs = document.hrefs();
const hrefsQuery = Query(hrefs);
await document.save();
indexSize += 1;
hrefs = hrefsQuery.individualize();
const hrefIncidences = Promise.all(hrefs.map(function (href) {
return new Promise(async function (resolve) {
const incidences = await Site.countDocuments({
url: href
});
resolve(incidences);
});
}));
hrefs = hrefs.filter(function (_, i) {
return hrefIncidences[i] === 0;
}).map(function (href) {
return {
_id: href
};
});
await Queue.insertMany(hrefs);
nextQueue = nextQueue.concat(hrefs);
};
await Queue.deleteOne({
_id: url
});
resolve("Success");
});
}));
process.exit(0);
};
});
queue = nextQueue;
};

amazon-chime-sdk-js display self video

I created a video application using chime js SDK with the help of the documentation https://aws.github.io/amazon-chime-sdk-js/index.html
const indexMap = {};
const acquireVideoElement = tileId => {
for (let i = 0; i < 16; i += 1) {
if (indexMap[i] === tileId) {
return videoElements[i];
}
}
for (let i = 0; i < 16; i += 1) {
if (!indexMap.hasOwnProperty(i)) {
indexMap[i] = tileId;
return videoElements[i];
}
}
throw new Error('no video element is available');
};
const releaseVideoElement = tileId => {
for (let i = 0; i < 16; i += 1) {
if (indexMap[i] === tileId) {
delete indexMap[i];
return;
}
}
};
const observer = {
videoTileDidUpdate: tileState => {
if (!tileState.boundAttendeeId || tileState.localTile || tileState.isContent) {
return;
}
meetingSession.audioVideo.bindVideoElement(tileState.tileId, acquireVideoElement(tileState.tileId));
},
videoTileWasRemoved: tileId => {
releaseVideoElement(tileId);
}
};
meetingSession.audioVideo.addObserver(observer);
const audioMix = document.getElementById('meeting-audio');
meetingSession.audioVideo.bindAudioElement(audioMix);
meetingSession.audioVideo.start();
meetingSession.audioVideo.startLocalVideoTile();
This is working good and I can see all the attendees who is joined in the meeting. But I need to show my video also in a tag. Is it possible?

In your videoTileDidUpdate, for your video tile to show, where are you binding the local tile, I see that if tileState.localTile is true you are returning and the bindVideoElement is not getting called hence your localTile is not showing up. Can you please remove the localTile check and see if that works as the initial step.

videoTileDidUpdate: tileState => {
// Ignore a tile without attendee ID, a local tile (your video), and a content share.
const { allUsers } = this.props;
if (!tileState.boundAttendeeId || tileState.isContent) {
return;
}
if( tileState.localTile ) {
if( Obj[tileState.boundExternalUserId]['tileId'] === tileState.tileId) {
return;
}
}
this.meetingSession.current.audioVideo.bindVideoElement( tileState.tileId,document.getElementById(tileState.boundExternalUserId) );
}

AngularFire2 iterate through array of objects and get data from realtime database

I have to go through an array of items and fetch the prices in the realtime database, but I want to show a loading message on start and after I finish loading I want to remove the loading message, but the way I did the message does not work its removed before the end of loading:
let loading = this.loadingCtrl.create({
content: 'Waiting...'
});
loading.present();
for (let i = 0; i <= this.Cart.length - 1; i++) {
let product = this.db.object('/menuItems/' + this.Cart[i].item.itemId);
product.valueChanges().subscribe((prodData: any) => {
if (prodData != null) {
this.Cart[i].item.price = prodData.price[0];
}
});
}
loading.dismiss();
How can I do this correctly?

You need to hide the loading indicator inside the subscriber. The simplest way is to hide it when the first data comes in:
for (let i = 0; i <= this.Cart.length - 1; i++) {
let product = this.db.object('/menuItems/' + this.Cart[i].item.itemId);
product.valueChanges().subscribe((prodData: any) => {
if (prodData != null) {
this.Cart[i].item.price = prodData.price[0];
}
loading.dismiss();
});
}
A bit more tricky is to hide it after all data comes back, but it doesn't have to be too hard. A simple way is to count how many responses you've gotten back. Once that is the same as the number of items in the cart, you can hide the loading indicator.
var responseCount = 0;
for (let i = 0; i <= this.Cart.length - 1; i++) {
let product = this.db.object('/menuItems/' + this.Cart[i].item.itemId);
product.valueChanges().subscribe((prodData: any) => {
if (prodData != null) {
this.Cart[i].item.price = prodData.price[0];
}
if (responseCount++ == this.Cart.length) loading.dismiss();
});
}

Object prints in the console but can't use it to set state - ReactJS

I'm making a request to an API for prices of a cryptocurrency for a date range & storing the result in an array. The goal is to populate a line graph with those prices when the page loads but I can't figure out for the life of my why pricesArr prints when I console.log it but if I try this.setState({dataArr: pricesArr}) then I get an TypeError: Cannot read property 'setState' of undefined.
The object in pricesArr prints in the console and looks like this:
[{USD: 7629.4, EUR: 6503.14}, {USD: 6754.4, EUR: 5432.14} ... ]
class App extends Component {
constructor(props) {
super(props);
this.state = {
dataArr:[]
}
}
getData(){
var url = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
let pricesArr = []
for (let i=1; i <= 30; i++) {
const num = i.toString()
const date = new Date('2018.06.' + num)
const unixTimestamp = date / 1000
const api_url = url + unixTimestamp
if (i % 5 === 0 || i === 1) {
axios.get(api_url)
.then(res => {
var obj = res.data['BTC']
pricesArr.push(obj)
})
}
}
this.setState({dataArr: pricesArr})
}
componentDidMount () {
this.getData()
}
render() {
return (
<div>
Hi
</div>
);
}
}
export default App;

Strange that your this is undefined. try this.
getData = () => {
var url = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
let pricesArr = []
for (let i=1; i <= 30; i++) {
const num = i.toString()
const date = new Date('2018.06.' + num)
const unixTimestamp = date / 1000
const api_url = url + unixTimestamp
if (i % 5 === 0 || i === 1) {
axios.get(api_url)
.then(res => {
var obj = res.data['BTC']
pricesArr.push(obj)
})
}
}
this.setState({dataArr: pricesArr})
}

Use Promise to get the return data

I am trying to pull the data from realms and then process it using the native call. Once I get the processed data I further filter it in react native code and then give the return data. But till the time I complete all the activity the return parameter is already executed. Please have a look at the below code snippet:
async getArray() {
realm = await this.getRealMObject();
favoriteArray = realm.objects(constants.SCHEMA_SAVE);
return favoriteArray;
}
async getLevel() {
let low = 'false';
let favoriteArray = await this.getAllArray();
let getDetails = [];
if (favoriteArray.length > 0) {
for (var y in favoriteArray) {
NativeModules.Get.GetAsyncTask(
brd[0].toUpperCase(),
strChk, '1',
kResponse => {
if (Platform.OS == 'ios') {
getDetails = kResponse;
} else {
getDetails = JSON.parse(kResponse);
}
for (let x = 0; x < getDetails.length; x++) {
const strLevel = this.getPercentage(getDetails[x]);
if (strLevel <= 20) {
low = true;
}
}
});
}
} else {
low = false;
}
return low;
}
Above is the code snippet. So when I call getlevel() before the processing is finished I get the return low value.
How can I handle, this? I want the data to be processed and then get the return value.