I'm trying to use phantomjs to get some metrics about the likelihood of a race condition affecting a page, I have 2 script files, some functionality hosted on my site is dependant on some globals set by a file coming from a third party.
I thought that using onResourceReceived in phantomjs I could log when each file loads and then run that test a bunch of times to get an idea of how often this race condition will cause issues, an example of my code is below (it's not the actual code and I'm not affiliated with the BBC):
(function (p, wp) {
"use strict";
var page, start,
count = 0, max = 10,
webpage = require('webpage'),
url = "http://www.bbc.co.uk";
function process() {
if (max == count) {
console.log('done processing!');
p.exit();
} else {
count++;
start = new Date();
page = wp.create();
page.onResourceReceived = onResourceReceived;
page.open(url, onOpen);
}
}
function onResourceReceived(response) {
var match, t = new Date(),
url = response.url,
status = response.status;
t = t.valueOf() - start.valueOf();
if (!!(match = url.match(/locator\.css/))) {
console.log(match[0] + ': ' + t + 'msecs status: ' + status);
}
if (!!(match = url.match(/en-GB\.json/))) {
console.log(match[0] + ': ' + t + 'msecs status: ' + status);
}
};
function onOpen() {
console.log('Test ' + count + ' done!!!');
page.close();
process();
}
process();
}(phantom, require('webpage')));
This kinda runs how I expected except that each file is logged twice, why is this?
Sometimes the time differences are very different.
locator.css: 323msecs status: 200
locator.css: 323msecs status: 200
en-GB.json: 2199msecs status: 200
en-GB.json: 2200msecs status: 200
Test 1 done!!!
You need to check for response.stage property. stage will have start and end. start gives the first byte arrived time and end give you the when you got the complete response.
please add a check in your function.
function onResourceReceived(response) {
if(response.stage == 'end') return;
//rest of your code from above example.
};
Related
I've got the code that retrieves couple of millions of rows from a db and a couple of millions through an api.
let closedOrdersStartDate;
preparedOrdersPromise = tickApiConnector.obtainToken().then(function () {
return ordersController.getAllTrades(0, [], 0, accountIdsToRun);
}).then(function (trades) {
closedOrdersStartDate = new Date();
return Promise.all([trades, fthsApiConnector.getData('closed_orders', '&sort=id', 10000, 0)]);
}).then(function (tradesAndClosedOrderIds) {
//stuck before getting there
console.log('now processing orders time spent from starting getting closed_orders till now is: ' +
((new Date().getTime() - closedOrdersStartDate.getTime())/ 1000) + ' seconds');
return ordersController.processOrders(tradesAndClosedOrderIds[0], tradesAndClosedOrderIds[1]);
});
The app gets stuck after calling that getData() function.
getData: async function (entityName, getParams = '', perRequest = 10000, skip = 0) {
if(getParams.indexOf('&take=') !== -1) {
return fthsApiRequest(entityName, getParams);
}
const totalCount = await fthsApiRequest(entityName, getParams + '&getCount');
let result = [];
let count = 0;
while(count < totalCount) {
result = result.concat(await fthsApiRequest(entityName, getParams + '&take=' + perRequest + '&skip=' + count));
count += perRequest;
}
return result;
}
The function executes till the last request(I see it in logs) and after that the script gets unresponsable. I thought that it could be a memory leak and I've rewritten that getData() function in different ways but still, there's enough memory on the server and the script doesn't consume even a bit of it. Still I get 100% of CPU load in a while after that last iteration of getData() is rant. After that the app gets stuck forever.
I've tried profiling it. And there are thousands of Code move event for unknown code: 0x2a5c24bfb4c0, I'm not sure what that means, but there could be a clue in that. Here's the V8.log
The possible problem maybe in block :
while(count < totalCount) { result = result.concat(await fthsApiRequest(entityName, getParams + '&take=' + perRequest + '&skip=' + count)); count += perRequest; }
Ensure that the api give response. And on the last statement :
return result;
In async function the better use :
return Promise.resolve(result);
I have a function firing two 'value' Firebase events. One is necessary to get the number of children, which is corresponding to the deepest path of the next one.
function myFunction(data){
//get last child
var lastchild = 0;
var data_child = firebase.database().ref('rooms/' + 633 + '/' + 630);
data_child.once('value', function(child) {
if(child.exists()){
lastchild = child.numChildren();
console.log('function 1');
}else{
console.log('error no child');
}
});
//use this number to read the path
var msgsStored = firebase.database().ref('rooms/' + 633 + '/' + 630 + '/' + lastchild);
msgsStored.orderByChild('datetime').once('value', function(snapshot) {
var store = (snapshot.val());
snapshot.forEach(function(child) {
console.log('function 2');
//do something
}
}
}//myFunction
Firebase will always fire the last 'value' event before the first one. Why?
That will result always on variable lastchild = 0; and 'function 2' will always print before 'function 1' on console.
I tried also creating a callback(); function to try control the order with JavaScript, but will not work.
I know different events of Firebase are fired in different order, but in my case I need to read stored data only.
Anybody knows what am I missing and how to address me to a solution?
Firebase, like most of the modern web, reads data from its database asynchronously. This is easiest to see by placing some log statements in your code:
console.log("1");
var data_child = firebase.database().ref('rooms/' + 633 + '/' + 630);
data_child.once('value', function(child) {
console.log("2");
});
console.log("3");
The output of this is:
1
3
2
This is probably not what you initially expected, but explains a lot on the behavior you're seeing. When log statement 3 executes, the data from the first listener hasn't been loaded yet. So you can't make a query based on that data.
For this reason you always need to move the code that requires the data into the callback:
var lastchild = 0;
var data_child = firebase.database().ref('rooms/' + 633 + '/' + 630);
data_child.once('value', function(child) {
if(child.exists()){
lastchild = child.numChildren();
//use this number to read the path
var msgsStored = firebase.database().ref('rooms/' + 633 + '/' + 630 + '/' + lastchild);
msgsStored.orderByChild('datetime').once('value', function(snapshot) {
var store = (snapshot.val());
snapshot.forEach(function(child) {
console.log('function 2');
//do something
}
}
}else{
console.log('error no child');
}
});
I'm working on a meteor react project and want to use load data from local storage which happens async. Unfortunately I wasn't able to get the data out of callback even with binds. I tried multiple ways but could not get any of them to work, surely I'm missing something.
I stripped as much away as I could but had to keep some for the context.
From my understanding it can only be related to the track object as setting those simple Integers, Booleans works fine.
render() {
const { audioCollection } = this.props; // database collection, async hence the following if check
if (audioCollection) {
this.tracks = audioCollection.audio(); // setting tracks for later use
// make sure tracks are loaded and only run once, as we do this in the react renderer
if (this.tracks && !this.state.tracksLoaded) {
var trackLoadedCount = 0;
this.tracks.forEach((track, i) => { // I tried to use forEach and map here
// now we try to load the data from local storage and if it fails fall back to the remote server
LocalForage.getItem(track.file_id).then(function(err, file) {
if (!err && file) {
console.log(track.file_id + ' from cache')
var blob = new Blob([file]);
fileURI = window.URL.createObjectURL(blob);
} else {
console.log(track.file_id + ' from database')
fileURI = audioCollection.audioLink(track.file_id);
}
track.fileURI = fileURI; // assigning the retrieved data uri to the track object, also tried to set it on the original parent object not the extracted one from the forEach/map
console.log(fileURI + ' ' + track.fileURI) // yes, both are set
trackLoadedCount++; // increasing the tracks loaded count, to calculate if all have been loaded and to test if it can write outside of the callback, which it does
// if all files have been processed, set state loaded, this works too.
if (trackLoadedCount == this.tracks.length) {
this.setState({
tracksLoaded: true,
})
}
}.bind(track, this))
});
}
}
// once all has been processed we try to retrieve the fileURI, but it isn't set :(
if (audioCollection && this.tracks && this.state.tracksLoaded) {
console.log('all loaded ' + this.tracks.length)
this.tracks.map(track => {
console.log('track: ' + track.file_id + ' ' + track.fileURI) // file_id is set, fileURI is not
})
}
// we only log
return (
<Content {...this.props}>
<div>just console output</div>
</Content>
);
}
I tried more approaches:
Writing the tracks as an array to state like tracksLoaded (didn't work work)
Defining a new var before the async call and setting its values from within the callback, like trackLoadedCount (with and without bind) (doesn't work)
Why isn't this working while its working for tracksLoaded and trackLoadedCount?
Update regarding Firice Nguyen Answer
render() {
const { audioCollection } = this.props;
if (audioCollection) {
this.tracks = audioCollection.audio();
if (this.tracks && !this.state.tracksLoaded) {
var trackLoadedCount = 0;
this.tracks.forEach((track, i, trackArray) => {
LocalForage.getItem(track.file_id).then(function(err, file) {
if (!err && file) {
console.log(track.file_id + ' from cache')
var blob = new Blob([file]);
fileURI = window.URL.createObjectURL(blob);
} else {
console.log(track.file_id + ' from database')
fileURI = audioCollection.audioLink(track.file_id);
}
track.fileURI = fileURI;
console.log('1. ' + track.file_id + ' ' + track.fileURI);
trackArray[i] = track;
console.log('2. ' + track.file_id + ' ' + trackArray[i].fileURI);
trackLoadedCount++;
if (trackLoadedCount == this.tracks.length) {
this.setState({
tracksLoaded: true,
})
}
}.bind(track, this))
});
}
}
if (audioCollection && this.tracks && this.state.tracksLoaded) {
console.log('all loaded ' + this.tracks.length)
this.tracks.map(track => {
console.log('3. ' + track.file_id + ' ' + track.fileURI) // file_id is set, fileURI is not
})
}
return (
<Content {...this.props}>
<div>just console output</div>
</Content>
);
}
returns
MXqniBNnq4zCfZz5Q from database
1. http://localhost:3000/cdn/storage/files/MXqniBNnq4zCfZz5Q/original/MXqniBNnq4zCfZz5Q.m4a
2. http://localhost:3000/cdn/storage/files/MXqniBNnq4zCfZz5Q/original/MXqniBNnq4zCfZz5Q.m4a
keBWP6xb9PyEJhEzo from database
1. http://localhost:3000/cdn/storage/files/keBWP6xb9PyEJhEzo/original/keBWP6xb9PyEJhEzo.m4a
2. http://localhost:3000/cdn/storage/files/keBWP6xb9PyEJhEzo/original/keBWP6xb9PyEJhEzo.m4a
K2J2W9W26DDBNoCcg from database
1. http://localhost:3000/cdn/storage/files/K2J2W9W26DDBNoCcg/original/K2J2W9W26DDBNoCcg.m4a
2. http://localhost:3000/cdn/storage/files/K2J2W9W26DDBNoCcg/original/K2J2W9W26DDBNoCcg.m4a
all loaded 3
3. MXqniBNnq4zCfZz5Q undefined
3. keBWP6xb9PyEJhEzo undefined
3. K2J2W9W26DDBNoCcg undefined
hence the issue persists.
The forEach give out a copy of the element. The track is just a copy, not the original one. It does not point to the element in your array. You can try this:
this.tracks.forEach(track, i, trackArray) => {
// change `track` value
...
trackArray[i] = track;
});
Or try map method
var prompt = ...
var connection = ...
prompt.start();
var property = {
name: 'yesno',
message: 'approve this screencast?',
validator: /y[es]*|n[o]?/,
warning: 'Must respond yes or no',
default: 'no'
};
connection.queryAsync('SELECT * FROM screencasts WHERE approved = 0')
.spread(function(screencasts) {
screencasts.forEach(function(screencast) {
console.log('Title: "' + screencast.title + '".');
prompt.get(property, function(err, res) {
console.log('Command-line input received:');
console.log(' result: ' + res.yesno);
});
});
});
Aim: To enumerate the screencasts and prompt the user to approve or deny them, interactively.
Problem: I think the problem is that, the loop does not block at all, resulting in the next element being processed too quickly:
How do I wait for the user to input a value before "processing" the next element in the sequence?
Someone mentioned mapSeries, and that got me thinking about the example that shows how to avoid mapSeries and instead use bluebird's map.
What I ended up with is this:
var promise = ...
var prompt = ...
var connection = ...
promise.promisifyAll(prompt);
...
connection.queryAsync('SELECT * FROM screencasts WHERE approved = 0')
.spread(function(screencasts) {
var current = promise.resolve();
promise.map(screencasts, function(screencast) {
current = current.then(function() {
console.log('Title: "' + screencast.title + '".');
return prompt.getAsync(property);
}).then(function(res) {
console.log(res);
});
return current;
});
});
It seems to work well. I am curious, though. Is map the most appropiate function for this?
Currently working on an application that does quite a bit of database work, and thought that one smart way of getting around this should be to use events. So I set up a events emitter for the application to use, and then using this as a way to control the flow. And I kind of assumed that the events emitter would work similar to a message queue or similar.
This was not the case, and for some reason I can not get the even emitter to work as expected on the second call to the same function.
I would have understood if the problem would be that it called multiple times, but as it is, the below code will run the second function twice, but will only get passed the dbEvents.once('Ready' once. And not only can I not figure out why, it seems to work like this constantly, and will never print out "Wrote Fourth"
var mongo = require('mongodb');
var events = require('events');
var dbEvents = new events.EventEmitter();
function writeStuff(first, second, status, cuID){
console.log('Wrote '+ first);
if (status === 1) {
console.log('\t\tReady for '+ cuID);
dbEvents.emit('Ready');
} else {
console.log('\t\tReady (but not 1) for ' + cuID);
dbEvents.emit('Ready');
}
console.log('\t\tjust about to kick of number ' + status);
dbEvents.once('Ready', function (condata) {
console.log('Wrote '+ second)
dbEvents.emit('Done'+cuID)
});
}
var cuID1 = mongo.ObjectID();
writeStuff('First', 'Second', 1, cuID1);
dbEvents.on('Done'+cuID1, function(){
console.log('\tFirst completed ' + cuID1);
});
var cuID2 = mongo.ObjectID();
writeStuff('Third', 'Forth', 2, cuID2);
dbEvents.on('Done'+cuID2, function(){
console.log("\tSecond completed " + cuID2)
});
var b = 0;
for(var i = 0; i < 100000000; i++){
b++;
}
console.log("And I counted to " + b);
The output when I run the above code looks like this
Wrote First
Ready for 5540c57cd8fa9555076f9aba
just about to kick of number 1
Wrote Third
Ready (but not 1) for 5540c57cd8fa9555076f9abb
Wrote Second
First completed 5540c57cd8fa9555076f9aba
just about to kick of number 2
And I counted to 100000000
================ UPDATE ==============
Why can't I generate an event, I.E. if I use a unique identifier it does not seem like I can send it as an event.In this case I change the event to be Ready+cuID (mongodb), and now the second write never gets called.
var mongo = require('mongodb');
var events = require('events');
var dbEvents = new events.EventEmitter();
function writeStuff(first, second, status, cuID){
console.log('Wrote '+ first);
if (status === 1) {
console.log('\t\tReady for '+ cuID);
var flag = 'Ready'+cuID
dbEvents.emit(flag);
} else {
console.log('\t\tReady (but not 1) for ' + cuID);
dbEvents.emit('Ready');
}
console.log('\t\tjust about to kick of number ' + status);
dbEvents.once(flag, function (condata) {
console.log('Wrote '+ second)
dbEvents.emit('Done'+cuID)
});
}
var cuID1 = mongo.ObjectID();
writeStuff('First', 'Second', 1, cuID1);
dbEvents.on('Done'+cuID1, function(){
console.log('\tFirst completed ' + cuID1);
});
var cuID2 = mongo.ObjectID();
writeStuff('Third', 'Forth', 2, cuID2);
dbEvents.on('Done'+cuID2, function(){
console.log("\tSecond completed " + cuID2)
});
var b = 0;
for(var i = 0; i < 1000000000; i++){
b++;
}
console.log("And I counted to " + b);
It seems that you're adding the 'Ready' listener after dispatching the 'Ready' event on the first run. Now when you fire of the second run, the first listener will be called. This explains why 'Wrote Second' is logged after 'Wrote Third' is called.
Move the addition of the listener to the beginning of writeStuff().
function writeStuff(first, second, status, cuID){
console.log('\t\tjust about to kick of number ' + status);
dbEvents.once('Ready', function (condata) {
console.log('Wrote '+ second)
dbEvents.emit('Done'+cuID)
});
console.log('Wrote '+ first);
if (status === 1) {
console.log('\t\tReady for '+ cuID);
dbEvents.emit('Ready');
} else {
console.log('\t\tReady (but not 1) for ' + cuID);
dbEvents.emit('Ready');
}
}