In an simple web app that i am building using underscore.js and jquery. For a list of all people ( js object ) I am filtering out list of all the places (js object) they visited. People list is a html table with a td having places image icon which on click displays list of all places they visited. Icon needs to be shown only for people who have visited at the least one place. The problem here is that people and places count comes around 2000, 100. So the code below executes 2000*100 combinations. The browser complains me of unresponsive script. Code is provided below
_.each(peopleList, function (person, index, list) {
//filter the respective places for people
var visitedPlaces = _.filter(places, function (place) {
return place.PeopleId == person.Id;
});
if (_.isEmpty(visitedPlaces)) {
$("a#" + place.PeopleId).remove();
}
});
Dead simple isn't it. For each person check if visited places has him tracked. How do i optimize the above code to unblocking and responsive. Tried putting in _.defer and _.delay at some places but no improvement
FWIW, here is how I would solve it in underscore.
function removeNonTravelers(people, visits) {
var travelers = _.pluck(visits, 'PeopleId'),
nonTravelers = _.reject(people, function (person) {
return _.contains(travelers, person.Id);
});
$(_.map(nonTravelers, document.getElementById)).remove();
}
http://jsfiddle.net/FWzeN/
My suggestion would be to drop underscore and use plain JS for this:
function removeNonTravelers(people, visits) {
var i, peopleId,
numPeople = people.length,
numVisits = visits.length,
index = {}, nonTravelers = [];
// index
for (i = 0; i < numVisits; i++) {
peopleId = visits[i].PeopleId;
if (!index.hasOwnProperty(peopleId)) {
index[peopleId] = 1;
} else {
index[peopleId]++;
}
}
// find HTML elements to remove
for (i = 0; i < numPeople; i++) {
peopleId = people[i].Id;
if (!index.hasOwnProperty(peopleId)) {
nonTravelers.push(document.getElementById(peopleId));
}
}
// remove them all at once
$(nonTravelers).remove();
}
This is reasonably fast. If I didn't make any mistake, your test case (2000 people, 100 places) times at more than 700 operations per second on my rather outdated laptop (DOM operations excluded).
Try for yourself: http://jsperf.com/where-not-exists-in-javascript
var hashMap = {};
_.each(places, function(place) {
hashMap[place.PeopleId] = place;
});
_.each(peopleList, function (person, index, list) {
//filter the respective project documents
var visitedPlaces = hashMap[person.id];
if (visitedPlaces) {
$("a#" + place.PeopleId).remove();
}
});
Related
I'm learning about DOM manipulation and, to practice, I'm trying to get the first 100 Twitter users who have twitted about #Javascript (see link). As, for now, Twitter doesn't allow you to use console.log() function in the browser console, I have managed to show any string visually in the HTML, in this case, under the search textbox.
This is my "custom" console.log()
function consoleLog(data) {
var searchTextBox = document.querySelector("#doc > div.topbar.js-topbar > div > div > div > div > div");
var p = document.createElement("p");
var innerText = document.createTextNode(data);
p.appendChild(innerText);
searchTextBox.appendChild(p);
}
For getting the usernames, I keep scrolling the page every 4 seconds and looking for usernames until I have 100 or more of them in my usernames variable.
var scrollPage = setInterval(function() {
window.scrollTo(0, document.body.scrollHeight);
}, 4000);
var usernames = [];
while (true) { // <------ PROBLEM
if (usernames.length < 100) {
consoleLog("Getting usernames again");
usernames = getUsernames();
}
else {
consoleLog("We have ENOUGH usernames. BREAK");
clearInterval(scrollPage);
printUsernames();
break;
}
}
function printUsernames() {
for(var user of usernames) {
consoleLog(user);
}
}
function getUsernames() {
var results = [];
var usernameNodes = document.getElementsByClassName("username u-dir u-textTruncate");
var username = usernameNodes[0].textContent;
for(var node of usernameNodes) {
results.push(node.textContent);
}
return results.filter(isUnique);
}
function isUnique(value, index, self) {
return self.indexOf(value) === index;
}
The problem is that the while loop enters in infinte loop and I don't know why. I think the logic of the code is correct. In fact, if I first copy and paste all the declared functions to the browser console, then start the scrollPage interval and, lastly, start the while loop, it works well. The problem comes when I copy and paste all the code at one time in the browser console. It is like the executions of the interval and the while loop conflict in some way. But I can't understand.
Its better to have while conditioned like this:
var usernames = [];
// This will automatically end when length is greater or equall 100
// no need to break
while (usernames.length < 100) {
consoleLog("Getting usernames again");
usernames = getUsernames();
}
consoleLog("We have ENOUGH usernames.");
clearInterval(scrollPage);
printUsernames();
This is all still pretty new to me but I am running into an interesting behavior issue when generating an array of numbers in a NodeJS application that handles .nessus result files. First, some details on what I am trying to accomplish. My application generates an array [1234,1223,1222] of entries from an uploaded "results" file that is then used to query a mongodb instance to determine if those entries are currently in the DB. If those entries are not currently in the mongodb instance, it redirects to a page where a user can edit them before being added. If there are no new entries, it goes to a page to generate a report on the entries.
When a file is uploaded, it stores the new entries. In .nessus files, sometimes there is more than one host with entries. That changes the json structure and the function needs to iterate a little differently. The following function is how those entries are stored. This is important as this is where the weird behavior originates (I think)
function parsePluginNumbers(json){
var pluginNumbers = []
//Let's check the number of hosts
var hostLength = json['NessusClientData_v2']['Report'].ReportHost.length
if (hostLength != undefined) {
for (var i = 0; i < hostLength; i++) { //Since there is more than 1, need to iterate over each host to find the findings.
var item_length = json['NessusClientData_v2']['Report'].ReportHost[i].ReportItem.length
for (var t = 0; t < item_length; t++) { //Iterate through each finding on each host
if (json['NessusClientData_v2']['Report'].ReportHost[i].ReportItem[t].risk_factor != 'None') {
var newEntry = json['NessusClientData_v2']['Report'].ReportHost[i].ReportItem[t].pluginID
if (pluginNumbers.indexOf(newEntry) == -1) {
pluginNumbers.push(newEntry)
}
else {
continue
}
} else {
continue
}
}
}
} else {
var item_length = json['NessusClientData_v2']['Report']['ReportHost'].ReportItem.length
for (var t = 0; t < item_length; t++) { //Iterate over findings
if (json['NessusClientData_v2']['Report']['ReportHost'].ReportItem[t].risk_factor != 'None') {
var newEntry = json['NessusClientData_v2']['Report']['ReportHost'].ReportItem[t].pluginID
if (pluginNumbers.indexOf(newEntry) == -1) {
pluginNumbers.push(newEntry)
}
else {
continue
}
} else {
continue
}
}
}
return pluginNumbers
}
Once those plugins are stored. Another function is called to look if those results are in the mongodbinstance. In this function, those plugins are in an array "pluginsTotal".
function queryForNewResultsInANessusFile(pluginsTotal, collectionname, filename){ //function to call mongodb query and send results to parseNewFindings and parseOldFindings.
var db = req.db;
var collection = db.get(collectionname);
collection.find({ 'PluginNumber' : { $in: pluginsTotal }}, 'FindingTitle FindingDescription Remediation Mitigation SeeAlso PluginFamily PluginNumber CVE Risk -_id', function(error, result){
var newPluginArray = parseOutFindingNumbersInMongoDB(result, pluginsTotal);
//IF statements go here with specific redirects as needed to check if there are new values not in the repo
}
During this collection.find call, there is a function parseOutFindingNumbersInMongoDB that is called to determine if there are plugins in the .nessus results file that are not in the repo. It compares the results from collection.find and pluginsTotal (generated from the first function) and returns an array of the new plugins that are not in the repo. The function details are below:
function parseOutFindingNumbersInMongoDB(repoResults, reportPlugins) {
for (var i = 0; i < repoResults.length; i++){
var index = reportPlugins.indexOf(repoResults[i].PluginNumber);
if (index != -1) {
reportPlugins.splice(index, 1);
}
else {
continue
}
}
return reportPlugins
}
Now to my question --- When I upload a .nessus file with more than one host, parseOutFindingNumberInMongoDB always returns empty even though there are new entries. What gives? Is it the way I parse out the numbers to begin with in the parsePluginNumbers function or is because it is called in the collection.find synchronous function (This seems unlikely as if there is one host, it returns the new plugin values as I want)? Any thoughts/ideas/review would be much appreciated as I cannot figure out what is wrong. I have checked the data types within the array before being passed into the functions and they all match up.
It's always returning an empty array because every element in the array matches the condition to be spliced.
You first retrieve elements that have a PluginNumber in pluginTotal array with this filter { $in: pluginsTotal }
collection.find({ 'PluginNumber' : { $in: pluginsTotal }}, 'FindingTitle FindingDescription Remediation Mitigation SeeAlso PluginFamily PluginNumber CVE Risk -_id', function(error, result){
var newPluginArray = parseOutFindingNumbersInMongoDB(result, pluginsTotal);
}
Then you remove all elements that have a PluginNumber in pluginTotal
var index = reportPlugins.indexOf(repoResults[i].PluginNumber);
if (index != -1) {
reportPlugins.splice(index, 1);
}
So the result is always an empty array.
I can't add new name and value ff. this given condition:
$.each(names, function (i, name) {
$.get('https://www.example.com/path/' + name, function (data) {
var arrNow = CSVToArray(data, ',');
allArr.push(arrNow);
counter++;
if (counter === names.length) {
for (var j = 0; j < allArr.length; j++) {
for (var k = 1; k < allArr[j].length; k++) {
//console.log(allArr[j][k][0] + ': ' + allArr[j][k][1]);
//var f = moment(allArr[j][k][0]).format('lll');
var f = allArr[j][k][0];
json.push({
"datetime": f
});
if (j == 0) {
if (json[k].datetime === allArr[0][k][0]) {
var newAtt = "water_actual";
var newValue = allArr[0][k][1];
json[k][newAtt] = newValue;
}
}
if (j == 1) {
if (json[k].datetime === allArr[1][k][0]) {
var newAtt = "rainfall_actual";
var newValue = allArr[1][k][1];
json[k][newAtt] = newValue;
}
}if (j == 2) {
if (json[k].datetime == allArr[2][k][0]) {
var newAtt = "forecast_water";
var newValue = allArr[2][k][1];
json[k][newAtt] = newValue;
}
}
}
}
};
});
});
I was able to add a new namewater_actual and its value using if statement. If the datetime from the json object matches to the array value(date and time), I'd like to add it with its specific name as stated above. But I can't seem to make it work.
Here's the fiddle.
If I may provide some general feedback: it's probably good practice to simplify your code to the minimum example that reproduces your problem. Not only can that drastically increase your chances of fixing it yourself, it also increases the odds that you'll get help here.
With that in mind, consider the basic structure of what you're trying here:
var someNames = ["foo", "bar"];
var allTheData = [{
"aardvark": true
}];
$.each(someNames, function (i, name) {
$.get('http://example.com/api/' + name, function (data) {
data.aNewProperty = 'wombat';
allTheData.push(data);
});
});
console.log(allTheData);
Here, $.each iterates through everything in someNames and then proceeds immediately to the console.log statement. For all we know, each individual API call ($.get) could take seconds, or minutes. By this time we've already tried to use the contents of allTheData, which may or may not have been modified.
To avoid this sort of thing in legacy JavaScript we can make use of the callback already provided by $.get:
$.get('http://example.com/api/' + name, function (data) {
data.aNewProperty = 'wombat';
console.log(data);
});
Inside the callback, we know for sure that the API request has already completed (although the above assumes that it succeeded, which is a whole other kettle of fish). This would output the result of each API request as the responses arrive, though not necessarily in the order you'd expect!
JavaScript's asynchronous nature tended to lead in the past to a whole lot of callbacks. With the advent of ES6 we have some more options available to us, especially promises.
I have an application that uses the DataTables jQuery library to render content in my target browser IE8. The problem is when I push a big array to be rendered, IE8 sometimes throws up the infamous long running script error.
After profiling the app it seems that the call to __fnAddData in the following code is causing the problem:
if (bUsePassedData) {
for (var i = 0, len = oInit.aaData.length; i < len; i++) {
_fnAddData(oSettings, oInit.aaData[i]);
}
} else if (oSettings.bDeferLoading ||
(oSettings.sAjaxSource === null && oSettings.ajax === null)) {
_fnAddTr(oSettings, $(oSettings.nTBody).children('tr'));
}
I was looking around for solutions and saw Nicholas Zakas' write up here and tons of other solutions that would work if the for loop wasn't inside of an if else if "block". When I tried, on my 1st attempt of many, to wrap it in a setTimeout function it of course didn't work because the 2nd part of the if else if resolves to true.
(oSettings.sAjaxSource === null && oSettings.ajax === null) // true
What is a good solution for this? Thanks in advance.
I think you might split up your function in 3 functions:
Before the if statement.
Processing the oInit.aaData
After the if statement
Here is the code split up in 3 functions:
function beforeIf(){
if (bUsePassedData) {
procesData(oSettings,oInit.aaData.concat());
} else if (oSettings.bDeferLoading ||
(oSettings.sAjaxSource === null && oSettings.ajax === null)) {
_fnAddTr(oSettings, $(oSettings.nTBody).children('tr'));
}
afterIF();
}
function processData(oSettings,arr){
//process in chuncks of 50;
// setTimeout takes a long time in IE
// it'll noticibly slow donw your script when
// only processing one item at the time
var tmp=arr.splice(0,50);
for (var i = 0, len = tmp.length; i < len; i++) {
_fnAddData(oSettings, tmp[i]);
}
if(arr.length!==0){
setTimeout(function(){
processData(oSettings,arr);
},0);
return;
}
afterIf();
}
function afterIf(){
//continue processing
}
Thanks #HMR. You helped to bring me closer to my goal. To solve the problem I worked my code down to this IIFE:
(function processData(oSettings, arr) {
var tmp = arr.splice(0, 50);
tickApp.$orders.dataTable().fnAddData(tmp);
if (arr.length !== 0) {
setTimeout(function () {
processData(oSettings, arr);
}, 0);
}
}(oSettings, oInit.aaData.concat()));
Instead of using the private _fnAddData function I opted for the DataTables public fnAddData (http://datatables.net/ref#fnAddData) function. This way I am able to push 50 rows at a time into the table which is stored in the tickApp.$orders object which I just a reference to my jQuery object that stores the table in memory:
tickApp.$orders = $('#orders');
In another part of my code. They way you had it it was still pushing 1 row at a time instead of the whole 50.
Thanks again.
If you are using ajax to fetch your data, you can override "fnServerData" in your datatables config object. This will allow you to fetch the data to be loaded and then process it however you want.
In my case, I have a generic datatables config object that I use for all my datatables. I override the default fnServerData function with one that passes rows to the datatable in sets of 200 using fnAddData and setTimeout to call the function again until all the data has been processed, finally I call fnDraw to draw the table.
var DEFAULT_CHUNK_SIZE = 200;
function feedDataToDataTableInChunks(startIndex, data, oSettings) {
var chunk = data.slice(startIndex, DEFAULT_CHUNK_SIZE);
oSettings.oInstance.fnAddData(chunk, false);
if((startIndex += DEFAULT_CHUNK_SIZE) < data.length) {
setTimeout(function () {
feedDataToDataTableInChunks(startIndex, data, oSettings);
});
} else {
oSettings.oApi._fnInitComplete(oSettings, data);
oSettings.oInstance.fnDraw();
}
}
var config = {fnServerData: function(){
oSettings.jqXHR = $.getJSON(sSource, aoData)
.done(function (result) {
feedDataToDataTableInChunks(0, result || [], oSettings);
});
}}
I am using datatables version 1.9.4
I have a web application where I load (via ajax) a dictionary file (1MB) into the javascript array. I found the reason why the Mobile Safari crashes after 10 seconds. But now what I'm wondering is how do I get around this issue?
On the link above the answer suggest using setInterval, but this would mean I would have to have a dictionary file chunked into pieces and have them loaded one by one. This surely could be done, but I would have to make a lot of chunks taking into account the internet speed and too many requests would take forever for the page to load (and if I make the chunks too big it could happen that some mobile users wouldn't be able to download the chunk in a given 10second period).
So, my question is: has anyone encountered this kind of problem and how did you go about it? A general push in the right direction is appreciated.
edit:
This is the js code which I use to load the dictionary:
var dict = new Trie();
$.ajax({
url: 'data/dictionary_342k_uppercase.txt',
async: true,
success: function (data) {
var words = data.split('\n');
for (var i = words.length - 1; i >= 0; i--) {
dict.insert(words[i]);
}
},
error: function(){
$('#loading-message').text("Problem s rječnikom");
}
});
Trie.js:
function Trie () {
var ALPHABET_SIZE = 30;
var ASCII_OFFSET = 'A'.charCodeAt();
this.children = null;
this.isEndOfWord = false;
this.contains = function (str) {
var curNode = this;
for (var i = 0; i < str.length; i++) {
var idx = str.charCodeAt(i) - ASCII_OFFSET;
if (curNode.children && curNode.children[idx]) {
curNode = curNode.children[idx];
} else {
return false;
}
}
return curNode.isEndOfWord;
}
this.has = function (ch) {
if (this.children) {
return this.children[ch.charCodeAt() - ASCII_OFFSET] != undefined;
}
return false;
}
this.next = function (ch) {
if (this.children) {
return this.children[ch.charCodeAt() - ASCII_OFFSET];
}
return undefined;
}
this.insert = function (str) {
var curNode = this;
for (var i = 0; i < str.length; i++) {
var idx = str.charCodeAt(i) - ASCII_OFFSET;
if (curNode.children == null) {
curNode.children = new Array(ALPHABET_SIZE);
curNode = curNode.children[idx] = new Trie();
} else if (curNode.children[idx]) {
curNode = curNode.children[idx];
} else {
curNode = curNode.children[idx] = new Trie();
}
}
curNode.isEndOfWord = true;
return curNode;
}
}
This is a very common issue once you start doing processing in JS. If the Mobile Safari issue is the cause then what you want to do is figure out where the CPU time is going here.
I'm assuming it's the dict.insert() loop and not the data.split() call (that would be a bit more difficult to manage).
The idea here is to split up the dict.insert() loop into functional blocks that can be called asynchronously in a sequenced loop (which is what the setupBuildActions function does). After the first block each subsequent block is called via setTimeout, which effectively resets the function-time counter in the JS runtime (which seems to be what's killing your process).
Using the Sequencer function means you also keep control of the order in which the functions are run (they always run in the sequence they are generated in here and no two or more functions are scheduled for execution at the same time). This is much more effective than firing off thousands of setTimeout calls without callbacks. Your code retains control over the order of execution (which also means you can make changes during execution) and the JS runtime isn't overloaded with scheduled execution requests.
You might also want to check the node project at https://github.com/michiel/sequencer-js for more sequencing examples and http://ejohn.org/blog/how-javascript-timers-work/ for an explanation on setTimeout on different platforms.
var dict = new Trie();
// These vars are accessible from all the other functions we're setting up and
// running here
var BLOCKSIZE = 500;
var words = [];
var buildActions = [];
function Sequencer(funcs) {
(function() {
if (funcs.length !== 0) {
funcs.shift()(arguments.callee);
}
})();
}
// Build an Array with functions that can be called async (using setTimeout)
function setupBuildActions() {
for (var offset=0; offset<words.length; offset+= BLOCKSIZE) {
buildActions.push((function(offset) {
return function(callback) {
for (var i=offset; i < offset + BLOCKSIZE ; i++) {
if (words[i] !== null) { // ugly check for code brevity
dict.insert(words[i]);
}
}
// This releases control before running the next dict.insert loop
setTimeout(callback, 0);
};
})(offset));
}
}
$.ajax({
url: 'data/dictionary_342k_uppercase.txt',
async: true,
success: function (data) {
// You might want to split and setup these calls
// in a setTimeout if the problem persists and you need to narrow it down
words = data.split('\n');
setupBuildActions();
new Sequencer(buildActions);
},
error: function(){
$('#loading-message').text("Problem s rječnikom");
}
});
Here's an example using setTimeout to defer the actual insertion of words into your trie. It breaks up the original string into batches, and uses setTimeout to defer processing of inserting each batch of words. The batch size in my example is 5 words.
The actual batch insertion happens as subsequent event handlers in the browser.
It's possible that just breaking the words up into batches might take too long. If you hit this problem, remember you can chain setTimeout() calls, eg iterating for a while then using setTimeout to schedule another event to iterate over some more, then setTimeout again, etc.
function addBatch(batch)
{
console.log("Processing batch:");
for (var i = 0; i < batch.length; i++)
console.log(batch[i]);
console.log("Return from processing batch");
}
var str = "alpha\nbravo\ncharlie\ndelta\necho\nfoxtrot\n" +
"golf\nhotel\nindia\njuliet\nkilo\nlima\n" +
"mike\nnovember\noscar\npapa\nquebec\n" +
"romeo\nsierra\ntango\nuniform\n" +
"victor\nwhiskey\nxray\nyankee\nzulu";
var batch = []
var wordend;
for (var wordstart = 0; wordstart < str.length; wordstart = wordend+1)
{
wordend = str.indexOf("\n", wordstart);
if (wordend < 0)
wordend = str.length;
var word = str.substring(wordstart, wordend);
batch.push(word);
if (batch.length > 5)
{
setTimeout(addBatch, 0, batch);
batch = [ ];
}
}
setTimeout(addBatch, 0, batch);
batch = [ ];