db.listCollections().toArray(function(err, collections){
for(i = 1; i < collections.length; i++){
if(i < parseInt(collections.length)){
var collect = db.collection(collections[i].name.toString(),function(){
collect.count(function(err,result){
console.log(collections[i].name.toString());
corrCount = corrCount + result;
});
});
}else{
collect = collections[i].name;
}
}
});
So my problem is that collect ends up being undefined and thus one can't count the amount of entries in the collection. Something tells me that I should solve this with callback but I keep failing. Also I don't understand why the console prints out 2 times before it shots of the error. I'm using nodejs with mongodb native driver.
You need to move the rest of the code out of the callback, like so:
db.listCollections().toArray(function(err, collections) {
for (i = 1; i < collections.length; i++) {
if (i < parseInt(collections.length)) {
var collect = db.collection(collections[i].name.toString());
collect.count(function(err, result) {
console.log(collections[i].name.toString());
corrCount = corrCount + result;
});
} else {
collect = collections[i].name;
}
}
});
Related
I am doing a modified version of collecting word co-occurrences, so I wrote my own javascript, and I am tracking the occurrences in three objects. However, once the objects get large (~8 million, 3 million, and 172000) a function that took 5 seconds per 100000 sentences now takes minutes to do one sentence with 30 words (30 tokens). I am nowhere near my RAM cap (I have 12 more GBs of RAM it could be using, and the program is only using 2.2GB). Using Node.js v17.3.1.
Why does my function take so long when the objects get bigger (even though the sentences remain the same length)? Should I be using a different object besides Javascript's default object, or is there a way improve the speed of access and setting these objects when they are so big?
Code:
let posCounts = {};
let negCounts = {};
// the number of times each word occurs
let wordCounts = {};
let tokens = // some function that gets tokens;
for (let k = 0; k < tokens.length; k++) {
// count word occurences
if (tokens[k] in wordCounts) {
wordCounts[tokens[k]] += 1;
} else {
wordCounts[tokens[k]] = 1;
}
for(let tok = k + 1; tok < tokens.length; tok++) {
if (tok == k) {
// avoid word to self cooccurrence
// should no longer be possible
continue;
} else {
// check which form of the cooccurence exists already in either count
actual_tok = (tokens[k] + "-" + tokens[tok]);
if(actual_tok in posCounts || actual_tok in negCounts) {
// no-op
} else {
actual_tok = (tokens[tok] + "-" + tokens[k]);
}
// condition set before this block of code
if(condition) {
if (actual_tok in posCounts) {
posCounts[actual_tok] += 1;
} else {
posCounts[actual_tok] = 1;
}
} else {
if (actual_tok in negCounts) {
negCounts[actual_tok] += 1;
} else {
negCounts[actual_tok] = 1;
}
}
}
}
}
Update: I've tried increasing the heap size via node train_matrices.js --max-old-space-size=12288 and node train_matrices.js --max_old_space_size=12288 (underline instead of dash), and that didn't work either.
Probably not the main issue in your code, but you can reduce the number of lookups by changing this structure from this:
if (tokens[k] in wordCounts) {
wordCounts[tokens[k]] += 1;
} else {
wordCounts[tokens[k]] = 1;
}
to this:
let token = tokens[k];
let cnt = wordCounts[token] || 0;
wordCounts[token] = cnt + 1;
And, as I said in a comment, I've read that a Map object with .get() and .set() is better suited when there are lots of dynamically created keys whereas plain objects are better suited when you have lots of objects with all the same keys (as the JS compiler can sometimes make a C-like struct for it), but this can't be done when you're regularly adding new keys.
The answer was to both use the increase memory flag node <YOUR_FILE_NAME>.js --max-old-space-size=12288 and change to using a Map instead of an object - thanks to #jfriend00 and #Norman Breau for the suggestions. That said, maps have a max capacity of 2^24 items or 1 GB, so I ended up using a modified version of the BigMap from this stackoverflow (modified to limit the total number of items still - ended up running completely out of RAM).
Modified code (you can replace BigMap with Map if you want):
let posCounts = new BigMap();
let negCounts = new BigMap();
let wordCounts = new BigMap();
let actual_tok;
tokens = // some code
// mark every cooccurrence
for (let k = 0; k < tokens.length; k++) {
// count word occurences
if (wordCounts.has(tokens[k])) {
wordCounts.set(tokens[k], wordCounts.get(tokens[k]) + 1);
} else {
wordCounts.set(tokens[k], 1);
}
for(let tok = k + 1; tok < tokens.length; tok++) {
if (tok == k) {
// avoid word to self cooccurrence
// should no longer be possible
continue;
} else {
// check which form of the cooccurence exists already in either count
actual_tok = (tokens[k] + "-" + tokens[tok]);
if(posCounts.has(actual_tok) || negCounts.has(actual_tok)) {
// no-op
} else {
actual_tok = (tokens[tok] + "-" + tokens[k]);
}
if(condition) {
if (posCounts.has(actual_tok)) {
posCounts.set(actual_tok, posCounts.get(actual_tok) + 1);
} else {
posCounts.set(actual_tok, 1);
}
} else {
if (negCounts.has(actual_tok)) {
negCounts.set(actual_tok, negCounts.get(actual_tok) + 1);
} else {
negCounts.set(actual_tok, 1);
}
}
}
}
}
}
I am pretty new to javascript and doing a course.I understand the return principle but when its nested in for example a nested for loop i get really confused.Would anyone mind to explain it to me?here is nothing nested but I got some example-code:
let generatePlayerBoard = (numberOfRows,numberOfColumns)=>{
const board = [];
for (var rowIndex = 0; rowIndex < numberOfRows.length; rowIndex++) {
const row = [];
for (var columnIndex = 0; columnIndex < numberOfColumns.length; columnIndex++) {
row.push(' ');
}
board.push(row);
}
return board;
};
console.log(generatePlayerBoard(2,3));
Thank you already for the help :D
The basics of nested returns is that a function can only return a value once, so once a function reaches its first return statement it ends the function.
Here's an example:
function hello () {
for(i = 0; i < 10; i++) {
if(i > 10) {
return 'Since i is never > 10 Im never reached so I never get to return anything'
} else {
return 'Hello!'
}
return 'The else statements returns before Im reached'
}
return 'Im never reached';
}
alert(hello());
Run that little script and you'll get an alert that says 'Hello!'
As mentioned before, in your script the return isn't nested, it's just run after all your other code has run.
Nesting is when you e.g. run:
if(1 < 2) {
if(2 < 3) {
// This second if is called a nested if because it's run inside another
// if statement
}
}
if(1 < 2 && 2 < 3) {
// This is also a form of nested if because it has the same effect as running
// the code above
}
I'm querying a mongo database to retrieve the tiles for the display in rougelike game. This is the function I use:
function get_display(){
var collections = ['austinsroom'];
var db = mongojs(uri, collections);
var temphtml = '';
for(var j = 0; j < 3; j++) {
console.log("y=" + String(j));
db.austinsroom.find({"y": j}, {}).sort({"x": 1}, function(err, records) {
if(err) {
console.log("There was an error executing the database query.");
return;
}
var i = records.length;
while(i--) {
temphtml += records[i].display;
}
temphtml += '<br>';
//console.log(temphtml);
//return temphtml;
//THE ONLY WAY I CAN GET ANYTHING TO PRINT IN THE CONSOLE IS IF I PUT IT INSIDE THE LOOP HERE
});
//console.log(temphtml);
//return temphtml;
//THIS DOES NOTHING
}
//console.log(temphtml);
//return temphtml;
//THIS DOES NOTHING
}
get_display();
If I put the console.log(temphtml) inside the loop, it prints out three times which isn't what I want. I only want the final string (i.e. ...<br>...<br>...<br>. Also I can't ultimately return the temphtml string, which is actually the important thing. Is this some quirk of javascript? Why would it not execute statements after the loop?
Also: is there a better way to retrieve every element of a grid that's stored in a mongo database, in order, so it can be displayed properly? Here's what the mongo documents look like:
{
"_id": {"$oid": "570a8ab0e4b050965a586957"},
"x": 0,
"y": 0,
"display": "."
}
Right now, the game is supposed to display a "." in all empty spaces using the x and y values for the coordinates. The database is indexed by "x" values.
See async.whilst. You want flow control of the for loop, for which this provides a callback to control each loop iteration.
var temphtml = "",
j = 0;
async.whilst(
function() { return j < 3 },
function(callback) {
db.austinsroom.find({"y": j }, {}).sort({"x": 1}, function(err, records)
temphtml += records.map(function(el) {
return el.display;
}).join("") + '<br>';
j++;
callback(err);
});
},
function(err) {
if (err) throw err;
console.log(temphtml);
}
)
Either that or use Promise.all() on collected promises to return "one big result". But you would also need to switch to promised-mongo from mongojs, as the nearest equivalent, since there are more mongodb drivers that actually support promises. That one is just the direct fork from mongojs:
var temphtml = "",
j = 0,
promises = [];
for ( var j=0; j < 3; j++ ) {
promises.push(db.austinsroom.find({"y": j }, {}).sort({"x": 1}).toArray());
promises.push('<br>'); // this will just join in the output
)
Promise.all(promises).then(function(records) {
temphtml += records.map(function(el) {
return el.display;
}).join("");
})
Not exactly the same thing, since it's one list output and not three, but the point is that the Promise objects defer until actually called to resolve, so you can feed the paramters in the loop, but execute later.
I do not use MongoDB but from what I am reading it is asynchronous. So what is happening is your db.austinsroom.find call fires another "thread" and returns to the for loop to continue the next iteration.
One way to do what you want is have a check at the end of your db.austinsroom.find function to see if you're done with the for loop. Something like:
function get_display()
{
var collections = ['austinsroom'];
var db = mongojs(uri, collections);
var temphtml = '';
var doneCounter = 0;
for(var j = 0; j < 3; j++)
{
console.log("y = " + String(j));
db.austinsroom.find({"y": j}, {}).sort({"x": 1}, function(err, records)
{
if(err)
{
console.log("There was an error executing the database query.");
return;
}
var i = records.length;
while(i--)
{
temphtml += records[i].display;
}
temphtml += '<br>';
// we're done with this find so add to the done counter
doneCounter++;
// at the end, check if the for loop is done
if(doneCounter == 3)
{
// done with all three DB calls
console.log(temphtml);
}
});
}
}
This is probably not the best way as I know nothing about MongoDB but it should put you on the right path.
I'm only crawling in JS so probably the solution is obvious.
I'm writing a Chrome extension which in browser action (after clicking on extension's button) reads several pages and from each of them it retrieves an integer. Then, it creates a table with these integers. I'm doing it qith AJAX so it's asynchronous and I want to have all integers before creating the table.
I've read these topics:
Pass in an array of Deferreds to $.when()
How to tell when multiple functions have completed with jQuery deferred
...and wrote a code that doesn't work.
var sum = 0;
document.addEventListener('DOMContentLoaded', function () {
var deferreds = [];
var users = [...], pages = [...];
for(i = 0; i < users.length; ++i) {
for(j = 0; j < pages.length; ++j)
deferreds.push(window[pages[j]](users[i], pages[j]));
}
$.when.apply(null, deferreds).done(function() {
alert("sum: " + sum);
});
});
function Name(user, page) {
$.get("http://page2/" + user, function(data) {
sum += 7;
});
return 7;
}
function Name2(user, page) {
$.get("http://page/" + user, function(data) {
sum += 84;
});
return 84;
}
So the alert prints 0 instead of 91. I cut the part with table as all fields are "undefined" anyway. If I get sum = 91, I'll probably get the table.
As I said - obvious mistake. The functions should look like:
function Name(user, page) {
return $.get("http://page2/" + user, function(data) {
sum += 7;
});
}
I've currently got a implementation which loops through an array within a json document (returned from mongoose) and looks for specific items as below
So what's happening is i'm passing an id in the request header to express and what i need to happen is for it to grab the associated story.users.id.name from the story.users array is returned and then once it has the name send do something with all the other items in the array.
I did try to do this like below:
for (var i = 0; i < story.users.length; i++) {
if (story.users[i].id._id == req.headers.id) {
var name = story.users[i].id.name
} else {
push.apns(story.users[i].id._id, name + " started a new story");
}
}
Where it would loop through grab the name and then do something with all the other users in the array, however sometimes the else argument fires first so the name variable is undefined.
So i resorted to running two if loops after each other like below:
for (var i = 0; i < story.users.length; i++) {
if (story.users[i].id._id == req.headers.id) {
var name = story.users[i].id.name
}
};
for (var i = 0; i < story.users.length; i++) {
if (story.users[i].id._id == req.headers.id) {
} else {
push.apns(story.users[i].id._id, name + " started a new story");
}
}
But there must be a better way to the above rather than looping through an array twice?
What you do looks like the right solution (with the goal you seem to have). There's no real simple way to do only one loop.
You could make it faster and cleaner, though :
var name; // this is just cleaner than to define it in the loop
for (var i = 0; i < story.users.length; i++) {
if (story.users[i].id._id == req.headers.id) {
name = story.users[i].id.name;
break; // don't loop over the other elements
}
};
for (var i = 0; i < story.users.length; i++) {
if (story.users[i].id._id !== req.headers.id) {
push.apns(story.users[i].id._id, name + " started a new story");
}
}