I'm trying to query a large set of results from a MongoDB over Python. I do this via JavaScript, because I want to get something like the grandchildren in a tree-like structure. My code looks like the following:
col = db.getCollection(...)
var res = new Array();
col.find( { "type" : ["example"] } ).forEach(
function(entry)
{
v1 = col.find( {"_id" : entry["..."]} )
... (walk through the structure) ...
vn = ...
res.push([v1["_id"], vn["data"]]);
}
);
return res;
Now, I'm having the problem, that the resulting array becomes very (too) large and the memory gets exceeded. Is there a way, to yield the results instead of pushing them into an array?
Alright, I think I know, what you mean. I created a structure like the following:
var bulksize = 1000;
var col = db.getCollection("..");
var queryRes = col.find( { ... } )
process = function(entity) { ... }
nextEntries = function()
{
var res = new Array();
for(var i=0; i<bulksize; i++)
{
if(hasNext())
res.push(process(queryRes.next()));
else
break;
}
return res;
}
hasNext = function()
{
return queryRes.hasNext();
}
The script separates the results into bulks of 1000 entries. And from Python side eval the noted script and then I do the following:
while database.eval('hasNext()'):
print "test"
for res in database.eval('return nextEntries()'):
doSth(res)
The interesting thing is, that the console always says:
test
test
test
test
test
test
Then I get the error:
pymongo.errors.OperationFailure: command SON([('$eval', Code('return nextEntries()', {})), ('args', ())]) failed: invoke failed: JS Error: ReferenceError: nextEntries is not defined nofile_a:0
This means, that the first calls of nextEntries() work, but then the function is not there, anymore. Could it be, that MongoDB does something like a clearing of the JavaScript cache? The problem does not depend on the bulksize (tested with 10, 100, 1000, 10000 and always the same result).
Alright, I found a line in the source code of MongoDB, which clears all JavaScripts that are used more than 10 times. So if no changes on the database server are wanted, it is necessary to query the database multiple times and send bulks to the client by selecting amounts of items with help of the skip() and limit() functions. This works surprisingly fast. Thanks for your help.
Related
I'm new to nodejs and jquery, and I'm trying to update one single html object using a script.
I am using a Raspberry pi 2 and a ultrasonic sensor, to measure distance. I want to measure continuous, and update the html document at the same time with the real time values.
When I try to run my code it behaves like a server and not a client. Everything that i console.log() prints in the cmd and not in the browesers' console. When I run my code now i do it with "sudo node surveyor.js", but nothing happens in the html-document. I have linked it properly to the script. I have also tried document.getElementsByTagName("h6").innerHTML = distance.toFixed(2), but the error is "document is not defiend".
Is there any easy way to fix this?
My code this far is:
var statistics = require('math-statistics');
var usonic = require('r-pi-usonic');
var fs = require("fs");
var path = require("path");
var jsdom = require("jsdom");
var htmlSource = fs.readFileSync("../index.html", "utf8");
var init = function(config) {
usonic.init(function (error) {
if (error) {
console.log('error');
} else {
var sensor = usonic.createSensor(config.echoPin, config.triggerPin, config.timeout);
//console.log(config);
var distances;
(function measure() {
if (!distances || distances.length === config.rate) {
if (distances) {
print(distances);
}
distances = [];
}
setTimeout(function() {
distances.push(sensor());
measure();
}, config.delay);
}());
}
});
};
var print = function(distances) {
var distance = statistics.median(distances);
process.stdout.clearLine();
process.stdout.cursorTo(0);
if (distance < 0) {
process.stdout.write('Error: Measurement timeout.\n');
} else {
process.stdout.write('Distance: ' + distance.toFixed(2) + ' cm');
call_jsdom(htmlSource, function (window) {
var $ = window.$;
$("h6").replaceWith(distance.toFixed(2));
console.log(documentToSource(window.document));
});
}
};
function documentToSource(doc) {
// The non-standard window.document.outerHTML also exists,
// but currently does not preserve source code structure as well
// The following two operations are non-standard
return doc.doctype.toString()+doc.innerHTML;
}
function call_jsdom(source, callback) {
jsdom.env(
source,
[ 'jquery-1.7.1.min.js' ],
function(errors, window) {
process.nextTick(
function () {
if (errors) {
throw new Error("There were errors: "+errors);
}
callback(window);
}
);
}
);
}
init({
echoPin: 15, //Echo pin
triggerPin: 14, //Trigger pin
timeout: 1000, //Measurement timeout in µs
delay: 60, //Measurement delay in ms
rate: 5 //Measurements per sample
});
Node.js is a server-side implementation of JavaScript. It's ok to do all the sensors operations and calculations on server-side, but you need some mechanism to provide the results to your clients. If they are going to use your application by using a web browser, you must run a HTTP server, like Express.js, and create a route (something like http://localhost/surveyor or just http://localhost/) that calls a method you have implemented on server-side and do something with the result. One possible way to return this resulting data to the clients is by rendering an HTML page that shows them. For that you should use a Template Engine.
Any DOM manipulation should be done on client-side (you could, for example, include a <script> tag inside your template HTML just to try and understand how it works, but it is not recommended to do this in production environments).
Try searching google for Node.js examples and tutorials and you will get it :)
i have a Meteor Application which is very "slow" as there are a lot of API-Calls.
What i try to do is to break apart the loading/calls.
What i just did is:
i have loading template via iron-router
i waitOn for the first API-Call has finished
then i start the next API-calls in the Template.myTemplate.rendered - function
This was already a big benefit for the speed of my Application, but i want to break it up even more as the second call is in fact more like 5-25 API-calls.
So what i try to do now is inside the rendered function is a self-calling function which calls itself as long as there are no more to do and saves the response inside a session. (Until now it just rewrites, but even to this point i can´t get)
Template.detail.rendered = function(){
//comma separated list of numbers for the API-Call
var cats = $(this.find(".extra")).attr('data-extra').split(',');
var shop = $(this.find(".extra")).attr('data-shop');
var counter = 0;
var callExtras = function(_counter){
var obj = {
categories : [cats[_counter]],
shop : shop
};
if(_counter <= cats.length){
Meteor.subscribe('extra', obj,function(result){
//TODO dickes todo... nochmal nachdenken und recherchieren
//console.log(_counter);
Session.set('extra',Extra.find('extra').fetch()[0].results);
counter++;
callExtras(counter);
});
}
};
callExtras(counter);
Session.set('loading_msg', '' );
};
Now i have again problems with my reactive parts of the app desscribed here - Meteor: iron-router => waitOn without subscribe As i can´t find a proper way to update my client-side per user base collection. Also in the docs it is described the publish method also creates a new collection. (The new document´s ID) here - http://docs.meteor.com/#/full/publish_added
here is the publish from server
Meteor.publish('extra', function(obj){
var that = this;
Meteor.call('extra', obj, function(error, result){
if (result){
//console.log(result);
that.added("extra", "extra", {results: result});
//that.changed('extra','extra',{results: result});
that.ready();
} else {
//that.ready();
}
});
});
So my question is: Is there from scratch a better way to structuring my code means solving the problem somehow different? If not how can i achive it the cleanest way? Because for my understanding this is just strange way to do it.
EDIT:
For example.
Can i do a per-user-collection (maybe only client-side like now) and push data from the server and just subscribe to this collection? But then how can i check when the async API-Call has finshed to start the next round. So the view gets data piece by piece. I am just confused right now.
My fault was simple as i thaught: You don´t need to use subscribe.
I just added "error,result" in the callback of Meteor.call
Only "result" leads to the result is always undefined.
var cats = $(this.find(".extra")).attr('data-extra').split(',');
var shop = $(this.find(".extra")).attr('data-shop');
var counter = 0;
var callExtras = function(_counter){
var obj = {
categories : [cats[_counter]],
shop : shop
};
if(_counter <= cats.length){
Meteor.call('extra', obj,function(error,result){
var actual_session = Session.get('extra');
if(actual_session === false){
actual_session = [];
}
actual_session = actual_session.concat(result);
Session.set('extra',actual_session);
counter++;
callExtras(counter);
});
}
};
callExtras(counter);
Then in the template helper
"extra" : function(){
return Session.get('extra');
},
I'm learning FRP using Bacon.js, and would like to assemble data from a paginated API in a stream.
The module that uses the data has a consumption API like this:
// UI module, displays unicorns as they arrive
beautifulUnicorns.property.onValue(function(allUnicorns){
console.log("Got "+ allUnicorns.length +" Unicorns");
// ... some real display work
});
The module that assembles the data requests sequential pages from an API and pushes onto the stream every time it gets a new data set:
// beautifulUnicorns module
var curPage = 1
var stream = new Bacon.Bus()
var property = stream.toProperty()
var property.onValue(function(){}) # You have to add an empty subscriber, otherwise future onValues will not receive the initial value. https://github.com/baconjs/bacon.js/wiki/FAQ#why-isnt-my-property-updated
var allUnicorns = [] // !!! stateful list of all unicorns ever received. Is this idiomatic for FRP?
var getNextPage = function(){
/* get data for subsequent pages.
Skipping for clarity */
}
var gotNextPage = function (resp) {
Array.prototype.push.apply(allUnicorns, resp) // just adds the responses to the existing array reference
stream.push(allUnicorns)
curPage++
if (curPage <= pageLimit) { getNextPage() }
}
How do I subscribe to the stream in a way that provides me a full list of all unicorns ever received? Is this flatMap or similar? I don't think I need a new stream out of it, but I don't know. I'm sorry, I'm new to the FRP way of thinking. To be clear, assembling the array works, it just feels like I'm not doing the idiomatic thing.
I'm not using jQuery or another ajax library for this, so that's why I'm not using Bacon.fromPromise
You also may wonder why my consuming module wants the whole set instead of just the incremental update. If it were just appending rows that could be ok, but in my case it's an infinite scroll and it should draw data if both: 1. data is available and 2. area is on screen.
This can be done with the .scan() method. And also you will need a stream that emits items of one page, you can create it with .repeat().
Here is a draft code (sorry not tested):
var itemsPerPage = Bacon.repeat(function(index) {
var pageNumber = index + 1;
if (pageNumber < PAGE_LIMIT) {
return Bacon.fromCallback(function(callback) {
// your method that talks to the server
getDataForAPage(pageNumber, callback);
});
} else {
return false;
}
});
var allItems = itemsPerPage.scan([], function(allItems, itemsFromAPage) {
return allItems.concat(itemsFromAPage);
});
// Here you go
allItems.onValue(function(allUnicorns){
console.log("Got "+ allUnicorns.length +" Unicorns");
// ... some real display work
});
As you noticed, you also won't need .onValue(function(){}) hack, and curPage external state.
Here is a solution using flatMap and fold. When dealing with network you have to remember that the data can come back in a different order than you sent the requests - that's why the combination of fold and map.
var pages = Bacon.fromArray([1,2,3,4,5])
var requests = pages.flatMap(function(page) {
return doAjax(page)
.map(function(value) {
return {
page: page,
value: value
}
})
}).log("Data received")
var allData = requests.fold([], function(arr, data) {
return arr.concat([data])
}).map(function(arr) {
// I would normally write this as a oneliner
var sorted = _.sortBy(arr, "page")
var onlyValues = _.pluck(sorted, "value")
var inOneArray = _.flatten(onlyValues)
return inOneArray
})
allData.log("All data")
function doAjax(page) {
// This would actually be Bacon.fromPromise($.ajax...)
// Math random to simulate the fact that requests can return out
// of order
return Bacon.later(Math.random() * 3000, [
"Page"+page+"Item1",
"Page"+page+"Item2"])
}
http://jsbin.com/damevu/4/edit
I have a Single Page Application that is working pretty well so far but I have run into an issue I am unable to figure out. I am using breeze to populate a list of projects to be displayed in a table. There is way more info than what I actually need so I am doing a projection on the data. I want to add a knockout computed onto the entity. So to accomplish this I registered and entity constructor like so...
metadataStore.registerEntityTypeCtor(entityNames.project, function () { this.isPartial = false; }, initializeProject);
The initializeProject function uses some of the values in the project to determine what the values should be for the computed. For example if the Project.Type == "P" then the rowClass should = "Red".
The problem I am having is that all the properties of Project are null except for the ProjNum which happens to be the key. I believe the issue is because I am doing the projection because I have registered other initializers for other types and they work just fine. Is there a way to make this work?
EDIT: I thought I would just add a little more detail for clarification. The values of all the properties are set to knockout observables, when I interrogate the properties using the javascript debugger in Chrome the _latestValue of any of the properties is null. The only property that is set is the ProjNum which is also the entity key.
EDIT2: Here is the client side code that does the projection
var getProjectPartials = function (projectObservable, username, forceRemote) {
var p1 = new breeze.Predicate("ProjManager", "==", username);
var p2 = new breeze.Predicate("ApprovalStatus", "!=", "X");
var p3 = new breeze.Predicate("ApprovalStatus", "!=", "C");
var select = 'ProjNum,Title,Type,ApprovalStatus,CurrentStep,StartDate,ProjTargetDate,CurTargDate';
var isQaUser = cookies.getCookie("IsQaUser");
if (isQaUser == "True") {
p1 = new breeze.Predicate("QAManager", "==", username);
select = select + ',QAManager';
} else {
select = select + ',ProjManager';
}
var query = entityQuery
.from('Projects')
.where(p1.and(p2).and(p3))
.select(select);
if (!forceRemote) {
var p = getLocal(query);
if (p.length > 1) {
projectObservable(p);
return Q.resolve();
}
}
return manager.executeQuery(query).then(querySucceeded).fail(queryFailed);
function querySucceeded(data) {
var list = partialMapper.mapDtosToEntities(
manager,
data.results,
model.entityNames.project,
'ProjNum'
);
if (projectObservable) {
projectObservable(list);
}
log('Retrieved projects using breeze', data, true);
}
};
and the code for the partialMapper.mapDtosToEntities function.
var defaultExtension = { isPartial: true };
function mapDtosToEntities(manager,dtos,entityName,keyName,extendWith) {
return dtos.map(dtoToEntityMapper);
function dtoToEntityMapper(dto) {
var keyValue = dto[keyName];
var entity = manager.getEntityByKey(entityName, keyValue);
if (!entity) {
extendWith = $.extend({}, extendWith || defaultExtension);
extendWith[keyName] = keyValue;
entity = manager.createEntity(entityName, extendWith);
}
mapToEntity(entity, dto);
entity.entityAspect.setUnchanged();
return entity;
}
function mapToEntity(entity, dto) {
for (var prop in dto) {
if (dto.hasOwnProperty(prop)) {
entity[prop](dto[prop]);
}
}
return entity;
}
}
EDIT3: Looks like it was my mistake. I found the error when I looked closer at initializeProject. Below is what the function looked like before i fixed it.
function initializeProject(project) {
project.rowClass = ko.computed(function() {
if (project.Type == "R") {
return "project-list-item info";
} else if (project.Type == "P") {
return "project-list-item error";
}
return "project-list-item";
});
}
the issue was with project.Type I should have used project.Type() since it is an observable. It is a silly mistake that I have made too many times since starting this project.
EDIT4: Inside initializeProject some parts are working and others aren't. When I try to access project.ProjTargetDate() I get null, same with project.StartDate(). Because of the Null value I get an error thrown from the moment library as I am working with these dates to determine when a project is late. I tried removing the select from the client query and the call to the partial entity mapper and when I did that everything worked fine.
You seem to be getting closer. I think a few more guard clauses in your initializeProject method would help and, when working with Knockout, one is constantly battling the issue of parentheses.
Btw, I highly recommend the Knockout Context Debugger plugin for Chrome for diagnosing binding problems.
Try toType()
You're working very hard with your DTO mapping, following along with John's code from his course. Since then there's a new way to get projection data into an entity: add toType(...) to the end of the query like this:
var query = entityQuery
.from('Projects')
.where(p1.and(p2).and(p3))
.select(select)
.toType('Project'); // cast to Project
It won't solve everything but you may be able to do away with the dto mapping.
Consider DTOs on the server
I should have pointed this out first. If you're always cutting this data down to size, why not define the client-facing model to suit your client. Create DTO classes of the right shape(s) and project into them on the server before sending data over the wire.
You can also build metadata to match those DTOs so that Project on the client has exactly the properties it should have there ... and no more.
I'm writing about this now. Should have a page on it in a week or so.
Our application currently shares messages between the Java and Javascript side. They are stored as resource bundles in the class path, and we have a custom controller that returns all the messages as Json. The client side code look like this:
// This calls the controller to get all the messages
var messages = MessageBundle();
var text = messages.get('my.message', 1);
This is great because we can mock "messages" in our unit tests.
I want to start using JAWR for this, since we already use it for other things. The problem is JAWR generates the following Javascript object:
var text = messages.my.message(1);
This means the code cannot be unit tested anymore unless the unit tests also define a global "messages" variable with the right nested objects. Is there a way around this? Any idea how to extend JAWR to make this unit-testable?
Currently my work around is:
function messages() {
var args = Array.prototype.slice.call(arguments);
var messageId = args.shift();
var messageFunc = window.messages;
messageId.split('.').forEach(function(part) {
messageFunc = messageFunc[part];
});
return messageFunc(args);
}
// Same syntax as the old one, but uses the JAWR object behind the scenes
// This function is easy to mock for a unit test
var text = messages('my.message', 1);
Thanks for any ideas!
Maybe next samples can help you.
1)
function messagesTester(funcPath,id) {
var args=funcPath.split('.'),root=window.messages;
for(var i=0;i<args.length;i++)root=root[args[i]];
return root(id);
// or if more that one parameter for *func*, then, for example:
// return root.apply(null,Array.prototype.slice(arguments,1));
}
var text = messagesTester('my.message',1);
2)
function messagesTester(funcPath) {
var args=funcPath.split('.'),root=window.messages;
for(var i=0;i<args.length;i++)root=root[args[i]];
return root;
}
// var text = messagesTester('my.message')( /*arguments list*/ );
var text = messagesTester('my.message')(1);