Develop Reference

Translate aggregation operation in MongoDB to MapReduce

I've been trying to translate this query into MapReduce for a few days. Specifically, I need to figure out how many different cars have driven "N" kilometers.
Query:
db.adsb.group({
"key": {
"KM": true
},
"initial": {
"countCar": 0
},
"reduce": function(obj, prev) {
if (obj.Matricula != null) if (obj.Matricula instanceof Array) prev.countCar += obj.Matricula.length;
else prev.countCar++;
},
"cond": {
"KM": {
"$gt": 10000,
"$lt": 45000
}
}
});
Each document in Mongo has this form:
{
"_id" : ObjectId("5a8843e7d79a740f272ccc0a"),
"KM" : 45782,
"Matricula" : "3687KTS",
}
I'm trying to get something like:
/* 0 */
{
“KM” : 45000,
“total” : 634
}
/* 1 */
{
“KM” : 46000,
“total” : 784
}
My code is below, and it compiles but does not give me the expected results.
In particular, every time I enter 'reduce' it seems to reset all the values to 0, which prevents me from accumulating the registrations.
One of my problems is that when handling large amounts of information, the function must iterate several times ' reduce'.
I also don't know if it could be done that way, or I would need to return a list of car plates and their counter together in 'reduce'; and then in finalize add it all up.
// Map function
var m = function() {
if (this.KM > 10000 && this.KM < 45000) { // So that i can get KM grouped together by thousands (10000, 20000, 30000...)
var fl = Math.round(this.KM / 1000) * 1000;
var car = this.Matricula
emit (fl, car);
//print("map KM=" + fl + " Matricula= " + car);
}
};
// Reduce function
var r = function(key, values) {
var ya_incluido = false;
var cars_totales = 0;
var lista_car = new Array();
//print( key + " ---- " + values);
for (var i=0; i < values.length;i++)
{
for (var j=0; j < lista_car.length;j++)
{
if(values[i] == lista_car[j]) { //If it is already included, don't aggregate it
ya_incluido = true;
}
} if (ya_incluido != true) { //If it is not included, add it to lista_av list.
lista_car.push(values[i]);
} ya_incluido = false;
}
cars_totales = lista_av.length; //The number of distinct cars is equal to the lenght of the list we created
return cars_totales;
};
// Finalize function
var f = function(key,value) {
// Sum up the results?
}
db.runCommand( {
mapReduce: "dealer",
map: m,
reduce: r,
finalize: f,
out: {replace : "result"}
} );
I found the answer and a really good explanation here: https://stackoverflow.com/a/27532153/13474284

I found the answer and a really good explanation here: https://stackoverflow.com/a/27532153/13474284
I couldn't find a way to return in 'reduce' the same thing that came from ' map' . And since it was run several times, it only got the results of the last iteration. The way it appears in the link, the problem is solved without any difficulty.

Trying to write a function for interestCalculator but I keep getting errors

let data =
[
{
principal: 2500,
time: 1.8
},
{
principal: 1000,
time: 5
},
{
principal: 3000,
time: 1
},
{
principal: 2000,
time: 3
}
]
console.log(data)
function interestCalculator() {
for (let i of data)
if (data[i].principal>=2500 && data[i].time>1 && data[i].time<3){
return "rate = 3"
} else if (data[i].principal>=2500 && data[i].time>=3){
return "rate = 4"
} else if (data[i].principal<2500 && data[i].time<=1 ){
return "rate = 2"
} else {
return "rate = 1"
}
}
I'm trying to write a function called "interestCalculator" that takes an array as a single argument and does the following:
Determine the rate applicable using the conditions:
If the principal is greater than or equal to 2500 and the time is greater than 1 and less than 3, then rate = 3
If the principal is greater than or equal to 2500 and the time is greater than or equal to 3, then rate = 4
If the principal is less than 2500 or the time is less than or equal to 1, then rate = 2
Otherwise, rate = 1;
But then I keep getting errors saying principal is undefined

Please update loop statement. Use in instead of of
for (let i in data)

instead of data[i].principial, you should write like i.principial.
Here i is not index.
it is just an element from the array.

you didnt have braces around your for loop and "for of" returns objects (ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...of)
function interestCalculator() {
for (let obj of data) {
if (obj.principal>=2500 && obj.time>1 && obj.time<3){
return "rate = 3"
}
else if (obj.principal>=2500 && obj.time>=3){
return "rate = 4"
}
else if (obj.principal<2500 && obj.time<=1 ){
return "rate = 2"
}
return "rate = 1"
}
}

when you loop over the array
for (let i of data) { ... }
i is already the object, so you need to use i.principal and i.time.
If you want instead to access the ith element with data[i] then you need to loop the array with
for (let i = 0, j = data.length; i++; i < j) { ... }
Finally as a side note it's worth to point that you have some return statements inside the loop, so the function will just check the first object in the array and will soon return, no matter how many elements may have data.

How to handle extremely large data in node.js array

I am working on a Node.js server that responds with JSON. I have a legacy server that contains the data. The Node.js server acts as a middle-man connecting the user to the data. The legacy server returns data as a csv with columns being keys and rows being objects. I want to turn this into an array of json objects. I am using this tool to do just that: https://github.com/Keyang/node-csvtojson. I can either have the tool construct the output for me or I can have a function called with each row passed in and construct it myself. I am currently doing the later.
The users of my application can pass in a search value which means only rows that contain this value should be returned. The user can also pass in filter keys and values which means only rows where key contains value for every key and value given should be returned. They can of course give both search and filter values or neither. They also give a sort key which is the key in the objects to sort the array by. They also give me a sort order: ascending or descending. They also give me a page size and offset to return, which is for pagination.
My question is, what is the best way to handle this with data sets that could contain millions of rows? I can't modify the legacy server. Here is what I have. It works but I would like to improve performance as much as possible. Is there a more efficient way to do the sorting, searching, filtering, pagination, etc? Would it be better to add to the array and then sort instead of using a binary search tree and sorting during add? Is there a way I could use streams to improve performance? I expect the server to be limited by CPU performance not RAM. I am looking for any recomendations for better performance. Thanks! :)
EDIT: Also, what part of the below code is blocking?
function searchRow(row, search) {
if (search) {
for (var key in row) {
if (row.hasOwnProperty(key) && row[key].toString().toLowerCase().indexOf(search.toLowerCase()) > -1) {
return true;
}
}
return false;
} else {
return true;
}
}
function filterByField(obj) {
return obj.field === this.valueOf();
}
function filterRow(row, filter) {
if (filter) {
filter = JSON.parse(decodeURIComponent(filter));
var passed = true;
for (var key in filter) {
if (filter.hasOwnProperty(key)) {
var index = row[key].toString().toLowerCase().indexOf(filter[key].toString().toLowerCase());
passed = passed && (index > -1);
}
if (!passed) {
break;
}
}
return passed;
} else {
return true;
}
}
function orderByKey(key, reverse, a, b) {
return function (a, b) {
var x = (a[key] || '').toString().toLowerCase();
var y = (b[key] || '').toString().toLowerCase();
if (x > y) {
return reverse ? -1 : 1;
} else if (y > x) {
return reverse ? 1 : -1;
} else if (hash(a) > hash(b)) {
return reverse ? -1 : 1;
} else if (hash(b) > hash(a)) {
return reverse ? 1 : -1;
} else {
return 0;
}
};
}
function sortedInsert(element, array, key, reverse) {
array.splice(locationOf(element, array, key, reverse) + 1, 0, element);
return array;
}
function locationOf(element, array, key, reverse, start, end) {
if (array.length === 0) {
return -1;
}
start = start || 0;
end = end || array.length;
var pivot = parseInt(start + (end - start) / 2, 10);
var c = orderByKey(key, reverse, element, array[pivot]);
if (end - start <= 1) {
return c == -1 ? pivot - 1 : pivot;
}
switch (c) {
case -1: return locationOf(element, array, key, reverse, start, pivot);
case 0: return pivot;
case 1: return locationOf(element, array, key, reverse, pivot, end);
}
}
function getTable(path, columns, delimiter, query) {
var deferred = q.defer();
var headers = [];
var data = [];
delimiter = delimiter ? delimiter : '\t';
var converter = new Converter({
delimiter: delimiter,
noheader: true,
headers: columns,
workerNum: os.cpus().length,
constructResult: false
});
converter.on("error", function(errMsg, errData) {
deferred.reject(errMsg);
});
converter.on("record_parsed", function(row) {
if (searchRow(row, query.search) && filterRow(row, query.filter)) {
sortedInsert(row, data, query.sort || headers[0].split("!").pop(), query.order === 'desc');
}
});
converter.on("end_parsed", function() {
var offset = parseInt(query.offset || 0);
var limit = parseInt(query.limit || data.length);
var total = data.length;
data = data.slice(offset, offset + limit);
deferred.resolve({
"total": total,
"rows": data
});
});
var options = {
url: config.url + path,
gzip: true,
method: 'GET'
};
request(options, function (error, response, body) {
if (error || response.statusCode != 200) {
deferred.reject(error);
}
}).pipe(converter);
return deferred.promise;
}

How to get the last N objects before a certain object in a Javascript array?

I'm trying to implement the endless loading of items in Javascript. Like the effect you get when you scroll through your messages in your favorite messaging application. I have a big array like this (+1000 objects):
var array = [
{ id : 1, text: "First"},
{ id : 2, text: "Second"},
{ id : 3, text: "Third"},
{ id : 4, text: "Forth"},
{ id : 5, text: "Fifth"},
{ id : 6, text: "Sixth"},
{ id : 7, text: "Seventh"},
...
];
Now I want to load only 10 items at a time. For example I'm showing only the items with the id of 30 to 39. Now the user wants to see the items before 30. What is the best way to select last 10 items before that object with the id of 30? As mentioned before, the array's size is big so performance does matter here.
EDIT
The example above is just one case. I should be able to filter my array 10 items at a time as many times as needed.
What I'm trying to achieve is loading a big array but not all at once. I want to load 10 items at a time. I'm keeping track of the first object in my filtered array (e.g 30) and then I want to get the last 10 objects before that particular object.
EDIT 2
This is my View:
<div ng-repeat="message in messages" class="message-wrapper">
// Show content of message
</div>
Now initially i'm showing the last 10 items in messages
$scope.init = function(){
var messages = Database.AllMessages(); // This function returns all my messages
$scope.messages = messages.length > 10 ? messages.slice(-10) : messages;
}
Now let's say the items returned by this function are the items with the Id of 30 to 39. The user scrolls up and wants to see the messages prior to 30. So how can i filter the whole array returned by AllMessages() to get 10 last items before the 30 message?
Thanks in advance for any insights.

You want to query your data source as efficient as possible. Make sure the array is sorted by id first:
array.sort(function(a, b) {
if (a.id === b.id) return 0;
if (a.id > b.id) return 1;
return -1;
});
Then, binary search can be performed to find the index of the element you're looking for:
var search = function(arr, val) {
if (arr.length === 0) {
return -1;
}
var max = arr.length - 1, min = 0;
while (max !== min) {
var index = Math.floor((max + min) / 2);
if (arr[index].id === val) {
return index;
}
else if (arr[index].id < val) {
min = index;
}
else {
max = index;
}
}
return arr[index].id === val ? index : -1;
}
// find index of element with desired id.
var index = search(array, 30);
Once we know the index, we simply have to select the elements before/after the index:
var rangeMin = index - 10; // Inclusive. 10 is the maximum number of elements you want to render.
var rangeMax = index; // Not inclusive.
if (rangeMin < 0) { rangeMin = 0; }
if (rangeMax > array.length) { rangeMax = array.length; }
var elementsToRender = array.slice(rangeMin, rangeMax);
elementsToRender will now contain all the elements you want to render.

Now let's say the items returned by this function are the items with the Id of 30 to 39. The user scrolls up and wants to see the messages prior to 30. So how can i filter the whole array returned by AllMessages() to get 10 last items before the 30 message?
(...) my view should show the items 20 to 39
Suppose allMessages contains all messages (very large array).
Suppose ctrl is a controller instance
Suppose ctrl.messages contains the items currently displayed.
Suppose currentIndex is 30
Suppose stepSize is 10
Then the following code will extend messages to contain items 20 to 39 and will set currentIndex to 20:
currentIndex -= stepSize; // the new first message will have index 20
var extraItems = allMessages.slice(currentIndex, currentIndex+stepSize);
Array.prototype.push.apply(extraItems, ctrl.messages); // append ctrl.messages to extraItems
ctrl.messages = extraItems;
For more information about Array.prototype.push.apply, see Mozilla (scroll down to 'Merging two arrays').
Here is small app to demonstrate it (you'll have to add logic to protect the user to cross the array boundaries):
<html>
<head>
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.2/angular.min.js"></script>
<script>
var myApp = angular.module('myApp', []);
myApp.controller('MyController', function() {
var ctrl = this;
var allMessages = Database.AllMessages();
var stepSize = 10;
var currentIndex = allMessages.length - stepSize;
// show the last 10 messages
ctrl.messages = allMessages.length > 10 ? allMessages.slice(-10) : allMessages;
// show ten more messages
ctrl.showMore = function () {
currentIndex -= stepSize;
var extraItems = allMessages.slice(currentIndex, currentIndex+stepSize);
Array.prototype.push.apply(extraItems, ctrl.messages);
ctrl.messages = extraItems;
};
});
</script>
</head>
<body ng-app="myApp">
<div ng-controller="MyController as ctrl">
<table>
<tr ng-repeat="message in ctrl.messages"><td>{{message.text}}</td></tr>
</table>
<button ng-click="ctrl.showMore();">show more</button>
</div>
</body>
</html>

You can try this to get the object with id 30 without a loop using the following code:
var array = [
{ id : 1, text: "First"},
{ id : 2, text: "Second"},
{ id : 3, text: "Third"},
{ id : 4, text: "Forth"},
{ id : 5, text: "Fifth"},
{ id : 6, text: "Sixth"},
{ id : 7, text: "Seventh"}
];
var result = $.grep(array, function(e){ return e.id == 5; });
console.log(result);

What is the best way to select last 10 items before that object with the id of 30?
var index = 30;
var count = 10;
// 10 items before 30 :
array.slice(index-count, index);

I knew this thing I made could be helpful to somebody. Here's the relevant piece:
$('button.getResult').on('click', function(e) {
e.preventDefault();
var limit = $('input.getResult').val();
if (limit != '' && !isNaN(limit)) {
var dots = $('ul li');
if (dots.length > limit) {
//show the limit as group
var foundSelected = false;
var counter = 0;
for (var i = dots.length - 1; i >= 0; i--) {
dots.eq(i).addClass('group');
dots.eq(i + parseInt(limit)).removeClass('group');
counter++;
if (i == $('li.selected').index()) {
foundSelected = true;
};
if (foundSelected && counter >= limit) {
i = -1;
};
};
} else {
//show all as group
dots.addClass('group');
};
};
return false;
});
dots is basically your array and the first conditional is just checking to make sure the number of results you want is smaller than the length of the array. Then a loop is performed through the array (backwards in my case) checking for the important item. If I find it, I change the boolean to true.
Additionally, I mark each as selected as I iterate and if the selected index is outside of the limit of pagination, then I remove the selected mark (to keep the pagination). Once I've both found the important item and have marked the correct number of items past the important one, I stop the loop.
Of course, you might not be able to mark things in your case but I figure you could look at this for some inspiration.

Javascript square function

I want to write a function that checks if the given number has a certain order.
The second number has to be the square of the previous number.
The first number can only be 0 - 9.
So for example 2439 would return 'true' because 4 is the square of 2 and 9 is the square of 3.
39416 would also give 'true', and for example 1624 would return 'false'.
I don't really have an idea how to do this. It should be a recursive function but an example of how to do it without recursion would be helpfull too.

I would try something like this:
function isOrdered(input){
var position = 0;
while(position<input.length-2)
{
var currentFirstNumber = parseInt(input[position]);
if(currentFirstNumber<=2) {
if (Math.sqrt(parseInt(input[position + 1])) !== currentFirstNumber)
return false;
else
position+=2;
}
if(currentFirstNumber>=4 && currentFirstNumber<=9)
{
var squared = input.substring(position+1,position+3);
if(Math.sqrt(parseInt(squared))!==currentFirstNumber)
return false;
else
position=position+3;
}
}
return true;
}
console.log(isOrdered("2439")); // outputs true
console.log(isOrdered("39416")); // outputs true
console.log(isOrdered("1624")); // outputs false
I pass the number to the function as a string.

Take a look at this recursive function
function detectOrder(input)
{
var input = input.toString()
var first = input.substr(0,1);
var power = Math.pow(parseInt(first), 2);
var powerLength = power.toString().length;
if ( parseInt(input.substr(1, powerLength)) == power )
{
if (input.length <= 1+powerLength)
{
return true;
}
else
{
return detectOrder(input.substr(1+powerLength));
}
}
else
{
return false;
}
}

As mention in the comment section, OP said that the 'firsts' are limited to 0..9. So the easiest way to accomplish this is by going through the power function instead of the square root function.
UPDATE: Sorry, you asked for JavaScript code. Be careful with the FIRST CALL. if you manually pass to the function the last position, it will return true.
function verification(number, position){
var str = String(number); // Cast number to String
if(str.length > position){ // Verify valid position
var value = str.substr(position, 1); // take the 'first' value
var pow = Math.pow(value, 2); // Calculate the power
// Verify if the next value is equivalent to the power
if(str.indexOf(pow, position) == position + 1){
// Recursive call to verify the next position
return verification(number, position + String(pow).length + 1);
} else {
// If not equivalent, you found an inconsistency.
return false;
}
// If you ran it until you reached the last position, you're good to go.
}else if(str.length == position){
return true;
}
}
console.log(verification(39416, 0)); // True
console.log(verification(39415, 0)); // True
console.log(verification(981524, 0)); // false
console.log(verification(981525, 0)); // true
console.log(verification(98525, 0)); // false