Develop Reference

How do I join separate json objects output from a for loop into an array?

I am scraping websites using CasperJS and one of the tasks involve crawling across url set by a for loop counter. The url looks like this
www.example.com/page/no=
where the no is any number from 0-10 set by the for loop counter. The scraper then goes through all the pages, scrapes the data into a JSON object and repeats until no=10.
The data that I am trying to get is stored in discrete groups in each page- what I would like to work with is a single JSON object by joining all the scraped output from each page.
Imagine Page1 has Expense 1 and the object I am getting is { expense1 } and Page 2 has Expense 2 and object that I am getting is { expense2 }. What I would like to have is one JSON at the end of scraping that looks like this:
scrapedData = {
"expense1": expense1,
"expense2": expense2,
}
What I am having trouble is joining all the JSON object into one array.
I initialized an empty array and then each object gets pushed to array.
I have tried a check where if iterator i in for loop is equal to 10, then the JSON object is printed out but that didnt seem to work. I looked up and it seems Object spread is an option but I am not sure how to use it this case.
Any pointers would be helpful. Should I be using any of the array functions like map?
casper.then(function(){
var url = "https:example.net/secure/SaFinShow?url=";
//We create a for loop to go open the urls
for (i=0; i<11; i++){
this.thenOpen(url+ i, function(response){
expense_amount = this.fetchText("td[headers='amount']");
Date = this.fetchText("td[headers='Date']");
Location = this.fetchText("td[headers='zipcode']");
id = this.fetchText("td[headers='id']");
singleExpense = {
"Expense_Amount": expense_amount,
"Date": Date,
"Location": Location,
"id": id
};
if (i ===10){
expenseArray.push(JSON.stringify(singleExpense, null, 2))
this.echo(expenseArray);
}
});
};
});

Taking your example and expanding on it, you should be able to do something like:
// Initialize empty object to hold all of the expenses
var scrapedData = {};
casper.then(function(){
var url = "https:example.net/secure/SaFinShow?url=";
//We create a for loop to go open the urls
for (i=0; i<11; i++){
this.thenOpen(url+ i, function(response){
expense_amount = this.fetchText("td[headers='amount']");
Date = this.fetchText("td[headers='Date']");
Location = this.fetchText("td[headers='zipcode']");
id = this.fetchText("td[headers='id']");
singleExpense = {
"Expense_Amount": expense_amount,
"Date": Date,
"Location": Location,
"id": id
};
// As we loop over each of the expenses add them to the object containing all of them
scrapedData['expense'+i] = singleExpense;
});
};
});
After this runs the scrapedData variable should be of the form:
scrapedData = {
"expense1": expense1,
"expense2": expense2
}
Updated code
One problem with the above code is that inside the for loop when you loop over the expenses, the variables should be local. The variable names also should not be Date and Location since those are built-in names in JavaScript.
// Initialize empty object to hold all of the expenses
var scrapedData = {};
casper.then(function(){
var url = "https:example.net/secure/SaFinShow?url=";
//We create a for loop to go open the urls
for (i=0; i<11; i++){
this.thenOpen(url+ i, function(response){
// Create our local variables to store data for this particular
// expense data
var expense_amount = this.fetchText("td[headers='amount']");
// Don't use `Date` it is a JS built-in name
var date = this.fetchText("td[headers='Date']");
// Don't use `Location` it is a JS built-in name
var location = this.fetchText("td[headers='zipcode']");
var id = this.fetchText("td[headers='id']");
singleExpense = {
"Expense_Amount": expense_amount,
"Date": date,
"Location": location,
"id": id
};
// As we loop over each of the expenses add them to the object containing all of them
scrapedData['expense'+i] = singleExpense;
});
};
});

Search for term in array and return array entry containing that term

I am making a tool that analyzes words and tries to identify when they were most used. I am using Google's Ngram datasets to do so. In my code, I am streaming this data (which is about 2 gigabytes). I am turning the stream data into an array, with each line of data as one entry. What I want to do is to search for a certain word in the data, and store all the array entries containing the word in a variable. I can find if the word is in the dataset, and print that word (or the position of it in the dataset) to the console. I am still learning to program, so please keep that in mind if my code is messy.
// imports fs (filesystem) package duh
const fs = require('fs');
// the data stream
const stream = fs.createReadStream("/Users/user/Desktop/authortest_nodejs/testdata/testdata - p");
// gonna use this to keep track of whether ive found the search term or not
let found = false;
// this is the term the program looks for in the data
var search = "proceeded";
// lovely beautiful unclean way of turning my search term into regular expression
var searchThing = `\\b${search}`
var searchRegExp = new RegExp(searchThing, "g");
// starts streaming the test data file
stream.on('data', function(data) {
// if found is false (my search term isn''t found in this data chunk), set the found variable to true or false depending on whether it found anything
if (!found) found = !!('' + data).match(searchRegExp);
// turns raw data to a string and tries to find the location of the search term within it
var dataLoc = data.toString().search(searchRegExp);
var dataStr = data.toString().match(searchRegExp);
// if the data search is null, continue streaming (gotta do this cuz if .match() turns up with no results it throws an error smh)
if (!dataStr) return;
// removes the null spots and line breaks, pretty up the displayed stuff
var dataDisplay = dataStr.toString().replace("null", " ");
var dataLocDisplay = dataLoc.toString().replace(/(\r\n|\n|\r)/gm,"");
// turns each line of raw data into array
var dataArray = data.toString().split("\n");
// log found instances of search term (dunno why the hell id wanna do that, should fix to something useful) edit: commented it out cuz its too annoying
//console.log(dataDisplay);
// log location of word in string (there, more useful now?)
console.log(dataDisplay);
});
// what happens when the stream thing returns an error
stream.on('error', function(err) {
console.log(err, found);
});
// what happens when the stream thing finishes streaming
stream.on('close', function(err) {
console.log(err, found, searchRegExp);
});
This currently outputs every instance of the search term in the data (basically one word repeated a hundred times or so), but I need an output of each entire line that contains the search term, not just the term. ("Proceeded 2006 5 3", not just "proceeded")

From what I understood, you're looking for something like this:
const fs = require('fs');
function grep(path, word) {
return new Promise((resolve) => {
let
stream = fs.createReadStream(path, {encoding: 'utf8'}),
buf = '',
out = [],
search = new RegExp(`\\b${word}\\b`, 'i');
function process(line) {
if (search.test(line))
out.push(line);
}
stream.on('data', (data) => {
let lines = data.split('\n');
lines[0] = buf + lines[0];
buf = lines.pop();
lines.forEach(process);
});
stream.on('end', () => {
process(buf);
resolve(out);
});
});
}
// works?
grep(__filename, 'stream').then(lines => console.log(lines))
I guess this is pretty straightforward, the buf stuff is needed to emulate line-by-line reading (you can also use readline or a dedicated module for the same).

how to query file field in parse javascript

I have 10 file fields in my UserAccount in parse table say file1. file2, file3 and son on. I want to query the UserAccount Table so its only fetch those records which has at least 1 file available. If all fields are empty for particular USerAccount then it should not be in the result.
Want a javascript parse query that do this.

The way to do this is (sadly, I wish there was a better way) to combine disjunctive queries with Parse.Query.or().
In your case, one query per file column, like this:
var queryFile1 = new Parse.Query(Parse.User);
queryFile1.exists("file1");
var queryFile2 = new Parse.Query(Parse.User);
queryFile2.exists("file2");
// and so on, then
Parse.Query.or([queryFile1, queryFile2, ...]).then(function(results) {
// results will be an array of users that have any one of file1,2,3 defined
});
This can be made a little more terse using underscore.js...
var _ = require('underscore');
var columns = ["file1", "file2", ... ];
var queries = _.map(columns, function(column) {
return new Parse.Query(Parse.User).exists(column);
});
Parse.Query.or(queries).then(...

From SQL one-to-many to Parse NoSQL

I have always used MySQL for database and seeing that joins are twisted with Parse API for NoSQL, I believe I have a flaw with the design of my database.
Here is what I use :
Game
---------
id
userA
userB
currentRound
RoundScore // A Game can have 0-3 RoundScore
---------
id
game -> Pointer field to Game
user
round
score
(There is also a default User collection with Parse and all user-related fields inside Game and RoundScore point to the User collection. This works great).
Seeing how the Parse API works, I found it difficult to make the query :
Get all Games and their (up to 3) Rounds' score where Game.userA = me or Game.userB = me.
I could easily get all Games but without their Rounds' score. I can't join both.
How should I handle this query or the design ?
Should I fuse the RoundScore into the Game collection ? If so, how should I declare the new field ?

I have read all these pages:
https://parse.com/docs/js_guide#queries
https://www.parse.com/questions/how-to-achieve-or-condition-in-query
https://parse.com/questions/combining-queries-or-not-and-modeling-relationships
https://www.parse.com/docs/js/symbols/Parse.Query.html
I am of the opinion that the following code snippet should work for you:
var RoundScoreQuery = new Parse.Query("RoundScore");
var userAQuery = new Parse.Query("Game");
userAQuery.equalTo("userA", "me");
var userBQuery = new Parse.Query("Game");
userBQuery.equalTo("userB", "me");
var gamesQuery = Parse.Query.or(userAQuery, userBQuery);
gamesQuery.find({
success: function(results) {
// results contains all games where "me" is a part of
for (var i = 0; i < results.length; i++) {
var gameId = results[i].get("id");
RoundScoreQuery.equalTo("game", gameId);
query.first({
success: function(object) {
// Successfully retrieved the object with the score
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
}
});
}
},
error: function(error) {
// There was an error.
}
});

Okay, that's a good point.
I would create the Game Object like this (https://parse.com/docs/js_guide#objects):
var id = 1; // game #1
var userA = "Name A";
var userB = "Name B";
var score1 = { user: "Name A", score: 3 }; // round one
var score2 = { user: "Name B", score: 5 }; // round two
var score3 = null; // round three
var Game = Parse.Object.extend("Game");
var game = new Game();
game.set("id", number);
game.set("userA", userA);
game.set("userB", userB);
game.set("ScoreR1", object);
game.set("ScoreR2", object);
game.set("ScoreR3", object);
game.save();
The flag "currentRound" is no longer needed
because you know what's the current round is
when you look at ScoreR1, ScoreR2 and ScoreR3.
Now you should only need this code to get all the games where "me" is part from:
var userAQuery = new Parse.Query("Game");
userAQuery.equalTo("userA", "me");
var userBQuery = new Parse.Query("Game");
userBQuery.equalTo("userB", "me");
var gamesQuery = Parse.Query.or(userAQuery, userBQuery);
gamesQuery.find({
success: function(results) {
// results contains all games where "me" is a part of
},
error: function(error) {
// There was an error.
}
});
For more info about "NoSQL Design Patterns for Relational Data", I would recommend this article:
http://robvolk.com/designing-for-relationships-on-a-no-sql-database/

2 ways of doing this:
use pointers from 'game' to 'score' so that you can flatten the query with 'include=' syntax.
Note they even use "games" in the docs example!
Or leverage noSql arrays to flatten your physical design:
Game
---------
id
userA -> type:Array:"scores":["123","234","345"] <-- handle null array
userB -> type:Array:"scores":["321","242","325"]
Just say the Game scores are an array that belong to "game/user"
Just say Game consists of 2 sets of "userScores" which are arrays
IMO you want to come up with a noSql compatible way of dealing with your models and their collections in your MVC mechanism so you can always template it neatly by coming out of your Parse.com API calls with JSON structs that you can easily parse and feed appropriate JSON Objects to whatever template you use for ( JSON to model Obj Classes & Collections ).
example i guess even tho java it may give idea for ios

retrieve unique values for key from parse.com when class has > 1000 objects

In parse I have a class named "TestScore". Each object has a key named "quizName".
I need to get an array of unique "quizName" values. I wrote the below which queries the "TestClass" and loops through the results looking for unique "quizName" values.
At first seemed to do the job. But then I realized that the maximum number of returned objects is 1000. Soon there will be more than 1000 objects stored which means that this method will not guarantee that I end up will all values.
function loadTests(){
//create an array to hold each unique test name as we find them
var uniqueEntries = [];
//query parse to return TestScore objects
var TestScore = Parse.Object.extend("TestScore");
var query = new Parse.Query(TestScore);
query.limit(1000) //added this after realizing that the default query limit is only 100
query.find({
success: function(testScore) {
$(testScore).each(function(index, score) {
//here I loop though all of the returned objects looking at the "quizName" for each
if($.inArray(score.get("quizName"), uniqueEntries) === -1) {
//if the quiz name is not already in the "uniqueEntries" array, I add it to the array
uniqueEntries.push(score.get("quizName"));
}
});
//do stuff with quiznames here...., add them as options in select boxes mostly
}
});
}
I looked at {Parse.Query} notContainedIn(key, values) which looks promising but cant figure out if I can add values to the array as I find them. It seems like I would have to have an array to start with (defeating the whole point.)
This part of the docs "{Parse.Query} Returns the query, so you can chain this call." makes me think I might be able to chain queries together to get what I need, but that doesn't seem very efficient.
How can I retrieve unique values for key "quizName" when my class has > 1000 objects?

I'm sure you're long past this by now, but only way I know of to do it is to use one query after another by using a .skip(#) value for each query. So get 1000, then query again with .skip(1000), concatenate the items from the first list and second, then query again with .skip(2000), etc...
Be aware that I think there's a limit on skip values of 10,000. Don't take my word on that, just pointing you to something that I think is right that you should confirm if you think it applies to your situation.

I eventually found a tutorial online that I was able to modify and came up with the below. This effectively sets the return limit to 10,000 instead of 1,000 and allows setting several different parameters for the query.
My changes could surely be written better, maybe as an options object or similar but it works for my needs.
You can see a working demo here
function getStuff(){
// here we will setup and call our helper functions with callbacks to handle the results
var scheme =['SOTest',true]; // return all objects with value `true` in the `SOTest` column
// var scheme =['descending','createdAt']; // return all objects with sort order applied
// var scheme =''; // or just return all objects
// see `findChunk()` below for more info
var Remark = Parse.Object.extend("Remark");
schemePromise(Remark, scheme).done(function (all) {
console.log('Found ' + all.length+' Remarks');
$.each( all, function(i, obj){
$('#test').append(obj.get('Remark') +'<br>');
});
})
.fail(function (error) {
console.log("error: " + JSON.stringify(error));
});
}
getStuff(); // call our function
// helper functions used to get around parse's 1000 query limit
// raises the limit to 10,000 by using promises
function findChunk(model, scheme, allData) {
// if `scheme` was an empty string, convert to an array
// this is the default and returns all objects in the called class
if(scheme==''){ ['scheme',''] };
// will return a promise
var limit = 1000;
var skip = allData.length;
var findPromise = $.Deferred();
var query = new Parse.Query(model);
// to get all objects from the queried Class then sort them by some column
// pass `scheme` as an array like [ sort method, column to sort ]
if (scheme[0]=='descending') query.descending(scheme[1]);
else if (scheme[0]=='ascending') query.ascending(scheme[1]);
// to limt results to objects that have a certain value in a specific column
// pass `scheme` as an array like [ column name, value ]
else query.equalTo(scheme[0], scheme[1]);
// more options can easily be built in here using `scheme`
query
.limit(limit)
.skip(skip)
.find()
.then(function (results) {
findPromise.resolve(allData.concat(results), !results.length);
}, function (results) {
findPromise.reject(error);
});
return findPromise.promise();
}
function schemePromise(model, scheme, allResults, allPromise) {
// find a scheme at a time
var promise = allPromise || $.Deferred();
findChunk(model, scheme, allResults || [])
.done(function (results, allOver) {
if (allOver) {
// we are done
promise.resolve(results);
} else {
// may be more
schemePromise(model, scheme, results, promise);
}
})
.fail(function (error) {
promise.reject(error);
});
return promise.promise();
}