Develop Reference

How to get "numFound" using JSON query for Solr?

I am simply trying to obtain the "numFound" figure from a Solr query in a piece of javascript.
At present the code I have outputs the number of responses, limited to X rows I specify, as well as X number of items.
What I want instead is the "numFound" value in the response, and to store it as a var in my javascript.
In the example below it would be 394.
{
"responseHeader":{
"zkConnected":true,
"status":0,
"QTime":31,
"params":{
"q":"names:\"Leo Varadkar\" AND region:\"ROI\""}},
"response":{"numFound":394,"start":0,"maxScore":11.911881,"docs":[
I don't want any info from the fields of a specific entry or anything like that. In my python code I can obtain such a figure by something like "solr.search(query).hits". However I have been unable to find an equivalent from poking around with this. I have tried to guess something like "data.response.hits" and the like but to no avail. I am really in the dark here!
I have been unable to find clear documentation on how to do this, or an example of someone doing the same thing, despite it seeming like quite an important aspect of the whole point of queries in Solr. Top 50 results are no use to me. I am dealing with tens of thousands of items. My confusion might suggest I am failing to understand some key aspect of the whole thing...but I don't think so?
All I want is that figure. Surely somebody knows how to get it? I bet it's very simple.
My javascript below:
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
<script>
function on_data(data) {
$('#results').empty();
var docs = data.response.docs;
$.each(docs, function(i, item) {
$('#results').prepend($('<div>' + item.name + '</div>'));
});
var hits = 'Found ' + data.response. + ' hits'
$('#hits').prepend('<div>' + hits + '</div>');
var total = 'Found ' + docs.length + ' results';
$('#results').prepend('<div>' + total + '</div>');
}
function on_search() {
var query = $('#query').val();
if (query.length == 0) {
return;
}
var url='http://localhost:8983/solr/articles/select?q=text:'+query+'&version=2.2&start=0&rows=10000&indent=on&wt=json&callback=?&json.wrf=on_data';
var urlB='http://localhost:8983/solr/articles/select?q=text:'+query+'&version=2.2&start=0&rows=50&indent=on&wt=json&callback=?&json.wrf=on_data';
$.getJSON(urlB);
}
function on_ready() {
$('#search').click(on_search);
/* Hook enter to search */
$('body').keypress(function(e) {
if (e.keyCode == '13') {
on_search();
}
});
}
$(document).ready(on_ready);

The structure is {"response":{"numFound":394 ... }}, which is loaded into your data variable.
var hits = 'Found ' + data.response.numFound + ' hits'
.. should give you the number you're looking for. The hits name is just given by various libraries (probably pysolr in your case) to the same value.
You can also use console.log(data) to see the parsed data structure in your developer tools' console window.

Javascript: sequential, not parallel

I'm working on a project and have trouble to make my javascript work in a sequential way. I know that javascript can execute tasks in parallel so it won't get stuck when you do a request to a server that doesn't respond. This has It's advantages and disadvantages. In my case, It's a pain in the bum.
The scenario is the following, I'm using Firebase to store how many people agree with an opinion. This looks as follows:
I like javascript
- I like it a lot
- I like it
- like nor dislike it
- don't like it
- I basically hate it
People can now select 1 of the options. A record of their selection is then placed under the option. The structure:
-dfd21DeH67 : {
Title : "I like javascript"
options : {
0 : {
option : "I like it a lot"
agrees : {
-dfd21DQH6sq7 : "Jhon"
-dfd21DQH6sq8 : "Luke"
-dfd21DQH6sq9 : "Bruce"
}
}
1 : {
option : "I like it"
agrees : {
-dfd21DQH6sqA : "Harry"
-dfd21DQH6sqB : "Jimmy"
}
}
2 : {
option : "like nor dislike it"
agrees : {
-dfd21DQH6sqC : "Timmy"
-dfd21DQH6sqD : "Paul"
-dfd21DQH6sqE : "Danny"
-dfd21DQH6sqF : "Robin"
-dfd21DQH6sqG : "Dan"
}
}
3 : {
option : "don't like it"
agrees : {
-dfd21DQH6sqH : "Nathan"
-dfd21DQH6sqI : "Jerry"
}
}
4 : {
option : "I basically hate it"
agrees : {
-dfd21DQH6sqJ : "Tony"
}
}
}
}
Now, I want to query it. The results I need for each option are:
- Option (ex. "I like it a lot")
- agree count (ex. n people agreed on this one)
- Percentage (ex. 3 out of 13 said: "I like javascript". this would be ~23%)
Now, getting the "option" and showing this in the browser: success
getting the "agree count": success thanks to numChildren()
getting the "percentage": challenge because I need the sum of them all to calculate this.
Any ideas on how to pull this off? Tried pulling off some callback tricks. Unfortunatly to no avail. Below is my vision:
//javascript
var hash = "-dfd21DeH67";
var fbDB = new Firebase("https://<MyDB>.firebaseio.com/");
var buffer = [];
var sum;
fbDB.child(hash).once("value", function(opinion) {
var thisOpinion = opinion.val();
// First section
$.each(thisOpinion.options, function(i) {
fbPoll.child(hash + "/options/" + i + "/agrees").once("value", function(agrees) {
votes.push(agrees.numChildren());
sum += agrees.numChildren();
});
});
// execute only if the First section is done
// this is currently not the case, this gets executed before
// the first one finishes => result: #ResultsPanel stays empty
$.each(buffer, function(i) {
$("#ResultsPanel").append(
"<div class=\"OptionStatsBlock\">" +
"Option: " + thisOpinion.options[i].option + "<br>" +
"Total: " + buffer[i] + "<br>" +
"Percentage: " + ((buffer[i] / sum) * 100) +
"</div>"
);
});
});
Thanks in advance,

As covered in the comments already: you seem to struggle with the asynchronous nature of Firebase (and most of the modern web). I would encourage you to spend as much time on understanding asynchronous programming as it takes to get rid of the pain it currently causes. It will prevent you far more pain down the line.
One possible solution
Firebase DataSnapshots always contain all data from under the node that you are requesting. So instead of doing a once in a loop (which is almost always a bad idea), you can just request the data one level higher and then loop and use child:
// First section
$.each(thisOpinion.options, function(i) {
var agrees = opinion.child("/options/" + i + "/agrees");
votes.push(agrees.numChildren());
sum += agrees.numChildren();
});
This code executes synchronously.
Note that I didn't test this, so there may be typos. It's basically just a slight modification of your code, that uses DataSnapshot.child: https://www.firebase.com/docs/web/api/datasnapshot/child.html

You can use jQuery Promise to handle this async nature of calls.
http://api.jquery.com/category/deferred-object/
Even this post might help you out -
http://joseoncode.com/2011/09/26/a-walkthrough-jquery-deferred-and-promise/

For one the second $.each is running on an empty buffer array. So it should never even run.
A better approach may be adding an event listener to the "child_added" event similar to below: Referenced from here https://www.firebase.com/docs/web/guide/retrieving-data.html#section-event-types
var ref = new Firebase("https://docs-examples.firebaseio.com/web/saving-data/fireblog/posts");
// Retrieve new posts as they are added to Firebase
ref.on("child_added", function(snapshot) {
var newPost = snapshot.val();
console.log("Author: " + newPost.author);
console.log("Title: " + newPost.title);
});

Searching nested object with Elastic Search and Firebase

Following the Flashlight GitHub example by Firebase.com, I got it to work as instructed to deploy on Heroku - but the way my data is structured, I think there might be a better way?
My data structure looks like this:
(root)pages
-->Firebase Unique ID
---->chat
---->content
------>Firebase Unique ID
-------->title
-------->description
-------->filter = article
---------->tags
------------tag 1
------------tag 2
I only want Elastic Search to return results based on matching the tags or the filter = article, but I get everything from pages down (all the chat, all the content regardless if it matched the result, but not content from other pages, etc...)
The path in config.js for the flashlight code is simply:
exports.paths = [
{
path: "pages",
index: "firebase",
type: "pages"
},
{
path: "event_entry",
index: "firebase",
type: "events"
}
];
I'm still early enough in development that I can change the structure of the data; I thought about:
(root) content
-->Firebase Unique ID
---->title
---->description
---->filter = article
And then simply storing the Firebase Unique ID in the pages object somewhere?
As it stands right now, I'm parsing the result as so to check the tags and only show what is actually tagged with what I searched for and it's just... ugly...
function showResults(snap) {
if( snap.val() === null ) { return; } // wait until we get data
var dat = snap.val();
var out = '';
$.each(dat["hits"]["0"]["_source"]["content"], function(i, v) {
for(var k in v.tags)
{
if(v['tags'][k]['text'] == $scope.search)
{
out += v.title + ": " + i + "<br>";
console.log(i);
}
}
});
$("#results").text(out);
}

From SQL one-to-many to Parse NoSQL

I have always used MySQL for database and seeing that joins are twisted with Parse API for NoSQL, I believe I have a flaw with the design of my database.
Here is what I use :
Game
---------
id
userA
userB
currentRound
RoundScore // A Game can have 0-3 RoundScore
---------
id
game -> Pointer field to Game
user
round
score
(There is also a default User collection with Parse and all user-related fields inside Game and RoundScore point to the User collection. This works great).
Seeing how the Parse API works, I found it difficult to make the query :
Get all Games and their (up to 3) Rounds' score where Game.userA = me or Game.userB = me.
I could easily get all Games but without their Rounds' score. I can't join both.
How should I handle this query or the design ?
Should I fuse the RoundScore into the Game collection ? If so, how should I declare the new field ?

I have read all these pages:
https://parse.com/docs/js_guide#queries
https://www.parse.com/questions/how-to-achieve-or-condition-in-query
https://parse.com/questions/combining-queries-or-not-and-modeling-relationships
https://www.parse.com/docs/js/symbols/Parse.Query.html
I am of the opinion that the following code snippet should work for you:
var RoundScoreQuery = new Parse.Query("RoundScore");
var userAQuery = new Parse.Query("Game");
userAQuery.equalTo("userA", "me");
var userBQuery = new Parse.Query("Game");
userBQuery.equalTo("userB", "me");
var gamesQuery = Parse.Query.or(userAQuery, userBQuery);
gamesQuery.find({
success: function(results) {
// results contains all games where "me" is a part of
for (var i = 0; i < results.length; i++) {
var gameId = results[i].get("id");
RoundScoreQuery.equalTo("game", gameId);
query.first({
success: function(object) {
// Successfully retrieved the object with the score
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
}
});
}
},
error: function(error) {
// There was an error.
}
});

Okay, that's a good point.
I would create the Game Object like this (https://parse.com/docs/js_guide#objects):
var id = 1; // game #1
var userA = "Name A";
var userB = "Name B";
var score1 = { user: "Name A", score: 3 }; // round one
var score2 = { user: "Name B", score: 5 }; // round two
var score3 = null; // round three
var Game = Parse.Object.extend("Game");
var game = new Game();
game.set("id", number);
game.set("userA", userA);
game.set("userB", userB);
game.set("ScoreR1", object);
game.set("ScoreR2", object);
game.set("ScoreR3", object);
game.save();
The flag "currentRound" is no longer needed
because you know what's the current round is
when you look at ScoreR1, ScoreR2 and ScoreR3.
Now you should only need this code to get all the games where "me" is part from:
var userAQuery = new Parse.Query("Game");
userAQuery.equalTo("userA", "me");
var userBQuery = new Parse.Query("Game");
userBQuery.equalTo("userB", "me");
var gamesQuery = Parse.Query.or(userAQuery, userBQuery);
gamesQuery.find({
success: function(results) {
// results contains all games where "me" is a part of
},
error: function(error) {
// There was an error.
}
});
For more info about "NoSQL Design Patterns for Relational Data", I would recommend this article:
http://robvolk.com/designing-for-relationships-on-a-no-sql-database/

2 ways of doing this:
use pointers from 'game' to 'score' so that you can flatten the query with 'include=' syntax.
Note they even use "games" in the docs example!
Or leverage noSql arrays to flatten your physical design:
Game
---------
id
userA -> type:Array:"scores":["123","234","345"] <-- handle null array
userB -> type:Array:"scores":["321","242","325"]
Just say the Game scores are an array that belong to "game/user"
Just say Game consists of 2 sets of "userScores" which are arrays
IMO you want to come up with a noSql compatible way of dealing with your models and their collections in your MVC mechanism so you can always template it neatly by coming out of your Parse.com API calls with JSON structs that you can easily parse and feed appropriate JSON Objects to whatever template you use for ( JSON to model Obj Classes & Collections ).
example i guess even tho java it may give idea for ios

Sorting a threaded conversation

Not sure how to frame this one, but here goes.
It's a sorting algorithm question
Tools I have to play with are PostgresSQL and python (2.6) on the server and javascript/jquery at the browser end
I need to move data that describes a threaded conversation from a postgres DB to a web page. The data starts off in chronological order, I want to display in threads
Record numbers are small - should never be more than 100 messages returned
So, as a simple example, imagine the table:
ID Reply_To Text
=== ======== =============================
1 0 Hello World
2 0 Make me a sandwich
3 1 Hello back
4 2 No chance
5 1 Who are you?
6 5 Me of course
The end point I want to reach is
Hello World
Hello Back
Who are you?
Me of course
Make me a sandwich
No chance
Or, to put it another way...
ID Reply_To Text
=== ======== =============================
1 0 Hello World
3 1 Hello back
5 1 Who are you?
6 5 Me of course
2 0 Make me a sandwich
4 2 No chance
I'm not after a full solution here, all the ajax, json and formatting stuff I'm happy to get on with.
I'm just having issues getting my head around the neatest way to manage the sort.
SQL? Python? Javascript?
I'm currently playing with array sorts in Javascript (for no better reason than the fact that my python skills are exceptionally weak)
EDIT
At the moment I'm at something like:
function byThread(a,b) {
if (a.reply > b.id && a.reply != 0){
console.log("Compared id=" + a.id + " with id=" + b.id + " and returned -1 ")
return -1;
}
if (a.id > b.reply && b.reply != 0 ){
console.log("Compared id=" + a.id + " with id=" + b.id + " and returned 1 ")
return 1;
}
console.log("Compared id=" + a.id + " with id=" + b.id + " and returned 0 ")
return 0;
}
msg.sort(byThread);
And it's frustratingly close

I've tried to do this in pure sql, because I think that's where the logic should belong. What you need to do is find the list of ids from parent to child, and order by that. Luckily, postgres has array types which can be ordered, so we can use a recursive CTE:
with recursive threaded(id, reply_to, message, order_path) as (
select
parent.id,
parent.reply_to,
parent.message,
NULL::int[] || parent.id -- create an 1-element array with the parent id
from conversation parent
where parent.reply_to is null
UNION ALL
select
reply.id,
reply.reply_to,
reply.message,
t.order_path || reply.id -- append the reply id to the current path
from threaded t
join conversation reply on t.id = reply.reply_to
where reply.reply_to is not null
)
select * from threaded order by order_path;
And the results:
1 NULL "Hello World" "{1}"
3 1 "Hello Back" "{1,3}"
5 1 "Who are you?" "{1,5}"
6 5 "Me of course" "{1,5,6}"
2 NULL "Make me a sandwich" "{2}"
4 2 "No Chance" "{2,4}"
I'm not sure how this will perform though, so you should definitely test and profile this on your real dataset to make sure it's fine. If it's not, perhaps you could look at restructuring your data, and investigating different ways of storing "tree" data in a database. There is a library for django called django-mptt that can efficiently store and retrieve trees. The concept applies to databases in general, but the algorithms for pulling out trees and making sure they stay intact require changes to your application logic, better handled by a library.
Edit:
I should mention that I was originally using just the top-level id for "order_path" as a single number. This answer led me to using an array of ids to guarantee the order all the way down.

You could try something like this on JS side. Since this have replies inside replies, it'd be easy to build a DOM from the convoSorted
var convo = [{id: 1, replyTo: 0, text: "hello world"},
{id: 2, replyTo: 0, text: "Make me a sandwich"},
{id: 3, replyTo: 1, text: "hello back"},
{id: 4, replyTo: 2, text: "no chance"},
{id: 5, replyTo: 1, text: "who are you?"},
{id: 6, replyTo: 5, text: "me of course"},
{id: 7, replyTo: 0, text: "new question"},
{id: 8, replyTo: 7, text: "new answer"}];
var convoSorted = [];
function addToReplies(obj, rply){ //recursive function to find the q. to reply to.
if (obj.id == rply.replyTo){
obj.replies.push({id: rply.id, text: rply.text, replies: []}); //add to the replies array
}
else{
for (k = 0; k < obj.replies.length; k++){
addToReplies(obj.replies[k], rply);
}
}
}
function sortConvo(){
for (i = 0; i < convo.length; i++){
if (convo[i].replyTo == 0){ //if it's not a reply, add to sorted array
convoSorted.push({ id : convo[i].id, text: convo[i].text, replies: [] });
}
else{ //it's a reply, find the question it's replying
for (j = 0; j < convoSorted.length; j++){
addToReplies(convoSorted[j], convo[i]);
}
}
}
}
sortConvo();
console.log(convoSorted);

I feel an idiot - I've been over-complicating this.
All it needed was a bit of lateral thinking (which this discussion has helped with)
msgs=[
{id:1,reply:0,msg:'Hello World'},
{id:2,reply:0,msg:'Make me a sandwich'},
{id:3,reply:1,msg:'Hello back'},
{id:4,reply:2,msg:'No chance'},
{id:5,reply:1,msg:'Who are you?'},
{id:6,reply:5,msg:'Me of course'}
];
msgs.sort(function(a,b){
return a.reply - b.reply;
});
var conversation=$("<div id='threaded'>");
var list = $("<ul>")
.attr("id","thread_" + msgs[0].reply);
var message = $("<li>")
.text(msgs[0].msg)
.attr("id","msg_" + msgs[0].id);
$(list).append(message);
$(conversation).append(list);
for (i=1; i<msgs.length; i++){
var message = $("<li>")
.text(msgs[i].msg)
.attr("id","msg_" + msgs[i].id);
if ($(conversation).find("#msg_" + msgs[i].reply).length == 0){
$(conversation).find("ul:eq(0)").append(message);
} else {
var list = $("<ul>")
.attr("id","thread_" + msgs[i].id);
$(list).append(message);
$(conversation).find("#msg_" + msgs[i].reply).append(list);
}
}
Sometimes I forget just what a powerful tool the dom is