I'm trying to set an unlimited query parameter in express js.But I couldn't figure out how should I implement that in my code. I'm using MongoDB aggeration
I want to build unlimited facets searched with multiple $match stage
Which works like this:
'http://localhost:4000/search?text=mango'
'http://localhost:4000/search?text=mango&key=brand&value=rasna' //unlimited facets.
'http://localhost:4000/search?text=mango&key=brand&value=rasna&key=color&value=yellow' //unlimited facet parameters
Here's my code to do this:
app.get("/search", async(request, response) => {
try {
const textsearch = request.query.text;
var keystore = request.query.key; //storing `key` in 'keystore'
var valuestore = request.query.value; //storing `value` in `valuestore`
if (keystore, valuestore) {
facetjson = [
{
'$match': {
[keystore]: `${valuestore}` //Storing key and value in $match
}
}
]
const Pipeline = [{
'$search': {
'text': {
'query': `${textsearch}`,
'path': 'title',
}
}
},
{
'$limit': 5
}
]
//Pushing 'facetjson' array into Pipeline array to make a filtered search possible.
const newitem = insert(Pipeline, Pipeline.length - 1, facetjson)
let result = collection.aggregate(newitem).toArray();
response.send(result);
} else {
const Pipeline = [{
'$search': {
'text': {
'query': `${textsearch}`,
'path': 'title',
}
}
},
{
'$limit': 5
}
]
let result = collection.aggregate(Pipeline).toArray();
response.send(result);
};
} catch (error) {
response.status(500).send({ message: error.message });
}
})
(JSFIDDLE code Example)[https://jsfiddle.net/divyanshuking/z0vo589e/]
==> I know that I've to pass $match in the Pipeline array each time for single Key , Value Pair. Doing many google searches I've figured out that I've to use the Rest Parameter (...keystore,...valuestore). But I didn't know how to implement this. Have you guys any better idea to do solve this problem? Pls help me:
Why don’t you use forEach or something
function endPoint (req, res) {
const queriesFound ={}
req.query.forEach(query=>{
queriesFound[query]=query;
}
QueriesFound will be an object
{ “Name”:”Name”, “AnotherParam”:”AnotherParam” }
}
//QueriesFound will be an object
{
“Name”:”Name”,
“AnotherParam”:”AnotherParam”
}
Your request URL has a wrong structure for query parameters. If you want to pass multiple kay/value pairs in URL, the correct structure is like this:
'http://localhost:4000/search?text=mango&brand=rasana&color=yellow
This code should work with this URL structure:
app.get("/search", async(request, response) => {
try {
//We need "search pipeline stage" in all conditions. whether we got a key/value pair in query or not.
//so we use "search stage" when declare pipeline array;
let pipeline = [{
'$search': {
'text': {
'query': `${request.query.text}`,
'path': 'title',
}
}
}];
//If there are keys/values pairs in the query parameters, we add match stage to our pipeline array;
if(request.query) {
let match = {}, hasMatchSatge = false;
for(let item in request.query){
if(item !=== 'text'){
match[item] = request.query[item];
hasMatchStage = true;
}
}
if(hasMatchStage) pipeline.push({'$match': match});
}
//Finally, we add our "limit stage" to the pipeline array;
pipeline.push({'$limit' : 5});
let result = collection.aggregate(pipeline).toArray();
response.status(200).send(result);
} catch (error) {
response.status(500).send({ message: error.message });
}
})
I am trying to do what should be a rather simple operation in my mongoDB/Node environment. Every document in the collection I'm targeting has a field "openBalance", which is a number value. All I want to do is find the totalOpenBalance by adding all of those together.
So far, in reviewing the MongoDB documentation, both $add and $sum seem to be used to perform an operation on the individual documents within the collection, rather than on the collection itself.
This leads me to wonder, is there a different way I should approach this? I've tried numerous constructions, but none work. Here is my function in full:
exports.getClientData = async function (req, res, next) {
let MongoClient = await require('../config/database')();
let db = MongoClient.connection.db;
let search, skip, pagesize, page, ioOnly = false, client;
let docs = [];
let records = 0;
if (_.isUndefined(req.params)) {
skip = parseInt(req.skip) || 0;
search = JSON.parse(req.search);
pagesize = parseInt(req.pagesize) || 0;
page = parseInt(req.page) || 0;
client = req.client || '';
ioOnly = true;
}
else {
skip = parseInt(req.query.skip) || 0;
search = req.body;
pagesize = parseInt(req.query.pagesize) || 0;
page = parseInt(req.query.page) || 0;
client = req.query.client || '';
}
search = {};
if (skip === 0) {
skip = page * pagesize;
}
if (client) {
let arrClient = [];
arrClient = client.split(",");
if (arrClient) {
// convert each ID to a mongo ID
let mongoArrClient = arrClient.map(client => new mongo.ObjectID(client));
if (mongoArrClient) {
search['client._id'] = { $in: mongoArrClient };
}
}
}
console.log(search);
let counter = 0;
let count = await db.collection('view_client_data').find(search).count();
let totalClients = await db.collection('view_client_data').find(search).count({ $sum: "client._id" });
console.log('totalClients', totalClients);
let totalOpenBalance = await db.collection('view_client_data').find(search).count({ $sum: { "$add" : "openBalance" } });
console.log('totalOpenBalance', totalOpenBalance);
db.collection('view_client_data').find(search).skip(skip).limit(pagesize).forEach(function (doc) {
counter ++; {
console.log(doc);
docs.push(doc);
}
}, function (err) {
if (err) {
if (!ioOnly) {
return next(err);
} else {
return res(err);
}
}
if (ioOnly) {
res({ sessionId: sessID, count: count, data: docs, totalClients: totalClients, totalOpenBalance: totalOpenBalance });
}
else {
res.send({ count: count, data: docs, totalClients: totalClients, totalOpenBalance: totalOpenBalance });
}
});
}
As you can see in the above code, I am getting the total number of clients with this code:
let totalClients = await db.collection('view_client_data').find(search).count({ $sum: "client._id" });
console.log('totalClients', totalClients);
That works perfectly, adding up the instances of a client and giving me the total.
Again, to be crystal clear, where I'm running into a problem is in summing up the numerical value for all of the openBalance values. Each document has a field, openBalance. All I want to do is add those up and output them in a variable titled totalOpenBalance and pass that along in the response I send, just like I do for totalClients. I have tried numerous options, including this:
let totalOpenBalance = await db.collection('view_client_data').find(search).count({ $sum: { "$add" : "openBalance" } });
console.log('totalOpenBalance', totalOpenBalance);
and this:
let totalOpenBalance = await db.collection('view_client_data').find(search).aggregate({ $sum: { "$add" : "openBalance" } });
console.log('totalOpenBalance', totalOpenBalance);
... but as I say, none work. Sometimes I get a circular reference error, sometimes an aggregate is not a function error, other times different errors. I've been wracking my brain trying to figure this out -- and I assume it shouldn't be that complicated once I understand the required syntax. How can I get my totalOpenBalance here?
By the way, the documents I'm targeting look something like this:
{
"_id": "3hu40890sf131d361f1ad908",
"client": {
"_id": "4ft9d366121j04563be0b01d6",
"name": {
"first": "John",
"last": "Smith"
}
},
"openBalance": 128,
"lastPurchaseDate": "2018-01-19T00:00:00.000Z"
},
$sum is an accumulator operator that must appear within a $group or $project aggregate pipeline stage. To also incorporate your search filter, you can include a $match stage in your pipeline.
let result = await db.collection('view_client_data').aggregate([
{$match: search},
{$group: {_id: null, totalOpenBalance: {$sum: '$openBalance'}}}
]).next();
console.log(result.totalOpenBalance);
I think $group is what you're looking for.
So for example to calculate all the openBalance fields
db.view_client_data.aggregate(
[
{
$group: {
_id : null
totalOpenBalance: { $sum: "$openBalance" },
}
}
]
)
this should give you an object back like {totalOpenBalance: 900}
Here is the mongodb documentation for some more examples
https://docs.mongodb.com/manual/reference/operator/aggregation/group/#pipe._S_group
I'd like to get the names of all the keys in a MongoDB collection.
For example, from this:
db.things.insert( { type : ['dog', 'cat'] } );
db.things.insert( { egg : ['cat'] } );
db.things.insert( { type : [] } );
db.things.insert( { hello : [] } );
I'd like to get the unique keys:
type, egg, hello
You could do this with MapReduce:
mr = db.runCommand({
"mapreduce" : "my_collection",
"map" : function() {
for (var key in this) { emit(key, null); }
},
"reduce" : function(key, stuff) { return null; },
"out": "my_collection" + "_keys"
})
Then run distinct on the resulting collection so as to find all the keys:
db[mr.result].distinct("_id")
["foo", "bar", "baz", "_id", ...]
With Kristina's answer as inspiration, I created an open source tool called Variety which does exactly this: https://github.com/variety/variety
You can use aggregation with the new $objectToArray aggregation operator in version 3.4.4 to convert all top key-value pairs into document arrays, followed by $unwind and $group with $addToSet to get distinct keys across the entire collection. (Use $$ROOT for referencing the top level document.)
db.things.aggregate([
{"$project":{"arrayofkeyvalue":{"$objectToArray":"$$ROOT"}}},
{"$unwind":"$arrayofkeyvalue"},
{"$group":{"_id":null,"allkeys":{"$addToSet":"$arrayofkeyvalue.k"}}}
])
You can use the following query for getting keys in a single document.
db.things.aggregate([
{"$match":{_id: "<<ID>>"}}, /* Replace with the document's ID */
{"$project":{"arrayofkeyvalue":{"$objectToArray":"$$ROOT"}}},
{"$project":{"keys":"$arrayofkeyvalue.k"}}
])
A cleaned up and reusable solution using pymongo:
from pymongo import MongoClient
from bson import Code
def get_keys(db, collection):
client = MongoClient()
db = client[db]
map = Code("function() { for (var key in this) { emit(key, null); } }")
reduce = Code("function(key, stuff) { return null; }")
result = db[collection].map_reduce(map, reduce, "myresults")
return result.distinct('_id')
Usage:
get_keys('dbname', 'collection')
>> ['key1', 'key2', ... ]
If your target collection is not too large, you can try this under mongo shell client:
var allKeys = {};
db.YOURCOLLECTION.find().forEach(function(doc){Object.keys(doc).forEach(function(key){allKeys[key]=1})});
allKeys;
If you are using mongodb 3.4.4 and above then you can use below aggregation using $objectToArray and $group aggregation
db.collection.aggregate([
{ "$project": {
"data": { "$objectToArray": "$$ROOT" }
}},
{ "$project": { "data": "$data.k" }},
{ "$unwind": "$data" },
{ "$group": {
"_id": null,
"keys": { "$addToSet": "$data" }
}}
])
Here is the working example
Try this:
doc=db.thinks.findOne();
for (key in doc) print(key);
Using python. Returns the set of all top-level keys in the collection:
#Using pymongo and connection named 'db'
reduce(
lambda all_keys, rec_keys: all_keys | set(rec_keys),
map(lambda d: d.keys(), db.things.find()),
set()
)
Here is the sample worked in Python:
This sample returns the results inline.
from pymongo import MongoClient
from bson.code import Code
mapper = Code("""
function() {
for (var key in this) { emit(key, null); }
}
""")
reducer = Code("""
function(key, stuff) { return null; }
""")
distinctThingFields = db.things.map_reduce(mapper, reducer
, out = {'inline' : 1}
, full_response = True)
## do something with distinctThingFields['results']
I am surprise, no one here has ans by using simple javascript and Set logic to automatically filter the duplicates values, simple example on mongo shellas below:
var allKeys = new Set()
db.collectionName.find().forEach( function (o) {for (key in o ) allKeys.add(key)})
for(let key of allKeys) print(key)
This will print all possible unique keys in the collection name: collectionName.
I think the best way do this as mentioned here is in mongod 3.4.4+ but without using the $unwind operator and using only two stages in the pipeline. Instead we can use the $mergeObjects and $objectToArray operators.
In the $group stage, we use the $mergeObjects operator to return a single document where key/value are from all documents in the collection.
Then comes the $project where we use $map and $objectToArray to return the keys.
let allTopLevelKeys = [
{
"$group": {
"_id": null,
"array": {
"$mergeObjects": "$$ROOT"
}
}
},
{
"$project": {
"keys": {
"$map": {
"input": { "$objectToArray": "$array" },
"in": "$$this.k"
}
}
}
}
];
Now if we have a nested documents and want to get the keys as well, this is doable. For simplicity, let consider a document with simple embedded document that look like this:
{field1: {field2: "abc"}, field3: "def"}
{field1: {field3: "abc"}, field4: "def"}
The following pipeline yield all keys (field1, field2, field3, field4).
let allFistSecondLevelKeys = [
{
"$group": {
"_id": null,
"array": {
"$mergeObjects": "$$ROOT"
}
}
},
{
"$project": {
"keys": {
"$setUnion": [
{
"$map": {
"input": {
"$reduce": {
"input": {
"$map": {
"input": {
"$objectToArray": "$array"
},
"in": {
"$cond": [
{
"$eq": [
{
"$type": "$$this.v"
},
"object"
]
},
{
"$objectToArray": "$$this.v"
},
[
"$$this"
]
]
}
}
},
"initialValue": [
],
"in": {
"$concatArrays": [
"$$this",
"$$value"
]
}
}
},
"in": "$$this.k"
}
}
]
}
}
}
]
With a little effort, we can get the key for all subdocument in an array field where the elements are object as well.
This works fine for me:
var arrayOfFieldNames = [];
var items = db.NAMECOLLECTION.find();
while(items.hasNext()) {
var item = items.next();
for(var index in item) {
arrayOfFieldNames[index] = index;
}
}
for (var index in arrayOfFieldNames) {
print(index);
}
Maybe slightly off-topic, but you can recursively pretty-print all keys/fields of an object:
function _printFields(item, level) {
if ((typeof item) != "object") {
return
}
for (var index in item) {
print(" ".repeat(level * 4) + index)
if ((typeof item[index]) == "object") {
_printFields(item[index], level + 1)
}
}
}
function printFields(item) {
_printFields(item, 0)
}
Useful when all objects in a collection has the same structure.
To get a list of all the keys minus _id, consider running the following aggregate pipeline:
var keys = db.collection.aggregate([
{ "$project": {
"hashmaps": { "$objectToArray": "$$ROOT" }
} },
{ "$group": {
"_id": null,
"fields": { "$addToSet": "$hashmaps.k" }
} },
{ "$project": {
"keys": {
"$setDifference": [
{
"$reduce": {
"input": "$fields",
"initialValue": [],
"in": { "$setUnion" : ["$$value", "$$this"] }
}
},
["_id"]
]
}
}
}
]).toArray()[0]["keys"];
I know I am late to the party, but if you want a quick solution in python finding all keys (even the nested ones) you could do with a recursive function:
def get_keys(dl, keys=None):
keys = keys or []
if isinstance(dl, dict):
keys += dl.keys()
list(map(lambda x: get_keys(x, keys), dl.values()))
elif isinstance(dl, list):
list(map(lambda x: get_keys(x, keys), dl))
return list(set(keys))
and use it like:
dl = db.things.find_one({})
get_keys(dl)
if your documents do not have identical keys you can do:
dl = db.things.find({})
list(set(list(map(get_keys, dl))[0]))
but this solution can for sure be optimized.
Generally this solution is basically solving finding keys in nested dicts, so this is not mongodb specific.
Based on #Wolkenarchitekt answer: https://stackoverflow.com/a/48117846/8808983, I write a script that can find patterns in all keys in the db and I think it can help others reading this thread:
"""
Python 3
This script get list of patterns and print the collections that contains fields with this patterns.
"""
import argparse
import pymongo
from bson import Code
# initialize mongo connection:
def get_db():
client = pymongo.MongoClient("172.17.0.2")
db = client["Data"]
return db
def get_commandline_options():
description = "To run use: python db_fields_pattern_finder.py -p <list_of_patterns>"
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-p', '--patterns', nargs="+", help='List of patterns to look for in the db.', required=True)
return parser.parse_args()
def report_matching_fields(relevant_fields_by_collection):
print("Matches:")
for collection_name in relevant_fields_by_collection:
if relevant_fields_by_collection[collection_name]:
print(f"{collection_name}: {relevant_fields_by_collection[collection_name]}")
# pprint(relevant_fields_by_collection)
def get_collections_names(db):
"""
:param pymongo.database.Database db:
:return list: collections names
"""
return db.list_collection_names()
def get_keys(db, collection):
"""
See: https://stackoverflow.com/a/48117846/8808983
:param db:
:param collection:
:return:
"""
map = Code("function() { for (var key in this) { emit(key, null); } }")
reduce = Code("function(key, stuff) { return null; }")
result = db[collection].map_reduce(map, reduce, "myresults")
return result.distinct('_id')
def get_fields(db, collection_names):
fields_by_collections = {}
for collection_name in collection_names:
fields_by_collections[collection_name] = get_keys(db, collection_name)
return fields_by_collections
def get_matches_fields(fields_by_collections, patterns):
relevant_fields_by_collection = {}
for collection_name in fields_by_collections:
relevant_fields = [field for field in fields_by_collections[collection_name] if
[pattern for pattern in patterns if
pattern in field]]
relevant_fields_by_collection[collection_name] = relevant_fields
return relevant_fields_by_collection
def main(patterns):
"""
:param list patterns: List of strings to look for in the db.
"""
db = get_db()
collection_names = get_collections_names(db)
fields_by_collections = get_fields(db, collection_names)
relevant_fields_by_collection = get_matches_fields(fields_by_collections, patterns)
report_matching_fields(relevant_fields_by_collection)
if __name__ == '__main__':
args = get_commandline_options()
main(args.patterns)
As per the mongoldb documentation, a combination of distinct
Finds the distinct values for a specified field across a single collection or view and returns the results in an array.
and indexes collection operations are what would return all possible values for a given key, or index:
Returns an array that holds a list of documents that identify and describe the existing indexes on the collection
So in a given method one could do use a method like the following one, in order to query a collection for all it's registered indexes, and return, say an object with the indexes for keys (this example uses async/await for NodeJS, but obviously you could use any other asynchronous approach):
async function GetFor(collection, index) {
let currentIndexes;
let indexNames = [];
let final = {};
let vals = [];
try {
currentIndexes = await collection.indexes();
await ParseIndexes();
//Check if a specific index was queried, otherwise, iterate for all existing indexes
if (index && typeof index === "string") return await ParseFor(index, indexNames);
await ParseDoc(indexNames);
await Promise.all(vals);
return final;
} catch (e) {
throw e;
}
function ParseIndexes() {
return new Promise(function (result) {
let err;
for (let ind in currentIndexes) {
let index = currentIndexes[ind];
if (!index) {
err = "No Key For Index "+index; break;
}
let Name = Object.keys(index.key);
if (Name.length === 0) {
err = "No Name For Index"; break;
}
indexNames.push(Name[0]);
}
return result(err ? Promise.reject(err) : Promise.resolve());
})
}
async function ParseFor(index, inDoc) {
if (inDoc.indexOf(index) === -1) throw "No Such Index In Collection";
try {
await DistinctFor(index);
return final;
} catch (e) {
throw e
}
}
function ParseDoc(doc) {
return new Promise(function (result) {
let err;
for (let index in doc) {
let key = doc[index];
if (!key) {
err = "No Key For Index "+index; break;
}
vals.push(new Promise(function (pushed) {
DistinctFor(key)
.then(pushed)
.catch(function (err) {
return pushed(Promise.resolve());
})
}))
}
return result(err ? Promise.reject(err) : Promise.resolve());
})
}
async function DistinctFor(key) {
if (!key) throw "Key Is Undefined";
try {
final[key] = await collection.distinct(key);
} catch (e) {
final[key] = 'failed';
throw e;
}
}
}
So querying a collection with the basic _id index, would return the following (test collection only has one document at the time of the test):
Mongo.MongoClient.connect(url, function (err, client) {
assert.equal(null, err);
let collection = client.db('my db').collection('the targeted collection');
GetFor(collection, '_id')
.then(function () {
//returns
// { _id: [ 5ae901e77e322342de1fb701 ] }
})
.catch(function (err) {
//manage your error..
})
});
Mind you, this uses methods native to the NodeJS Driver. As some other answers have suggested, there are other approaches, such as the aggregate framework. I personally find this approach more flexible, as you can easily create and fine-tune how to return the results. Obviously, this only addresses top-level attributes, not nested ones.
Also, to guarantee that all documents are represented should there be secondary indexes (other than the main _id one), those indexes should be set as required.
We can achieve this by Using mongo js file. Add below code in your getCollectionName.js file and run js file in the console of Linux as given below :
mongo --host 192.168.1.135 getCollectionName.js
db_set = connect("192.168.1.135:27017/database_set_name"); // for Local testing
// db_set.auth("username_of_db", "password_of_db"); // if required
db_set.getMongo().setSlaveOk();
var collectionArray = db_set.getCollectionNames();
collectionArray.forEach(function(collectionName){
if ( collectionName == 'system.indexes' || collectionName == 'system.profile' || collectionName == 'system.users' ) {
return;
}
print("\nCollection Name = "+collectionName);
print("All Fields :\n");
var arrayOfFieldNames = [];
var items = db_set[collectionName].find();
// var items = db_set[collectionName].find().sort({'_id':-1}).limit(100); // if you want fast & scan only last 100 records of each collection
while(items.hasNext()) {
var item = items.next();
for(var index in item) {
arrayOfFieldNames[index] = index;
}
}
for (var index in arrayOfFieldNames) {
print(index);
}
});
quit();
Thanks #ackuser
Following the thread from #James Cropcho's answer, I landed on the following which I found to be super easy to use. It is a binary tool, which is exactly what I was looking for:
mongoeye.
Using this tool it took about 2 minutes to get my schema exported from command line.
I know this question is 10 years old but there is no C# solution and this took me hours to figure out. I'm using the .NET driver and System.Linq to return a list of the keys.
var map = new BsonJavaScript("function() { for (var key in this) { emit(key, null); } }");
var reduce = new BsonJavaScript("function(key, stuff) { return null; }");
var options = new MapReduceOptions<BsonDocument, BsonDocument>();
var result = await collection.MapReduceAsync(map, reduce, options);
var list = result.ToEnumerable().Select(item => item["_id"].ToString());
This one lines extracts all keys from a collection into a comma separated sorted string:
db.<collection>.find().map((x) => Object.keys(x)).reduce((a, e) => {for (el of e) { if(!a.includes(el)) { a.push(el) } }; return a}, []).sort((a, b) => a.toLowerCase() > b.toLowerCase()).join(", ")
The result of this query typically looks like this:
_class, _id, address, city, companyName, country, emailId, firstName, isAssigned, isLoggedIn, lastLoggedIn, lastName, location, mobile, printName, roleName, route, state, status, token
I extended Carlos LM's solution a bit so it's more detailed.
Example of a schema:
var schema = {
_id: 123,
id: 12,
t: 'title',
p: 4.5,
ls: [{
l: 'lemma',
p: {
pp: 8.9
}
},
{
l: 'lemma2',
p: {
pp: 8.3
}
}
]
};
Type into the console:
var schemafy = function(schema, i, limit) {
var i = (typeof i !== 'undefined') ? i : 1;
var limit = (typeof limit !== 'undefined') ? limit : false;
var type = '';
var array = false;
for (key in schema) {
type = typeof schema[key];
array = (schema[key] instanceof Array) ? true : false;
if (type === 'object') {
print(Array(i).join(' ') + key+' <'+((array) ? 'array' : type)+'>:');
schemafy(schema[key], i+1, array);
} else {
print(Array(i).join(' ') + key+' <'+type+'>');
}
if (limit) {
break;
}
}
}
Run:
schemafy(db.collection.findOne());
Output
_id <number>
id <number>
t <string>
p <number>
ls <object>:
0 <object>:
l <string>
p <object>:
pp <number>
I was trying to write in nodejs and finally came up with this:
db.collection('collectionName').mapReduce(
function() {
for (var key in this) {
emit(key, null);
}
},
function(key, stuff) {
return null;
}, {
"out": "allFieldNames"
},
function(err, results) {
var fields = db.collection('allFieldNames').distinct('_id');
fields
.then(function(data) {
var finalData = {
"status": "success",
"fields": data
};
res.send(finalData);
delteCollection(db, 'allFieldNames');
})
.catch(function(err) {
res.send(err);
delteCollection(db, 'allFieldNames');
});
});
After reading the newly created collection "allFieldNames", delete it.
db.collection("allFieldNames").remove({}, function (err,result) {
db.close();
return;
});
I have 1 simpler work around...
What you can do is while inserting data/document into your main collection "things" you must insert the attributes in 1 separate collection lets say "things_attributes".
so every time you insert in "things", you do get from "things_attributes" compare values of that document with your new document keys if any new key present append it in that document and again re-insert it.
So things_attributes will have only 1 document of unique keys which you can easily get when ever you require by using findOne()
What I want to do is have the Parse background job loop through a function repeatedly for every User in the database, using that Users respective matchCenterItem, MComparisonArray, and other object instances as criteria.
When I run this, what happens is that rather than doing what I just described, it runs the function once, using all instances of things like matchCenterItem and MComparisonArray, so that functions aren't able to differentiate between users.
How can I properly associate all the functions within this query to each individual user, so that when they save a new object for instance:
var mComparisonArray = Parse.Object.extend("MComparisonArray");
var newMComparisonArray = new mComparisonArray();
newMComparisonArray.set('Name', 'MatchCenter');
newMComparisonArray.set('MCItems', eBayResults);
newMComparisonArray.set("parent", Parse.User());
it'll show that user as the parent, rather than it saving it without a parent.
Parse.Cloud.job("MatchCenterBackground", function(request, status) {
console.log('background task started');
//Query through all users
var usersQuery = new Parse.Query(Parse.User);
//For every user, do the following:
usersQuery.each(function(user) {
//query through all their matchCenterItems
var matchCenterItem = Parse.Object.extend("matchCenterItem");
var query = new Parse.Query(matchCenterItem);
// promise and searchterm arrays to be filled
var promises = [];
var searchTerms = [];
//setting the limit of items at 10 for now
query.limit(10);
console.log('about to start the matchCenterItem query');
return query.find().then(function(results) {
console.log('matchCenterItem query results:' + results);
if (results.length > 0) {
console.log('we have entered the matchcenteritem query');
for (i = 0; i < results.length; i++) {
console.log('we have also entered the loop inside the matchCenterItem query');
// later in your loop where you populate promises:
var searchTerm = results[i].get('searchTerm');
// add it to the array just like you add the promises:
searchTerms.push(searchTerm);
url = 'http://svcs.ebay.com/services/search/FindingService/v1';
//push function containing criteria for every matchCenterItem into promises array
promises.push((function() {
if (results[i].get('itemLocation') == 'US')
{
console.log('americuh!');
var httpRequestPromise = Parse.Cloud.httpRequest({
url: url,
params: {
'OPERATION-NAME': 'findItemsByKeywords',
'SERVICE-VERSION': '1.12.0',
'SECURITY-APPNAME': '*APP ID GOES HERE*',
'GLOBAL-ID': 'EBAY-US',
'RESPONSE-DATA-FORMAT': 'JSON',
'REST-PAYLOAD&sortOrder': 'BestMatch',
'paginationInput.entriesPerPage': '3',
'outputSelector=AspectHistogram&itemFilter(0).name=Condition&itemFilter(0).value(0)': 'New',
'itemFilter(0).value(1)': results[i].get('itemCondition'),
'itemFilter(1).name=MaxPrice&itemFilter(1).value': results[i].get('maxPrice'),
'itemFilter(1).paramName=Currency&itemFilter(1).paramValue': 'USD',
'itemFilter(2).name=MinPrice&itemFilter(2).value': results[i].get('minPrice'),
'itemFilter(2).paramName=Currency&itemFilter(2).paramValue': 'USD',
'itemFilter(3).name=LocatedIn&itemFilter(3).value': 'US',
'itemFilter(4).name=ListingType&itemFilter(4).value': 'FixedPrice',
'keywords': results[i].get('searchTerm'),
}
});
}
else if (results[i].get('itemLocation') == 'WorldWide')
{
console.log('Mr worlwide!');
var httpRequestPromise = Parse.Cloud.httpRequest({
url: url,
params: {
'OPERATION-NAME': 'findItemsByKeywords',
'SERVICE-VERSION': '1.12.0',
'SECURITY-APPNAME': '*APP ID GOES HERE*',
'GLOBAL-ID': 'EBAY-US',
'RESPONSE-DATA-FORMAT': 'JSON',
'REST-PAYLOAD&sortOrder': 'BestMatch',
'paginationInput.entriesPerPage': '3',
'outputSelector=AspectHistogram&itemFilter(0).name=Condition&itemFilter(0).value(0)': 'New',
'itemFilter(0).value(1)': results[i].get('itemCondition'),
'itemFilter(1).name=MaxPrice&itemFilter(1).value': results[i].get('maxPrice'),
'itemFilter(1).paramName=Currency&itemFilter(1).paramValue': 'USD',
'itemFilter(2).name=MinPrice&itemFilter(2).value': results[i].get('minPrice'),
'itemFilter(2).paramName=Currency&itemFilter(2).paramValue': 'USD',
// 'itemFilter(3).name=LocatedIn&itemFilter(3).value' : 'US',
'itemFilter(3).name=ListingType&itemFilter(3).value': 'FixedPrice',
'keywords': results[i].get('searchTerm'),
}
});
}
return httpRequestPromise;
})());
}
}
//when finished pushing all the httpRequest functions into promise array, do the following
return Parse.Promise.when(promises).then(function(results) {
console.log('were in the when.then of promise');
var eBayResults = [];
// piece together eBayResults
for (var i = 0; i < arguments.length; i++) {
var httpResponse = arguments[i];
// since they're in the same order, this is OK:
var searchTerm = searchTerms[i];
// pass it as a param:
var top3 = collectEbayResults(httpResponse.text, searchTerm);
eBayResults.push(top3);
}
// collects ebay responses for every item in the form of top3
function collectEbayResults(eBayResponseText, searchTerm) {
var ebayResponse = JSON.parse(eBayResponseText);
//console.log('lets check eBayResults here:' + eBayResults);
var matchCenterItems = [];
//Parses through ebay's response, pushes each individual item and its properties into an array
ebayResponse.findItemsByKeywordsResponse.forEach(function(itemByKeywordsResponse) {
itemByKeywordsResponse.searchResult.forEach(function(result) {
result.item.forEach(function(item) {
matchCenterItems.push(item);
});
});
});
var top3Titles = [];
var top3Prices = [];
var top3ImgURLS = [];
var top3ItemURLS = [];
//where the title, price, and img url are sent over to the app
matchCenterItems.forEach(function(item) {
var title = item.title[0];
var price = item.sellingStatus[0].convertedCurrentPrice[0].__value__;
var imgURL = item.galleryURL[0];
var itemURL = item.viewItemURL[0];
top3Titles.push(title);
top3Prices.push(price);
top3ImgURLS.push(imgURL);
top3ItemURLS.push(itemURL);
});
console.log('about to define top3 value');
console.log('btw ebay results is:' + eBayResults);
var top3 = {
"Top 3": [{
"Title": top3Titles[0],
"Price": top3Prices[0],
"Image URL": top3ImgURLS[0],
"Item URL": top3ItemURLS[0]
},
{
"Title": top3Titles[1],
"Price": top3Prices[1],
"Image URL": top3ImgURLS[1],
"Item URL": top3ItemURLS[1]
},
{
"Title": top3Titles[2],
"Price": top3Prices[2],
"Image URL": top3ImgURLS[2],
"Item URL": top3ItemURLS[2]
},
{
"Search Term": searchTerm
}
]
};
// return top3
}
//After all the above is done, eBayResults has presumably been constructed, and we will now make the comparisons
//MatchCenter update checking goes here:
console.log('the eBayResults length is:' + eBayResults.length);
console.log('the eBayResults are:' + eBayResults);
// Only check for new matches if user has matchCenterItems
if (eBayResults.length > 0) {
console.log('yes the ebay results be longer than 0');
//Query users MComparisonArray with the following criteria:
var mComparisonArray = Parse.Object.extend("MComparisonArray");
var mComparisonQuery = new Parse.Query(mComparisonArray);
mComparisonQuery.contains('Name', 'MatchCenter');
// mComparisonQuery.contains("MCItems", eBayResults);
console.log('setup query criteria, about to run it');
return mComparisonQuery.find().then(function(results) {
console.log('eh2:' + results);
// No new items
if (results.length > 0) {
console.log("No new items, you're good to go!");
}
// New items found
else if (results.length == 0) {
console.log('no matching mComparisonArray, lets push some new shit');
//replace MCItems array with contents of eBayResults
Parse.Object.destroyAll(mComparisonArray);
var newMComparisonArray = new mComparisonArray();
newMComparisonArray.set('Name', 'MatchCenter');
newMComparisonArray.set('MCItems', eBayResults);
newMComparisonArray.set("parent", Parse.User());
console.log('yala han save il hagat');
// Save updated MComparisonArray
newMComparisonArray.save().then({
success: function() {
console.log('MComparisonArray successfully created!');
//status.success('MComparisonArray successfully created!');
},
error: function() {
console.log('MComparisonArray error!!!');
//status.error('Request failed');
}
});
//send push notification
}
// status.success('MatchCenter Comparison Success!');
},
function(err) {
console.log('nah no results for you bro:' + err);
});
}
});
});
}).then(function() {
// Set the job's success status
status.success("background job worked brah!");
}, function(error) {
// Set the job's error status
status.error('DAMN IT MAN');
});
});
At a quick glance I would guess it is an issue with closures, I see you've protected part of your code with the inline function but not all of it.
I would suggest breaking your code into functions and calling them, not only does this make your code easier to read and maintain, it has the side-benefit of also protecting you from issues with closures, e.g.:
Parse.Cloud.job("MatchCenterBackground", function(request, status) {
// ... other code to setup usersQuery ...
usersQuery.each(function (user) {
return processUser(user);
}).then(function() {
status.success("background job worked brah!");
}, function(error) {
status.error(error);
});
});
// process user, return promise
function processUser(user) {
// ... code to setup per-user query ...
// easy way to share multiple arrays
var shared = {
promises: [],
searchTerms: [],
};
return query.find().then(function(results) {
// process results, populate shared data (promises and searchTerms)
buildEbayRequestPromises(results, shared);
}).then(function() {
// process promises, return query promise
return Parse.Promise.when(shared.promises).then(function() {
// process the results of the promises, returning a query promise
// ... code here ...
});
});
}
// process matchCenterItem results to build eBay promises
function buildEbayRequestPromises(results, shared) {
// ... code that pushes items into shared.promises and shared.searchTerms ...
}
I haven't included all the levels, but you should be able to get an idea from that sample of how to make it easier to work out what is going on.
As a general rule I like to keep each function to no more than one screen full of code, if it gets to be more than that I try to break it up into functions.