Develop Reference

Convert the simple fetch API code to ".then" notation code

How can I covert following code into .then notation. I wanted to strictly use ".then" notation. That is what I observed with my system.
I raised one question with similar kind of request however, I got the code using async/await. Rather than asking the new requirement over the same thread I initiated this new thread.
Apology for inconvenience. I should have posted this it in first thread itself. Kindly help.
var obj = [{"Id":"10101","descr":"server1.com"},{"Id":"10102","descr":"server2.com"},{"Id":"10103","descr":"server3.com"},{"Id":"10104","descr":"server4.com"},{"Id":"10105","descr":"server5.com"},{"Id":"10106","descr":"server6.com"},{"Id":"10107","descr":"server7.com"}];
var temp = [];
for (var i = 0; i < obj.length; i++){
var id = obj[i].Id;
let response = await fetch('https://abced.com/api/'+id+'/value', {method : "GET", headers: {"Authorization": "xyz"}});
var data = await response.json();
var stats = data.status;
if (stat != "OK")
{
temp.push({Id:obj[i].Id, descr:obj[i].descr, value:"ERROR"})
}
}
console.log(temp);
My expected output is, (values of Id and descr will depends on "if statement" in the code)
[{"Id": "10101","descr": "server1.com","status": "ERROR"},
{"Id": "10103","descr": "server3.com","status": "ERROR"},
{"Id": "10104","descr": "server4.com","status": "ERROR"}]
I tried following but in my system compiler says, "Function declared within loops referencing an outer scope variable mat lead to confusing semantics (Id, descr)"
function fetchMock(url) {
let id = url.split('/')[4];
if ([10101, 10103, 10104].includes(+id)) {
return Promise.resolve({
json() {
return Promise.resolve({
status: 'BAD'
});
}
});
} else {
return Promise.resolve({
json() {
return Promise.resolve({
status: 'OK'
});
}
});
}
}
var obj = [{
"Id": "10101",
"descr": "server1.com"
},
{
"Id": "10102",
"descr": "server2.com"
},
{
"Id": "10103",
"descr": "server3.com"
},
{
"Id": "10104",
"descr": "server4.com"
},
{
"Id": "10105",
"descr": "server5.com"
},
{
"Id": "10106",
"descr": "server6.com"
},
{
"Id": "10107",
"descr": "server7.com"
}
];
function getResults() {
const results = [];
for (let {
Id,
descr
} of obj) {
fetchMock('https://abced.com/api/' + Id + '/value', {
method: "GET",
headers: {
"Authorization": "xyz"
}
}).then(res => res.json()).then(function(data) {
if (data.status !== 'OK') {
results.push({
Id,
descr,
value: 'ERROR'
});
}
});
}
return results;
}
function test() {
const results = getResults();
return results;
}
test();

MongoDB retrieve all keys with Node.js [duplicate]

I'd like to get the names of all the keys in a MongoDB collection.
For example, from this:
db.things.insert( { type : ['dog', 'cat'] } );
db.things.insert( { egg : ['cat'] } );
db.things.insert( { type : [] } );
db.things.insert( { hello : [] } );
I'd like to get the unique keys:
type, egg, hello

You could do this with MapReduce:
mr = db.runCommand({
"mapreduce" : "my_collection",
"map" : function() {
for (var key in this) { emit(key, null); }
},
"reduce" : function(key, stuff) { return null; },
"out": "my_collection" + "_keys"
})
Then run distinct on the resulting collection so as to find all the keys:
db[mr.result].distinct("_id")
["foo", "bar", "baz", "_id", ...]

With Kristina's answer as inspiration, I created an open source tool called Variety which does exactly this: https://github.com/variety/variety

You can use aggregation with the new $objectToArray aggregation operator in version 3.4.4 to convert all top key-value pairs into document arrays, followed by $unwind and $group with $addToSet to get distinct keys across the entire collection. (Use $$ROOT for referencing the top level document.)
db.things.aggregate([
{"$project":{"arrayofkeyvalue":{"$objectToArray":"$$ROOT"}}},
{"$unwind":"$arrayofkeyvalue"},
{"$group":{"_id":null,"allkeys":{"$addToSet":"$arrayofkeyvalue.k"}}}
])
You can use the following query for getting keys in a single document.
db.things.aggregate([
{"$match":{_id: "<<ID>>"}}, /* Replace with the document's ID */
{"$project":{"arrayofkeyvalue":{"$objectToArray":"$$ROOT"}}},
{"$project":{"keys":"$arrayofkeyvalue.k"}}
])

A cleaned up and reusable solution using pymongo:
from pymongo import MongoClient
from bson import Code
def get_keys(db, collection):
client = MongoClient()
db = client[db]
map = Code("function() { for (var key in this) { emit(key, null); } }")
reduce = Code("function(key, stuff) { return null; }")
result = db[collection].map_reduce(map, reduce, "myresults")
return result.distinct('_id')
Usage:
get_keys('dbname', 'collection')
>> ['key1', 'key2', ... ]

If your target collection is not too large, you can try this under mongo shell client:
var allKeys = {};
db.YOURCOLLECTION.find().forEach(function(doc){Object.keys(doc).forEach(function(key){allKeys[key]=1})});
allKeys;

If you are using mongodb 3.4.4 and above then you can use below aggregation using $objectToArray and $group aggregation
db.collection.aggregate([
{ "$project": {
"data": { "$objectToArray": "$$ROOT" }
}},
{ "$project": { "data": "$data.k" }},
{ "$unwind": "$data" },
{ "$group": {
"_id": null,
"keys": { "$addToSet": "$data" }
}}
])
Here is the working example

Try this:
doc=db.thinks.findOne();
for (key in doc) print(key);

Using python. Returns the set of all top-level keys in the collection:
#Using pymongo and connection named 'db'
reduce(
lambda all_keys, rec_keys: all_keys | set(rec_keys),
map(lambda d: d.keys(), db.things.find()),
set()
)

Here is the sample worked in Python:
This sample returns the results inline.
from pymongo import MongoClient
from bson.code import Code
mapper = Code("""
function() {
for (var key in this) { emit(key, null); }
}
""")
reducer = Code("""
function(key, stuff) { return null; }
""")
distinctThingFields = db.things.map_reduce(mapper, reducer
, out = {'inline' : 1}
, full_response = True)
## do something with distinctThingFields['results']

I am surprise, no one here has ans by using simple javascript and Set logic to automatically filter the duplicates values, simple example on mongo shellas below:
var allKeys = new Set()
db.collectionName.find().forEach( function (o) {for (key in o ) allKeys.add(key)})
for(let key of allKeys) print(key)
This will print all possible unique keys in the collection name: collectionName.

I think the best way do this as mentioned here is in mongod 3.4.4+ but without using the $unwind operator and using only two stages in the pipeline. Instead we can use the $mergeObjects and $objectToArray operators.
In the $group stage, we use the $mergeObjects operator to return a single document where key/value are from all documents in the collection.
Then comes the $project where we use $map and $objectToArray to return the keys.
let allTopLevelKeys = [
{
"$group": {
"_id": null,
"array": {
"$mergeObjects": "$$ROOT"
}
}
},
{
"$project": {
"keys": {
"$map": {
"input": { "$objectToArray": "$array" },
"in": "$$this.k"
}
}
}
}
];
Now if we have a nested documents and want to get the keys as well, this is doable. For simplicity, let consider a document with simple embedded document that look like this:
{field1: {field2: "abc"}, field3: "def"}
{field1: {field3: "abc"}, field4: "def"}
The following pipeline yield all keys (field1, field2, field3, field4).
let allFistSecondLevelKeys = [
{
"$group": {
"_id": null,
"array": {
"$mergeObjects": "$$ROOT"
}
}
},
{
"$project": {
"keys": {
"$setUnion": [
{
"$map": {
"input": {
"$reduce": {
"input": {
"$map": {
"input": {
"$objectToArray": "$array"
},
"in": {
"$cond": [
{
"$eq": [
{
"$type": "$$this.v"
},
"object"
]
},
{
"$objectToArray": "$$this.v"
},
[
"$$this"
]
]
}
}
},
"initialValue": [
],
"in": {
"$concatArrays": [
"$$this",
"$$value"
]
}
}
},
"in": "$$this.k"
}
}
]
}
}
}
]
With a little effort, we can get the key for all subdocument in an array field where the elements are object as well.

This works fine for me:
var arrayOfFieldNames = [];
var items = db.NAMECOLLECTION.find();
while(items.hasNext()) {
var item = items.next();
for(var index in item) {
arrayOfFieldNames[index] = index;
}
}
for (var index in arrayOfFieldNames) {
print(index);
}

Maybe slightly off-topic, but you can recursively pretty-print all keys/fields of an object:
function _printFields(item, level) {
if ((typeof item) != "object") {
return
}
for (var index in item) {
print(" ".repeat(level * 4) + index)
if ((typeof item[index]) == "object") {
_printFields(item[index], level + 1)
}
}
}
function printFields(item) {
_printFields(item, 0)
}
Useful when all objects in a collection has the same structure.

To get a list of all the keys minus _id, consider running the following aggregate pipeline:
var keys = db.collection.aggregate([
{ "$project": {
"hashmaps": { "$objectToArray": "$$ROOT" }
} },
{ "$group": {
"_id": null,
"fields": { "$addToSet": "$hashmaps.k" }
} },
{ "$project": {
"keys": {
"$setDifference": [
{
"$reduce": {
"input": "$fields",
"initialValue": [],
"in": { "$setUnion" : ["$$value", "$$this"] }
}
},
["_id"]
]
}
}
}
]).toArray()[0]["keys"];

I know I am late to the party, but if you want a quick solution in python finding all keys (even the nested ones) you could do with a recursive function:
def get_keys(dl, keys=None):
keys = keys or []
if isinstance(dl, dict):
keys += dl.keys()
list(map(lambda x: get_keys(x, keys), dl.values()))
elif isinstance(dl, list):
list(map(lambda x: get_keys(x, keys), dl))
return list(set(keys))
and use it like:
dl = db.things.find_one({})
get_keys(dl)
if your documents do not have identical keys you can do:
dl = db.things.find({})
list(set(list(map(get_keys, dl))[0]))
but this solution can for sure be optimized.
Generally this solution is basically solving finding keys in nested dicts, so this is not mongodb specific.

Based on #Wolkenarchitekt answer: https://stackoverflow.com/a/48117846/8808983, I write a script that can find patterns in all keys in the db and I think it can help others reading this thread:
"""
Python 3
This script get list of patterns and print the collections that contains fields with this patterns.
"""
import argparse
import pymongo
from bson import Code
# initialize mongo connection:
def get_db():
client = pymongo.MongoClient("172.17.0.2")
db = client["Data"]
return db
def get_commandline_options():
description = "To run use: python db_fields_pattern_finder.py -p <list_of_patterns>"
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-p', '--patterns', nargs="+", help='List of patterns to look for in the db.', required=True)
return parser.parse_args()
def report_matching_fields(relevant_fields_by_collection):
print("Matches:")
for collection_name in relevant_fields_by_collection:
if relevant_fields_by_collection[collection_name]:
print(f"{collection_name}: {relevant_fields_by_collection[collection_name]}")
# pprint(relevant_fields_by_collection)
def get_collections_names(db):
"""
:param pymongo.database.Database db:
:return list: collections names
"""
return db.list_collection_names()
def get_keys(db, collection):
"""
See: https://stackoverflow.com/a/48117846/8808983
:param db:
:param collection:
:return:
"""
map = Code("function() { for (var key in this) { emit(key, null); } }")
reduce = Code("function(key, stuff) { return null; }")
result = db[collection].map_reduce(map, reduce, "myresults")
return result.distinct('_id')
def get_fields(db, collection_names):
fields_by_collections = {}
for collection_name in collection_names:
fields_by_collections[collection_name] = get_keys(db, collection_name)
return fields_by_collections
def get_matches_fields(fields_by_collections, patterns):
relevant_fields_by_collection = {}
for collection_name in fields_by_collections:
relevant_fields = [field for field in fields_by_collections[collection_name] if
[pattern for pattern in patterns if
pattern in field]]
relevant_fields_by_collection[collection_name] = relevant_fields
return relevant_fields_by_collection
def main(patterns):
"""
:param list patterns: List of strings to look for in the db.
"""
db = get_db()
collection_names = get_collections_names(db)
fields_by_collections = get_fields(db, collection_names)
relevant_fields_by_collection = get_matches_fields(fields_by_collections, patterns)
report_matching_fields(relevant_fields_by_collection)
if __name__ == '__main__':
args = get_commandline_options()
main(args.patterns)

As per the mongoldb documentation, a combination of distinct
Finds the distinct values for a specified field across a single collection or view and returns the results in an array.
and indexes collection operations are what would return all possible values for a given key, or index:
Returns an array that holds a list of documents that identify and describe the existing indexes on the collection
So in a given method one could do use a method like the following one, in order to query a collection for all it's registered indexes, and return, say an object with the indexes for keys (this example uses async/await for NodeJS, but obviously you could use any other asynchronous approach):
async function GetFor(collection, index) {
let currentIndexes;
let indexNames = [];
let final = {};
let vals = [];
try {
currentIndexes = await collection.indexes();
await ParseIndexes();
//Check if a specific index was queried, otherwise, iterate for all existing indexes
if (index && typeof index === "string") return await ParseFor(index, indexNames);
await ParseDoc(indexNames);
await Promise.all(vals);
return final;
} catch (e) {
throw e;
}
function ParseIndexes() {
return new Promise(function (result) {
let err;
for (let ind in currentIndexes) {
let index = currentIndexes[ind];
if (!index) {
err = "No Key For Index "+index; break;
}
let Name = Object.keys(index.key);
if (Name.length === 0) {
err = "No Name For Index"; break;
}
indexNames.push(Name[0]);
}
return result(err ? Promise.reject(err) : Promise.resolve());
})
}
async function ParseFor(index, inDoc) {
if (inDoc.indexOf(index) === -1) throw "No Such Index In Collection";
try {
await DistinctFor(index);
return final;
} catch (e) {
throw e
}
}
function ParseDoc(doc) {
return new Promise(function (result) {
let err;
for (let index in doc) {
let key = doc[index];
if (!key) {
err = "No Key For Index "+index; break;
}
vals.push(new Promise(function (pushed) {
DistinctFor(key)
.then(pushed)
.catch(function (err) {
return pushed(Promise.resolve());
})
}))
}
return result(err ? Promise.reject(err) : Promise.resolve());
})
}
async function DistinctFor(key) {
if (!key) throw "Key Is Undefined";
try {
final[key] = await collection.distinct(key);
} catch (e) {
final[key] = 'failed';
throw e;
}
}
}
So querying a collection with the basic _id index, would return the following (test collection only has one document at the time of the test):
Mongo.MongoClient.connect(url, function (err, client) {
assert.equal(null, err);
let collection = client.db('my db').collection('the targeted collection');
GetFor(collection, '_id')
.then(function () {
//returns
// { _id: [ 5ae901e77e322342de1fb701 ] }
})
.catch(function (err) {
//manage your error..
})
});
Mind you, this uses methods native to the NodeJS Driver. As some other answers have suggested, there are other approaches, such as the aggregate framework. I personally find this approach more flexible, as you can easily create and fine-tune how to return the results. Obviously, this only addresses top-level attributes, not nested ones.
Also, to guarantee that all documents are represented should there be secondary indexes (other than the main _id one), those indexes should be set as required.

We can achieve this by Using mongo js file. Add below code in your getCollectionName.js file and run js file in the console of Linux as given below :
mongo --host 192.168.1.135 getCollectionName.js
db_set = connect("192.168.1.135:27017/database_set_name"); // for Local testing
// db_set.auth("username_of_db", "password_of_db"); // if required
db_set.getMongo().setSlaveOk();
var collectionArray = db_set.getCollectionNames();
collectionArray.forEach(function(collectionName){
if ( collectionName == 'system.indexes' || collectionName == 'system.profile' || collectionName == 'system.users' ) {
return;
}
print("\nCollection Name = "+collectionName);
print("All Fields :\n");
var arrayOfFieldNames = [];
var items = db_set[collectionName].find();
// var items = db_set[collectionName].find().sort({'_id':-1}).limit(100); // if you want fast & scan only last 100 records of each collection
while(items.hasNext()) {
var item = items.next();
for(var index in item) {
arrayOfFieldNames[index] = index;
}
}
for (var index in arrayOfFieldNames) {
print(index);
}
});
quit();
Thanks #ackuser

Following the thread from #James Cropcho's answer, I landed on the following which I found to be super easy to use. It is a binary tool, which is exactly what I was looking for:
mongoeye.
Using this tool it took about 2 minutes to get my schema exported from command line.

I know this question is 10 years old but there is no C# solution and this took me hours to figure out. I'm using the .NET driver and System.Linq to return a list of the keys.
var map = new BsonJavaScript("function() { for (var key in this) { emit(key, null); } }");
var reduce = new BsonJavaScript("function(key, stuff) { return null; }");
var options = new MapReduceOptions<BsonDocument, BsonDocument>();
var result = await collection.MapReduceAsync(map, reduce, options);
var list = result.ToEnumerable().Select(item => item["_id"].ToString());

This one lines extracts all keys from a collection into a comma separated sorted string:
db.<collection>.find().map((x) => Object.keys(x)).reduce((a, e) => {for (el of e) { if(!a.includes(el)) { a.push(el) } }; return a}, []).sort((a, b) => a.toLowerCase() > b.toLowerCase()).join(", ")
The result of this query typically looks like this:
_class, _id, address, city, companyName, country, emailId, firstName, isAssigned, isLoggedIn, lastLoggedIn, lastName, location, mobile, printName, roleName, route, state, status, token

I extended Carlos LM's solution a bit so it's more detailed.
Example of a schema:
var schema = {
_id: 123,
id: 12,
t: 'title',
p: 4.5,
ls: [{
l: 'lemma',
p: {
pp: 8.9
}
},
{
l: 'lemma2',
p: {
pp: 8.3
}
}
]
};
Type into the console:
var schemafy = function(schema, i, limit) {
var i = (typeof i !== 'undefined') ? i : 1;
var limit = (typeof limit !== 'undefined') ? limit : false;
var type = '';
var array = false;
for (key in schema) {
type = typeof schema[key];
array = (schema[key] instanceof Array) ? true : false;
if (type === 'object') {
print(Array(i).join(' ') + key+' <'+((array) ? 'array' : type)+'>:');
schemafy(schema[key], i+1, array);
} else {
print(Array(i).join(' ') + key+' <'+type+'>');
}
if (limit) {
break;
}
}
}
Run:
schemafy(db.collection.findOne());
Output
_id <number>
id <number>
t <string>
p <number>
ls <object>:
0 <object>:
l <string>
p <object>:
pp <number>

I was trying to write in nodejs and finally came up with this:
db.collection('collectionName').mapReduce(
function() {
for (var key in this) {
emit(key, null);
}
},
function(key, stuff) {
return null;
}, {
"out": "allFieldNames"
},
function(err, results) {
var fields = db.collection('allFieldNames').distinct('_id');
fields
.then(function(data) {
var finalData = {
"status": "success",
"fields": data
};
res.send(finalData);
delteCollection(db, 'allFieldNames');
})
.catch(function(err) {
res.send(err);
delteCollection(db, 'allFieldNames');
});
});
After reading the newly created collection "allFieldNames", delete it.
db.collection("allFieldNames").remove({}, function (err,result) {
db.close();
return;
});

I have 1 simpler work around...
What you can do is while inserting data/document into your main collection "things" you must insert the attributes in 1 separate collection lets say "things_attributes".
so every time you insert in "things", you do get from "things_attributes" compare values of that document with your new document keys if any new key present append it in that document and again re-insert it.
So things_attributes will have only 1 document of unique keys which you can easily get when ever you require by using findOne()

pg-promise task and map with multiple same level nested queries

I am using node and pg-promise to create a basic rest API and am having some issue querying all data for a particular user. Below is what the data returned should look like. Address, Phone Number, and Skills all live in separate tables. I am having no issue retrieving addresses or phone numbers its just skills I can't seem to get. Not quite sure how to have multiple queries after the main query that gets the user to get all these other fields, please see the attached code for reference and I will be happy to answer any questions.
{
"user_id": 1,
"first_name": "Eugene",
"last_name": "Hanson",
"display_name": "Eugene Hanson",
"email": "ehanson0#typepad.com",
"hash": "88a6aa27235d2e39dd9cb854cc246487147050f265578a3e1aee35be5db218ef",
"privilege_id": 14,
"seniority": 1,
"birthday": "19-11-1940 00:00:00.0",
"shift_count_total": 587,
"shift_count_year": 62,
"address_id": 1,
"street": "92 Schmedeman Lane",
"city": "Fort Smith",
"state": "AR",
"zip": 72905,
"phone_numbers": [
{
"phone_number": "62-(705)636-2916",
"name": "PRIMARY"
}
],
"skills": [
"Head Audio",
"Head Video",
"Head Electrician",
"Carpenter",
"rigger"
]
}
function getAllUsers() {
// console.time("answer time")
var deferred = Q.defer();
db.task(t => {
return t.map('SELECT * \
FROM users \
JOIN addresses \
ON users.address_id = addresses.address_id',[], user => {
var user_id = user.user_id;
// console.log(user_id)
console.time("answer time")
return t.manyOrNone('SELECT phone_numbers.phone_number, phone_types.name \
FROM users \
JOIN users_phone_numbers \
ON users.user_id = users_phone_numbers.user_id \
JOIN phone_numbers \
ON users_phone_numbers.phone_id = phone_numbers.phone_id \
JOIN phone_types \
ON phone_numbers.phone_type_id = phone_types.phone_type_id \
WHERE users.user_id = $1', user.user_id)
.then(phone_numbers=> {
// logger.log('info', phone_numbers)
user.phone_numbers = phone_numbers;
return user;
})
}).then(t.batch);
})
.then(data => {
// console.log(data)
console.timeEnd("answer time");
var response = {code: "200",
message: "",
payload: data};
deferred.resolve(response);
})
.catch(error => {
var response = {code: error.code,
message: error.message,
payload: ""};
logger.log('error', error)
deferred.reject(response)
});

I'm the author of pg-promise.
Simplified version of your function would be:
function getAllUsers() {
return db.task(t => {
return t.map('SELECT * FROM users', [], user => {
return t.batch([
t.any('SELECT * FROM phones'), // plus formatting params
t.any('SELECT * FROM skills'), // plus formatting params
])
.then(data => {
user.phones = data[0];
user.skills = data[1];
return user;
});
}).then(t.batch);
});
}
getAllUsers()
.then(data => {
// data tree
})
.catch(error => {
// error
});
And if you are using bluebird as the promise library, then you can replace this code:
.then(data => {
user.phones = data[0];
user.skills = data[1];
return user;
});
with this one:
.spread((phones, skills) => {
user.phones = phones;
user.skills = skills;
return user;
});
And do not use things like var deferred = Q.defer();, it is not needed there. The library is already promise-based.
For a high-performance alternative see: get JOIN table as array of results with PostgreSQL/NodeJS.

Better pattern for accessing nested data in Javascript

I am writing some code which iterates first through an array, and then further iterates through an array contained in the original array.
I am ending up with this weird pattern which I am repeating and I am certain is not optimized. While iterating through the last rssFeeds array item, it changes the value of 'triggerCallback' to true. Later, while iterating through the item array, a conditional checks if both triggerCallback is true and the items array is iterating through its last item, at which point it triggers a callback to be in used in async.js's waterfall pattern.
function countSuccessPosts(rssFeeds, cb){
var successCounter = 0;
var triggerCallback = ''
rssFeeds.forEach(function(feed, index, array){
if(index == array.length - 1){
triggerCallback = 'true'
}
feed.itemsToPost.forEach(function(item, itemIndex, itemArray){
if(item.response.id){
++successCounter
}
if(itemIndex == itemArray.length - 1 && triggerCallback == 'true'){
cb(null, rssFeeds, successCounter)
}
})
})
}
What's a more optimal way to structure this pattern?
Data Structure: RssFeeds will have up to 5 different itemsToPost elements.
[
{
"_id": "55808127b8f552c8157f74a7",
"name": "",
"imageUrl": "",
"url": "http://www.taxheaven.gr/bibliothiki/soft/xml/soft_law.xml",
"latestDate": "1434056400000",
"endpoints": [
{
"_id": "554f9319bc479deb1757bd2e",
"name": "Wise Individ",
"id": 26413291125,
"type": "Group",
"__v": 0
}
],
"__v": 1,
"itemsToPost": [
{
"title": "Aριθμ.: Υ194/12.6.2015 Τροποποίηση απόφασης ανάθεσης αρμοδιοτήτων στον Αναπληρωτή Υπουργό Οικονομικών Δημήτριο Μάρδα.",
"summary": "Τροποποίηση απόφασης ανάθεσης αρμοδιοτήτων στον Αναπληρωτή Υπουργό Οικονομικών Δημήτριο Μάρδα.",
"url": "http://www.taxheaven.gr/laws/circular/view/id/21113",
"published_at": 1434056400000,
"time_ago": "5 days ago",
"guid": {
"link": "http://www.taxheaven.gr/laws/circular/view/id/21113",
"isPermaLink": "true"
}
}
]
},
{
"_id": "558093013106203517f96d9c",
"name": "",
"imageUrl": "",
"url": "http://www.taxheaven.gr/bibliothiki/soft/xml/soft_new.xml",
"latestDate": "1434489601236",
"endpoints": [],
"__v": 0,
"itemsToPost": [
{
"title": "Taxheaven - Άμεση ενημέρωση - Έγκαιρη επιστημονική κωδικοποίηση - Καινοτομικά εργαλεία. Κωδικοποιήθηκαν όλοι οι νόμοι στους οποίους επιφέρει αλλαγές ο νόμος 4330/2015",
"summary": {},
"url": "http://www.taxheaven.gr/news/news/view/id/24088",
"published_at": 1434494400000,
"time_ago": "about 4 hours ago",
"guid": {
"link": "http://www.taxheaven.gr/news/news/view/id/24088",
"isPermaLink": "true"
}
}
]
}
]

I didn't check this but it is pretty similar to what I'm currently using in my project:
function countSuccessPosts(rssFeeds, cb){
async.each(rssFeeds, function(eachFeed, outerCallback) {
async(eachFeed.itemToPost, function(eachItem, innerCallback) {
if(item.response.id) {
//Do Something That Is Actually Async. Could be asking the server for success flag, for instance.
doSomethingThatIsActuallyAsync(item.response.id).then(function (err) {
if (!err) {
successCounter = successCounter + 1;
}
innerCallback();
});
} else { //This could be to skip "empty responses" without the need to go to the server, right?
innerCallback();
}
}, outerCallback);
}, function() {
//All done
cb(null, rssFeeds, successCounter);
});
}
As others mentioned, you need this only if you have actual async methods calls inside the inner loop.

You don't need to keep track of the last item. Just call the callback after both loops exit. I also changed the .forEach to for loops as these execute faster.
function countSuccessPosts(rssFeeds, cb) {
var index, itemIndex, feed, item;
for (index = 0; index < rssFeeds.length; index++) {
feed = rssFeeds[index];
for (itemIndex = 0; itemIndex < feed.itemsToPost.length; itemIndex++) {
item = feed.itemsToPost[itemIndex];
if(item.response && item.response.id) {
successCounter++;
}
}
}
cb(null, rssFeeds, successCounter);
}
Of course, if you'd rather call countSuccessPosts without a callback the calling code can look like:
var successPosts = countSuccessPosts(rssFeeds);
And you can reformat the function to look like this:
function countSuccessPosts(rssFeeds) {
var index, itemIndex, feed, item, successCounter = 0;
for (index = 0; index < rssFeeds.length; index++) {
feed = rssFeeds[index];
for (itemIndex = 0; itemIndex < feed.itemsToPost.length; itemIndex++) {
item = feed.itemsToPost[itemIndex];
if(item.response && item.response.id) {
successCounter++;
}
}
}
return successCounter;
}

Wait, why are you using a callback when you can read the data synchronously?
After you've updated your question, it looks like you're just summing a
number of elements in an array
Here's a fully synchronous version that counts the number of itemsToPost that have a valid response.id set.
function countSuccessPosts(rssFeeds) {
return rssFeeds.reduce(function(sum, x) {
return sum + x.itemsToPost.filter(function(y) {
return !!y.response.id;
}).length;
}, 0);
}
If you're required to inject this into an async control flow, you can easily put a wrapper on it
function(rssFeeds, done) {
done(null, rssFeeds, countSuccessPosts(rssFeeds));
}
The point tho, is that countSuccessPosts has a synchronous API because everything that happens within that function is synchronous.

Missing Children

Using Mongoose with MongoDB, my Schema is as follows:
var PartSchema = new Schema({
partcode: String,
children: [String]
});
And the data looks like the following:
[{"partcode":"A1","children":["B1","B2","B3","B4"]},
{"partcode":"B1","children":["C11","C21","C31","C41"]},
{"partcode":"B3","children":["C13","C23","C33","C43"]},
I can query for A1's children field by using the following static call:
PartSchema.static('getChildren', function (partcode, callback) {
var self = this;
self.findOne({ partcode: partcode }, childrenOnly)
.exec(function (err, doc) {
return self.find({"partcode": {"$in": doc.children} }, exclId, callback);
});
});
This returns (via express)
[{"partcode":"B1","children":["C11","C21","C31","C41"]},
{"partcode":"B3","children":["C13","C23","C33","C43"]}]
What I need is to return all children not found, for example:
[{"children":["B2","B4"}]

You could use the _.difference() method from the lodash library to calculate the array set difference:
var _ = require("lodash");
PartSchema.static('getChildren', function (partcode, callback) {
var self = this;
self.findOne({ partcode: partcode }, childrenOnly)
.exec(function (err, doc) {
var promise = self.find({"partcode": {"$in": doc.children} }, exclId).lean().exec();
promise.then(function (res){
var codes = res.map(function (m) {
return m.children;
}),
obj = {"children": _.difference(doc.children, codes)},
result = [];
result.push(obj);
return result;
});
});
});
-- UPDATE --
With MongoDB's aggregation framework, you can achieve the desired result. Let's demonstrate this in mongoshell first.
Suppose you insert the following test documents in the parts collection:
db.part.insert([
{"partcode":"A1","children":["B1","B2","B3","B4"]},
{"partcode":"B1","children":["C11","C21","C31","C41"]},
{"partcode":"B3","children":["C13","C23","C33","C43"]}
])
The aggregation can be useful here given that you have an array of the children partcodes for a given partcode, say "A1", which is ["B1","B2","B3","B4"]. In this instance, your aggregation pipeline would consist of the following aggregation pipeline stages:
$match - You need this to filter those documents whose children partcodes are not in the ["B1","B2","B3","B4"] array. This is achieved using the $nin operator.
$group - This groups all the documents from the previous stream and creates an additional array field that has the parent partcodes. Made possible by using the $addToSet accumulator operator.
$project - Reshapes each document in the stream by adding a new field partcode (which will eventually become part of the result object) and suppresses the _id field. This is where you can get the array difference between the parent partcode in the criteria and those not in the pipeline documents, made possible using the $setDifference set operator.
Your final aggregation operator would look like this (using mongoshell):
var children = ["B1","B2","B3","B4"];
db.part.aggregate([
{
"$match": {
"children": { "$nin": children }
}
},
{
"$group": {
"_id": null,
"parents": {
"$addToSet": "$partcode"
}
}
},
{
"$project": {
"_id": 0,
"partcode": {
"$setDifference": [ children, "$parents" ]
}
}
}
]).pretty()
Output:
/* 0 */
{
"result" : [
{
"partcode" : ["B2","B4"]
}
],
"ok" : 1
}
Using the same concept in your Mongoose schema method:
PartSchema.static('getChildren', function (partcode, callback) {
var self = this;
self.findOne({ partcode: partcode }, childrenOnly)
.exec(function (err, doc) {
var pipeline = [
{
"$match": {
"children": { "$nin": doc.children }
}
},
{
"$group": {
"_id": null,
"parents": {
"$addToSet": "$partcode"
}
}
},
{
"$project": {
"_id": 0,
"partcode": {
"$setDifference": [ doc.children, "$parents" ]
}
}
}
],
self.aggregate(pipeline).exec(callback);
});
});
Or using Mongoose aggregation pipeline builder for a fluent call:
PartSchema.static('getMissedChildren', function (partcode, callback) {
var self = this;
self.findOne({ partcode: partcode }, childrenOnly)
.exec(function (err, doc) {
var promise = self.aggregate()
.match({"children": { "$nin": doc.children }})
.group({"_id": null,"parents": {"$addToSet": "$partcode"}})
.project({"_id": 0,"partcode": {"$setDifference": [ doc.children, "$parents" ]}})
.exec(callback);
});
});