Develop Reference

Making an api call inside a loop in node.js

I am trying to scan the top 100 movie torrents on the pirate bay using node and add a movie poster for each result.
I am using these libraries
thepiratebay
imdb-api
I am able to find top 100 and return the results with no problems
app.get('/movies', function(req, res){
tpb.topTorrents(207).then(function(topMovies){
async.map(topMovies, tpb.getTorrent, function(err, results){
res.send(results);
})
})
});
I am also able to look up movies via an IMDB ID and return the results
app.get('/imdb', function(req, res){
imdb.getReq({ id: 'tt2660888' }, function(err, things) {
res.send(things);
});
});
What I am trying to do is loop over the top 100 results pull the imdb id out of the description field out and query imdb replacing the picture field with result.
app.get('/movies', function(req, res){
tpb.topTorrents(207).then(function(topMovies){
async.map(topMovies, tpb.getTorrent, function(err, results){
for (var value of results) {
if (S(value.description).contains('www.imdb.com/title/')) {
var imdbId = S(value.description).between('www.imdb.com/title/', '/').s
imdb.getReq({ id: imdbId }, function(err, movie) {
value["picture"] = movie.poster
});
}
}
res.send(results);
})
})
});
This isn't working for some reason but it makes sense to me intuitively. If I remove the imdb-api call and replace it with value["picture"] = "foo". It does work. I'm not sure if this is related to how node handles loops. I'm new to the JS world and have a ruby background
Thanks in advance

You are on the right track with the async module but the imdb requests are also asynchronous so res.send just gets called with the initial result of async.map
You can use another async.map for the imdb calls and them chain them with async.waterfall which will pass the results of the first function as an argument to the second (async.apply just invokes the tpb function with your topMovies).
function tpb (topMovies, done) {
async.map(topMovies, tpb.getTorrent, done);
}
function imdb (movies, done) {
function lookup (value, callback) {
if (S(value.description).contains('www.imdb.com/title/')) {
var imdbId = S(value.description).between('www.imdb.com/title/', '/').s
imdb.getReq({ id: imdbId }, function(err, movie) {
value["picture"] = movie.poster
return cb(err, value);
});
} else {
return callback(null);
}
}
async.map(movies, lookup, done);
}
app.get('/movies', function(req, res){
tpb.topTorrents(207).then(function(topMovies){
async.waterfall([async.apply(tpb, topMovies), imdb], function (err, results) {
if (err) {
// do error handling
}
return res.send(results);
});
});
});

How to wait for moongose to finish query?

router.post('/logout', function(req, res) {
if(req.body.newList){
var aux= JSON.parse(req.body.newList);
var aux1;
var i=0;
while(i<aux.length){
aux1=aux[i];
Task.findOne({username:aux1.username,text:aux1.text},function(err,res){
if(res){
i++;
}else{
task= new Task({username:aux1.username,text:aux1.text,state:aux1.state});
task.save(function(err){
i++;
});
}
});
}
}
Well, I have this so far. aux stores an array of json objects, they dont have an _id, so.. I need to iterate through them saving one by one when needed. Problem is (I figure) async, cause this code is saving in the data base only the last element of the array n times, being n the amount of diferent elements which should be saved**(see example). So, I think the async behavior of moongose querys is giving me the headache. Probably keeps iterating waiting for the result of the query, and then save with the last value of the array.
** for instance, if I have these values for save [{username:'x',text:'a'},{username:'x',text:'aa'},{username:'x',text:'aaa'}]..
It saves [{username:'x',text:'aaa'},{username:'x',text:'aaa'},{username:'x',text:'aaa'}]
I tried using promises, but it didn´t work.. Probably used them wrong

For such things I prefer using the caolan's async library. Simply do:
npm install async to add the module.
Then replace your code with:
router.post('/logout', function(req, res) {
if (req.body.newList) {
var aux = JSON.parse(req.body.newList);
var async = require("async");
async.each(aux, function(aux1, callback) {
Task.findOne({
username: aux1.username,
text: aux1.text
}, function(err, res) {
if (res) {
callback();
} else {
task = new Task({
username: aux1.username,
text: aux1.text,
state: aux1.state
});
task.save(function(err) {
callback();
});
}
});
}, function(err) {
// if any of the query processing produced an error, err would equal that error
if (err) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('A query failed to process');
} else {
console.log('All queries have been processed successfully');
//do other things here
}
});
}
});

nodejs and MongoDB's collection.find() does not respond

I have about 30,000 documents in a MongoDB collection. And have been stuck in developing a node.js script to retrieve only the records with a specific string key-value pair.
this query on MongoDB server returns me the exact results I've been looking for:
db.getCollection('posts').find({authorName: "Ashwin-kumar"})
Returns me about 33 documents instantly. Likewise I've about 40 authors with different names.
Here's my node.js script to retrieve posts by authorName (Yes, it is based on Name, a string, as there is no ID for these authors :( ):
var fs = require('fs'),
request = require('request'),
async = require("async"),
assert = require('assert');
_ = require('lodash'),
MongoClient = require('mongodb').MongoClient;
var db, postsCollection, postCol;
async.series([dbConnect, checkCollection, createMeta, dbClose], function(){
console.log("Executed all calls in series.");
process.exit(0);
});
function dbConnect(callback){
MongoClient.connect("mongodb://localhost:27017/jPosts", function(pErr, pDb) {
if(pErr) {
console.dir(pDb);
return 0;
}
db = pDb;
callback();
});
}
function dbClose(callback){
db.close(true, function (err) {
if (err) console.error(err);
else console.log("close complete");
callback();
});
}
function checkCollection(callback) {
db.collection('posts', function(err, collection) {});
postsCollection = db.collection('posts');
postCol = db.collection('posts');
callback();
}
function createMeta(callback){
var meta = [];
postsCollection.aggregate([
{
$group : {_id : "$authorName"}
}]).toArray(function(err, result) {
assert.equal(err, null);
async.forEachLimit(result, 1, function(pPost, callback) {
getPosts(pPost._id, callback);
}, function(err) {
console.log(err);
callback();
});
});
}
function getPosts(pAuthor, callback){
var cursor = postCol.find({ "authorName": pAuthor});
cursor.toArray(function(err,items){
if(err)
callback(err);
else
callback(null, items);
});
}
This does not seem to work for me. cursor.toArray() does nothing but wait forever. Is it because of too many fields in each document?
I tried to get the count of the documents the cursor fetched and it works well.
function getPosts(pAuthor, callback){
var cursor = postCol.find({ "authourName": pAuthor});
cursor.count().then(function(items_count) {
console.log(items_count);
callback();
});
}
Also, I tried the cursor's .each method to iterate the documents fetched. But no luck yet.
function getPosts(pAuthor, callback){
var cursor = postCol.find({ "authourName": pAuthor});
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
console.dir(doc);
} else {
console.log(err);
}
});
}
Am I missing something here? What else can be done to make this work? Is there any issues with the way I'm using async?
P.S: The idea here is to query the dump and generate the PDF's for authours in the jPost collection.
P.S 2: Here's a sample document
{
"_id" : ObjectId("571d36b55672f713fe346a66"),
"id" : 56517,
"authorName" : "Ashwin-kumar",
"comment_count" : 380,
"tagline" : "... Opinions you don't really need",
"vote_count" : 5152,
"exclusive" : null,
"post": [
],
"post_comments" : [
//comment_count objects
],
"date" : "2016-03-27"
}
(I've omitted post & post_comments parts for brevity.)

try this:
var collection = db.collection("collection_name");
collection.find({authourName: "Ashwin-kumar"}).toArray(function (err,items) {
if (err) {
console.dir(err);
} else {
//do something with items array
console.dir(items);
}
});

Did you check what is the value of pAuthor in getPosts? Because when you do aggregation, you receive a collection of objects with _id field (not authourName), so you should do:
// not sure why you need meta array, at least it's not used in the code you provided
meta.push({
author: pPost._id
});
getPosts(pPost._id, callback);

mongoose "Find" with multiple conditions

I am trying to get data from my mongoDB database by using mongoose filters. The scenario is that each user object in the database has certain fields like "Region" or "Sector".
Currently I am getting all the users that contain the keyword "region" in there object like so:
// Filter all healthcare bios by region
app.get('/user',function(req, res) {
// use mongoose to get all users in the database
User.find({region: "NA"}, function(err, user)
{
// if there is an error retrieving, send the error. nothing after res.send(err) will execute
if (err)
{
res.send(err);
}
// return all todos in JSON format
console.log(user);
res.json(user);
});
});
How can put some conditions in mongoose that it return users that contain both "region" && "Sector" in their objects. Currently its only returning the user which have the region keyword in them.
I have tried using $and operator but I couldn't get it to work.

app.get('/user',function(req, res) {
User.find({region: "NA",sector:"Some Sector"}, function(err, user)
{
if (err)
{
res.send(err);
}
console.log(user);
res.json(user);
});
});
If you want data with either region:"NA" or sector:"Some Sector". you can use $or operator.
User.find({$or:[{region: "NA"},{sector:"Some Sector"}]}, function(err, user)
{
if (err)
{
res.send(err);
}
console.log(user);
res.json(user);
});

If you want results that contain any region or sector as long as both are present at the same time you need the following query in your User.find:
{region: {$exists:true},sector: {$exists:true}}
, is the equivalent of $and as long as you are searching different fields.

const dateBetweenDates = await Model.find({
$or: [
{
$and: [
{ From: { $gte: DateFrom } },
{ To: { $lte: DateTo } },
], // and operator body finishes
},
{ _id: req.user.id},
], //Or operator body finishes
})

For anyone else trying to find with multiple conditions using mongoose, here is the code using async/await.
app.get('/user', async (req, res) {
const user = await User.find({region: "NA",sector:"Some Sector"});
if (user) {
// DO YOUR THING
}
});

How to wait mongoose .exec function to be done?

I am some cofused by asychronous nodejs and mongoose. Simplily, I want to post an array of usernames and check, if a username is in database, then I put it in the valid array, otherwise, put it in the invalid array.
Here is my current code:
var User = require('../../db/models/user');
api.post('/userlist', function(req, res) {
var invalid = []; // usernames which can not be found in database
var valid = []; // usernames which can be found in database
(req.body.userlist).forEach(function(username) {
User
.findOne({username: username})
.exec(function(err, user) {
if (err) {
res.send(err);
return;
} else if (!user) {
invalid.push(username);
} else {
valid.push(req.params.item);
}
});
});
res.send({
Invalid: invalid,
Valid: valid
});
});
When I executed the above code, it outputs the intial empty array directly.
Invalid: [],
Valid: []
I know it is because nodejs first execute this res.send then execute function .exec(function(err, user), but i do not know how to get the right invalid and valid array, pls advise.

Your best bet is to use a promise:
api.post('/userlist', (req, res) => {
// Takes a username and returns a promise for information on that username.
function findByUsername(username) {
return new Promise((resolve, reject) =>
User.findOne({username}).exec((err, user) =>
err ? reject(err) : resolve(user)
)
);
}
// Iterate the array and transform each user to a promise for data on that user.
Promise.all(req.body.userlist.map(findByUsername))
// Then, when all of the promises in that new array resolve
.then(allUserDataInOrder => {
// Find all the valid ones (if (user))
let Valid = allUserDataInOrder.filter(Boolean); // Only those who are truthy
// And all the invalid ones (if (!user))
let Invalid = allUserDataInOrder.filter(userData => !userData); // Sadly, no convenient function here :(
// And send both away
res.send({Valid, Invalid}); // Short syntax FTW!
})
.catch(res.send); // Called with error object if any.
});

While these other solutions solve what you're trying to accomplish, they still incorporate bad design by iterating findOne(). Executing 1 query for every item in your list is incredibly inefficient. Using an $in query and a basic map, you can use a single query:
var User = require('../../db/models/user');
api.post('/userlist', function(req, res) {
User.find({username: {$in: req.body.userlist}}, function(err, users) {
if (err) {
return res.send(err);
}
// create a map of all the users in your list that exist in your database
var dbUserMap = {};
users.forEach(function(user) {
dbUserMap[user.username] = true;
});
var valid = [];
var invalid = [];
// check your POST list against the database map
req.body.userlist.forEach(function(username){
if (dbUserMap[username]) {
valid.push(username);
}
else {
invalid.push(username);
}
});
res.send({
valid: valid,
invalid: invalid
});
});
});

Try to use async module:
var invalid = [];
var valid = [];
async.each(req.body.userlist, function(name, next) {
User.findOne({username: name}, function(err, user) {
if (err) {
return next(err);
}
if (!user) {
invalid.push(name);
} else {
valid.push(name);
}
next();
)};
}, function(err) {
if (err) {
return res.send(err);
}
res.send({
Invalid: invalid,
Valid: valid
});
});