I am trying to work with twitter data in node and am running into some road blocks that I think should be related to node style coding. The block of code is meant to grab the tweets, check if the text is in mongo and if not insert it.
The first stumbling block I find is that on trying to print out i to the console it will always iterate through each i before it starts iterating through the cursor. I think if I could clear that up it may help me going forward. Is this enough info to help me out?
What I have is:
T.get('statuses/user_timeline', options , function(err, data) {
var db = MongoClient.connect('mongodb://127.0.0.1:27017/test', function(err, db) {
if(err)
throw err;
console.log("connected to the mongoDB !");
myCollection = db.collection('test_collection2');
for (var i = 0; i < data.length ; i++) {
//let's wrap this in a loop
docu = data[i];
//console.dir(data);
console.dir(i);
var cursor = myCollection.find({text : data[i].text}).limit(1);
cursor.each(function(err, doc) {
if (doc != null) {
console.dir('doc is not null');
console.dir(doc.text);
} else {
console.dir('doc is null - inserting');
myCollection.insert(docu, function(err, records){
console.log("Record added as "+records.text);
});
}
})
}
});
})
The problem is just because Javascript is async. The loop is finished before the find function in Mongo gives you a return value.
I would do the Following, or something similar - just to explaine the concept:
T.get('statuses/user_timeline', options , function(err, data) {
var db = MongoClient.connect('mongodb://127.0.0.1:27017/test', function(err, db) {
if(err)
throw err;
console.log("connected to the mongoDB !");
myCollection = db.collection('test_collection2');
var myfunction = function(correct_i,docu){
var cursor = myCollection.find({text : data[correct_i].text}).limit(1);
cursor.each(function(err, doc) {
if (doc != null) {
console.dir('doc is not null');
console.dir(doc.text);
} else {
console.dir('doc is null - inserting');
myCollection.insert(docu, function(err, records){
console.log("Record added as "+records.text);
});
}
})
};
for (var i = 0; i < data.length ; i++) {
//let's wrap this in a loop
docu = data[i];
//console.dir(data);
console.dir(i);
myfunction(i,docu);
}
});
})
This way each lookup/find to Mongo will have the correct i. because the function gets it as an parameter
Related
im making posts, comment etc i want to redirect upper route.
im making delete post's comment.
after delete, i want to redirect upper route, without a specific route
router.route('/post/comment/destroy/:postroot').post(function (req, res) {
var database = req.app.get('database')
var postroot = path.parse(req.params.postroot).base;
var filterd = req.body.commentid;
database.PostModel.findOne({
"_id": postroot
}, function (err, rawContent) {
if (err) throw err;
var idx;
for (var i = 0; i < rawContent.comments.length ; i++){
if (rawContent.comments[i]._id == filterd) {
idx = i;
break;
}
}
rawContent.comment.commentcount-=1;
rawContent.comments.splice(idx, 1);
rawContent.save(function (err) {
if (err) throw err;
});
});
res.redirect(`/post/${postroot}`);
});
simply use your params inside the router
res.redirect('/post/'+req.params.postroot);
I've been pointed towards using the async module, but I'm not quite sure how to use waterfall to solve my problem.
My original code had problems with asynchronicity.
var Image = require('./models/image');
var User = require('./models/user');
var query = Image.find({});
query.limit(10);
query.sort('-date')
query.exec(function (err, collected) {
if (err) return console.error(err);
var i = 0;
var authors = [];
while (i < 8) {
var search = User.find({'twitter.id' : collected[i].author});
search.exec(function (err, user){
if (err) return console.error(err);
var result = (user[0].twitter.username);
authors.push(result);
});
i = i + 1;
}
}
console.log(authors);
I want the authors array to hold all the found usernames. However when that last console.log() call returns '[]'
So, you want to wait for all of the searches to complete first. You should put all your async calls into an array, and then use an async library to chain them together (waterfall) or execute simultaneously (parallel). Parallel tends to execute "faster":
var searches = [];
while (i < 8) {
var search = User.find({'twitter.id' : collected[i].author});
searches.push(function(cb) {
search.exec(function (err, user){
if (err) cb(err, null);
else cb(null, user[0].twitter.username);
});
});
i++;
}
async.parallel(searches, function( err, authors ) {
if ( err ) return console.error( err );
console.log(authors);
// Authors only defined here.
// TODO: More code
});
// Authors not defined here.
So i have a csv file containing my information, i need to do a mass add/update
exports.add_questions_from_file = function (file_path, surveyid, callback)
{
var U = [{}];
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
console.log(data);
d = data.split(/\r\n|\n/);
for (x=0;x <d.length;x++)
{
line = d[x].split(',');
if (line[0] == "") {return};
RQuestion.add_by_line (line,function (err, question)
{
U.push({id:question.id});
console.log(U);
});
}
}
});
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":U}},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
};
But even though im using callback functions the update seems to happen with the same object always, even the console.log here
U.push({id:question.id});
console.log(U);
returns the same object (even that all the other were created)
Im doing something wrong?
I see a few issues.
First for:
if (line[0] == "") {return};
Don't you mean to use a break or continue instead? Otherwise the entire function will quit if there is a blank line anywhere in the file. This is very important because Survey.update won't get called either.
Second: I assumed that RQuestion.add_by_line and Survey.update are doing something async like updating a database. Your code needs to be restructured to wait for those async items to complete before moving on to the next step. I'd recommend an npm package named async for that.
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
d = data.split(/\r\n|\n/);
async.map(d, function(line, callback) {
//this function is called for each line
add_by_line (line,function (err, question)
{
callback(err,{id:question.id});
});
}, function(err, results) {
//this function is called when all of the items are done
console.log("done with async");
console.dir(results);
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":results},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
});
}
});
How do you access mongodb count results in nodejs so the result can be accessible to the asynchronous request? I can get the result and update the database but the asynchronous request fails to access the vars or the vars are empty and the vars appear to be updated when the next asynchronous request is made. The request must not be waiting for the query to finish and the next request is filled with the previous request's variables.
testOne.increment = function(request) {
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/bbb_tracking', function(err, db) {
if (err) throw err;
collection = db.collection('bbb_tio');
collection.count({vio_domain:dom}, function(err, docs) {
if (err) throw err;
if (docs > 0) {
var vio_val = 3;
} else {
var vio_val = 0;
}
if (vio_val === 3) {
event = "New_Event";
var inf = 3;
}
db.close();
console.log("docs " + docs);
});
});
};
In the above, even when the vars are set in scope they are not defined asynchronously. Can I get some guidance on structuring this properly so the vars are populated in the callback. Thank you!
Since the count function is asynchronous, you'll need to pass a callback to the increment function so that when the count is returned from the database, the code can call the callback.
testOne.increment = function(request, callback) {
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/bbb_tracking', function(err, db) {
if (err) throw err;
var collection = db.collection('bbb_tio');
// not sure where the dom value comes from ?
collection.count({vio_domain:dom}, function(err, count) {
var vio_val = 0;
if (err) throw err;
if (count > 0) {
vio_val = 3;
event = "New_Event";
var inf = 3;
}
db.close();
console.log("docs count: " + count);
// call the callback here (err as the first parameter, and the value as the second)
callback(null, count);
});
});
};
testOne.increment({}, function(err, count) {
// the count would be here...
});
(I don't understand what the variables you've used mean or why they're not used later, so I just did a bit of a clean-up. Variables are scoped to function blocks and hoisted to the function, so you don't need to redeclare them in each if block like you had done with vio_val).
You could use the 'async' module. It makes the code a lot cleaner and easier to debug. Take a look at the code in GitHub for adduser.js & deleteuser.js in the following post
http://gigadom.wordpress.com/2014/11/05/bend-it-like-bluemix-mongodb-using-auto-scaling-part-2/
Regards
Ganesh
length give you count of result array
const userdata = await User.find({ role: role, 'name': new RegExp(searchkey, 'i') },{date: 0,__v:0,password:0}).
sort(orderObj)
.limit(limit)
.skip(skip);
console.log(userdata.length);
I have updated the post with the actual code.
The problem is that the node app hangs and does not exit unless I comment out the query in addArticle. I am wonder what I'm doing wrong here (in regards to the hanging problem).
function addArticle(title, text, date, link) {
connection.query("SELECT * FROM articles WHERE link LIKE "+connection.escape(link), function(error, rows, fields) {
if(rows.length == 0) {
console.log("article not in database");
console.log(connection.escape(title));
var values = [connection.escape(title), connection.escape(text), date, connection.escape(link), '{}'];
connection.query("INSERT INTO articles (title, text, date, link, topics) VALUES ?", [[values]], function(err) {
if(err) throw err;
});
}
});
}
function scrapeReuters() {
var url = 'http://www.reuters.com/news/archive/technologyNews?date=10092013';
request(url, function(err, resp, body){
$ = cheerio.load(body);
links = $('a');
$(links).each(function(i, link){
var addr = $(link).attr('href');
if(addr != undefined && addr.indexOf('article') != -1 && addr.indexOf('http') == -1 ) {
var full_link = "http://www.reuters.com"+addr;
var title = $(link).text();
request(full_link, function(err, resp, body){
$ = cheerio.load(body);
para = $('p').text();
addArticle(title, para,new Date().getTime(), full_link);
});
}
});
});
}
You probably need to close the connection after all the queries have finished. You can try using the https://github.com/caolan/async library to run the queries in sequence and then in a master callback, close the connection.
Its a little tricky, but first you need to define an array of functions to execute. Then you run async.sequence(arrayOfFns,masterCallback). The master callback gets errs and results (notice plural, its from all the functions). In that master callback, terminate the mysql connection/and or end the process.
To do this, I would rewrite the addArticle query to just return the query string. Then before your $(links).each loop, I would make an array called toInsert
In each loop I would say
toInsert.push(function(callback) {
connection.query(addArticle(...),function(err) {
if(err) callback(err);
else callback(null,true);
});
});
Then after the loop run
async.sequence(toInsert,function(errs,results) {
connection.close() //not sure if correct
process.exit(); //maybe, if needed?
});