I have two functions that query the twitter api. The query is done through a web interface and I call each function when a checkbox is on. My problem now is after all the querying has been done, I want to be able to store the data and send it back to the web interface. How do I do this ?
if (string.checktweet1 == 'on') {
tweets(string.teamname)
}
if (string.checkmentions1 == 'on'){
mentions(string.teamname)
}
if (string.checktweet2 == 'on'){
tweets(string.playername)
}
if (string.checkmentions2 == 'on'){
mentions(string.playername)
}
function mentions(x){
client.get('search/tweets', {q:x, count:1},
function (err,data,){
for(var index in data.statuses){
var tweet = data.statuses[index];
console.log(tweet.text);
}
})
}
My code is only sending the data for the function "tweets"
json = {};
function tweets(y){
client.get('statuses/user_timeline', {screen_name:y, count:1},
function(err,data) {
for(var index in data){
var tweet = data[index];
console.log(tweet.text);
}
json[index] = tweet
res.end(JSON.stringify(json));
})
}
As I understand you are not looking to save the data, but just collect the results of multiple asynchronous calls and once all are completed deliver the data to your client? If so, you could use async or promises.
There are already examples of this in Stack Overflow, eg. this Node.js: Best way to perform multiple async operations, then do something else? but here anyways simplified implementations for both.
Using async
var async = require('async');
// ...
var tweets = function(y) {
return function(cb) {
client.get('statuses/user_timeline', {screen_name: y, count: 1},
function(err, data) {
// Process the data ...
cb(null, processedTweets);
}
);
}
};
var mentions = function(x) {
return function(cb) {
client.get('search/tweets', {q: x , count: 1},
function(err, data) {
// Process the data ...
cb(null, processedMentions);
}
);
}
};
app.get('/mytweetsapi', function(req, res) {
var tasks = [];
if (string.checktweet1 == 'on') {
tasks.push(tweets(string.teamname));
}
if (string.checkmentions1 == 'on'){
tasks.push(mentions(string.teamname));
}
if (string.checktweet2 == 'on'){
tasks.put(tweets(string.playername));
}
if (string.checkmentions2 == 'on'){
tasks.put(mentions(string.playername));
}
async.parallel(tasks, function(err, results) {
// Process the results array ...
res.json(processedResults);
});
});
Using promises
var Promise = require('bluebird');
// ...
var tweets = function(y) {
return new Promise(function(resolve, reject) {
client.get('statuses/user_timeline', {screen_name: y, count: 1},
function(err, data) {
// Process the data ...
resolve(processedTweets);
}
);
});
};
var mentions = function(x) {
return new Promise(function(resolve, reject) {
client.get('search/tweets', {q: x , count: 1},
function(err, data) {
// Process the data ...
resolve(processedMentions);
}
);
});
};
app.get('/mytweetsapi', function(req, res) {
var tasks = [];
// Same if this then tasks.push as in async example here
Promse.all(tasks).then(function(results) {
// Process the results array ...
res.json(processedResults);
});
});
I don't know what HTTP client you are using, but you can maybe use var client = Promise.promisifyAll(require('your-client-lib')); to convert the methods to return promises, and then you could convert the tweets and mentions functions to
var tweets = function(y) {
return client.get('statuses/user_timeline', {screen_name: y, count: 1});
};
This way though the results in Promise.all are mixed responses and you would need to identify which are tweets and which are mentions to process them properly.
For saving the data to a file without a database, you can use fs.writeFile():
fs.writeFile('message.txt', 'Hello Node.js', (err) => {
if (err) throw err;
console.log('It\'s saved!');
});
https://nodejs.org/api/fs.html#fs_fs_writefile_file_data_options_callback
Server/Node:
var app = require('http').createServer(handler)
var io = require('socket.io')(app);
var fs = require('fs');
app.listen(80);
function handler (req, res) {
fs.readFile(__dirname + '/index.html',
function (err, data) {
if (err) {
res.writeHead(500);
return res.end('Error loading index.html');
}
res.writeHead(200);
res.end(data);
});
}
io.on('connection', function (socket) {
if(twitterInfoReady){
socket.emit('twitterInfoIncoming', {
twitterInfo1: 'blah',
twitterInfo2: 'more data',
twitterInfo3: 'and more data'
});
}
});
Client
<script src="/socket.io/socket.io.js"></script>
<script>
var socket = io('http://localhost');
socket.on('twitterInfoIncoming', function (data) {
console.log(data.twitterInfo1);
});
</script>
Example taken from Socket.io docs, just modified a tiny bit http://socket.io/docs/
Related
I am trying to use async with node.js to handle multiple incoming POST requests to edit a JSON file. No matter how I refactor it, it will always make one of the edits and not the other. I though that using async.queue would force the operations to handle sequentially? What am I doing wrong?
My code:
var editHandler = function(task, done) {
var req = task.req;
var res = task.res;
fs.stat( "./app//public/json/" + "data.json", function(err, stat) {
if(err == null) {
console.log('File exists');
} else if(err.code == 'ENOENT') {
console.log("Error");
} else {
console.log('Some other error: ', err.code);
}
});
console.log(req.params.id);
console.log(req.body);
fs.readFile( "./app//public/json/" + "data.json", 'utf8', function (err, data) {
data = JSON.parse( data );
data[req.params.id] = req.body.school;
//console.log( data );
fs.writeFile("./app//public/json/" + "data.json", JSON.stringify(data), function (err){
if(err) {
return console.log(err);
}
})
res.redirect('/');
});
};
//Make a queue for the services
var serviceQ = async.queue(editHandler, 20);
serviceQ.drain = function() {
console.log('all services have been processed');
}
app.post('/edit_school/:id', function(req, res) {
serviceQ.push({req: req, res: res })
})
Thanks in advance for any insights! I am really new to using node.js for anything other than npm/webpack.
I would like to make http call synchronously using Q Promises, I have 100 students that I need for each of them to take some data from another platform and to do that I was trying via Q Promises but it does not seem like it is doing synchronously.
How do I make sure that another call is not being made once one is finished with parsing it's response and insertion into mongodb:
my code so far looks like this:
var startDate = new Date("February 20, 2016 00:00:00"); //Start from February
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.eachSeries(students, function iteratee(student, callback) {
if (student.worksnap.user != null) {
var worksnapOptions = {
hostname: 'worksnaps.com',
path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic xxxx='
},
method: 'GET'
};
promisedRequest(worksnapOptions)
.then(function (response) { //callback invoked on deferred.resolve
parser.parseString(response, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
saveTimeEntry(item);
});
});
});
callback();
}, function (newsError) { //callback invoked on deferred.reject
console.log(newsError);
});
}
});
function saveTimeEntry(item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
throw err;
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('item inserted...');
}
});
});
}
function promisedRequest(requestOptions) {
//create a deferred object from Q
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var deferred = Q.defer();
var req = http.request(requestOptions, function (response) {
//set the response encoding to parse json string
response.setEncoding('utf8');
var responseData = '';
//append data to responseData variable on the 'data' event emission
response.on('data', function (data) {
responseData += data;
});
//listen to the 'end' event
response.on('end', function () {
//resolve the deferred object with the response
console.log('http call finished');
deferred.resolve(responseData);
});
});
//listen to the 'error' event
req.on('error', function (err) {
//if an error occurs reject the deferred
deferred.reject(err);
});
req.end();
//we are returning a promise object
//if we returned the deferred object
//deferred object reject and resolve could potentially be modified
//violating the expected behavior of this function
return deferred.promise;
}
Anyone could tell me what do I need to do to achieve such things?
Is it also possible so that I know when all of the http calls are finished and the insertion is done for all...
I would abandon your current approach and use the npm module request-promise.
https://www.npmjs.com/package/request-promise
It's very popular and mature.
rp('http://your/url1').then(function (response1) {
// response1 access here
return rp('http://your/url2')
}).then(function (response2) {
// response2 access here
return rp('http://your/url3')
}).then(function (response3) {
// response3 access here
}).catch(function (err) {
});
Now you just need to convert this to some kind of iteration for the 100 requests you want and the job will be done.
I understand well the concepts of OOP and prototypal inheritance in JavaScript, but sometimes, I wonder how to make use of these in real world applications.
I'll take as an exemple a simple(istic) contact management web application I pushed on GitHub a couple of months ago.
In the main handler mainly reside functions:
var UserModel = require('../models/userModel.js');
var checkObjectId = new RegExp('^[0-9a-fA-F]{24}$');
var root;
exports.getContacts = function(request, response) {
var id = JSON.parse(request.params.user)[0];
// validate
if (!checkObjectId.test(id)) {
return res.status(400).json({error: 'Not a user id'});
}
UserModel.findById(id, function(err, user) {
if (err) {
return console.log(err);
}
response.send(user.contacts);
});
};
exports.addContact = function(request, response) {
var id = JSON.parse(request.params.user)[0];
// validate
if (!checkObjectId.test(id)) {
return res.status(400).json({error: 'Not a user id'});
}
UserModel.findById(id, function(err, user) {
if (err) {
return console.error(err);
}
var contact = {};
// avoid to save empty info
if (request.body.first.length > 1) {contact.first = request.body.first;}
if (request.body.last.length > 1) {contact.last = request.body.last;}
if (request.body.mobile.length > 1) {contact.mobile = request.body.mobile;}
if (request.body.home.length > 1) {contact.home = request.body.home;}
if (request.body.office.length > 1) {contact.office = request.body.office;}
if (request.body.email.length > 1) {contact.email = request.body.email;}
if (request.body.company.length > 1) {contact.company = request.body.company;}
if (request.body.description.length > 1) {contact.description = request.body.description;}
if (request.body.keywords.length > 1) {contact.keywords = request.body.keywords;}
user.contacts.push(contact);
user.save(function(err) {
if (err) {
return console.error(err);
}
console.log('contact saved');
response.send(user.contacts);
});
});
};
exports.updateContact = function(request, response) {
var id = JSON.parse(request.params.user)[0];
// validate
if (!checkObjectId.test(id)) {
return res.status(400).json({error: 'Not a user id'});
}
var contact = {
_id: request.body._id,
first: request.body.first,
last: request.body.last,
mobile: request.body.mobile,
home: request.body.home,
office: request.body.office,
email: request.body.email,
company: request.body.company,
description: request.body.description,
keywords: request.body.keywords
};
UserModel.update({_id: id, "contacts._id": request.body._id}, {$set: {"contacts.$": contact}}, function(err, user) {
if (err) {
return console.error(err);
}
response.sendStatus(user);
});
};
exports.deleteContact = function(request, response) {
var id = JSON.parse(request.params.user)[0];
// validate
if (!checkObjectId.test(id)) {
return res.status(400).json({error: 'Not a user id'});
}
return UserModel.update({_id: id}, {$pull: {contacts: {_id: request.params.id}}}, function(err, user) {
if (err) {
return console.error(err);
}
console.log('contact removed');
console.log(user);
response.sendStatus(user);
});
};
It doesn't do much: fetch data from DB and return them or take data from user and save them to DB.
If it was a bit more complexe, I would surely place some logic in separate functions to reuse them and break down complexity.
Nevertheless, this code looks rather procedural, so does the hypothetical more complex version with separate functions. How would it be organized in a OOP way and how would I gain from it?
For instance, would I benefit from a User constructor?
I think the first thing you could do is nest the instance of your constructor inside an initializing function so you wouldn't have to repeat your validation code.
var connection = (function() {
var UserModel = require('../models/userModel.js');
var notAUser = {error: 'Not a user id'};
function init(request, response) {
var status = validate(JSON.parse(request.params.user)[0]);
if (!status.id) return response.status(400).json(status);
return new Connect(request, response, status.id);
}
function Connect(request, response, id) {
this.request = request;
this.response = response;
this.id = id;
this.info = { _id: id, "contacts._id": request.body._id };
}
function validate(id) {
if (!/^[0-9a-fA-F]{24}$/.test(id)) return notAUser;
else return {id: id};
}
Connect.prototype.getContact = function() {}
//etc...
return init;
})();
module.exports = connection;
Then in your actual application
var connection = require("./connection.js");
someAsync(data, function(req, res) {
var query = connection(req, res); //returned instance of constructor
query.getContact(someData, callback);
});
I would start by encapsulating the request and response since every method needs those. Like:
var contact = function (request, response) {
return {
add: add
}
function add() {
// add() gets access request and response for free
}
};
OR, if you are keen on the new operator:
function Contact(request, response) {
this.request = request;
this.response = response;
}
Contact.prototype.add = function () {
this.request;
}
Then move repeated code and callbacks to private methods you can reuse inside the object.
I'm using mongodb for pretty much everything in my node.js application, and now i want create a restful application, so, i did that:
I'm trying to do just the get method, for now:
restApi.js:
var restAPI = {
get: function(method, model, sort, limit, options) {
if (method !== 'get') {
return;
}
model.find(options).sort(sort).limit(3).exec(function (error, result) {
if (error) {
return error;
} else {
return result;
}
});
},
};
And now i can require this in my route:
var restApi = require('restApi');
and use like this:
app.get('/', function(req, res, next) {
var result = restAPI.get('get', Event, 'date', 3, {'isActive': true});
res.render('/', {
result: result
});
});
Is not working, the result is undefined. Why??
How can i transform this in a async function with callback? This is possible?
Thanks! :)
You're not returning anything from restApi.get. If you're using mongoose, you could return a Promise easily enough:
var restAPI = {
get: function(method, model, sort, limit, options) {
if (method !== 'get') {
return;
}
return model.find(options).sort(sort).limit(3).exec();
},
};
Then you can use it like this:
app.get('/', function(req, res, next) {
restAPI.get('get', Event, 'date', 3, {'isActive': true}).then( function ( result ) {
res.render('/', {
result: result
});
}).catch( error ) {
// Render error page and log error
});
});
It is because your model is async. You have to pass callbacks.
Using async way is better because it is not blocking your application while waiting for response.
Example on your case:
restApi.js:
var restAPI = {
get: function(method, model, sort, limit, options, cb) {
if (method !== 'get') {
return cb("Method must be GET");
}
model.find(options).sort(sort).limit(3).exec(function (error, result) {
if (error) {
return cb(error);
} else {
return cb(null, result);
}
});
},
};
And now i can require this in my route:
var restApi = require('restApi');
and use like this:
app.get('/', function(req, res, next) {
restAPI.get('get', Event, 'date', 3, {'isActive': true}, function(err, result){
if(err)
return res.render("Error:" + err)
res.render('/', {
result: result
});
});
});
I've added cb argument to your REST API function so it is called when model async operation is done.
Router handler passes it's callback and prints output when operation is finished.
In "view" method within my controller was previously using node-async but I wanted to try out using q.
I'm currently trying to convert this
exports.view = function (req, res) {
var category = req.params.category,
id = req.params.id,
ip = req.connection.remoteAddress,
slug = req.params.slug,
submission,
userId = typeof req.session.user !== 'undefined' && req.session.user.id ? req.session.user.id : null,
views;
var getSubmission = function (submissionId, callback) {
Submission.getSubmission({
id: submissionId
}, function (err, submission) {
if (err) {
callback(err);
} else if (submission) {
callback(null, submission);
} else {
callback(err);
}
});
};
async.waterfall([
function (callback) {
getSubmission(id, callback);
},
function (submission, callback) {
res.render('submission', {
title: submission.title + ' -',
submission: submission
});
}]);
To using q... I started doing something like:
var getSubmission = function(id) {
return Submission.getSubmission({
id : submissionId
}).then(function(submission) {
return submission;
});
};
q.fcall(getSubmission).then(function(submission) {
console.log(submission);
});
But it's not quite working as I intended. Am I doing something wrong here? How can I do this?
Is Submission.getSubmission a call to a database? Then you can't "chain" promises to that. You'll have to use the deferred method:
var getSubmission = function(id) {
var deferred = Q.defer();
Submission.getSubmission({
id: id
}, function(err, data){
if (err) {
deferred.reject(err);
} else {
deferred.resolve(data);
}
});
return deferred.promise;
}
getSubmission(some_id).then(successCallback, failureCallback);
You can also use Q#denodeify to convert a function using nodejs-style callbacks (function(err, data)) into a promise based function. Thus, the above can also be achieved by the following:
getSubmissionPromise = Q.denodeify(Submission.getSubmission);
getSubmissionPromise({id: some_id}).then(successCallback, failureCallback);