I want to make http requests to an API-s to collect for each user it's data and insert into mongodb.
The problem I am having is, it is doing all the requests at once, and seems it gets stuck somewhere and I don't know what is going on.
Al thou I am using async library and add the request() method inside each iteration, and I dont know if this is the right way, here is the code:
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.forEach(students, function iteratee(student, callback) {
if (student.worksnap.user != null) {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
}
};
request(options, getTimeEntriesFromWorksnap);
}
callback(); // tell async that the iterator has completed
}, function (err) {
console.log('iterating done');
});
});
}
function getTimeEntriesFromWorksnap(error, response, body) {
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
saveTimeEntry(item);
});
});
});
}
}
function saveTimeEntry(item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
throw err;
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('item inserted...');
}
});
});
}
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
I am new to JavaScript, especially when dealing with async.
Any help?
Use Async.eachLimit() to make batched request to the api...Try this iterateThruAllStudents() function.
I already had same question before here
See tutorial of limiting here.
Though i am making the limit as 5 but you can do whatever you want(10,50 etc).
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.eachLimit(students,5,function iteratee(student, callback) {
if (student.worksnap.user != null) {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
}
};
request(options,getTimeEntriesFromWorksnap(callback));
}
}, function (err) {
console.log(err);
console.log('iterating done');
});
});
}
function getTimeEntriesFromWorksnap(cb) {
return function(error, response, body){
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
async.each(timeEntries,function(timeEntry,cb1){
async.each(timeEntry,function(item,cb2){
saveTimeEntry(item,cb2);
},function(err){
if(err)
cb1(err);
else
cb1();
})
},function(err){
if(err)
cb(err);
else
cb();
});
//_.forEach(timeEntries, function (timeEntry) {
// _.forEach(timeEntry, function (item) {
// saveTimeEntry(item);
// });
//});
});
}
cb(null);
}
}
function saveTimeEntry(item,cb2) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
return cb2(err);
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
//return cb2(err);//Do it if you wanna throw an error.
} else {
console.log('item inserted...');
}
cb2();
});
});
}
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
In your example you missed iteratee param in the each method of async - iteratee(item, callback). Look at this example here.
You need to call callback each time inside your iteratee function to tell async continue doing its processing.
each(collection, iteratee, [callback])
collection - collection to iterate over.
iteratee(item, callback) - function to apply to each item in coll. The iteratee is passed a callback(err) which must be called once it has completed. If no error has occurred, the callback should be run without arguments or with an explicit null argument. The array index is not passed to the iteratee. If you need the index, use forEachOf.
callback(err) - Optional callback which is called when all iteratee functions have finished, or an error occurs.
If you need synchronous behavior, no probs! There is also eachSeries method with the same signature except every collection item will be iterated synchronously.
UPDATE:
Changes should be implemented:
Pass async callback:
request(options, getTimeEntriesFromWorksnap(callback));
Return necessary for request callback function:
function getTimeEntriesFromWorksnap(callback) {
return function(error, response, body) {
// ...
saveTimeEntry(item, callback);
// ...
}
}
Call callback only after record is saved in database:
function saveTimeEntry(item, callback) {
// ..
student.save(callback);
// ..
}
Refactor nested loops (not sure what timeEntries, timeEntry are, so use appropriate async method to iterate these data structures):
async.each(timeEntries, function (timeEntry, callback) {
async.each(timeEntry, function (item, callback) {
saveTimeEntry(item, callback);
}, callback);
}, callback);
Related
Bit new to node.js and javascript. Attempting to have my foreach call a function that in turn does a remote call. I'd like there to be a delay between each but can't seem to work out where to put the set timeout.
I know I have my setTimeout in the wrong place below but putting it in there as an example.
var likeRecommendation = function (recommendation, callback) {
context.Client.like(recommendation._id, function (error, data) {
recommendation['drupal_user_uid'] = context.message.uid;
recommendation['drupal_user_uuid'] = context.message.uuid;
if (error) return callback(new Error('Could not like recommendations'));
context.broker.publish('saves_swipes_publication', recommendation, function (err, publication) {
if (err) return callback(new Error('Could queue swipes to save'));
publication.on('error', console.error);
});
console.log('Liked!');
return callback()
});
}
async.forEach(context.recommendations, likeRecommendation, function (error) {
if (!error) return done(null);
done(new Error('Could not like recommendations'));
});
}
See timeout, warning one of your callback was missing and one other was not well placed.
var likeRecommendation = function (recommendation, callback) {
context.Client.like(recommendation._id, function (error, data) {
recommendation['drupal_user_uid'] = context.message.uid;
recommendation['drupal_user_uuid'] = context.message.uuid;
if (error)
setTimeout(function(){return callback(new Error('Could not like recommendations'))}, 100);
else {
context.broker.publish('saves_swipes_publication', recommendation, function (err, publication) {
if (err)
setTimeout(function(){return callback(new Error('Could queue swipes to save'))}, 100);
else
{
publication.on('error', console.error);
setTimeout(function(){ return callback();}, 100);
}
});
}
});
}
async.forEach(context.recommendations, likeRecommendation, function (error) {
if (!error) return done(null);
done(new Error('Could not like recommendations'));
});
}
The setTimeout will have to be placed at the end of the callback code.
Here is a possible solution (note that it doesn't use async.forEach)
var likeRecommendation = function (recommendation, callback) {
context.Client.like(recommendation._id, function (error, data) {
recommendation['drupal_user_uid'] = context.message.uid;
recommendation['drupal_user_uuid'] = context.message.uuid;
console.log(recommendation);
if (error) return done(new Error('Could not like recommendations'));
context.broker.publish('saves_swipes_publication', recommendation, function (err, publication) {
if (err) throw err
publication.on('error', console.error);
});
console.log('Liked!');
return callback()
});
}
function iterateWithTimeout(list, ctx, timeoutDuration) {
var currentIndex = 0;
(function invoke() {
list[currentIndex](ctx, function(error) {
if (!error) return done(null);
done(new Error('Could not like recommendations'));
setTimeout(function() {
if (++currentIndex < list.length) {
invoke();
}
}, timeoutDuration);
});
})();
}
// Interval is 1 second for now
iterateWithTimeout(context.recommendations, likeRecommendation, 1000);
So using "async.eachSeries" rather than "async.forEach" was the solution!
As always thanks for the help!
I am currently trying to run a set of MySQL query's in order using async.js series control flow function. But I keep receiving the following error:
throw err; // Rethrow non-MySQL errors
^
TypeError: Cannot read property 'status' of undefined
I have tested the query's in seperate functions outside the async.series and they are fine and give me back the data, the only reason I can think for the error is due to the async nature it doesn't have the data at that time hence the error E.G when I log the rows I get:
[]
[]
[]
Below is the Async function:
function SQLuserDataAsync() {
connection.getConnection(function (err, connection) {
async.series([
function (callback) {
connection.query('SELECT status FROM users WHERE name= ?;',
[userval],
function (err, rows) {
if (rows[0]['status']) {
console.log("Account Status: " + accountval);
} else {
console.log(err);
}
callback(null, 'one');
});
},
function (callback) {
connection.query('SELECT account_type FROM settings_tbl WHERE id=(SELECT id FROM users WHERE name= ?);',
[userval],
function (err, rows) {
if (rows[0]['account_type']) {
var acctype = rows[0]['account_type'];
console.log("Account Type: " + acctype);
} else {
console.log(err);
}
callback(null, 'two');
});
},
function (callback) {
connection.query('SELECT type FROM settings_tbl WHERE id=(SELECT id FROM users WHERE name= ?);',
[userval],
function (err, rows) {
if (rows[0]['type']) {
var type = rows[0]['type'];
console.log("Type: " + type);
} else {
console.log(err);
}
callback(null, 'three');
});
}
]);
connection.release();
});
}
Any suggestions as the reason for the error or what am doing wrong here?
You've missed the main callback function to the async.series function.
function SQLuserDataAsync() {
connection.getConnection(function (err, connection) {
async.series([
function (callback) {
// YOUR CODE
},
function (callback) {
// YOUR CODE
},
function (callback) {
// YOUR CODE
}
], function(error, results) { // <--- this is the main callback
connection.release();
});
});
}
You should call connection.release() inside the main callback, otherwise, the MySQL connection will be released/terminated before the queries are executed (due to the asynchronous nature the code).
if there is a user with defined in userval name it will work.
But let's simplify our code:
function SQLuserDataAsync(userval) {
connection.getConnection(function (err, connection) {
async.waterfall([
// getting user
function (next) {
connection.query(
'SELECT * FROM users WHERE name = ? LIMIT 1',
[userval],
function (err, result) {
next(err, result[0]); // passing user to next function
});
},
// getting settings of user, maybe user_id (not id) in query below
function (user, next) {
connection.query(
'SELECT * FROM settings_tbl WHERE id = ? LIMIT 1',
[user.id],
function (err, result) {
next(err, user, result[0]);
});
},
// handling both user and settings
function (user, settings, next) {
console.log('User: ', user);
console.log('Settings: ', settings);
connection.release();
}
]);
});
}
SQLuserDataAsync('someone');
I have a program where user first create a file once file is created i am appending data to the file that is coming from client consistently.The below code is working as expected. I am new to nodejs so just want to get an expert opinion in case when multiple users creating and recording files on their machines at same time, will it work asynchronously or do i need to make some changes to the code ?
io.js
socket.on('createlogfile', function() {
logsRecording.userLogs(function(filename) {
socket.emit('filename', filename);
});
});
socket.on('startrecording', function(obj) {
logsRecording.recordLogs(obj);
});
server.js
userLogs: function (callback) {
var filename = uuid.v4() + '.log';
var file = filePath + '/' + filename;
fs.openSync(file, 'a',function () {
console.log('file created');
});
console.log('userLogs');
callback(filename);
},
recordLogs: function (obj) {
var dir = './app/records/templogs'
var fileAppend = dir + '/'+ obj.file;
console.log('data from recording', obj.data);
fs.readdir(dir, function(err, items) {
items.forEach(function(file){
if(obj.file === file){
fs.appendFile(fileAppend, obj.data+ "\r\n", null, 'utf8', function (err) {
if (err) throw err;
});
console.log('filename in records',obj.file);
}
});
});
}
You are using fs.openSync, which is synchronous and as such can hang the event loop.
You should be using fs.open and callback inside it:
userLogs: function (callback) {
var filename = uuid.v4() + '.log';
var file = filePath + '/' + filename;
fs.open(file, 'a', function (err) {
console.log('file created');
console.log('userLogs');
callback(err, filename);
});
},
And you can flatten recordLogs using async.
Also, it is bad practice to throw error in synchronous function, you should be passing the error in the callback.
As a last tip, Array.forEach is synchronous, and can hang the process, you should be using async.each
recordLogs: function (obj, callback) {
var dir = './app/records/templogs'
var fileAppend = dir + '/'+ obj.file;
console.log('data from recording', obj.data);
async.waterfall([
(callback) => {
fs.readdir(dir, (err, items) => {
callback(err, items);
});
},
(items, callback) => {
async.each(items, (file, callback) => {
if(obj.file === file) {
fs.appendFile(fileAppend, obj.data+ "\r\n", null, 'utf8', function (err) {
callback(err);
});
console.log('filename in records',obj.file);
} else {
callback();
}
}, (err) => {
callback(err);
});
}
], (err, file) => {
if(callback) {
callback(err);
}
});
}
I would like to make http call synchronously using Q Promises, I have 100 students that I need for each of them to take some data from another platform and to do that I was trying via Q Promises but it does not seem like it is doing synchronously.
How do I make sure that another call is not being made once one is finished with parsing it's response and insertion into mongodb:
my code so far looks like this:
var startDate = new Date("February 20, 2016 00:00:00"); //Start from February
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.eachSeries(students, function iteratee(student, callback) {
if (student.worksnap.user != null) {
var worksnapOptions = {
hostname: 'worksnaps.com',
path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic xxxx='
},
method: 'GET'
};
promisedRequest(worksnapOptions)
.then(function (response) { //callback invoked on deferred.resolve
parser.parseString(response, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
saveTimeEntry(item);
});
});
});
callback();
}, function (newsError) { //callback invoked on deferred.reject
console.log(newsError);
});
}
});
function saveTimeEntry(item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
throw err;
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('item inserted...');
}
});
});
}
function promisedRequest(requestOptions) {
//create a deferred object from Q
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var deferred = Q.defer();
var req = http.request(requestOptions, function (response) {
//set the response encoding to parse json string
response.setEncoding('utf8');
var responseData = '';
//append data to responseData variable on the 'data' event emission
response.on('data', function (data) {
responseData += data;
});
//listen to the 'end' event
response.on('end', function () {
//resolve the deferred object with the response
console.log('http call finished');
deferred.resolve(responseData);
});
});
//listen to the 'error' event
req.on('error', function (err) {
//if an error occurs reject the deferred
deferred.reject(err);
});
req.end();
//we are returning a promise object
//if we returned the deferred object
//deferred object reject and resolve could potentially be modified
//violating the expected behavior of this function
return deferred.promise;
}
Anyone could tell me what do I need to do to achieve such things?
Is it also possible so that I know when all of the http calls are finished and the insertion is done for all...
I would abandon your current approach and use the npm module request-promise.
https://www.npmjs.com/package/request-promise
It's very popular and mature.
rp('http://your/url1').then(function (response1) {
// response1 access here
return rp('http://your/url2')
}).then(function (response2) {
// response2 access here
return rp('http://your/url3')
}).then(function (response3) {
// response3 access here
}).catch(function (err) {
});
Now you just need to convert this to some kind of iteration for the 100 requests you want and the job will be done.
sails v0.11.0 (http://sailsjs.org/)
I have tried unsuccessfully to use .exec callback, promises (http://sailsjs.org/documentation/reference/waterline-orm/queries) and now async.js (https://github.com/caolan/async) to control the asynchronous flow revolving around looping over a find query. The async.each log output does not have the parallel work in it (although parallel does populate).
So if your solution works using .exec callback, promises or async.js - I will gladly take it!
I found this link gave some helpful examples of async.js (http://www.sebastianseilund.com/nodejs-async-in-practice)
Thank you for your time and assistance.
Below is my code using async:
/**
* #module :: Service
* #type {{findProfile: Function}}
*/
require('async');
module.exports = {
getProfile: function (userId, callback) {
var json = {};
json.notFound = false;
json.locations = {};
json.sports = {};
User.findOne({id: userId}).exec(function (err, user) {
if (err) {
json.notFound = true;
json.err = err;
}
if (!err) {
json.user = user;
UserSport.find({user_id: user.id}).exec(function (err, userSports) {
if (err) {
sails.log.info("userSports error: " + userSports);
}
async.each(userSports, function (userSport, callback) {
LocationSport.findOne({id:userSport.locationsport_id}).exec(function (err, locationSport) {
if (locationSport instanceof Error) {
sails.log.info(locationSport);
}
async.parallel(
[
function (callback) {
Location.findOne({id:locationSport.location_id}).exec(function (err, location) {
if (location instanceof Error) {
sails.log.info(location);
}
callback(null, location);
});
},
function (callback) {
Sport.findOne({id:locationSport.sport_id}).exec(function (err, sport) {
if (sport instanceof Error) {
sails.log.info(sport);
}
callback(null, sport);
});
}
],
function (err, results) {
if (!(results[0].id in json.locations)) {
json.locations[results[0].id] = results[0];
}
if (!(results[1].id in json.sports)) {
json.sports[results[1].id] = results[1];
}
}
); // async.parallel
}); // locationSport
callback();
}, function (err) {
sails.log.info('each');
sails.log.info(json);
}); // async.each
}); // UserSport
}
}); // User
}
}
The structure of your code is the following :
async.each(userSports, function (userSport, callback) {
// Whatever happen here, it runs asyncly
callback();
}, function (err) {
sails.log.info('each');
sails.log.info(json);
}); // async.each
You are calling the callback method, but the processing on your data is not yet done (it is running async-ly). As a result, sails.log.info is called immediatly.
You should modify your code so the callback is called once the process is done. i.e in the result of your async.parallel :
async.each(userSports, function (userSport, outer_callback) {
LocationSport.findOne({id:userSport.locationsport_id}).exec(function (err, locationSport) {
//...
async.parallel(
[
function (callback) {
// ...
},
function (callback) {
// ...
}
],
function (err, results) {
// ...
outer_callback();
}
); // async.parallel
}); // locationSport
}, function (err) {
sails.log.info('each');
sails.log.info(json);
}); // async.each