Develop Reference

How to perform file write with increment data in node js?

I have a list of 80 items and I get 10 item on each page.Now my idea is to write all the data into file in such a way that first 10 will be in one file and then next 10 in another and so on i get 8 pages for my 80 blogs of 10 in each page.The problem is I am getting only one file got written with 10 blogs what about the else.Can anyone please find the error.Thanks.For that I wrote the script as follows,
I find that the loop is not getting incremented.
exports.getBlogsTest = function(req, res) {
helper.logs('getBlogs', 'blog');
var pages = ['undefined', '2', '3', '4', '5', '6', '7', '8'],
pageNum = '';
pages.forEach(function(i, v) {
try {
var currentPage = Number(i);
var itemsPerPage = 10;
var startItem = (currentPage - 1) * itemsPerPage;
async.waterfall([
function(done) {
try {
if (currentPage === 1) {
blogs.count().exec(function(err, count) {
if (err) {
helper.logs('getBlogs', 'blog', err, 'console');
throw err;
} else {
done(err, count);
}
});
} else {
done('', 'page');
}
} catch (e) {
helper.logs('getBlogs', 'blog', e.message);
}
},
function(count, done) {
try {
if (count) {
if (count && count !== 'page') {
res.setHeader('totalItems', count);
}
blogs.find().sort({ date: -1 }).select('-text').skip(startItem).limit(itemsPerPage).exec(function(err, data) {
if (err) {
helper.logs('getBlogs', 'blog', err, 'console');
throw err;
}
if (data && data.length > 0) {
res.send(data);
console.log('reached###################')
if (i === 'undefined') {
pageNum = '';
} else {
pageNum = i;
}
var fileName = 'public/amp/test2/amp-blog-list' + pageNum + '.html';
var modData = data;
fs.writeFile(fileName, modData, function(err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
} else {
res.send([]);
}
});
} else {
res.send([]);
}
} catch (e) {
helper.logs('getBlogs', 'blog', e.message);
}
}
],
function(err) {
helper.logs('getBlogs', 'blog', err, 'console');
throw err;
});
} catch (e) {
helper.logs('getBlogs', 'blog', e.message);
}
})
};

Why are you using so many try catch?
While it might be useful in many cases you should avoid using it when not necessary.
A few points where you had an issues:
var currentPage = Number(i);
when i is 'undefined' currentPage === NaN
same for var startItem = (currentPage - 1) * itemsPerPage;
when currentPage is 'NaN' startItem === NaN
What i guess is that you thought the params in forEach are (index, value) but it's the other way round (value, index)
I tried to improve your code a bit but haven't actualy run it.
exports.getBlogsTest = function(req, res) {
helper.logs('getBlogs', 'blog');
var itemsPerPage = 10;
for (var i = 1; i < 11; i++) { // pages 1-10
(function(currentPage){ // currentPage = i
var startItem = (currentPage - 1) * itemsPerPage;
async.waterfall([
function(done) {
if (currentPage === 1) {
blogs.count().exec(function(err, count) {
if (err) {
helper.logs('getBlogs', 'blog', err, 'console');
//throw err;
done(err); // the done callback will be called with the error
} else {
done(null, count); // no error so send null
}
});
} else {
done(null, 'page'); // no error so send null
}
},
function(count, done) {
if (count) {
if (count !== 'page') {
res.setHeader('totalItems', count);
}
blogs.find().sort({ date: -1 }).select('-text').skip(startItem).limit(itemsPerPage).exec(function(err, data) {
if (err) {
helper.logs('getBlogs', 'blog', err, 'console');
done(err); // you never called done
//throw err;
}
if (data && data.length > 0) {
res.send(data);
var fileName = 'public/amp/test2/amp-blog-list' + (currentPage === 1 ? '' : currentPage) + '.html';
var modData = data;
fs.writeFile(fileName, modData, function(err) {
if (err) {
done(err);
return console.log(err);
}
console.log("The file was saved!");
done(); // you never called done
});
} else {
res.send([]);
done(); // you never called done
}
});
} else {
res.send([]);
done(); // you never called done
}
}
], function done(err) {
// this is called when waterfall is done or in case of error
// it would always throw if you didn't check for error
// also is it necessary to throw?
// if (err) throw err;
helper.logs('getBlogs', 'blog', err, 'console');
});
})(i);
}
};

how to make synchronous http calls within async.each in nodejs

I want to make http requests to an API-s to collect for each user it's data and insert into mongodb.
The problem I am having is, it is doing all the requests at once, and seems it gets stuck somewhere and I don't know what is going on.
Al thou I am using async library and add the request() method inside each iteration, and I dont know if this is the right way, here is the code:
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.forEach(students, function iteratee(student, callback) {
if (student.worksnap.user != null) {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
}
};
request(options, getTimeEntriesFromWorksnap);
}
callback(); // tell async that the iterator has completed
}, function (err) {
console.log('iterating done');
});
});
}
function getTimeEntriesFromWorksnap(error, response, body) {
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
saveTimeEntry(item);
});
});
});
}
}
function saveTimeEntry(item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
throw err;
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('item inserted...');
}
});
});
}
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
I am new to JavaScript, especially when dealing with async.
Any help?

Use Async.eachLimit() to make batched request to the api...Try this iterateThruAllStudents() function.
I already had same question before here
See tutorial of limiting here.
Though i am making the limit as 5 but you can do whatever you want(10,50 etc).
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.eachLimit(students,5,function iteratee(student, callback) {
if (student.worksnap.user != null) {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
}
};
request(options,getTimeEntriesFromWorksnap(callback));
}
}, function (err) {
console.log(err);
console.log('iterating done');
});
});
}
function getTimeEntriesFromWorksnap(cb) {
return function(error, response, body){
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
async.each(timeEntries,function(timeEntry,cb1){
async.each(timeEntry,function(item,cb2){
saveTimeEntry(item,cb2);
},function(err){
if(err)
cb1(err);
else
cb1();
})
},function(err){
if(err)
cb(err);
else
cb();
});
//_.forEach(timeEntries, function (timeEntry) {
// _.forEach(timeEntry, function (item) {
// saveTimeEntry(item);
// });
//});
});
}
cb(null);
}
}
function saveTimeEntry(item,cb2) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
return cb2(err);
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
//return cb2(err);//Do it if you wanna throw an error.
} else {
console.log('item inserted...');
}
cb2();
});
});
}
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);

In your example you missed iteratee param in the each method of async - iteratee(item, callback). Look at this example here.
You need to call callback each time inside your iteratee function to tell async continue doing its processing.
each(collection, iteratee, [callback])
collection - collection to iterate over.
iteratee(item, callback) - function to apply to each item in coll. The iteratee is passed a callback(err) which must be called once it has completed. If no error has occurred, the callback should be run without arguments or with an explicit null argument. The array index is not passed to the iteratee. If you need the index, use forEachOf.
callback(err) - Optional callback which is called when all iteratee functions have finished, or an error occurs.
If you need synchronous behavior, no probs! There is also eachSeries method with the same signature except every collection item will be iterated synchronously.
UPDATE:
Changes should be implemented:
Pass async callback:
request(options, getTimeEntriesFromWorksnap(callback));
Return necessary for request callback function:
function getTimeEntriesFromWorksnap(callback) {
return function(error, response, body) {
// ...
saveTimeEntry(item, callback);
// ...
}
}
Call callback only after record is saved in database:
function saveTimeEntry(item, callback) {
// ..
student.save(callback);
// ..
}
Refactor nested loops (not sure what timeEntries, timeEntry are, so use appropriate async method to iterate these data structures):
async.each(timeEntries, function (timeEntry, callback) {
async.each(timeEntry, function (item, callback) {
saveTimeEntry(item, callback);
}, callback);
}, callback);

Node JS: Request Loop Until Status Code 200

I currently have working code that does a request and checks if it receives a successful status code of 200. I would like to grow on this and loop it where it will keep sending requests until the status code is 200. I tried using a while loop but was not receiving the correct results. Thanks for the help!
request('http://0.0.0.0:9200', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log('success');
do(something);
}
else {
console.log('fail');
}
});

Would be something like:
let retry = (function() {
let count = 0;
return function(max, timeout, next) {
request('http://0.0.0.0:9200', function (error, response, body) {
if (error || response.statusCode !== 200) {
console.log('fail');
if (count++ < max) {
return setTimeout(function() {
retry(max, timeout, next);
}, timeout);
} else {
return next(new Error('max retries reached'));
}
}
console.log('success');
next(null, body);
});
}
})();
retry(20, 1000, function(err, body) {
do(something);
});
You can set a max number of retries and a timeout between retries. So that you do not introduce an infinite loop, and you do not deliver the final punch to an overloaded request target ^^

I wanted a little more intuitive answer including promises. I build on top of miggs answer within a try/catch the code below with promises and axios.
Based on a simple example of recursive functions
const throwNumbers = (count = 0) => {
console.log(count);
if (count++ < 10) {
throwNumbers(count);
} else {
console.log('max reached');
};
};
You can put anything else on the try part and handle error codes in the catch part. You have to set a max number of retries, which is 10 in my case.
let getResponse = async(count = 0) => {
try {
const axiosResponse = await axios.get(someURL, {
params: {
parameter1: parameter1,
},
});
return axiosResponse;
} catch (error) {
if (error || error.status != 200) {
console.error('failed, retry');
if (count++ < 10) {
return getResponse(count);
} else {
throw new Error('max retries reached');
};
} else {
throw error;
};
};
};
You would call the function with the following and handle the body or whatever with the response value.
let response = await getResponse();
console.log('This is the response:', response);
Has no timeout but works for me.

Mutable variable is accessible from closure with promise and loop

I use the following code to generate a unique token for a user. Data for users is stored on MongoDB so I use promise to handle asynchronous talking to the db. In WebStorm I receive this warning : Mutable variable is accessible from closure with promise and loop. and I know there have been posts on SO about this thing, but I my case is more complicated. I know I may not even need to worry about it as I only use the last value of token but what I want to solve this issue in a correct way ?
var generateToken = function(userId) {
User.findOne({userId: userId}, function(err, user) {
if (user !== null) {
var loop = true;
while (loop) {
var token = Common.randomGenerator(20);
User.find({tokens: token}, function(err, result) {
if (err) {
loop = false;
return Promise.reject('Error querying the database');
} else {
if (result.length === 0) {
if (user.tokens === undefined){
user.tokens.push(token);
}
loop = false;
return Promise.resolve(token);
}
}
});
}
} else {
return Promise.reject('UserNotFound');
}
});
};
I came up with the following solution , is it correct ?
var generateToken = function(userId) {
User.findOne({userId: userId}, function(err, user) {
if (user !== null) {
var loop = true;
while (loop) {
var token = Common.randomGenerator(20);
(function(e){
User.find({tokens: e}, function(err, result) {
if (err) {
return Promise.reject('Error querying the database');
} else {
if (result.length === 0) {
if (user.tokens === undefined){
user.tokens = [];
}
user.tokens.push(e)
return Promise.resolve(e);
}
}
});
})(token);
}
} else {
return Promise.reject('UserNotFound');
}
});
};
NOTE As #Alex Nikulin suggested , I flipped loop to false before sending back the result of the promise. But still it's an infinite loop as it doesn't go into the User.find({tokens: e})....

Your problem is that, return statement is within function, not within loop.Your loop is infinite. I has decomposited your code.Or simply make loop false, when you resolve/reject promise. Also my code will wait each answer from User, and error will gone. And your variant with wrap function is correct (function(e){})(token).
var generateToken = function(userId) {
return new Promise(function(resolve, reject) {
User.findOne({userId: userId}, function(err, user) {
if (user !== null) {
userTokenIterator(user,resolve, reject);
} else {
reject('UserNotFound');
}
});
});
};
var addTokenToUser = function(token,user){
return new Promise(function(resolve, reject) {
User.find({tokens: token}, function(err, result) {
if (err) {
reject('Error querying the database');
} else {
var result = result.length === 0;
if (result) {
if(!user.tokens) {
user.tokens = []
}
user.tokens.push(token);
}
resolve(result);
}
});
});
};
var userTokenIterator = function (user, resolve, reject){
var token = Common.randomGenerator(20);
addTokenToUser(token, user).then(function(result){
if(result) {
resolve(token);
}else{
userTokenIterator(user,resolve, reject)
}
},function(error){
reject(error);
});
};

Callback was already called async parallel

Hi I am trying to use the Async module to retrieve two users and do some processing after they have both been retrieved however I keep getting the error message: Callback was already called. Below is the code i currently have:
app.get('/api/addfriend/:id/:email', function(req, res) {
var id = req.params.id;
var friendEmail = req.params.email;
async.parallel([
//get account
function(callback) {
accountsDB.find({
'_id': ObjectId(id)
}, function(err, account) {
console.log(id);
if (err || account.length === 0) {
callback(err);
}
console.log(account[0]);
callback(null, account[0]);
});
},
//get friend
function(callback) {
accountsDB.find({
'email': friendEmail
}, function(err, friend) {
console.log(friendEmail);
if (err || friend.length === 0 || friend[0].resId === undefined) {
callback(err);
}
console.log(friend[0]);
callback(null, friend[0].resId);
});
}
],
//Compute all results
function(err, results) {
if (err) {
console.log(err);
return res.send(400);
}
if (results === null || results[0] === null || results[1] === null) {
return res.send(400);
}
//results contains [sheets, Friends, Expenses]
var account = results[0];
var friend = results[1];
if (account.friends_list !== undefined) {
account.friends_list = account.friends_list + ',' + friend;
}
else {
account.friends_list = friend;
}
// sheetData.friends = results[1];
accountsDB.save(
account,
function(err, saved) {
if (err || !saved) {
console.log("Record not saved");
}
else {
console.log("Record saved");
return res.send(200, "friend added");
}
}
);
}
);
});
Any help would be appreciated.

Add else statement to your code, because if you get error, your callback executes twice
if (err || account.length === 0) {
callback(err);
} else {
callback(null, account[0]);
}

The docs from async actually say:
Make sure to always return when calling a callback early, otherwise
you will cause multiple callbacks and unpredictable behavior in many
cases.
So you can do:
return callback(err);