In short.
I need to access two or more remote resource feeds, combine it and show it as one result from my nodejs service.
In detail
I need to fetch the feeds from multiple providers (which may vary in number according to what is stored in dashboard object)
Concatenate them, do some other data manipulations and show the content as one array at the end.
var allFeeds = [];
dashboard.providers.forEach(function(provider) {
if (provider.source === 'facebook') {
...
fb.getFeeds(provider.data.id, function(feeds) {
...
Array.prototype.push.apply(allFeeds, feeds);
});
} else if (provider.source === 'google') {
...
google.getFeeds(provider.data.id, function(feeds) {
...
Array.prototype.push.apply(allFeeds, feeds);
});
} else if (provider.source === 'twitter') {
...
twitter.getFeeds(provider.data.id, function(feeds) {
...
Array.prototype.push.apply(allFeeds, feeds);
});
}
});
...
// other data manipulations
...
res.json(allFeeds);
As nodejs is having asynchronous network calls how can I achieve this?
You can use async.
var async = require('async');
var allFeeds = [];
var tasks = [];
dashboard.providers.forEach(function (provider) {
if (provider.source === 'facebook') {
...
tasks.push(function (done) {
fb.getFeeds(provider.data.id, function (feeds) {
...
Array.prototype.push.apply(allFeeds, feeds);
done();
});
});
} else if (provider.source === 'google') {
...
tasks.push(function (done) {
google.getFeeds(provider.data.id, function (feeds) {
...
Array.prototype.push.apply(allFeeds, feeds);
done();
});
});
} else if (provider.source === 'twitter') {
...
tasks.push(function (done) {
twitter.getFeeds(provider.data.id, function (feeds) {
...
Array.prototype.push.apply(allFeeds, feeds);
done();
});
});
}
});
async.parallel(tasks, function () {
...
// other data manupulations
...
res.json(allFeeds);
});
You can also check out this post I wrote to structure your code to better manage async operations
you can achieve that with promises I will show you with bluebird.js
var Promise = require('bluebird');
var fbFeedAsync = Promise.promisify(fb.getFeeds);
var googleFeedAsync = Promise.promisify(google.getFeeds);
var twitterFeedAsync = Promise.promisify(twitter.getFeeds);
function getFeedFor(type, id) {
if (type === 'twitter') {
return twitterFeedAsync(id);
} else if (type === 'google') {
return googleFeedAsync(id);
} else if (type === 'facebook') {
return fbFeedAsync(id);
}
}
var feedRequests = dashboard.providers.map(function(provider) {
return getFeedFor(provider.source, provider.data.id);
});
Promise.all(feedRequests).then(function(allFeeds) { // you can use Promise.settle (depending on your use case)
console.log(allFeeds);
});
Related
I am working in node js express framework and I have a scenario where I have to call 2-3 nested callback functions inside for loop.
Below is my code:
for (i in jdp_tb_trades) {
var jdp_response_json_parsed = JSON.parse(jdp_tb_trades[i].jdp_response);
processConsign(jdp_tb_trades[i].tid, function(err_process_consign, lpnumber) {
if (err_process_consign) {
console.log("Some error occurred in processConsign. Error is:" + err_process_consign);
//Check if generate XML is enabled from admin end.
configuration.getOneByKey('generateXml', function(err_configuration, result_configuration) {
if (err_configuration) {
console.log('[generateXml]: generate xml enabled/disabled - No response.');
return callback(null, lpnumber);
} else {
if (result_configuration.value == 'true') {
console.log('[generateXml]: generate xml enabled.')
generateXml(jdp_tb_trades[i].tid, jdp_response_json_parsed, lpnumber, function(err_generate_xml, success_generate_xml);
if (err_generate_xml) {
return callback(err_generate_xml);
} else {
return callback(null, success_generate_xml);
}
});
} else {
console.log('[generateXml]: generate xml disabled.');
return callback(null, lpnumber);
}
}
});
} else {
//Check if generate XML is enabled.
configuration.getOneByKey(
'generateXml',
function(err_configuration, result_configuration) {
if (err_configuration) {
console.log('[generateXml]: generate xml enabled/disabled - No response.');
return callback(null, lpnumber);
} else {
if (result_configuration.value == 'true') {
console.log('[generateXml]: generate xml enabled.')
generateXml(jdp_tb_trades[i].tid, jdp_response_json_parsed, lpnumber, function(err_generate_xml, success_generate_xml) {
if (err_generate_xml) {
return callback(err_generate_xml);
} else {
return callback(null, success_generate_xml);
}
});
} else {
console.log('[generateXml]: generate xml disabled.');
return callback(null, lpnumber);
}
}
});
});
}
Update
The above code is part of a function named getOrders which is called as:
module.exports = {
getOrders: function (callback) {
getOrders(function(err_getOrders, getOrdersResponse){
if(err_getOrders){
console.log("generate Order process error:"+err_getOrders);
return callback(err_getOrders);
}
else{
console.log("generate Order process success:"+getOrdersResponse);
return callback(null, getOrdersResponse);
}
});
},
}
I have made multiple callbacks because function ends in multiple scenarios. I am not concerned about output of getOrders because I am not going to consume that output anywhere.
Here I have two functions processConsign and generateXml. generateXml is called in callback of processConsign. But I think forloop does not wait for these two tasks to get complete and keep increment the loop without waiting for processing of these two functions.
Is there any way by which I can make for loop wait for completion of these two processes and then executing the next loop?
you can use the async.each
async.each(jdp_tb_trades, (jdp_tb_trade, callback) => {
// do manipulation here
// return callback() after the process. pass err if error
}, loop_ended (err) => {
if (err) {
// Error in loop | err callback returned with err
}
// loop already ended here
});
Kindly check this
const async = require('async');
function getOrders (callback) {
async.each(jdp_tb_trades, generate_xml, (err) => {
if (err) {
// the callback return err using callback(err)
}
else {
// check the jdp_tb_trades. no error found
}
});
}
function generate_xml (jdp_tb_trade, callback) {
let jdp_response_json_parsed;
try {
jdp_response_json_parsed = JSON.parse(jdp_tb_trade.jdp_response);
} catch (err) {
return callback(err);
}
processConsign(jdp_tb_trade.tid, (err_process_consign, lpnumber) => {
if (err_process_consign) {
console.log(`Some error occurred in processConsign. Error is: ${err_process_consign}`);
//Check if generate XML is enabled from admin end.
configuration.getOneByKey('generateXml', (err_configuration, result_configuration) => {
if (err_configuration) {
console.log('[generateXml]: generate xml enabled/disabled - No response.');
// base on your callback it still a success response
// return callback(null, lpnumber);
jdp_tb_trade.lpnumber = lpnumber;
return callback();
}
else {
if (result_configuration.value == 'true') {
console.log('[generateXml]: generate xml enabled.')
generateXml(jdp_tb_trade.tid, jdp_response_json_parsed, lpnumber, (err_generate_xml, success_generate_xml) => {
if (err_generate_xml) {
jdp_tb_trade.err_generate_xml = err_generate_xml;
// return error
return callback(err_generate_xml);
} else {
jdp_tb_trade.success_generate_xml = success_generate_xml;
return callback();
// return callback(null, success_generate_xml);
}
});
}
else {
console.log('[generateXml]: generate xml disabled.');
return callback(null, lpnumber);
}
}
});
}
else {
if (result_configuration.value == 'true') {
console.log('[generateXml]: generate xml enabled.')
generateXml(jdp_tb_trades[i].tid, jdp_response_json_parsed, lpnumber, (err_generate_xml, success_generate_xml) => {
if (err_generate_xml) {
return callback(err_generate_xml);
} else {
jdp_tb_trade.success_generate_xml = success_generate_xml;
// return callback(null, success_generate_xml);
return callback();
}
});
}
else {
console.log('[generateXml]: generate xml disabled.');
jdp_tb_trade.lpnumber = lpnumber;
return callback();
// return callback(null, lpnumber);
}
}
});
}
I have a factory object that contain private object, which is used to cache result retrieved from the api using the factory available functions.
global.mainApp.factory('SessionFactory', function (UserEndpointsResource, SiteEndpointsResource) {
var data = {
/**
* #type {boolean|null}
*/
isLoggedIn: null
};
return {
isUserLoggedIn: function (callback) {
if (data.isLoggedIn != null) {
callback(data.isLoggedIn);
}
else {
UserEndpointsResource.isLoggedIn().$promise.then(function (res) {
var isUserLoggedIn = res.status == 1;
// Trying to set the result to the outer scope data variable
data.isLoggedIn = isUserLoggedIn;
callback(isUserLoggedIn);
}, function (failureData) {
data.isLoggedIn = false;
callback(false);
});
}
},
...
};
});
The problem that every time I call the isUserLoggedIn function, data.isLoggedIn is always null.
How can I alter the factory data object inside the promise then function?
Thanks.
Using the suggestion supplied in the comments, aka do not store promise results, store promises themselves, this is the modified working code!
global.mainApp.factory('SessionFactory', function (UserEndpointsResource, SiteEndpointsResource) {
var data = {
/**
* #type {boolean|null}
*/
isLoggedIn: null
};
return {
isUserLoggedIn: function (callback) {
if (data.isLoggedIn != null) {
data.isLoggedIn.then(function (isLoggedIn) {
callback(isLoggedIn.status == 1);
});
}
else {
data.isLoggedIn = UserEndpointsResource.isLoggedIn().$promise.then(function (res) {
var isUserLoggedIn = res.status == 1;
callback(isUserLoggedIn);
return isUserLoggedIn;
}, function (failureData) {
data.isLoggedIn = false;
callback(false);
return null;
});
}
}
};
});
Here is the function with in sql queries call.
I need return callback only after all queries done.
But it return an empty array
How to return array with data after all?
`
function getUserSales(days, callback){
getUserByLastLoginDay(days, function (users) {
var userArray = [];
_.each(users, function (user) {
getMostFavoredCat(user.id, function (cat) {
if(!cat || cat.length == 0){
return false;
} else {
user.mostFavoredCat = takeMostRepeatingObj(cat);
}
getRelatedSaleByCat(user.id, user.mostFavoredCat.id, function (sales) {
user.sales = sales;
userArray.push(user)
})
})
})
callback(userArray);
})
}
`
callback function first parameter is always an error
callback(null,userArray)
you can make use of async.js for the better control flow
npm i async --save
const async = require('async');
function getUserSales(days, callback){
getUserByLastLoginDay(days, function (users) {
var userArray = [];
async.each(users, function (user, cb) {
getMostFavoredCat(user.id, function (cat) {
if(!cat || cat.length == 0){
return false;
} else {
user.mostFavoredCat = takeMostRepeatingObj(cat);
}
getRelatedSaleByCat(user.id, user.mostFavoredCat.id, function (sales) {
user.sales = sales;
userArray.push(user)
cb();
})
})
}, (err) => {
if (err) {
return callback(err);
} else {
callback(null, userArray);
}
})
})
}
I think it will Works:
function getUserSales(days, callback){
getUserByLastLoginDay(days, function (users) {
var userArray = [];
_.each(users, function (user) {
getMostFavoredCat(user.id, function (cat) {
if(!cat || cat.length == 0){
return false;
} else {
user.mostFavoredCat = takeMostRepeatingObj(cat);
}
getRelatedSaleByCat(user.id, user.mostFavoredCat.id, function (sales) {
user.sales = sales;
userArray.push(user)
})
})
callback(userArray);
})
})
}
So considering I have a seeder like so:
var seedDatabase = function () {
var promises = [];
promises.push(createProductTypes());
promises.push(createCompensationTypes());
promises.push(createDataSources());
promises.push(createPeriodsAndStages());
promises.push(createExportDestinations());
return Promise.all(promises);
};
function createCompensationTypes() {
return models.sequelize
.transaction(
{
isolationLevel: models.sequelize.Transaction.ISOLATION_LEVELS.READ_COMMITTED
},
function (q) {
return models.CompensationType
.findAll()
.then(function (result) {
if (!result || result.length == 0) {
return models.CompensationType
.bulkCreate(compensationTypes, { transaction: q })
.then(function () {
console.log("CompensationTypes created");
})
}
else {
console.log("CompensationTypes seeder skipped, already objects in the database...");
return;
}
});
})
.then(function (result) {
console.log("CompensationTypes seeder finished...");
return;
})
.catch(function (error) {
console.log("CompensationTypes seeder exited with error: " + error.message);
return;
});
};
For some reason this hangs on execution. Yet, when I comment any one of the promises added to the array everything goes smoothly as it should.
Also when I do the following in my seedDatabase function, it works too:
var seedDatabase = function () {
var promises = [];
promises.push(createProductTypes());
promises.push(createDataSources());
promises.push(createPeriodsAndStages());
promises.push(createExportDestinations());
return Promise.all(promises).then(function() {
return createCompensationTypes()
});
};
which is bizarre since obviously my code is working but still hangs on an extra promise. Hence my question, is there a maximum of concurrent promises or transactions when using sequelizeJS? Or am I overlooking something else here?
I have a function that needs to return a list of favorite locations. Something like this
LocationsFactory.getFavoriteLocations().then(function($favoriteLocations) {
});
The getFavoriteLocations looks something like this
getFavoriteLocations: function() {
if (favorite_locations.length == 0)
{
var deff = $q.defer();
obj.getDeviceId().then(function(device_id) {
$http.get('url?token=' + device_id).then(function(response) {
favorite_locations = response.data;
deff.resolve(favorite_locations);
return deff.promise;
})
})
} else {
return favorite_locations;
}
}
The getDeviceId again, it's a function based on promise.
getDeviceId: function() {
var deff = $q.defer();
deff.resolve(Keychain.getKey());
return deff.promise;
}
The error that I got is TypeError: Cannot read property 'then' of undefined. Please help!
You can chain promises:
getFavoriteLocations: function () {
if (favorite_locations.length === 0) {
return obj.getDeviceId().then(function (device_id) {
return $http.get('url?token=' + device_id).then(function (response) {
favorite_locations = response.data;
return favorite_locations;
});
});
}
return $q.resolve(favorite_locations);
}
And improve this:
getDeviceId: function() {
return $q.resolve(Keychain.getKey());
}
$q in not necessary here:
if (favorite_locations.length == 0)
{
return obj.getDeviceId() // you have to return a promise here
.then(function(device_id) {
return $http.get('url?token=' + device_id) // you will access the response below
})
.then(function(response) {
favorite_locations = response.data;
return favorite_locations
});
})
}
Now it should work.