my openstates.billDetail function has a call back function inside of it, and I'm getting an error: 'callback must be a function', but my callback appears to be a function!
app.get('/search/:searchTerm', function(req, response) {
var nameArray = req.params.searchTerm.split('_');
var bills = []
var billIds = []
openstates.legSearch({
first_name: nameArray[0],
last_name: nameArray[1]
}, function(err, data) {
if (!err) {
openstates.billSearch({
state: 'CA',
chamber: 'lower',
page: '1'
}, function(err, data) {
for (var billIndex = 0; billIndex < data.length; billIndex++) {
billIds.push(data[billIndex].id)
}
for (var billIdIndex = 0; billIdIndex < billIds.length; billIdIndex++) {
openstates.billDetail(billIds[billIdIndex], function(err, data) {
console.log(data);
})
}
})
}
})
})
Anyone have any thoughts on this? my other callbacks work fine...
I tested your code, and it works perfectly fine.
I added the following to test it:
var openstates = {legSearch: function(a, cb){ cb(null, 'aa'); },
billSearch: function(a, cb){ cb(null, 'bb'); },
billDetail: function(a, cb){ cb(null, 'cc'); }};
Got no error, and the last function print 'cc' as expected.
BTW, you do not send any response back, I am not sure if this is on purpose.
Related
I am having trouble implementing Q promises with recursive dynamodb call, new to nodejs and q, considering the limitations of the dynamodb to retrieve results, we need to run recursive query to get the required results.
normally we use the query with Q implementation something like this as
function getDBResults(){
var q = Q.defer();
var params = {
TableName: mytable,
IndexName: 'mytable-index',
KeyConditionExpression: 'id = :id',
FilterExpression: 'deliveryTime between :startTime and :endTime',
ExpressionAttributeValues: {
':startTime': {
N: startTime.toString()
},
":endTime": {
N: endTime.toString()
},
":id": {
S: id.toString()
}
},
Select: 'ALL_ATTRIBUTES',
ScanIndexForward: false,
};
dynamodb.query(params, function(err, data) {
if (err) {
console.log('Dynamo fail ' + err);
q.reject(err);
} else {
console.log('DATA'+ data);
var results = data.Items;
q.resolve(results);
}
});
return q.promise;
}
getDBResults.then(
function(data) {
// handle data
},
function(err) {
//handle error
}
);
Using recursive query I can get the results but I need those results to be used in another function, but because of nodejs async nature,the next function calls happens already before the recursive query function finishes its job, now I want that I get all the results from the recursive query function and then get as a promise to a new function and finally handle all the data.
recursive query for dynamodb looks like this.
function getDBResults(){
//var q = Q.defer();
params = {
TableName: mytable,
IndexName: 'mytable-index',
KeyConditionExpression: 'id = :id',
FilterExpression: 'deliveryTime between :startTime and :endTime',
ExpressionAttributeValues: {
':startTime': {
N: startTime.toString()
},
":endTime": {
N: endTime.toString()
},
":id": {
S: id.toString()
}
},
Select: 'ALL_ATTRIBUTES',
ScanIndexForward: false,
};
dynamodb.query(params, onQueryCallBack);
}
function onQueryCallBack(err, data) {
if (err) {
console.log('Dynamo fail ' + err);
console.error("Could not query db" + err);
} else {
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("query for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
dynamodb.query(params, onQueryCallBack);
}
data.Items.forEach(function(item) {
allResults.push(item);
});
//console.log('NO:OF Results:' + allResults.length);
//q.resolve(tickets);
//});
}
Now I want that I can get the results as promise finally so I can handle them in the next function like this.
getDBResults.then(
function(data) {
// handle data
},
function(err) {
//handle error
}
);
Please help me on this, sorry if its a stupid question but recursive calls with promises have made a hurdle for me.
Thanks
First of all, keep the promisified function you already have. Use it as a building block for the recursive solution, instead of trying to alter it!
It might need two small adjustments though:
function getDBResults(startKey){
// ^^^^^^^^
var q = Q.defer();
var params = {
ExclusiveStartKey: startKey,
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
… // rest as before
};
dynamodb.query(params, function(err, data) {
if (err) {
q.reject(err);
} else {
q.resolve(data);
// ^^^^ Not `data.Items`
}
});
return q.promise;
}
Now we can use that to trivially implement the recursive solution:
function getRecursiveDBResults(key) {
return getDBResults(key).then(function(data) {
if (typeof data.LastEvaluatedKey != "undefined") {
return getRecursiveDBResults(data.LastEvaluatedKey).then(items) {
return data.Items.concat(items);
});
} else {
return data.Items
}
});
}
Here is how i solve the problem, Thanks Bergi for your solution as well
function getDBResults() {
var q = Q.defer();
var dynamodb = core.getDynamoDB();
params = {
TableName: mytable,
IndexName: 'mytable-index',
KeyConditionExpression: 'id = :id',
FilterExpression: 'deliveryTime between :startTime and :endTime',
ExpressionAttributeValues: {
':startTime': {
N: startTime.toString()
},
":endTime": {
N: endTime.toString()
},
":id": {
S: id.toString()
}
},
Select: 'ALL_ATTRIBUTES',
ScanIndexForward: false,
};
var results = [];
var callback = function(err, data) {
if (err) {
console.log('Dynamo fail ' + err);
q.reject(err);
} else if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
dynamodb.query(params, callback);
} else {
q.resolve(results);
}
data.Items.forEach(function(item) {
results.push(item);
});
}
dynamodb.query(params, callback);
return q.promise;
}
I have a big issue with my function in sails.js (v12). I'm trying to get all userDetail using async (v2.3) for deep populating my user info:
UserController.js:
userDetail: function (req, res) {
var currentUserID = authToken.getUserIDFromToken(req);
async.auto({
//Find the User
user: function (cb) {
User
.findOne({ id: req.params.id })
.populate('userFollowing')
.populate('userFollower')
.populate('trips', { sort: 'createdAt DESC' })
.exec(function (err, foundedUser) {
if (err) {
return res.negotiate(err);
}
if (!foundedUser) {
return res.badRequest();
}
// console.log('foundedUser :', foundedUser);
cb(null, foundedUser);
});
},
//Find me
me: function (cb) {
User
.findOne({ id: currentUserID })
.populate('myLikedTrips')
.populate('userFollowing')
.exec(function (err, user) {
var likedTripIDs = _.pluck(user.myLikedTrips, 'id');
var followingUserIDs = _.pluck(user.userFollowing, 'id');
cb(null, { likedTripIDs, followingUserIDs });
});
},
populatedTrip: ['user', function (results, cb) {
Trip.find({ id: _.pluck(results.user.trips, 'id') })
.populate('comments')
.populate('likes')
.exec(function (err, tripsResults) {
if (err) {
return res.negotiate(err);
}
if (!tripsResults) {
return res.badRequest();
}
cb(null, _.indexBy(tripsResults, 'id'));
});
}],
isLiked: ['populatedTrip', 'me', 'user', function (results, cb) {
var me = results.me;
async.map(results.user.trips, function (trip, callback) {
trip = results.populatedTrip[trip.id];
if (_.contains(me.likedTripIDs, trip.id)) {
trip.hasLiked = true;
} else {
trip.hasLiked = false;
}
callback(null, trip);
}, function (err, isLikedTrip) {
if (err) {
return res.negotiate(err);
}
cb(null, isLikedTrip);
});
}]
},
function finish(err, data) {
if (err) {
console.log('err = ', err);
return res.serverError(err);
}
var userFinal = data.user;
//userFinal.trips = data.isLiked;
userFinal.trips = "test";
return res.json(userFinal);
}
);
},
I tried almost everthing to get this fix but nothing is working...
I am able to get my array of trips(data.isLiked) but I couldn't get my userFInal trips.
I try to set string value on the userFinal.trips:
JSON response
{
"trips": [], // <-- my pb is here !!
"userFollower": [
{
"user": "5777fce1eeef472a1d69bafb",
"follower": "57e44a8997974abc646b29ca",
"id": "57efa5cf605b94666aca0f11"
}
],
"userFollowing": [
{
"user": "57e44a8997974abc646b29ca",
"follower": "5777fce1eeef472a1d69bafb",
"id": "5882099b9c0c9543706d74f6"
}
],
"email": "test2#test.com",
"userName": "dany",
"isPrivate": false,
"bio": "Hello",
"id": "5777fce1eeef472a1d69bafb"
}
Question
How should I do to get my array of trips (isLiked) paste to my user trips array?
Why my results is not what I'm expecting to have?
Thank you for your answers.
Use .toJSON() before overwriting any association in model.
Otherwise default toJSON implementation overrides any changes made to model associated data.
var userFinal = data.user.toJSON(); // Use of toJSON
userFinal.trips = data.isLiked;
return res.json(userFinal);
On another note, use JS .map or _.map in place of async.map as there is not asynchronous operation in inside function. Otherwise you may face RangeError: Maximum call stack size exceeded issue.
Also, it might be better to return any response from final callback only. (Remove res.negotiate, res.badRequest from async.auto's first argument). It allows to make response method terminal
I've made a webscraper with cheerio and request and I'm trying now to implement a loop on an array of url.
Unfortunately I'm doing something wrong with my calls and callback but I can not figure out what.
This is my code :
var getWebData = function(url) {
var i = 1;
var data = [];
for (c = 0; c < url.length; c++) {
data[i] = request(url[c], function(err, resp, body) {
console.log('ok');
if (!err) {
console.log('there');
var $ = cheerio.load(body);
$('.text').each(function(i, element) {
var jsObject = { name : "", description : "", price: "", categorie: "", pricePerKg: "", capacity: "", weight: "", scrapingDate : "", url: ""};
var name = 'TESTOK';
jsObject.name = name;
data.push(jsObject);
})
return data;
}
console.log('but');
});
i++;
}
var json = JSON.stringify(data);
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
console.log('File successfully written!');
})
}
getWebData(url);
app.listen('8080');
Note than any of my debugs print are not printed.
Does anyone know what's wrong in my code and how can I do to make it work ?
request is Aysnc
var json = JSON.stringify(data);
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
console.log('File successfully written!');
})
This above code runs before the for loop completetes execution and populates data object.
Try executing this piece of code when loop complete execution.
run this command first npm install async --save
var async = require('async');
var getWebData = function(url){
var data = [];
async.eachSeries(url, function(urlSingle , cb){
request(urlSingle, function(err, resp, body) {
//write your logic here and push data in to data object
cb();
})
},function(){
// this will rum when loop is done
var json = JSON.stringify(data);
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
console.log('File successfully written!');
});
});
}
I have been reading Asif's answer and the comments. That implementation is correct but you dont have to increment the c variable, also, if you initiate c=0 before, all the requests will be to url[0].
note that async.eachSeries callbacks each element of the array url in "urlsingle" callback, so you should use
request(urlsingle, ...
or consider using async.eachOf which gives you the index of each element in the array.
check for async documentation for any doubts http://caolan.github.io/async/
for (c = 0; c < url.length; c++) {
……
}
you should change like this：
var async = require('asycn');
async.map(url,
function(item, callback) {
data[i] = request(url[c],
function(err, resp, body) {
console.log('ok');
if (!err) {
console.log('there');
var $ = cheerio.load(body);
$('.text').each(function(i, element) {
var jsObject = {
name: "",
description: "",
price: "",
categorie: "",
pricePerKg: "",
capacity: "",
weight: "",
scrapingDate: "",
url: ""
};
var name = 'TESTOK';
jsObject.name = name;
data.push(jsObject);
}) callback(err, data);
}
console.log('but');
});
i++;
},function(err, results) {
if(err){
console.log(err);
}
});
in the loop is a time consuming operation.you should use asynchronous operation.
I'm brand new to express.js and API calls, and I can't figure out why this is crashing my server? Essentially it will run the first time through and render the page, but then crash the server saying:
TypeError: Cannot read property 'length' of undefined
for (var i = 0; i < data.businesses.length; i++) {
relevant section of code:
router.get('/:term/:radius/:lat/:lng', function (req, res) {
var yelp = require("yelp").createClient({
consumer_key: "xxxx",
consumer_secret: "xxxx",
token: "xxxx",
token_secret: "xxxx"
});
yelp.search({
term: req.params.term,
radius_filter: req.params.radius,
ll: req.params.lat + ',' + req.params.lng
},
function (error, data) {
var businessesArr = [];
if (typeof data) {
for (var i = 0; i < data.businesses.length; i++) {
businessesArr.push({
name: data.businesses[i].name,
image_url: data.businesses[i].image_url
});
}
res.render('selection', {
businesses: businessesArr
});
// console.log(data);
} else {
console.log(error);
}
});
});
This line:
if (typeof data) {
is always going to evaluate to true since it actually returns a string no matter what.
Replace it with something like:
if (data && data.businesses) {
In "view" method within my controller was previously using node-async but I wanted to try out using q.
I'm currently trying to convert this
exports.view = function (req, res) {
var category = req.params.category,
id = req.params.id,
ip = req.connection.remoteAddress,
slug = req.params.slug,
submission,
userId = typeof req.session.user !== 'undefined' && req.session.user.id ? req.session.user.id : null,
views;
var getSubmission = function (submissionId, callback) {
Submission.getSubmission({
id: submissionId
}, function (err, submission) {
if (err) {
callback(err);
} else if (submission) {
callback(null, submission);
} else {
callback(err);
}
});
};
async.waterfall([
function (callback) {
getSubmission(id, callback);
},
function (submission, callback) {
res.render('submission', {
title: submission.title + ' -',
submission: submission
});
}]);
To using q... I started doing something like:
var getSubmission = function(id) {
return Submission.getSubmission({
id : submissionId
}).then(function(submission) {
return submission;
});
};
q.fcall(getSubmission).then(function(submission) {
console.log(submission);
});
But it's not quite working as I intended. Am I doing something wrong here? How can I do this?
Is Submission.getSubmission a call to a database? Then you can't "chain" promises to that. You'll have to use the deferred method:
var getSubmission = function(id) {
var deferred = Q.defer();
Submission.getSubmission({
id: id
}, function(err, data){
if (err) {
deferred.reject(err);
} else {
deferred.resolve(data);
}
});
return deferred.promise;
}
getSubmission(some_id).then(successCallback, failureCallback);
You can also use Q#denodeify to convert a function using nodejs-style callbacks (function(err, data)) into a promise based function. Thus, the above can also be achieved by the following:
getSubmissionPromise = Q.denodeify(Submission.getSubmission);
getSubmissionPromise({id: some_id}).then(successCallback, failureCallback);