Hi i'm trying to access the elements returned from a find in mongoose and having some trouble with the asynchronous and callback situation.
Here is the code for better understanding.
function retrieveBudgets(email, callback) {
models.User.find({email: email},{budget:true}, function(err,budgets) {
if (err) {
callback(err, null);
} else {
callback(null, budgets);
}
});
};
retrieveBudgets(user.email, function(err, budgets) {
if (err) {
console.log(err);
}
budgets.forEach(function(budget){
console.log(JSON.stringify(budget, null, 4));
});
});
So this line console.log(JSON.stringify(budget, null, 4)); is working correctly and printing the objects to screen in json format but how do I store each to an array of objects from here? if I try to push to an array at this same line I get an error.
I have seen some questions that are similar but i am not getting any headway with them.
EDIT:____________________________________________________________
I did a little hack to get it working, i moved res.render up, so that rendering the page was done at the same time as the callback but I cant see this being the right solution any thoughts
var user=req.session.user;
res.locals.budgets=[];
function retrieveBudgets(email, callback) {
models.User.find({email: email},{budget:true}, function(err, budgets) {
if (err) {
callback(err, null);
} else {
callback(null, budgets);
}
});
};
retrieveBudgets(user.email, function(err, budgets) {
if (err) {
console.log(err);
}
res.locals.budgets = budgets.map((function(b){ return b; });
res.render('budget/budget.jade',{ csrfToken: req.csrfToken() });
});
This works I can access budgets through locals so any feedback on this would be great I doubt its the right way to do it?
the budgets return value that you get from the retrieveBudgets call is already an array.
this is evidenced by your call to budgets.forEach which is a method on arrays.
is there a specific need to create a new array from the items? that can be easily done:
var myNewArray = budgets.map((function(b){ return b; });
this one line of code will map the original budgets array into a new array containing each of the budget items.
there are other methods of creating a new array, depending on what you need to do exactly
update from comments below
what i really want to do to is use the budgets outside of the query so I can pass it to the view
in that case, you need to render the view from within the callback and pass the budgets to the view:
router.get("/foo", function(req, res, next){
retrieveBudgets(user.email, function(err, budgets) {
if (err) { return next(err); }
res.render('budget/budget.jade',{
budgets: budgets,
csrfToken: req.csrfToken()
});
});
});
This is the only, and correct, way to make this work.
If you tried to do it without waiting for the callback to finish, you would not have any data in your budgets array. Therefore, you must wait for the callback to be executed and then render your view with the budgets (or single budget or whatever) passed to the view.
(There are variations of this using promises, but I find callbacks to be the easier way to handle this.)
Turn the line in which you're passing an object to your jade file
res.render('budget/budget.jade',{
csrfToken: req.csrfToken(),
budgets: budgets.map(function(b) {return b;})
});
This will pass budgets to your jade file, and you should be able to access it there.
Related
I am kind of struggling with my JavaScript. Still getting my head around the callbacks. I have a function which recursively fetches the next link page from an REST endpoint.which works fine for a single url
Now i need to use the same recrusive function to do same for other end points. thats where iam struggling now. It is obvious that iam missing a point or i badly understood callback concept.
please excuse the formatting What i have done so far ..
function getFullLoad(rest_url,callback){
mongoose.connect(url, function(err) {
console.log(url);
if (err) throw err;
console.log("successfully connected");
getToken(function(err, token) {
if (err) throw console.error(err);
console.log('using access token to retrieve data :', token[0]
['access_token']);
var options = {
'auth': {
'bearer': token[0]['access_token']
}
}
request.get(rest_url, options, function(error, response, body) {
if (!error && response.statusCode === 200) {
var data = JSON.parse(body);
//save data in mongo
});
}
var next_page = JSON.parse(body)["_links"]["next"]
if (next_page) {
var next_page_url = JSON.parse(body)["_links"]["next"]["href"];
console.log("Fetching data from ", next_page);
//recrusively call back with next url
getFullLoad(next_page_url,callback);
} else {
callback();
} else {
if (error) throw error;
}
});
rest_url= "http://myrest_url"
//this bit is working if there is a single url
getFullLoad(rest_url, function() {
console.log('done , got all data');
});
this works great ,now i need to call the same function with different urls . It may be some urls may not have not pagination.
so far this is my effort , it only takes the first url and then does nothing.
This bit is not working . what iam doing wrong? i tried creating another callback function. i guess my knowledge is quite limited on callbacks. any help very much appreciated
api_endpoints =[ {"url1": "http://url1"},{"url2": "http://url2"}]
api_endpoints.forEach(function(Item, endpoint) {
var endpoint_name = Object.keys(api_endpoints[endpoint]).toString());
var rest_url = api_config.BASE_URL + api_endpoints[endpoint]
[endpoint_name];
getFullLoad(rest_url, function) {
console.log('done');
});
});
Thank you for looking my question
With a bit of refactoring I think you can get something to work.
I would change your getFullLoad method to return a promise (the request.get) then in your foreach loop I would build up an array of promises that call the getFullLoad with the different urls.
Then use promises.all(arrayOfPromises) to execute them and manage the results.
It will be easier to understand and maintain long term.
Check out the promise documentation here:
Promise.all
In my controller called MapController I'm doing a function to do a parse of remote json files, and from an if-else structure add some values in an array called "parsewebservice", apparently everything is working fine but console.log ( parsewebservice); is not returning the values that were passed to the array "parsewebservice" in the place where it is returning it empty. But when I put it inside the forEach it returns, but everything cluttered and repeated then is not the right way.
I wanted to know why the values that were passed to the array "parsewebservice" are not going along with the variable after populada and what would be the correct way to do it?
Here is my code below:
/**
* MapController
*
* #description :: Server-side logic for managing Maps
* #help :: See http://sailsjs.org/#!/documentation/concepts/Controllers
*/
module.exports = {
index: function(req, res, next) {
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
datas.forEach(function(data, index) {
var req = require("request");
var url = data.address + "?f=pjson";
req(url, function(err, res, retorno) {
if (err) {
console.log(err);
} else {
var camadas = JSON.parse(retorno);
if (camadas.mapName) {
camadas.layers.forEach(function(campo, i) {
if (campo.subLayerIds != null) {
} else if (campo.subLayerIds == null) {
parsewebservice.push([i, "dynamicMapLayer", campo.name, data.address]);
}
});
} else if (camadas.serviceDataType) {
parsewebservice.push([null, "imageMapLayer", camadas.name, data.address]);
} else if (camadas.type) {
parsewebservice.push([null, "featureLayer", camadas.name, data.address]);
}
}
});
});
console.log(parsewebservice);
});
},
};
My first comment has to be that you should not combine function(req, res) with var req = require('request')... you lose your access to the original req object!
So, you need to run a list of async tasks, and do something when they are all complete. That will never be entirely easy, and no matter what, you will have to get used to the idea that your code does not run from top to bottom as you've written it. Your console.log at the bottom runs before any of the callbacks (functions you pass in) you pass to your external requests.
The right way to do this is to use promises. It looks like you are using this request library, whose returned requests can only accept callbacks, not be returned as promises. You can create your own promise wrapper for them, or use an alternative library (several are recommended on the page).
I don't want to write a whole intro-to-promises right here, so what I will do is give you a less pretty, but maybe more understandable way to run some code at the completion of all your requests.
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
// here we will write some code that we will run once per returned data
var processResponse = function(resp) {
parsewebservice.push(resp);
if(parsewebservice.length >= datas.length) {
// we are done, that was the final request
console.log(parsewebservice);
return res.send({data: parsewebservice)}); // or whatever
}
};
datas.forEach(function(data, index) {
var request = require("request");
var url = data.address + "?f=pjson";
request(url, function(err, res, retorno) {
// do some processing of retorno...
// call our function to handle the result
processResponse(retorno);
});
});
console.log(parsewebservice); // still an empty array here
});
I solved the problem.
the "request" module is asynchronous so we need to wait for it to respond and then send the response to the view.
To do this we created a function called "foo" to contain the foreach and the request, we made a callback of that function and finally we made the response (res.view) within that function, so that the controller response would only be sent after the response of the "foo" function to the callback. So we were able to parse.json the data from the "data" collection using foreach and the "request" module and send the objects to the view.
Many thanks to all who have helped me, my sincere thanks.
I have a situation where I need to perform logic on a distinct set of values from a mongo collection (A) and then save result to another collection (B). However the contents of (A) will change over time and so I only want to perform the logic on those documents in (A) where there is not a corresponding document in (B). As joins aren't supported, I am trying to do this at the Node level. I am querying all items in collection (A) and using findOne to look for the corresponding entry in collection (B). If I find it, I would like to remove it from the array, but I am stuck because findOne uses an asynchronous callback which doesn't seem to work with the array filter method. Is there a better way to do this:
function loadNewDocumentsFromDB(callback){
db.collection('A', function(err, acollection){
bcollection.find().toArray(function(err, arr){
if(arr){
// toQuery is the global array to be used elsewhere
toQuery = arr.map(function(config){
transformed =
{
args: config._id, // these args are a doc representing a unique entry in 'B'
listings: config.value.id.split(',') // used by other functions
};
return transformed;
})
.filter(function(transformed){
db.collection('B', function(err, bcollection){
bcollection.findOne(transformed.args, function(err, doc){
// I want these values returned from the filter function not the callback
if(doc){
return false; // want to remove this from list of toQuery
}else{
return true; // want to keep in my list
});
});
}
callback();
});
});
}
This was how I managed to get it working:
function loadOptionsFromDB(callback){
toQuery = [];
db.collection('list', function(err, list){
db.collection('data', function(err, data){
list.find().each(function(err, doc){
if(doc){
transformed =
{
args: doc._id,
listings: doc.value.id.split(',')
};
(function(obj){
data.findOne(obj.args, function(err, found){
if(found){}
else{
toQuery.push(obj);
}
});
})(transformed);
}else{
//Done finding
setTimeout(callback, 20000);
}
});
});
});
}
A better way would be to do this on the database. Check if 2 executions of http://docs.mongodb.org/manual/core/map-reduce/ would be of any use to you.
See Merging two collections in MongoDB for more information
I'm new to mongoose/mongodb and I am trying to do some sort of error handling with my document save.
I am trying to create a stub id to store into the db for easier data retrieval later on (and also to put into the url bar so people can send links to my website to that particular page more easily -- like jsfiddle or codepen).
Basically I want to search for a document with a page_id and if it exists, I want to regenerate that page_id and search until it gets to one that's unused like this:
while(!done){
Model.findOne({'page_id': some_hex}, function (err, doc) {
if(doc){
some_hex = generate_hex();
}
else
{
done = true;
}
});
}
model.page_id = some_hex;
model.save();
However, since mongoose is asynchronous, the while loop will pretty much run indefinitely while the find works in the background until it finds something. This will kill the resources on the server.
I'm looking for an efficient way to retry save() when it fails (with a change to page_id). Or to try and find an unused page_id. I have page_id marked as unique:true in my schema.
Retrying should be performed asynchronously:
var tryToSave = function(doc, callback) {
var instance = new Model(doc);
instance.page_id = generate_hex();
instance.save(function(err) {
if (err)
if (err.code === 11000) { // 'duplicate key error'
// retry
return tryToSave(doc, callback);
} else {
// another error
return callback(err);
}
}
// it worked!
callback(null, instance);
});
};
// And somewhere else:
tryToSave(doc, function(err, instance) {
if (err) ...; // handle errors
...
});
I have frustrating problem with learning to work with callback style of programming in Node.js. I have a query to a MongoDB database. If I pass in a function to execute on the result it works but I'd rather flatten it out and have it return the value. Any help or direction on how to do this correctly is appreciated. Here's my code:
var getLots = function(response){
db.open(function(err, db){
db.collection('lots', function(err, collection){
collection.find(function(err, cursor){
cursor.toArray(function(err, items){
response(items);
})
})
})
})
}
I want something more like this:
lots = function(){
console.log("Getting lots")
return db.open(openCollection(err, db));
}
openCollection = function(err, db){
console.log("Connected to lots");
return (db.collection('lots',findLots(err, collection))
);
}
findLots = function(err, collection){
console.log("querying 2");
return collection.find(getLots(err, cursor));
}
getLots = function(err, cursor) {
console.log("Getting lots");
return cursor.toArray();
}
Where the final set of data would bubble back up through the function calls.
The problem is that I get an error from Node.js saying that err is not defined or that the collection is not defined. For some reason when I nest the callbacks the correct object is getting passed down. When I try going to this flattened style it complains that things are not defined. I don't know how to get it to pass the necessary objects.
What you need is one of the many control flow libraries available for node via npm and catalogued on the Node.js wiki. My specific recommendation is caolan/async, and you would use the async.waterfall function to accomplish this type of flow where each async operation must be executed in order and each requires the results from the previous operation.
Pseudocode example:
function getLots(db, callback) {
db.collection("lots", callback);
}
function findLots(collection, callback) {
collection.find(callback);
}
function toArray(cursor, callback) {
cursor.toArray(callback);
}
async.waterfall([db.open, getLots, find, toArray], function (err, items) {
//items is the array of results
//Do whatever you need here
response(items);
});
async is a good flow control library. Frame.js offers some specific advantages like better debugging, and better arrangement for synchronous function execution. (though it is not currently in npm like async is)
Here is what it would look like in Frame:
Frame(function(next){
db.open(next);
});
Frame(function(next, err, db){
db.collection('lots', next);
});
Frame(function(next, err, collection){
collection.find(next);
});
Frame(function(next, err, cursor){
cursor.toArray(next);
});
Frame(function(next, err, items){
response(items);
next();
});
Frame.init();