Develop Reference

Empty object returned from gRPC Server

Basically, the JSON object that's returned from a callback in my gRPC server is empty no matter what I do.
For the most part I'm following this tutorial, except I'm using a SQLite3 server instead of knex, and I've worked to the listProducts method. I haven't tried working on the other product methods yet.
In server.js I get some data from a SQLite3 database, and try to return it in a callback (at the bottom of the method). I also print out the data from the DB to confirm I'm actually getting valid data.
gRPC server.js
function listProducts(call, callback) {
console.log("******** Listed the products *********");
var data = "";
let db = new sqlite3.Database('../data/testDB.db', sqlite3.OPEN_READONLY, (err) => {
if(err){
console.error(err.message);
}
console.log("connected to DB");
});
db.serialize(() => {
db.get('SELECT NAME as name FROM PEEPS', (err, row) => {
if(err){
console.error(err.message);
}
console.log(row.name);
data.name = row.name;
});
});
db.close((err) => {
if(err) {
console.error(err.message);
}
console.log('closed db');
});
callback(null, { products: data.name });
}
Out put from gRPC server.js
******** Listed the products *********
connected to DB
Jeff // Correct data from DB.
closed db
The callback returns to client.js, where it was called. However, the object is always empty.
If I uncomment res.json({ name: "jessie" }); and comment res.json(result);, the code works as expected; name: jessie is sent to the browser as a JSON object.
So that tells me that from the client to the browser the data is being handled correctly. Therefore the problem is when the data is passed from the server.js to client.js.
gRPC client.js
// requirements
const path = require('path');
const protoLoader = require('#grpc/proto-loader');
const grpc = require('grpc');
// gRPC client
const productProtoPath = path.join(__dirname, '..', '..', 'protos', 'product.proto');
const productProtoDefinition = protoLoader.loadSync(productProtoPath);
const productPackageDefinition = grpc.loadPackageDefinition(productProtoDefinition).product;
const client = new productPackageDefinition.ProductService('localhost:50051', grpc.credentials.createInsecure());
// handlers
const listProducts = (req, res) => {
client.listProducts({}, (err, result) => {
console.log(result);
console.log(typeof result);
// console.log(res.json(result));
res.json(result);
// res.json({ name: "jessie" });
console.log("*******************");
});
};
Output from gRPC client.js
Server listing on port 3000
{} //Oh no! An empty JSON object!
object
*******************
Edit
Here is a link to my repository: https://github.com/burke212/grpc-node

The main problem here is that in your server code, your db methods are asynchronous but you are trying to access the result synchronously. You need to call the main callback for listProducts in the callback for db.get to ensure that you have the result of that database request before trying to use it. After making this change your listProducts method implementation should look more like this:
function listProducts(call, callback) {
let db = new sqlite3.Database('../data/testDB.db', sqlite3.OPEN_READONLY);
db.serialize(() => {
db.get('SELECT NAME as name FROM PEEPS', (err, row) => {
if(err){
console.error(err.message);
}
// Call the callback here to use the result of db.get
callback(null, { products: row.name });
});
});
db.close();
}
For simplicity I omitted the logging. Also, the sqlite3.Database constructor and db.close do not have callbacks in the example in the sqlite3 README. I suggest checking again whether those functions actually take callbacks.
In addition to that, now that you have shared the product.proto file that defines your service, there is another problem. The listProducts method in the ProductService service is declared as returning a ProductList object. In that message type, the products field must be an array of Product objects. All of the code in your method implementation is directed towards returning a string in that field, and that does not result in a compatible object.

NodeJS is asynchronous and my code doesn't run in the order I am expecting

postRegistrationHandler: function (account, req, res, next) {
console.log('postRegistrationHandler activated');
account.getCustomData(function(err, data) {
if (err) {
console.log(err.toString, "error string");
return next(err);
} else {
data.mongo_id = userCreationCtrl(account);
data.save();
next();
}
});
},
This function almost works properly, but the line:
data.save();
runs before the previous line finishes which means that the data I want to save isn't present at the appropriate time.
data.mongo_id = userCreationCtrl(account);
This line calls a function that creates a mongoDB document with information in the account object and then returns the _id (which is what I am trying to save.
I thought maybe using a .then() would help but that seems to be unavailable here for some reason. If anyone sees something I'm missing, that would be quite helpful. Thank you!
Here is the userCreationCtrl file as requested:
var UserSchema = require('./../models/UserModel.js');
var createNewUser = function (account, res, next){
// We will return mongoId after it is created by submitting a newUser
var mongoId = "";
// Save StormpathID (last 22 characters of account.href property)
var newStormpathId = account.href.slice(account.href.length - 22);
console.log('stormpath ID:', newStormpathId, 'just registered!');
console.log(account);
// Create new user from model by recycling info from the Stormpath registration form and include the stormpathId as well.
var newUser = new UserSchema({
stormpathId: newStormpathId,
firstName: account.givenName,
lastName: account.surname,
email: account.email,
street: account.street,
city: account.city,
zip: account.zip
});
// This saves the user we just created in MongoDB
newUser.save(function(err, result){
console.log(result);
if (err) {
console.error(err);
}
else {
console.log("User created in MongoDB, attempting to return mongoDB _id to stormpath customData");
// Keep track of the new user's mongo _id so we can return it to the previous function and save it as Stormpath custom data.
mongoId = result._id;
console.log(mongoId, "mongoid");
return result._id;
}
});
};
module.exports = createNewUser;

You have userCreationCtrl expecting 3 arguments, account, res, and next. next is the callback that should be called after the user is created so instead of return result._id you should call next like so:
// inside of createNewUser()
newUser.save(function(err, result){
console.log(result);
if (err) {
console.error(err);
}
else {
console.log("User created in MongoDB, attempting to return mongoDB _id to stormpath customData");
// Keep track of the new user's mongo _id so we can return it to the previous function and save it as Stormpath custom data.
mongoId = result._id;
console.log(mongoId, "mongoid");
// IMPORTANT change to make it all work...
// get rid of return result._id because its not doing anything
// pass the value to your callback function instead of returning the value
next(null, result._id);
}
});
then calling code in postRegistrationHandler should look like this:
account.getCustomData(function(err, data) {
if (err) {
console.log(err.toString, "error string");
return next(err);
} else {
// pass in a callback as the 3rd parameter that will be called by newUser.save() when its finished
userCreationCtrl(account, null, function(err, resultId) {
data.save();
next();
});
}
});

Can't list documents in mongodb collection using MongoClient in node.js

I can connect to my database from node.js using the MongoClient. I amb able to write, update and remove docs from a collection. But I am not able to retrieve data from it. This is my code:
var mongoClient=require('mongodb').MongoClient;
var mongoDbObj;
mongoClient.connect('mongodb://127.0.0.1:27017/trendoa', function(err, db){
if(err){
console.log(err);
}else{
global.db = db;
};
var col = global.db.collection('twitter_accounts_mon');
// create
var doc1 = {'hola':'sushi'};
col.insert(doc1, function(err, result) {
callback(err);
});
// update
col.update({hola:'jordi'}, {$set:{hola:'joan'}}, {w:1}, function(err, result) {});
// delete
col.remove({hola:'jordi'}, function(err, result) {
callback(err);
});
// read
col.find().toArray(function(err, docs) {
console.log(docs);
});
What I'm trying to do in the last lines of code is to get all the documents using find() but it doesn't return any results.
Through the mongo shell, using this command I get data on screen:
db.twitter_accounts_mon.find()
I don't know what I'm doing wrong. Thanks!

the nodejs callbacks must nest, ie only search the collection once the db is open
mongoClient.connect('mongodb://127.0.0.1:27017/trendoa', function(err, db){
var col = db.collection('twitter_accounts_mon');
coll.find({}, function(err, cursor) {
cursor.toArray(function(err, data) {
// process the data array
}
}
}

According to the MongoDB documentation on the Node.js driver, the find method does not execute the actual query, it builds an instance of a cursor that you then use to retrieve data. thus, you need to handle the result of the query.
var entireCollectionArray = col.find().toArray(function(err, items) {});
entireCollectionArray.forEach(function (element) {
console.log(element);
});

Closing a Mongoose connection in a script which does not run forever

I have a JavaScript program which is supposed to run for a brief period of time, insert rows into a MongoDB database, and then exit. Here is the cut down version of the application:
var mongoose = require('mongoose');
var models = require('./models');
mongoose.connect('mongodb://localhost/test')
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function callback() {
var row = models('testschema')({
name : 'test'
});
row.save(function (err, obj) {
if (err) {
console.log(err);
} else {
console.log('Saved item.');
}
});
console.log('Closing DB');
db.close();
});
Now the above doesn't work properly, as the item never gets into the database. My feeling is that because save() is async, the db.close() is happening first and the item never gets saved. If I move the db.close() call into the callback for save, as so:
row.save(function (err, obj) {
if (err) {
console.log(err);
} else {
console.log('Saved meeting details');
}
console.log('Closing DB');
db.close();
});
Then it works fine. However this isn't much practical help, as it means I can only write one row before needing to close the database. My question is, how do I close the Mongoose connection properly when I am in this situation:
var mongoose = require('mongoose');
var models = require('./models');
mongoose.connect('mongodb://localhost/test')
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function callback() {
itemsToSave.forEach(function(item) {
var row = models('testschema')({
name : item.name
});
row.save(function (err, obj) {
if (err) {
console.log(err);
} else {
console.log('Saved meeting details');
}
// Can't do this here.
//console.log('Closing DB');
//db.close();
});
});
// Nor here.
//console.log('Closing DB');
//db.close();
});
Edit: Here is the final version using C Blanchard's answer below. I should note, while it does achieve the desired result, I feel it has lost the convenience of mongoose at this point. If you are going to batch up calls to save() like this, you might as well take advantage of MongoDB's underlying bulk insert functionality and just use that to do the insert. I will probably do this task in another language as the async nature of node.js seems to make it nearly impossible to write elegant code to do something such as "open text file, for each line insert it into a database, close connection and exit". Anyhow, without further adieu, the final program:
var mongoose = require('mongoose');
var models = require('./models');
var async = require("async");
var rowsToSave = [];
var saveRow = function(item) {
return function (callback) {
console.log('Saving meeting details');
item.save(callback);
};
}
mongoose.connect('mongodb://localhost/test')
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function callback() {
rowsToSave.push(saveRow(models('testschema')({ name: 'name1' })));
rowsToSave.push(saveRow(models('testschema')({ name: 'name2' })));
rowsToSave.push(saveRow(models('testschema')({ name: 'name3' })));
rowsToSave.push(saveRow(models('testschema')({ name: 'name4' })));
console.log(JSON.stringify(rowsToSave));
async.series(rowsToSave, function (err, res) {
if (err) {
console.log(err);
} else {
console.log('Saved meeting details');
}
console.log('Closing DB');
db.close();
});
});
The other approach, which in some ways is nicer when looking at the code, but is also a horrible horrible hack to get around this deficiency, is to simply guesstimate the required time for the script and then close the db after this time has elapsed:
setTimeout(function() { db.close(); }, 5000);
I wouldn't blame anyone for doing this, MongoDB & Mongoose have forced you into a terrible position.

You can manage the control flow by using the async library. By using one of the control flow methods you can call the disconnect on Mongoose once all your saves are done.
What you could do is store all your save operations inside of an array and then pass it to async.series. async.series takes two arguments. 1) An array of functions to invoke in series. 2) A function which is invoked when all the functions in argument 1 are called.
Here's a sketch of a solution which follows the method described above:
// Bring in async library
var async = require("async");
// Create an array to store your save operations
var rowsToSave = [];
// Returns a function that will save a row when invoked
var saveRowMethod = function (item) {
var row = models('testschema')({
name : item.name
});
return function (callback) {
row.save(callback);
};
}
db.once('open', function callback() {
itemsToSave.forEach(function(item) {
// Store the row to save
rowsToSave.push(saveRowMethod(item))
});
// This will invoke each save operation in rowsToSave (in series)
async.series(rowsToSave, function (error, result) {
if (error) {
// Handle an error if one of the rows fails to save
}
console.log('Closing DB');
db.close();
})
});
Note that the functions you pass into rowsToSave must accept and invoke a callback when the save is complete. That's how async is able to track when the operations are done.
Update: I gave it more thought after your comment. Found a neater solution which relies only on Model#create - however Mongoose does not support bulk inserts for now (Issue #723)
Model#create accepts an array of objects and will do something similar to the solution I outlined above except it does not require async.
var rowsToSave = [];
rowsToSave.push({ name: 'name1' });
rowsToSave.push({ name: 'name2' });
rowsToSave.push({ name: 'name3' });
rowsToSave.push({ name: 'name4' });
TestModel.create(rowsToSave, function (err, name1, name2, name3, name4) {
if (err) {
console.log(err);
} else {
console.log('Saved meeting details');
}
console.log('Closing DB');
db.close();
});

Cannot return values to response with mongoose/mongodb and nodejs

I am using Nodejs, ExpressJs, MongoDB via Mongoose. I have created a simple UserSchema . I have my code separated into multiple files because I foresee them getting complex.
The url '/api/users' is configured to call the list function in 'routes/user.js' which happens as expected. The list function of UserSchema does get called, but it fails to return anything to the calling function and hence no result goes out.
What am I doing wrong ?
I tried to model it based on http://pixelhandler.com/blog/2012/02/09/develop-a-restful-api-using-node-js-with-express-and-mongoose/
I think I am doing something wrong with the function definition of userSchema.statics.list
app.js
users_module = require('./custom_modules/users.js'); // I have separated the actual DB code into another file
mongoose.connect('mongodb:// ******************');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function callback() {
users_module.init_users();
});
app.get('/api/users', user.list);
custom_modules/users.js
function init_users() {
userSchema = mongoose.Schema({
usernamename: String,
hash: String,
});
userSchema.statics.list = function () {
this.find(function (err, users) {
if (!err) {
console.log("Got some data"); // this gets printed
return users; // the result remains the same if I replace this with return "hello"
} else {
return console.log(err);
}
});
}
UserModel = mongoose.model('User', userSchema);
} // end of init_users
exports.init_users = init_users;
routes/user.js
exports.list = function (req, res) {
UserModel.list(function (users) {
// this code never gets executed
console.log("Yay ");
return res.json(users);
});
}

Actually in your code you are passing a callback, which is never handled in function userSchema.statics.list
You can try the following code:
userSchema.statics.list = function (calbck) {
this.find(function (err, users) {
if (!err) {
calbck(null, users); // this is firing the call back and first parameter should be always error object (according to guidelines). Here no error, so pass null (we can't skip)
} else {
return calbck(err, null); //here no result. But error object. (Here second parameter is optional if skipped by default it will be undefined in callback function)
}
});
}
Accordingly, you should change the callback which is passed to this function. i.e.
exports.list = function (req, res){
UserModel.list(function(err, users) {
if(err) {return console.log(err);}
return res.json(users);
});
}