I always have multiple operations in one route or endpoint. Take an example below, when a user deletes an item, I want the related file be deleted in s3 too besides deleting related collection from the database.
So is the code below ok? Does it matter if I put the first function (delete file from s3) inside the DeleteItem function?
router.post('/item/delete', function(req, res) {
if(req.body.dlt_item){
var tempArray = [];
tempArray.push({"Key":req.body.dlt_item});
s3Bucket.deleteObjects({
Bucket: 'myS3',
Delete: {
Objects: req.body.dlt_item
}
}, function(err, data) {
if (err)
return console.log(err);
});
}
Item.DeleteItem(req.body.item_id, function(err,result){
if(err){console.log(err)}
res.send({result:1});
})
});
You should organise your code like this. This will ensure that s3 deletion will start only when mongodb deletion has finished.
In your code both things happen simultaneously. this may cause issue in some cases.
If one fails and other succeeds then there will be trouble. Suppose s3 files get deleted successfully and mongo deletion fails. Then you will have many references to non existing resources.
router.post('/item/delete', function(req, res) {
if(req.body.dlt_item){
var tempArray = [];
tempArray.push({"Key":req.body.dlt_item});
Item.DeleteItem(req.body.item_id, function(err,result){
if(err)
{
console.log(err)
res.send(err);
}
else
{
//deletion from mongodb is succesful now delete from s3
s3Bucket.deleteObjects({
Bucket: 'myS3',
Delete: {
Objects: req.body.dlt_item
}
},function(err, data) {
if (err)
{
// deletion from s3 failed you should handle this case
res.send({result:1});
return console.log(err);
}
else
{
// successful deletion from both s3 and mongo.
// If you do not want to wait for this then send the response before this function.
res.send({result:1});
}
});
}
})
});
Related
I'm calling three functions, after the completion of these functions I want my script to close on it's own but it just hangs.
I've tried making the functions async/promise based, closing the database after each 'mongodb' type function, and using process.exit() within a function as a callback to the last called function.
Connecting to the (local - not Atlas) Database:
MongoClient.connect(local, {useNewUrlParser: true, useUnifiedTopology: true}, function(err, db) {
if (err) {
console.log(err)
}
else {
console.log('Connected to MongoDB...')
//Read in data from jsonfiles and store each file's contents into the database : This is where the functions are being called... within a successful connect to the MongoDB
insertJSON(db, jsonfiles, 'requests', jsonfilesSource)
insertJSON(db, issuedfiles, 'issuedLicenses', isssuedfilesSource)
insertLicenses(db)
}
db.close()
})
Function 1:
function insertJSON(db, dirBuf,collection, sourceFolder) {
var database = db.db('license-server')
var collection = database.collection(collection)
fs.readdir(dirBuf, function(err, files) {
if (err) {
console.log(err.message)
}
else {
files.forEach(function(filename) {
var text = fs.readFileSync(sourceFolder + filename);
var filecontents = JSON.parse(text)
//collection.insertOne(filecontents)
collection.findOne({"DisplayTitle" : filecontents.DisplayTitle, "NodeInformation" : filecontents.NodeInformation, "Date": filecontents.Date})
.then(function(result) {
if(result) {
console.log(`An Item could already be in the database: A file is unique if its display title, nodeinformation, and date are different.
the items display title is ${result.DisplayTitle}`)
return
}
else {
collection.insertOne(filecontents)
console.log(`Added ${filecontents.DisplayTitle} to database`)
}
})
.catch(function(error) {
console.log(error)
})
})
}
})
}
Function 2:
function insertLicenses(db) {
// Set up GridFS to import .lic and .licx files into the database
var database = db.db('license-server')
var collection = database.collection('fs.files')
var bucket = new mongodb.GridFSBucket(database);
var dirBuf = Buffer.from('../license-server/private/licenses')
fs.readdir(dirBuf, function(err, files) {
if (err) {
console.log(err.message)
}
else {
files.forEach(function(filename) {
collection.findOne({"filename": filename}).
then(function(result) {
if(result) {
console.log(`The file ${filename} is already in the database`)
return
}
else {
fs.createReadStream('./private/licenses/' + filename).
pipe(bucket.openUploadStream(filename)).
on('error', function(error) {
assert.ifError(error)
}).
on('finish', function() {
console.log(`Uploaded ${filename}`)
})
}
})
})
}
})
// I tried calling db.close() here since this is the last function to be called. No luck.
}
I'm guessing it has something to do with the mongodb functions having their own way to close themselves but I couldn't seem to find what I was looking for in previous attempts to resolve this issue.
The expected result should be the script closing itself, the actual result is a handing script.
All of these database calls are asynchronous -- the result of this code running is to immediately call db.close and then do the work in insertJSON and insertLicenses. If you were to rewrite this to use async/await (and you'd need to update your other functions as well) the db.close call would close the db, and that would allow the script to exit:
await insertJSON(db, jsonfiles, 'requests', jsonfilesSource)
await insertJSON(db, issuedfiles, 'issuedLicenses', isssuedfilesSource)
await insertLicenses(db)
db.close()
https://developer.mozilla.org/en-US/docs/Learn/JavaScript/Asynchronous/Introducing
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
Basically, the JSON object that's returned from a callback in my gRPC server is empty no matter what I do.
For the most part I'm following this tutorial, except I'm using a SQLite3 server instead of knex, and I've worked to the listProducts method. I haven't tried working on the other product methods yet.
In server.js I get some data from a SQLite3 database, and try to return it in a callback (at the bottom of the method). I also print out the data from the DB to confirm I'm actually getting valid data.
gRPC server.js
function listProducts(call, callback) {
console.log("******** Listed the products *********");
var data = "";
let db = new sqlite3.Database('../data/testDB.db', sqlite3.OPEN_READONLY, (err) => {
if(err){
console.error(err.message);
}
console.log("connected to DB");
});
db.serialize(() => {
db.get('SELECT NAME as name FROM PEEPS', (err, row) => {
if(err){
console.error(err.message);
}
console.log(row.name);
data.name = row.name;
});
});
db.close((err) => {
if(err) {
console.error(err.message);
}
console.log('closed db');
});
callback(null, { products: data.name });
}
Out put from gRPC server.js
******** Listed the products *********
connected to DB
Jeff // Correct data from DB.
closed db
The callback returns to client.js, where it was called. However, the object is always empty.
If I uncomment res.json({ name: "jessie" }); and comment res.json(result);, the code works as expected; name: jessie is sent to the browser as a JSON object.
So that tells me that from the client to the browser the data is being handled correctly. Therefore the problem is when the data is passed from the server.js to client.js.
gRPC client.js
// requirements
const path = require('path');
const protoLoader = require('#grpc/proto-loader');
const grpc = require('grpc');
// gRPC client
const productProtoPath = path.join(__dirname, '..', '..', 'protos', 'product.proto');
const productProtoDefinition = protoLoader.loadSync(productProtoPath);
const productPackageDefinition = grpc.loadPackageDefinition(productProtoDefinition).product;
const client = new productPackageDefinition.ProductService('localhost:50051', grpc.credentials.createInsecure());
// handlers
const listProducts = (req, res) => {
client.listProducts({}, (err, result) => {
console.log(result);
console.log(typeof result);
// console.log(res.json(result));
res.json(result);
// res.json({ name: "jessie" });
console.log("*******************");
});
};
Output from gRPC client.js
Server listing on port 3000
{} //Oh no! An empty JSON object!
object
*******************
Edit
Here is a link to my repository: https://github.com/burke212/grpc-node
The main problem here is that in your server code, your db methods are asynchronous but you are trying to access the result synchronously. You need to call the main callback for listProducts in the callback for db.get to ensure that you have the result of that database request before trying to use it. After making this change your listProducts method implementation should look more like this:
function listProducts(call, callback) {
let db = new sqlite3.Database('../data/testDB.db', sqlite3.OPEN_READONLY);
db.serialize(() => {
db.get('SELECT NAME as name FROM PEEPS', (err, row) => {
if(err){
console.error(err.message);
}
// Call the callback here to use the result of db.get
callback(null, { products: row.name });
});
});
db.close();
}
For simplicity I omitted the logging. Also, the sqlite3.Database constructor and db.close do not have callbacks in the example in the sqlite3 README. I suggest checking again whether those functions actually take callbacks.
In addition to that, now that you have shared the product.proto file that defines your service, there is another problem. The listProducts method in the ProductService service is declared as returning a ProductList object. In that message type, the products field must be an array of Product objects. All of the code in your method implementation is directed towards returning a string in that field, and that does not result in a compatible object.
I've been working on an application which allows me to add companies to a database. Originally my code was pure spaghetti, so I wanted to modularize it properly. For this purpose, I added routes, a controller and a dao.
This is how my code looks right now
Routes
app.post('/loadcompanies', (req, res)=> {
companiesController.loadcompany(req.body, (results)=>{
console.log(results);
res.send(200, "working!");
})
})
Controller
module.exports.loadCompany = (body, callback)=>{
companiesDao.loadCompany(body, callback);
}
Dao
module.exports.loadCompany = (company, callback)=>{
MongoClient.connect(conexionString, (err, database) => {
if (err) console.log(err);
db = database;
console.log(company);
db.collection('companies').insert(company, (err, result)=>{
callback({message:"Succesfully loaded company", company:result});
});
})
}
My current concern is that working with errors when modularizing like this is confusing. I tried adding a try-catch method around the db insert and throwing and error if there is one, but that doesn't seem to work. Other things I've tried is returning the error in the callback, like this:
if (err) callback (err, null);
but I end up getting a "Can't set headers after they are sent." error.
How would you handle errors in this situation? For example, in the case that someone tries to add a duplicate entry in an unique element.
You should be able to simply do the error checking inside the callback for the insert function:
db.collection('companies').insert(company, (err, result)=>{
if (err) {
callback(err, null);
return;
}
callback(null, {message:"Succesfully loaded company", company:result});
});
If you get an error like you say, that's probably because the database is actually returning an error. You could also make your errors more specific, like:
module.exports.loadCompany = (company, callback)=>{
MongoClient.connect(conexionString, (err, database) => {
if (err) {
callback(new Error('Connection error: ' + err.Error());
return;
}
db = database;
console.log(company);
db.collection('companies').insert(company, (err, result)=>{
if (err) {
callback(new Error('Insertion error: ' + err.Error());
return;
}
callback(null, {message:"Succesfully loaded company", company:result});
});
})
Here is your loadCompany done in async / await format.
Notise there is no need for error checking, errors will propagate as expected up the promise chain.
Note I've also changed loadCompany to be an async function too, so to call it you can simply do var ret = await loadCompany(conpanyInfo)
module.exports.loadCompany = async (company)=>{
let db = await MongoClient.connect(conexionString);
console.log(company);
let result = await db.collection('companies').insert(company);
return {message:"Succesfully loaded company", company:result};
}
router.post('/logout', function(req, res) {
if(req.body.newList){
var aux= JSON.parse(req.body.newList);
var aux1;
var i=0;
while(i<aux.length){
aux1=aux[i];
Task.findOne({username:aux1.username,text:aux1.text},function(err,res){
if(res){
i++;
}else{
task= new Task({username:aux1.username,text:aux1.text,state:aux1.state});
task.save(function(err){
i++;
});
}
});
}
}
Well, I have this so far. aux stores an array of json objects, they dont have an _id, so.. I need to iterate through them saving one by one when needed. Problem is (I figure) async, cause this code is saving in the data base only the last element of the array n times, being n the amount of diferent elements which should be saved**(see example). So, I think the async behavior of moongose querys is giving me the headache. Probably keeps iterating waiting for the result of the query, and then save with the last value of the array.
** for instance, if I have these values for save [{username:'x',text:'a'},{username:'x',text:'aa'},{username:'x',text:'aaa'}]..
It saves [{username:'x',text:'aaa'},{username:'x',text:'aaa'},{username:'x',text:'aaa'}]
I tried using promises, but it didn´t work.. Probably used them wrong
For such things I prefer using the caolan's async library. Simply do:
npm install async to add the module.
Then replace your code with:
router.post('/logout', function(req, res) {
if (req.body.newList) {
var aux = JSON.parse(req.body.newList);
var async = require("async");
async.each(aux, function(aux1, callback) {
Task.findOne({
username: aux1.username,
text: aux1.text
}, function(err, res) {
if (res) {
callback();
} else {
task = new Task({
username: aux1.username,
text: aux1.text,
state: aux1.state
});
task.save(function(err) {
callback();
});
}
});
}, function(err) {
// if any of the query processing produced an error, err would equal that error
if (err) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('A query failed to process');
} else {
console.log('All queries have been processed successfully');
//do other things here
}
});
}
});
so i have this problem i am working on 'following' feature in my application. What's important, i have two models:
Follows and Notifications
When I hit follow button in front-end I run function from follow.client.controller.js which POSTs to API endpoint /api/follows which corresponds to follow.server.controller.js and then update action on Follows model is performed - easy. AFAIK thats how it works (and it works for me).
But in follows.server.controller.js I want also invoke post to API endpoint at /api/notifications which corresponds to notifications.server.controller.js but I can't find a proper way to do that. Any help will be appreciated.
I don't want another call from front-end to add notification because it should be automatic = if user starts following someone, information is saved in both models at once.
You can add middleware in your server route.
app.route('/api/follows')
.post(notification.firstFunction, follows.secondFunction);
And now add 2 methods in your contollers. First makes the call to db and add's some result's data to request object which will be forwarded to second method.
exports.firstFunction= function(req, res, next) {
Notification.doSometing({
}).exec(function(err, result) {
if (err) return next(err);
req.yourValueToPassForward = result
next(); // <-- important
});
};
exports.secondFunction= function(req, res) {
//...
};
Or you can make few database calls in one api method, joining this calls with promises. Example:
var promise = Meetups.find({ tags: 'javascript' }).select('_id').exec();
promise.then(function (meetups) {
var ids = meetups.map(function (m) {
return m._id;
});
return People.find({ meetups: { $in: ids }).exec();
}).then(function (people) {
if (people.length < 10000) {
throw new Error('Too few people!!!');
} else {
throw new Error('Still need more people!!!');
}
}).then(null, function (err) {
assert.ok(err instanceof Error);
});