I have a database query (function) which is asynchronous, and as a result I need to use a callback function (no problem with that). However, in Node.js I need to make two separate queries in the same POST function. Both are asynchronous, so I'm having trouble on how to continue with the execution of the POST.
Goal:
Validate form entries for malformations, etc.
Check if username exists in db before saving (must be unique)
Check if email exists in db before saving (must be unique)
Save user if everything checks out, else throw some errors
Normally, I would have something like this (oversimplified):
postSearch = function(req, res, next) {
var searchCallback = function(err, results) {
// Do stuff with the results (render page, console log, whatever)
}
// This is the db query - async. Passes search results to callback
defaultSearch(input, searchCallback);
}
Which only has one async query, so only one callback. Normally I would just get the db results and render a page. Now I have to validate some form data, so my POST function looks something like this:
postUser = function(req, res, next) {
// Some static form validation (works, no issues)
var usernameExistsCallback = function(err, exists) {
// Does the username exist? True/false
}
// DB query - passes true or false to the callback
usernameExists(username, usernameExistsCallback);
var emailExistsCallback = function(err, exists) {
// Does the email exist? True/false
}
// DB query - passes true or false to the callback
emailExists(email, emailExistsCallback);
// Check if ALL validation constraints check out, implement error logic
}
The node-postgres module is async, and as a result the queries need callbacks (if I want to return any value, otherwise I can just run the query and disconnect). I have no problem executing both of those queries. I can console.log() the correct results in the callbacks. But now I don't know how to access those results later on in my postUser function.
I've read all about async JavaScript functions, but I've been scratching my head on this one for three hours now trying ridiculous things (like setting global variables [oh my!]) to no avail.
The results I need from these two queries are simply true or false. How can I organize this code to be able to use these results in the postUser function? It seems to me that I need something like a third callback, but I have no clue how to implement something like that. Is it necessary for me to start using async? Would it be a good idea? Nothing in this application is super complex thus far, and I'd like to keep dependencies low <-> it makes sense.
How about this:
postUser = function(req, res, next) {
// Some static form validation (works, no issues)
var emailExistsCallback = function(err, exists) {
// Does the email exist? True/false
var usernameExistsCallback = function(err, exists) {
// Does the username exist? True/false
// DO STUFF HERE
}
// DB query - passes true or false to the callback
usernameExists(username, usernameExistsCallback);
}
// DB query - passes true or false to the callback
emailExists(email, emailExistsCallback);
// Check if ALL validation constraints check out, implement error logic
}
Simplest way is to nest functions like this:
postUser = function(req, res, next) {
var emailExistsCallback = function(err, exists) {
// Does the email exist? True/false
// Check if ALL validation constraints check out, implement error logic
next(); // <= you should finally call "next" callback in order to proceed
}
var usernameExistsCallback = function(err, exists) {
// Does the username exist? True/false
emailExists(email, emailExistsCallback); // <= note this
}
usernameExists(username, usernameExistsCallback);
}
Or you can use async, Q, seq or yaff(which is Seq reimplemented). There are number of libs to make your life easier. Better to try them all and decide which one is right for you, your style, requirements and so on.
you can use a common var to keep track of how many responses you got back. if you have them both, you can then do the stuff that needs them in a third "callback", which i called done():
postUser = function(req, res, next) {
var hops=0, total=2;
function done(){
// do stuff with both username and email
}
// Some static form validation (works, no issues)
var usernameExistsCallback = function(err, exists) {
if(++hops>=total && exists ){ done(); }
// Does the username exist? True/false
}
// DB query - passes true or false to the callback
usernameExists(username, usernameExistsCallback);
var emailExistsCallback = function(err, exists) {
if(++hops>=total && exists){ done(); }
// Does the email exist? True/false
}
// DB query - passes true or false to the callback
emailExists(email, emailExistsCallback);
// Check if ALL validation constraints check out, implement error logic
}
you should probably add error handling as needed by your app, specifically in both of the SQL callbacks, but this is a nice parallel IO ajax pattern that should do what you need.
Related
I need a function that takes the request body and conditionally creates the validators that will be used on the request. I figured the best way to do this is by creating middleware but I'm running into a problem with express-validator. For express-validator to work the middleware has to return an array of my validators but if I do this I can't call next() and the request doesn't get handled.
// This will fail because I can't call next
const validatorMiddleware = (req, res, next) => {
const {
isFooRequired,
isBarRequired,
} = req.body;
return concat(
isFooRequired && createValidator('foo'),
isBarRequired && createValidator('bar'),
)
}
With the way I'm going about this I basically need a function that calls next() AND returns the array of validators.
I understand express-validator has a way to conditionally add validation rules but there doesn't seem to be an easy way to do this in bulk. I'd have to do this individually with each validation chain which is a huge pain and kind of gross if you ask me. I have several optional fields which are dependent on a lot of validation and I want to do something like this:
const addressValidators = (service) => [
body(`${service}.address.lineOne`).notEmpty().withMessage('Address cant be empty'),
body(`${service}.address.city`).notEmpty().withMessage('city cant be empty'),
body(`${service}.address.state`).isIn(allowedStates).withMessage('invalid state'),
body(`${service}.address.zipCode`).isPostalCode('US').withMessage('invalid zip code'),
];
In case anyone has this same problem I do, here's the solution I found. With express-validator you can run validations imperatively so I created the middleware you see below. Inside my constructValidators function I use the request body to see which fields are present and create my validation chain accordingly.
const validateMiddleware = async (req, res, next) => {
const validationList = constructValidators(req.body);
await Promise.all(validationList.map((validation) => validation.run(req)));
const errors = validationResult(req);
if (errors.isEmpty()) {
return next();
}
res.status(400).json({ errors: errors.array() });
return null;
};
Remember, 3 hours of troubleshooting could save you 10 minutes of reading documentation!
This code works like it should work, but after fifth GET request it does what it should do on the backend(stores the data in db) but it's not logging anything on the server and no changes on frontend(reactjs)
const express = require('express');
const router = express.Router();
const mongoose = require('mongoose');
const User = require('./login').User;
mongoose.connect('mongodb://localhost:27017/animationsdb');
router.get('/', async(req, res) => {
await User.findOne({ username: req.query.username }, (err, result) => {
if (result) {
// when user goes to his profile we send him the list of animations he liked
// list is stored in array at db, field likedAnimations
res.send({ animationList: result.likedAnimations });
console.log("Lajkovane animacije:", result.likedAnimations);
} else {
console.log("no result found");
res.sendStatus(404)
}
});
});
router.put('/', async(req, res) => {
console.log("username:", req.body.username);
console.log("link:", req.body.link);
// if animation is already liked, then dislike it
// if it's not liked, then store it in db
const user = await User.findOne({ username: req.body.username });
if (user.likedAnimations.indexOf(req.body.link) === -1) {
user.likedAnimations.push(req.body.link);
} else {
user.likedAnimations = arrayRemove(user.likedAnimations, user.likedAnimations[user.likedAnimations.indexOf(req.body.link)]);
}
user.save();
});
function arrayRemove(arr, value) {
return arr.filter((item) => {
return item != value;
});
}
module.exports = router;
For first five requests I get this output:
Liked animations: ["/animations/animated-button.html"]
GET /animation-list/?username=marko 200 5.152 ms - 54
Liked animations: ["/animations/animated-button.html"]
GET /animation-list/?username=marko 304 3.915 ms - -
After that I don't get any output on server console and no changes on front end untill I refresh the page, even though db operations still work and data is saved.
It appears you have a couple issues going on. First, this request handler is not properly coded to handle errors and thus it leaves requests as pending and does not send a response and the connection will stay as pending until the client eventually times it out. Second, you likely have some sort of database concurrency usage error that is the root issue here. Third, you're not using await properly with your database. You either use await or you pass a callback to your database, not both. You need to fix all three of these.
To address the first and third issues:
router.get('/', async(req, res) => {
try {
let result = await User.findOne({ username: req.query.username };
if (result) {
console.log("Liked animations:", result.likedAnimations);
res.send({ animationList: result.likedAnimations });
} else {
console.log("no database result found");
res.sendStatus(404);
}
} catch(e) {
console.log(e);
res.sendStatus(500);
}
});
For the second issue, the particular database error you mention appears to be some sort of concurrency/locking issue internal to the database and is triggered by the sequence of database operations your code executes. You can read more about that error in the discussion here. Since the code you show us only shows a single read operation, we would need to see a much larger context of relevant code including the code related to this operation that writes to the database in order to be able to offer any ideas on how to fix the root cause of this issue.
We can't see the whole flow here, but you need to use atomic update operations in your database. Your PUT handler you show is an immediate race condition. In multi-client databases, you don't get a value, modify it and then write it back. That's an opportunity for a race condition because someone else could modify the value while you're sitting their holding it. When you then modify your held value, you overwrite the change that the other client just made. That's a race condition. Instead, you use an atomic operation that updates the operation directly in one database call or you use transactions to make a multi-step operation into a safe operation.
I'd suggest you read this article on atomic operations in mongodb. And, probably you want to use something like .findAndModify() so you can find and change an item in the database in one atomic operation. If you search for "atomic operations in mongodb", there are many other articles on the topic.
Right now i am creating a very large application in Node JS. I am trying to make my code clean and short (Just like most of the developer). I've create my own js file to handle connection to mysql. Please see code below.
var mysql = require('mysql');
var config = {
'default' : {
connectionLimit : process.env.DB_CONN_LIMIT,
host : process.env.DB_HOST,
user : process.env.DB_USER,
password : process.env.DB_PASS,
database : process.env.DB_NAME,
debug : false,
socketPath : process.env.DB_SOCKET
}
};
function connectionFunc(query,parameters,callback,configName) {
configName = configName || "default";
callback = callback || null;
parameters = parameters;
if(typeof parameters == 'function'){
callback = parameters;
parameters = [];
}
//console.log("Server is starting to connect to "+configName+" configuration");
var dbConnection = mysql.createConnection(config[configName]);
dbConnection.connect();
dbConnection.query(query,parameters, function(err, rows, fields) {
//if (!err)
callback(err,rows,fields);
//else
//console.log('Error while performing Query.');
});
dbConnection.end();
}
module.exports.query = connectionFunc;
I am using the above file in my models, like below :
var database = require('../../config/database.js');
module.exports.getData = function(successCallBack){
database.query('SAMPLE QUERY GOES HERE', function(err, result){
if(err) {console.log(err)}
//My statements here
});
}
Using this coding style, everything works fine but when i am trying to create a function that will loop my model's method for some reason. Please see sample below :
for (i = 0; i < 10000; i++) {
myModel.getData(param, function(result){
return res.json({data : result });
});
}
It gives me an ER_CON_COUNT_ERROR : Too Many Conenction. The question is why i still get an error like these when my connection always been ended by this dbConnection.end();? I'm still not sure if i am missing something. I am still stuck on this.
My connection limit is 100 and i think adding more connection is a bad idea.
Because query data form the database is async.
In your loop the myModel.getData (or more precisely the underling query) will not halt/paus your code until the query is finished, but send the query to the database server and as soon as the database response the callback will be called.
The calling end on dbConnection will not close the connection immediately, it will just mark the connection to be closed as soon as all queries that where created with that connection are finished.
mysql: Terminating connections
Terminating a connection gracefully is done by calling the end() method. This will make sure all previously enqueued queries are still before sending a COM_QUIT packet to the MySQL server.
An alternative way to end the connection is to call the destroy() method. This will cause an immediate termination of the underlying socket. Additionally destroy() guarantees that no more events or callbacks will be triggered for the connection.
But with destroy the library will not wait for the result so the results are lost, destroy is rarely useful.
So with your given code you try to create 10000 connections at one time.
You should only use on connection by task, e.g. if a user requests data using the browser, then you should use one connection for this given request. The same is for timed task, if you have some task that is done in certain intervals.
Here an example code:
var database = require('./config/database.js');
function someTask( callback ) {
var conn = database.getConnection();
myModel.getData(conn, paramsA, dataReceivedA)
function dataReceivedA(err, data) {
myModel.getData(conn, paramsB, dataReceivedB)
}
function dataReceivedB(err, data) {
conn.end()
callback();
}
}
If you want to entirely hide your database connection in your model code. Then you would need to doe something like that:
var conn = myModel.connect();
conn.getData(params, function(err, data) {
conn.end();
})
How to actually solve this depends only many factors so it is only possible to give you hints here.
I'm writing my first (non tutorial) node application and am at a point where I'm writing a function that should take the username and password as parameters and query them against the user table of my database to return either true or false. The database is setup, and the app is connecting to it successfully.
However, I haven't worked with SQL very much, nor node, and I'm unsure how to proceed with this function (and short surrounding script). Here it is:
console.log('validator module initialized');
var login = require("./db_connect");
function validate(username, password){
connection.connect();
console.log('Connection with the officeball MySQL database openned...');
connection.query(' //SQL query ', function(err, rows, fields) {
//code to execute
});
connection.end();
console.log('...Connection with the officeball MySQL database closed.');
if(){ //not exactly sure how this should be set up
return true;
}
else{ //not exactly sure how this should be set up
return false;
}
}
exports.validate = validate;
This is using node-mysql. I'm looking for a basic example of how I might set the query and validation up.
I think you'll want to rethink your app into a more node-like way (i.e. one that recognizes that many/most things happen asynchronously, so you're not usually "returning" from a function like this, but doing a callback from it. Not sure what you plan to get from node-mysql, but I would probably just use the plain mysql module. The following code is still most likely not entirely what you want, but will hopefully get you thinking about it correctly.
Note that the use of 'return' below is not actually returning a result (the callback itself should not return anything, and thus its like returning undefined. The return statements are there so you exit the function, which saves a lot of tedious if/else blocks.
Hope this helps, but I'd suggest looking at various node projects on github to get a better feel for the asynchronous nature of writing for node.
function validate(username, password, callback){
var connection = mysql.createConnection({ user:'foo',
password: 'bar',
database: 'test',
host:'127.0.0.1'});
connection.connect(function (err){
if (err) return callback(new Error('Failed to connect'), null);
// if no error, you can do things now.
connection.query('select username,password from usertable where username=?',
username,
function(err,rows,fields) {
// we are done with the connection at this point), so can close it
connection.end();
// here is where you process results
if (err)
return callback(new Error ('Error while performing query'), null);
if (rows.length !== 1)
return callback(new Error ('Failed to find exactly one user'), null);
// test the password you provided against the one in the DB.
// note this is terrible practice - you should not store in the
// passwords in the clear, obviously. You should store a hash,
// but this is trying to get you on the right general path
if (rows[0].password === password) {
// you would probably want a more useful callback result than
// just returning the username, but again - an example
return callback(null, rows[0].username);
} else {
return callback(new Error ('Bad Password'), null);
}
});
});
};
I'm using the Node Express framework to build an API and I run into a problem regarding the Basic Auth functionality. I need to run an SQL query to retrieve information about a user and validate their credentials. The issue occurs after the query has been completed. The SQL data is sent into a callback function as shown below. I want to do all the validation inside that callback but I want to break out of the SQL callback and return true/false from the express.basicAuth() function. I have tried setting a global variable and then accessing it outside of the SQL callback but sometimes the query might not have finished by the time that it gets the block that accesses that global variable. Thanks in advance for your help.
var auth = express.basicAuth(function(user, pass) {
// Query to select information regarding the requested user
/* Query here to find the requested user's data */
client.query(query, function (err, rows, fields) {
if (err) throw err;
GLOBAL.sql = rows;
/* I want to break out of this query callback function and return true from this auth variable */
});
// Sometimes the query isn't completed when it gets to this stage
console.log(JSON.stringify(GLOBAL.sql));
});
express.basicAuth also supports asynchronous operation:
var auth = express.basicAuth(function(user, pass, next) {
...
client.query(query, function (err, rows, fields) {
if (err)
next(err);
else
if (/* authentication successful */)
next(null, user); // <-- sets 'req.remoteUser' to the username
else
next(null, false);
});
});