I wrote a code for running insert queries in parallel in Node.js and I am also using Promise.js.
But the code fails and raises an exception of "Duplicate Primary Key" entry.
The code is as follows,
var Promise = require("promise");
var mySql = require("mysql");
var _ = require("underscore");
var connection = mySql.createConnection({
host : "localhost",
user : "root",
password : "rahul",
database : "testDb" //schema
});
connection.connect();
function insertDept(name){
return new Promise(fn);
function fn(resolve,reject){
getMaxDept().then(function(rows){
var deptId = rows[0]["DeptId"];
deptId = (_.isNull(deptId) === true) ? 125 : deptId;
var sql = "insert into departmentTbl values("+deptId+",'"+name+"')";
console.log(sql);
connection.query(sql,function(err,rows,fields){
if(err){
console.log(err);
return reject(err);
}else{
return resolve(rows);
}
});
}).catch(function(error){
return reject(error);
});
}//fn
}//insertDept
function getMaxDept(){
return new Promise(fn);
function fn(resolve,reject){
var sql = "select max(deptId) + 1 as 'DeptId' from departmentTbl";
connection.query(sql,function(err,rows,fields){
if(err){
console.log(err.stack);
return reject(err);
}else{
return resolve(rows);
}
});
}// fn
} //getMaxDept
function createDeptForAll(){
var promiseObj = [];
if(arguments.length > 0){
_.each(arguments,callback);
}else{
throw "No departments passed";
}
function callback(deptName){
promiseObj.push(insertDept(deptName))
}
return Promise.all(promiseObj);
}//createDeptForAll
createDeptForAll("Archiology","Anthropology").then(function(createDepartment){
createDepartment.then(function(rows){
console.log("Rows inserted "+rows["affectedRows"]);
}).catch(function(error){
console.log(error);
}).done(function(){
connection.end();
});
});
When I run the above code code,
the output is
rahul#rahul:~/myPractise/NodeWebApp/NodeMySqlv1.0$ node queryUsingPromise02.js
insert into departmentTbl values(125,'Archiology')
insert into departmentTbl values(125,'Anthropology')
{ [Error: ER_DUP_ENTRY: Duplicate entry '125' for key 'PRIMARY'] code: 'ER_DUP_ENTRY', errno: 1062, sqlState: '23000', index: 0 }
As Department Id is Primary key and the promises run in parallel,
the primary key for second Department's insert query fails.
As you can see, before any insert query, I fetch the max of departments + 1.
if the above query fails, I assign '125'.
Now, what should I change so that my above written code runs.
Should I use trigger of "before insert" for calculating next value of primary key of "department ID" at the database level itself or should I do something in my own Node.js code?
This issue is not restricted to node or JavaScript, but you will face this problem with any technology that tries to write to an SQL database in parallel. Unique id generation in a scenario like this is not trivial.
If you have the option to do so, make your id field in your database AUTO_INCREMENT, this will save you a lot of headaches in situations like this.
More about AUTO_INCREMENT.
Advice on AUTO_INCREMENT looks good.
You might also consider writing a promisifier for connection.query(), allowing for the remaining code to be tidied up.
So with getMaxDept() purged and a connection.queryAsync() utility in place, you might end up with something like this :
var Promise = require("promise");
var mySql = require("mysql");
var connection = mySql.createConnection({
host: "localhost",
user: "root",
password: "rahul",
database: "testDb" //schema
});
connection.connect();
// promisifier for connection.query()
connection.queryAsync = function(sql) {
return new Promise((resolve, reject) => {
connection.query(sql, (err, rows, fields) => {
if(err) { reject(err); }
else { resolve({'rows':rows, 'fields':fields}); }
});
});
};
function insertDept(name) {
var sql = "insert into departmentTbl values(" + name + "')"; // assumed - needs checking
return connection.queryAsync(sql);
}
function createDeptForAll(departments) {
if(departments.length > 0) {
return Promise.all(departments.map(insertDept));
} else {
return Promise.reject(new Error('No departments passed'));
}
}
createDeptForAll(['Archiology', 'Anthropology']).then((results) => {
results.forEach((result) => {
console.log("Rows inserted " + result.rows.affectedRows);
connection.end();
});
}).catch((error) => {
console.log(error);
connection.end();
});
Related
I am developing an application in NodeJS that queries two different MySQL databases, I have a problem with the asynchronous behavior of the language.
What happens is I make a query to the database, but the application continues before the database returns the response of the query.
How can I place an await or make the function return when the query has finished?
for the database connection I am using the mysql package
var mysql = require('mysql');
var connection_dblink = mysql.createConnection({
host: 'localhost',
user: 'root',
password: '',
database: 'carritos_lead_ac_test'
});
connection_dblink.connect(function (error) {
if (error) {
console.log('conexion dblink --> N O T O K <-- ');
throw error;
} else {
console.log('conexion dblink OK');
}
})
It works correctly, it brings the data but after the return.
The structure of the function that is giving the problem
async function getContact(contact) {
let responseGetContactFunc = connection_dblink.query(`SELECT * FROM contacto WHERE rut = "${contact.rut}";`, function (err, result, fields) {
if (err) {
console.log("ERROR");
throw err;
} else {
return result[0];
}
});
console.log(responseGetContactFunc._ended);
}
return result[0]; it is the value that I need to wait for before continuing with the execution of the rest of the application
console.log(responseGetContactFunc._ended); returns false because logically the query has not finished yet.
I have fixed this issue by promisifying the query connection using utils. So the connection has to be modified as bellow.
var mysql = require("mysql");
const util = require("util"); // used to promisify
var connection_dblink = mysql.createConnection({
host: "localhost",
user: "root",
password: "",
database: "carritos_lead_ac_test",
});
connection_dblink.connect(function (error) {
if (error) {
console.log("conexion dblink --> N O T O K <-- ");
throw error;
} else {
console.log("conexion dblink OK");
}
});
var modified_connection_dblink = util.promisify(connection_dblink.query).bind(connection_dblink);
Then it can be used as bellows.
async function getContact(contact) {
try{
let responseGetContactFunc = await modified.connection_dblink(`SELECT * FROM contacto WHERE rut = "${contact.rut}";`)
return responseGetContactFunc[0];
}
catch(e){
return e;
}
}
I use try catch blocks to handle errors. Hope it fixes your issue.
I've searched on how to create a sqlite3 database with a callback in Node.js and have not been able to find any links. Can someone point me towards documentation or provide a 2-3 line code sample to achieve the following:
Create a sqlite3 database and catch an error if the creation fails for any reason.
Here is what I've tried:
let dbCreate = new sqlite3.Database("./user1.db", sqlite3.OPEN_CREATE, function(err){
if(!err){
logger.infoLog("Successfully created DB file: " + dbFileForUser + " for user: " + username );
} else {
logger.infoLog("Failed to create DB file: " + dbFileForUser + ". Error: " + err );
}
});
dbHandler[username] = dbCreate;
When I execute this, I get the following error:
"Failed to create DB file: ./database/user1.db. Error: Error: SQLITE_MISUSE: bad parameter or other API misuse"
This call without callback works just fine.
var customDB = new sqlite3.Database("./custom.db", sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE);
But in this, I will not know if I run into any errors while creating the Database.
Try this:
let userDB = new sqlite3.Database("./user1.db",
sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE,
(err) => {
// do your thing
});
Example.
#Irvin is correct, we can have a look at http://www.sqlitetutorial.net/sqlite-nodejs/connect/ and
check it says if you skip the 2nd parameter, it takes default value as sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE
and in this case if database does not exist new database will be created with connection.
sqlite3.OPEN_READWRITE: It is to open database connection and perform read and write operation.
sqlite3.OPEN_CREATE : It is to create database (if it does not exist) and open connection.
So here is the first way where you have to skip the 2nd parameter and close the problem without an extra effort.
const sqlite3 = require("sqlite3").verbose();
let db = new sqlite3.Database('./user1.db', (err) => {
if (err) {
console.error(err.message);
} else {
console.log('Connected to the chinook database.|');
}
});
db.close((err) => {
if (err) {
return console.error(err.message);
}
console.log('Close the database connection.');
});
And this is the 2nd way to connect with database (already answered by #Irvin).
const sqlite3 = require("sqlite3").verbose();
let db = new sqlite3.Database('./user1.db', sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE
, (err) => {
if (err) {
console.error(err.message);
} else {
console.log('Connected to the chinook database.');
}
});
db.close((err) => {
if (err) {
return console.error(err.message);
}
console.log('Close the database connection.');
});
I looked at similar questions first, but couldn't figure out how to change my code to get it working. I'm new to Node.js and MySQL. I'm running into this error:
My code is as follows (I am connecting okay, just changed host/pw info):
var mySQLpointer, connObj;
mySQLpointer = require("mysql");
connObj = mySQLpointer.createConnection( {
host: "host",
user: "user",
password: "pw",
database: "db"
} );
connObj.connect( function(err) {
if (err)
// throw err; or use the follwowing command
console.log("Connection Error: " + err.stack);
else
// console.log("Connected to DB. :-)");
console.log("Connection OK! ID = " + connObj.threadId);
});
let sqlStmt = "SELECT * FROM Product-Service";
connObj.query(sqlStmt,
function(err, dataSet, fields) {
if (err)
throw err;
else {
console.log(dataSet);
}
}
);
connObj.end();
What I'm trying to do is to display all rows and columns in my SQL table, I only have 3 rows in there:
Ideally, I'm trying to get them to display like this:
Any help would be appreciated.
You need backticks around your tablename, because of the -:
let sqlStmt = "SELECT * FROM `Product-Service`";
I'm making a logging system in NodeJS with MySQL DB. First I do the connection like this:
const con = mysql.createConnection({
host : 'localhost',
user : 'dbuser',
password : 'dbpass',
database : 'dbname',
port : 3306,
multipleStatements : true
});
Then when I do a query to get users data I do the following query.
var user;
con.query('SELECT * FROM users WHERE email = ?', email, function(err, rows) {
if (err) throw err;
else {
user = rows[0];
}
});
But when I finally compare any of the fields of the user returned I get an error:
if (tools.hashPassword(password) == user.hash) {
// Do stuff
}
The error is TypeError: Cannot read property 'hash' of undefined. Any suggestion?
con.query("SELECT * FROM users WHERE email = ?", email, function (err, rows) {
if (err) {
throw err;
} else {
if (!rows.length) {
throw new Error("User not found");
}
const user = rows[0];
if (tools.hashPassword(password) == user.hash) {
// Do stuff
}
}
});
The fact is that you are getting the result, but it is asynchronous. Therefore at the time you check for user's property hash, the object itself has not loaded yet. You should put your comparison in the callback like this:
con.query('SELECT * FROM users WHERE email = ?', email, function(err, rows) {
if (err) throw err;
else {
if (tools.hashPassword(password) === rows[0].hash) {
// Do stuff
}
}
});
// This stuff happens usually BEFORE the query's callback,
// due to Node.js asynchronous nature
I have a simple node-based API, which needs to parse some JSON, save some data into Postgres, and then, send the appropriate response code (like http 201).
My code looks like this:
router.route('/customer')
.post(function(req, res) {
Customers = req.body;
var numberOfCustomers = Customers.length;
for(var i = 0; i < Customers.length; i++){
Customer = Customers[i];
console.log(Customer.Name + " " + Customer.Address);
var date = moment(new Date()).unix();
client.query(
'INSERT into customer (name, address, date_modified) VALUES($1, $2, $3) RETURNING id',
[Customer.Name, Customer.Address, date],
function(err, result) {
if (err) {
console.log(err);
status = 1;
} else {
console.log('row inserted with id: ' + result.rows[0].id);
if(numberOfCustomers === i) {
res.status(201).send({ message: "created" });
}
}
});
}
})
I'm getting this error:
_
http_outgoing.js:344
throw new Error('Can\'t set headers after they are sent.');
^
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:344:11)
I need to account for the fact, that I'm performing my Postgres insert multiple times within a loop, so I can not send my response headers after just the first insert is done.
What is the most appropriate place within my 'POST' handler to put my res.status(201).send({ message: "created" });
?
Architectural decisions aside (for example, you might want a separate module that acts as an HTTP adapter to handle logic for sending response codes as opposed to doing it inside of your route controller), you can use promises to wait for all the inserts to finish and then send a single response code. For example, something like this:
var Promise = require('bluebird');
var query = Promise.promisify(client.query);
router.route('/customer')
.post(function(req, res) {
// all your logic, and then
return Promise.all(Customers.map(function() {
return query(sql, [Customer.Name, Customer.Address, date]);
})
.then(function() {
res.status(201).send({ message: 'Created' });
});
});
Check out the the bluebird docs for the API used in this example.
I'm unfamiliar with Postgres's API, but the concept should be similar: you need to wait for all the requests to your DB to be resolved first.
As stated above: Yes, async helpers such as Promises and async are beneficial for such matters. However, I do believe the 'best' way to solve this problem is to only use a single query. Instead of only performing one insert per query, batch them all up in to a single query like so:
INSERT into customer (name, address, date_modified)
VALUES
($1, $2, $3),
($4, $5, $6),
($7, $8, $9),
...
RETURNING id'
Suggestion
router.route('/customer').post(function(req, res) {
//Fetch customers
var customers = req.body;
//Store parameters and query inserts for db-query.
var params = [];
var inserts = [];
//For each customer
// - Add parameters for query
// - Build insert string
customers.forEach(function(customer){
inserts.push(
[
"($",
params.push(customer.Name),
", $",
params.push(customer.Address),
", ",
NOW(), //unnecessary to generate timestamp in js
")",
].join('')
)
});
//Build query
var query = 'INSERT into customer (name, address, date_modified) VALUES '+ inserts +' RETURNING id';
//Query database in a more simple fashion.
client.query(query, params, function(err, result) {
if (err) {
console.log(err);
status = 1;
} else {
res.status(201).send({ message: "created" });
});
}
})
If you're using ES6 you are able to simplify the string build operations by using string templating.
customers.forEach(function(customer){
var query = `($${params.push(customer.Name)}, $${params.push(customer.Address)}, NOW())`
inserts.push(query);
});
//and
var query = `
INSERT into customer (name, address, date_modified)
VALUES ${inserts}
RETURNING id
`;
The proper way be to do, I would also recommend you look into Async or lodash lib.
router.route('/customer')
.post(function(req, res) {
var Customers = req.body,
numberOfCustomers = Customers.length;
for(var i = 0; i < Customers.length; i++){
var Customer = Customers[i];
console.log(Customer.Name + " " + Customer.Address);
var date = moment(new Date()).unix(),
sql = 'INSERT into customer (name, address, date_modified) VALUES($1, $2, $3) RETURNING id';
client.query(sql, [Customer.Name, Customer.Address, date],
function(err, result) {
if (err) {
console.log(err);
res.status(500).json({message: "Server Error", err: err});
} else {
console.log('row inserted with id: ' + result.rows[0].id);
if (numberOfCustomers === i) {
res.status(201).send({ message: "Created" });
}
}
});
}
})