I need to Update MYSQL data using JS after I receive an AJAX Post request
I made a variable for the MYSQL Update Query and I'm passing in the field to be updated, new value, row to be updated as an array. But for some reason those variables are read with single quotes(') which, I believe, is causing me a syntax error.
var express = require('express');
var app = express();
var bodyParser = require('body-parser');
var MYSQL = require('mysql');
var server = require('http').createServer(app);
//declaring var 'conn' for MYSQL.createPool
let columns = new Array();
// Piece of code Starting the Server
// Routing
app.use(bodyParser.json()); // for parsing application/json
app.use(bodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded
app.use(express.static(path.join(__dirname, 'public')));
app.post('/', function (req, res) {
updateWorkbook(req.body);
res.send('Thanks for the data.');
});
//This is the function extracts the row, field value that need to be updated from the AJAX request
function updateWorkbook( data ){
getcolumns().then( function (columns) {
console.log("Columns got returned to Updateworkbook function")
for (let d = 0; d < data.length; d++) {
let rowToUpdate = data[d].id.replace('row_', '').split('_')[0];
let fieldToUpdate = data[d].id.replace('row_', '').split('_')[1];
let newValue = data[d].value;
console.log('row,field,value: ' + rowToUpdate + '|' + fieldToUpdate + '|' + newValue);
let key_to_replace;
for(let i = 0; i < columns.length; i++) {
let looper = columns[i].toLowerCase()
if (looper === fieldToUpdate) {
key_to_replace = columns[i]
}
}
let field_to_replace = key_to_replace.toString();
console.log(field_to_replace) //It prints out a normal string value here
updatemysql(field_to_replace, newValue, rowToUpdate);
}
});
};
//This is the function which updates MYSQL data
function updatemysql(field, newval, row) {
var sql = "UPDATE mydb.mytable SET ? = ? WHERE ROW_ID = ?;";
conn.getConnection( function (err, connection) {
if (err){
return cb(err);
connection.release();
}
console.log("Connection got established")
conn.query(sql, [field, newval, row], function (error, results){
if (error){
throw error;
connection.release();
}
console.log('Data Updated');
connection.release();
});
});
}
//Function to extract all columns from MYSQL and stores them in an array
function getcolumns() {
return new Promise(function(resolve, reject) {
console.log("getcolumns got initiated")
conn.getConnection( function (err, connection) {
if (err){
return cb(err);
connection.release();
return reject(err);
}
else {
var sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = 'mydb' AND TABLE_NAME = 'mytable';"
conn.query(sql, function (error, results){
for (let i = 0; i < results.length; i++) {
columns.push(results[i]['COLUMN_NAME'])
}
resolve(columns);
console.log("Extracted columns")
connection.release();
});
}
});
});
};
Here's the error I receive:
Error: ER_PARSE_ERROR: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ''Source_of_Phone_Number_' = 'Test' WHERE ROW_ID = '1'' at line 1`
Source_of_Phone_Number_ is the key_to_replace.
Test is the newValue.
1 is the Row_ID.
There is a problem in function updatemysql(), which uses the following SQL :
var sql = "UPDATE mydb.mytable SET ? = ? WHERE ROW_ID = ?;";
You cannot pass a column name as a parameter.
You would need to change this to :
var sql = "UPDATE mydb.mytable SET " + field + " = ? WHERE ROW_ID = ?;";
Accordingly, only two parameters should be passed to the query :
conn.query(sql, [newval, row], function (error, results){ ... });
Related
First of all, i'm new in JS. I have a function that possibly can use multiple requests to get the final data. How can i do this in the right way? In this example participants won't pushed to the dialogs array because it's in async call.
function getDialogs(token, callback) {
//getting user id
con.query("SELECT user_id FROM users_tokens WHERE user_token = '" + token + "'", function(error, results) {
if (error) {
throw error;
}
var userId = results[0].user_id;
//getting all conversation
con.query("SELECT cc.id as conversation_id, cc.type FROM chat_conversations cc INNER JOIN chat_participants cp ON cc.id = cp.conversation_id WHERE cp.user_id = " + userId + " GROUP BY cc.id", function (error, results) {
if (error) {
throw error;
}
var dialogs = [];
for (let i = 0; i < results.length; i++) {
var dialog = {id: results[i].conversation_id};
//getting chat participants
con.query("SELECT user_id FROM chat_participants WHERE conversation_id = " + results[i].conversation_id + " AND user_id != " + userId, function (error, results) {
var participants = [];
for (let j = 0; j< results.length; j++) {
participants.push(results[j].user_id);
}
dialogs[participants] = participants;
});
dialogs.push(dialog);
}
callback(dialogs);
});
});
}
Technically you can use a single request like this
SELECT user_id FROM chat_participants WHERE conversation_id IN (
SELECT
cc.id as conversation_id,
cc.type
FROM
chat_conversations cc
INNER JOIN chat_participants cp ON cc.id = cp.conversation_id
WHERE
cp.user_id IN (
SELECT
user_id
FROM
users_tokens
WHERE
user_token = "TOKEN"
)
GROUP BY
cc.id
)
but there are a few problems with this approach as well.
First of all, it seems like you are only using the user_id of your first row, so please use LIMIT 1 in such cases.
Second of all, it seems like user_id won't ever have a duplicate, so make it a primary key.
Third of all, don't concat your token, node mysql supports having placeholders using ? in your query, like this:
con.query("SELECT * FROM ? WHERE user_id = ?", ["table_name", "userid"])
Fourth of all, promisify your requests so you don't have a callback hell, do something like:
function promiseRequest(query, placeholders) {
return new Promise((res, rej) => {
con.query(query, placeholders, (err, data) => {
if (err) rej(err);
res(data);
});
});
}
I have a piece of code that needs to do the following :
For each Sensor objects in the array, get the Template ID first.
Search the Template Schema DB, get the zipcode of the corresponsing Template ID got.
From the zipcode, generate the URI
Make the requestAPI call
Get the output of the API and store it in the DB for that sensor object.
I am having problem with step 5, storing the result in the DB for each SenObject. I guess the problem is since newSensorObjectRes is defined locally in the test(), it can't be used outside. How else can I store the results for each object?
var arr = [];
function SenObj (id)
{
this.Objnum = 10;
this.Template = id;
this.Type = "AirFlow";
this.UserID = "Jessi";
}
// To store the results from the AIR API
var sensorResults = new Schema({
//reqId: {type: Number, required: true, unique: true},
//sensorId: {type: Number},
SenObj: {type: Number},
status: {type: String, default: 'Undefined'}
})
var SensorObjectRes = connUserSensors.model('SensorObjectRes', sensorResults)
//API:To create Sensor Objects when user requests for template
// When user makes a request to create a new Template,we get the tempate ID he has requested.
// We use the ID to create a Sensor Object.We might need the request id too here ??
// The array arr[] holds all the sensor obects..
app.get('/ProvisionTemplate/:id', function (req, res) {
var id = req.params.id;
console.log("Server recieved a GET /ProvisionTemplate/" + id + " request");
Template.findOne({templateId: parseInt(id)},function (err, data) {
if (err) return console.error(err);
console.log(data);
res.json(data);
// Create an Object here
var user1 = new SenObj(id);
console.log(user1.UserID);
arr.push(user1);
});
});
console.log("The array objects are :");
for (i = 0; i < arr.length; i++)
{
console.log(arr[i]);
}
var output = ""
function test()
{
var zip = "";
console.log("Interval reached");
for (i = 0; i < arr.length; i++)
{
// For each Sensor objects in the array, get the Template ID first.
// Search the Template Schema DB, get the zipcode of the corresponsing Template ID got.
// From the zipcode, generate the URI
// Make the requestAPI call
// Get the output of the API and store it in the DB for that sensor object.
console.log(arr[i]);
console.log(arr[i].Template);
var tem = arr[i].Template;
Template.findOne({templateId: tem},function (err, data)
{
if (err) return console.error(err);
console.log(data.zipcode);
zip = data.zipcode;
var uri_1 = "http://www.airnowapi.org/aq/observation/zipCode/current/?format=application/json&zipCode=";
var uri_2 = "&distance=25&API_KEY=1035C2AC-CDB8-4540-97E4-0E8D82BA335A";
var url = uri_1 + zip + uri_2;
console.log(url);
requestApi(url, function (error, response, body)
{
if (!error && response.statusCode == 200)
{
console.log(body);
//console.log(arr[i])
var newSensorObjectRes = new SensorObjectRes({"SenObj": 1 ,"status": body});
newSensorObjectRes.save(function (err, data)
{
if (err) return console.log("error updating");
console.log(data);
})
}
})
});
}
}
var interval = setInterval(test, 10000);
newSensorObjectRes.find(function (err, data)
{
if (err)
{
console.log("Could not find EC2Server db");
return;
}
console.log(data)
})
I believe the issue is having a for loop in your test function. You could be making all your Template requests before you get a response. Try using recursion instead. This will assure that your requests are made in an orderly fashion. Something like this...`
function test(i)
{
if(i == arr.length){
console.log("done");
return;
}
var zip = "";
console.log("Interval reached");
console.log(arr[i]);
console.log(arr[i].Template);
var tem = arr[i].Template;
Template.findOne({templateId: tem},function (err, data)
{
if (err) return console.error(err);
console.log(data.zipcode);
zip = data.zipcode;
var uri_1 = "http://www.airnowapi.org/aq/observation/zipCode/current/?format=application/json&zipCode=";
var uri_2 = "&distance=25&API_KEY=1035C2AC-CDB8-4540-97E4-0E8D82BA335A";
var url = uri_1 + zip + uri_2;
console.log(url);
requestApi(url, function (error, response, body)
{
if (!error && response.statusCode == 200)
{
console.log(body);
var newSensorObjectRes = new SensorObjectRes({"SenObj": 1 ,"status": body});
newSensorObjectRes.save(function (err, data)
{
if (err) return console.log("error updating");
console.log(data);
test(i+1);
})
}
})
});
}
test(0);
I have a file which is tab separated. It has thousands of data. How can I use nodeJs to read the file, line by line, parse them and create an object and insert them in a mongo DB.
I am just learning node and mongo. I come from different background. So how can this be done.
Finally the Mongo DB has to be populated with proper data.
I searched in net but I could not find the complete solution.
Thanks.
I had an issue with the answer by Juvenik. My problem was that the database would not be populated by the time readline had completed. The lines were being read synchronously, but the DB insertion was asynchronous.
Instead, I found a simpler solution with the line-reader package. It reads the lines and waits for a callback before continuing.
var MongoClient = require('mongodb').MongoClient
var dbName = 'yourDbName'
var url = 'mongodb://localhost:27017/' + dbName
var collectionName = 'yourCollectionName'
var filename = 'yourFileName.txt'
var printLine = 1000
MongoClient.connect(url, function(err, db) {
if (err) {
console.error('Problem connecting to database')
} else {
console.log('Connected correctly to server.')
var lineReader = require('line-reader')
var collection = db.collection(collectionName)
var lineNum = -1
var headers = []
lineReader.eachLine(filename, function(line, last, cb) {
lineNum++
try {
var split = line.split('\t')
var object = {}
if (lineNum > 0) {
for (var i = 0; i < split.length; i += 1) {
object[headers[i]] = split[i]
}
collection.insert(object, function (insertErr, insertObj) {
if (insertErr) console.error(insertErr)
if (lineNum % printLine === 0) console.log('Line ' + lineNum)
if (last) {
console.log('Done with ' + filename + ' (' + lineNum + ' records)')
process.exit(0)
} else {
cb()
}
})
} else {
headers = line.split('\t')
cb()
}
} catch (lineError) {
console.error(lineError)
}
})
}
})
I came across similar problem. This approach worked for me.
Have a look, it might be helpful.
var mongoDb = require('mongodb');
var mongoClient = mongoDb.MongoClient;
var dbname = 'YOUR_DB_NAME';
var collectionName = 'YOUR_COLLECTION_NAME';
var url = 'mongodb://localhost:27017/'+dbname;
var filename = 'FIle_Name.txt';
console.log('***************Process started');
mongoClient.connect(url,function(err,db){
if(err){
console.log('error on connection '+err);
}
else{
console.log('***************Successfully connected to mongodb');
var collection = db.collection(collectionName);
var fs = require('fs');
var readline = require('readline');
var stream = require('stream');
var instream = fs.createReadStream(filename);
var outstream = new stream;
var rl = readline.createInterface(instream,outstream);
console.log('***************Parsing, please wait ...');
rl.on('line',function(line){
try{
var arr = line.split('\t');
var object = {};
//Parse them here
//Example
object['name'] = arr[0]; //Just an example
var res = collection.insert(object);
}
catch (err){
console.log(err);
}
});
rl.on('close',function(){
db.close();
console.log('***************completed');
});
}
});
I am a learner too. If someone can make it better, it will be good.
Here is a more performant (inserting batches of objects) and updated version (using async and latest mongo driver) of frank-0's answer
const lineReader = require('line-reader');
async function readFileAndInsertInMongo(file) {
let total = 0;
return new Promise((resolve, reject) => {
let buffer = [];
lineReader.eachLine(file, (line, last, cb) => {
// prepare your object based on the line content
let insertObject = {'some_content': 'some_value'};
if (total % 10000 === 0 || last) {
collection.insertMany(buffer, function(err, res){
if (last) {
if (err) {
reject(err);
} else {
resolve(res);
}
} else {
buffer = [];
return cb();
}
});
} else {
buffer.push(insertObject);
return cb();
}
});
});
}
This really is the best solution I have found to parse huge files and insert them in the database without exploding Node's memory. Hope this can help ;)
I'm trying to switch from single mysql-queries to mysql-pool connection, so users can share one mysql-connection, but I'm not familiar with this at all (also new to nodejs/socket.io).
The following code is what I've done so far to send data every second to the socket in an array:
var
port = process.env.OPENSHIFT_NODEJS_PORT || 8000,
ip = process.env.OPENSHIFT_NODEJS_IP || '127.0.0.1',
app = require('http').createServer(handler),
fs = require('fs'),
request = require('request'),
mysql = require('mysql'),
moment = require('moment'),
tz = require('moment-timezone'),
pool = mysql.createPool({
connectionLimit: 100,
host: 'xxx',
user: 'xxx',
password: 'xxx',
database: 'xxx',
debug: false,
port: 3306}),
socketArray = [],
POLLING_INTERVAL = 1000,
pollingTimer;
moment.tz.setDefault("Europe/Berlin");
var io = require('socket.io').listen(app);
io.set('origins', '*:*');
function time()
{
output = new Date();
output = moment().format('(H:mm:ss.SS) ');
return output;
}
function handler(req,res)
{
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
res.statusCode = 200;
res.connection.setTimeout(0);
res.end();
}
app.listen(port,ip);
function pollingLoop () {
if (socketArray.length === 0) {
// no connections, wait and try again
setTimeout(pollingLoop, POLLING_INTERVAL);
return; // continue without sending mysql query
}
pool.getConnection(function(err,connection){
if (err) { console.log({"code" : 100, "status" : "connection-db error"}); return; }
console.log('connected as id ' + connection.threadId);
console.log('socketArray length: ' + socketArray.length);
var selection =
"SELECT\
a.`id`,a.`product_id` AS pid,a.`random` AS nr,a.`price`,a.`price_end` AS pe,\
TIMESTAMPDIFF(SECOND,NOW(),a.`datetime`) AS duration,\
ABS(TIMESTAMPDIFF(SECOND,NOW(),b.`date`)) AS hb\
FROM `auctions` AS a\
LEFT JOIN `auctions_bids` AS b ON b.`auction_id` = a.`id`\
WHERE TIMESTAMPDIFF(SECOND,NOW(),a.`datetime`) > '-1'\
GROUP BY a.`id`\
ORDER BY `duration` DESC,`id` DESC LIMIT 15";
var streamArray = [], lg = '';
var query = connection.query(selection, function(err, results, rows){
lg += ('id: '+results[0].id+' ('+results[0].duration+') ');
if
(
((results[0].duration < 2 || results[0].duration <= results[0].nr) && (results[0].price <= results[0].pe))
||
((results[0].duration < 2 || results[0].duration <= results[0].nr) && (results[0].hb > 0 && results[0].hb < 30))
)
{
min = 3;
max = 5;
rand = Math.floor(Math.random()*(max-min+1)+min);
price = results[0].price+0.01;
price = price.toFixed(2);
pool.query('UPDATE `auctions` SET `random` = ?,`price` = ?, `datetime` = DATE_ADD(`datetime`,INTERVAL(17-TIMESTAMPDIFF(SECOND,NOW(),`datetime`))SECOND) WHERE `id` = ?',[rand, price, results[0].id]);
console.log(time()+'UPDATED id '+results[0].id+': random ('+rand+') price ('+price+'€)');
}
streamArray.push(results[0]);
updateSockets({ streamArray: streamArray });
console.log("auctions pushed: " + streamArray);
connection.release();
setTimeout(pollingLoop, POLLING_INTERVAL);
});
console.log(time()+lg+' C: '+socketArray.length);
});
}
pollingLoop();
io.sockets.on('connection', function(socket) {
socket.on('disconnect', function() {
clearTimeout(pollingTimer);
var socketIndex = socketArray.indexOf(socket);
console.log(time()+'SOCKET-ID = %s DISCONNECTED', socketIndex);
if (~socketIndex) { socketArray.splice(socketIndex, 1); }
});
console.log(time()+'NEW SOCKET CONNECTED!');
socketArray.push(socket);
});
var updateSockets = function(data) {
socketArray.forEach(function(tmpSocket) { tmpSocket.volatile.emit('stream', data); });
};
console.log(time()+'server.js executed\n');
But this doesn't send me any data to the WebSocket. Is this approach (code-structure) even correct? Previously I used query.on('results') to get data like this:
var selection = "SELECT * FROM auctions";
var query = mysql.query(selection), auctions = [];
query.on('result', function(auction) {
console.log('id: '+auction.id+' ('+auction.duration+') ');
});
This worked fine showing data with auction.row but how to do this in my mysql pool connection?
Also after some seconds I'm getting an error that release() isn't even defined, but it's listed in the mysql-module documentation... so I think my whole logical process is somehow incorrect.
Should I use connection.end() and .release() at all? Because the
connection should never end.
Should I still use setInterval(function () { mysql.query('SELECT
1'); }, 5000); as answered in another StackOverflow question to keep
the connection alive here? (nodejs mysql Error: Connection lost The server closed the connection)
(Appreciate any tips or answers to even some of my questions! Better some answers than none, because I experienced that this topic isn't answered much at all.)
EDIT:
Updated my whole code (see above). Output looks like this now: http://s21.postimg.org/avsxa87rb/output.jpg
So the stream gets the data, but in the console.log is nothing and there's this javascript error?
You should be creating a pool, and using getConnection on that pool. Then, when you're done with the connection, release it. Additionally, you do not need to stop the pollingLoop or start it for each connection, one loop is enough.
I didn't understand the if statement with conditions, so i omitted it. It likely needs to go somewhere else.
var socketArr = [];
function handler(req, res) {
res.statusCode = 200;
res.connection.setTimeout(0);
res.end();
}
app.listen(port, ip);
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
function pollingLoop () {
if (socketArr.length === 0) {
// no connections, wait and try again
setTimeout(pollingLoop, 1000);
return; // continue without sending mysql query
}
pool.getConnection(function (err, connection) {
if (err) {
console.log({
"code": 100,
"status": "Error in connection database"
});
return;
}
console.log('connected as id ' + connection.threadId);
var selection = "SELECT * FROM auctions";
var streamArray = [],
lg = '';
var query = connection.query(selection, function (err, results, fields, rows) {
lg += ('id: ' + results[0].id + ' (' + results[0].duration + ') ');
/*if (conditions) {
var query_update = connection.query('UPDATE `auctions` SET `price` = ? WHERE `id` = ?', [price, auction.id]);
console.log(time() + 'UPDATED id ' + auction.id + ': price (' + price + '€)');
}*/
streamArray.push(results);
updateSockets({
streamArray: streamArray
});
console.log("auctions pushed: " + streamArray);
connection.release();
setTimeout(pollingLoop, 1000);
});
console.log(time() + lg + ' C: ' + socketArr.length);
});
}
// start loop
pollingLoop();
io.sockets.on('connection', function (socket) {
socket.on('disconnect', function () {
var socketIndex = socketArr.indexOf(socket);
console.log(time() + 'SOCKET-ID = %s DISCONNECTED', socketIndex);
if (~socketIndex) {
socketArr.splice(socketIndex, 1);
}
});
console.log(time() + 'NEW SOCKET CONNECTED!');
socketArr.push(socket);
});
var updateSockets = function (data) {
socketArr.forEach(function (tmpSocket) {
tmpSocket.volatile.emit('stream', data);
});
};
I used the closures in this loop. But its only print the right data on the console log, and the sql query did not work. The inserted data on MySQL is the last variable of the loop.
I thought this is because of writing speed of MySQL. But don't know how to fix it. Any idea?
Thanks
module.exports = function (callback) {
queryGetForSend = "SELECT * FROM image WHERE send_request is NULL AND post_request is NOT NULL AND year(moderate_time) = year(curdate()) AND month(moderate_time) = month(curdate()) AND (time(moderate_time) < (curtime() - 15));";
conn.query(queryGetForSend, function(err, rows, fields){
for (i in rows) {
if (rows[i].post_request == 'approve') {
resultSend = 1
} else {
resultSend = 2
}
var fileID = rows[i].img_md5;
queryString = fileID + "=" + resultSend;
// Request url: "http://im-api1.webpurify.com/image_queue/results/?key="
var d = new Date();
Date.masks.default = 'YYYY-MM-DD hh:mm:ss';
sendTime = d.format();
(function(queryString, sendTime) {
querySent = "UPDATE image SET send_request=1,result_sent='"+queryString+"',send_time='"+sendTime+"' WHERE send_request is NULL AND post_request is NOT NULL AND year(moderate_time) = year(curdate()) AND month(moderate_time) = month(curdate()) AND (time(moderate_time) < (curtime() - 15));";
conn.query(querySent, function (err, rows, fields) {
if (err) throw err;
console.log("http://google.com?key=" + key + "&" + queryString);
});
})(queryString, sendTime);
(function(queryString){
request.get("http://google.com" + key + "&" + queryString, function(err, res, body) {
});
})(queryString);
}
// callback(rows);
});
};
Two suggestions:
Avoid concatenation in SQL queries, use placeholders or prepared statements instead (if you care about security).
Use array.forEach() instead of a regular for-loop with a closure to avoid accidental use of variables set inside the for-loop:
conn.query(queryGetForSend, function(err, rows, fields) {
if (err) throw err;
rows.forEach(function(row) {
var resultSend;
if (row.post_request == 'approve') {
resultSend = 1
} else {
resultSend = 2
}
var fileID = row.img_md5;
var queryString = fileID + '=' + resultSend;
var d = new Date();
Date.masks.default = 'YYYY-MM-DD hh:mm:ss';
var sendTime = d.format();
var querySent = 'UPDATE image SET send_request=1, result_sent=?, send_time=? WHERE send_request IS NULL AND post_request IS NOT NULL AND year(moderate_time) = year(curdate()) AND month(moderate_time) = month(curdate()) AND (time(moderate_time) < (curtime() - 15));';
conn.query(querySent, [queryString, sendTime], function (err) {
if (err) throw err;
console.log('http://google.com?key=' + key + '&' + queryString);
});
request.get('http://google.com?key=' + key + '&' + queryString, function(err, res, body) {
});
});
});