Develop Reference

How to insert json output into mysql with node js

var express = require('express');
var fs = require('fs');
var mysql = require('mysql');
var request = require('request');
var cheerio = require('cheerio');
var bodyParser = require('body-parser');
var app = express();
var output;
app.use(bodyParser.json())
app.get('/scrape', function(req, res){
url = 'https://raitalumni.dypatil.edu/events/?tag=live';
request(url, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
var json = {
title : [],
date : [],
month : [],
venue : [],
link : []
};
output = {
events : []
};
$('p.event_name').each(function(){
json.title.push($(this).text());
});
$('p.calendar_date').each(function(){
json.date.push($(this).text());
});
$('span.calendar_month').each(function(){
json.month.push($(this).text());
});
//var fulldate = $('p.calendar_date').concat($('p.calendar_day')).text();
//console.log('all records: ' + fulldate);
$('p.event_venue').each(function(){
json.venue.push($(this).text());
});
// var title = $('p.event_name').each(function(){$(this).text()});
for(var i=0; i<json.title.length; i++){
output.events[i] = {
title : json.title[i],
date : json.date[i],
month : json.month[i],
venue : json.venue[i],
link : url
}
}
var connection = mysql.createConnection({
host: '127.0.0.1',
port: 3306,
user: 'root',
password: '',
database: 'raithub'
});
connection.connect(function(error){
if(!!error){
console.log('Error');
}else{
console.log('Connected to the database!');
}
});
var scrape = JSON.stringify(output, null, 4);
console.log(scrape);
var query = connection.query("INSERT INTO scrapped ('title','date','month','venue','link') VALUES ('" + output.title + "', '" + output.date + "', '" + output.month + "', '" + output.venue + "', '" + output.link + "');", scrape, function(err, result) {
if(err) throw err;
console.log('data inserted');
});
fs.writeFile('output4.json', JSON.stringify(output, null, 4), function(err){
console.log('File successfully written! - Check your project directory for the output.json file');
})
res.send('Check your console!')
}
else {
console.log("Network Error, please try again later")
}
})
})
app.listen('8000')
console.log('Server running on port 8081');
exports = module.exports = app;
Where am I going wrong?
Getting this error,
You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ''title','date','month','venue','li
nk') VALUES ('undefined', 'undefined', 'undefi' at line 1 error

The INSERT statement should look this:
INSERT INTO table_name (column1, column2, column3, ...)
VALUES (value1, value2, value3, ...);
where the column names should not be in quotes. Your specific statement would like this:
var query = connection.query("INSERT INTO scrapped (title,date,month,venue,link) VALUES ('" + output.title + "', '" + output.date + "', '" + output.month + "', '" + output.venue + "', '" + output.link + "');", scrape, function(err, result) {
if(err) throw err;
console.log('data inserted');
});
See this for the right syntax

There is a JSON datatype in MySQL.
mysql> CREATE TABLE t1 (jdoc JSON);
Query OK, 0 rows affected (0.20 sec)
mysql> INSERT INTO t1 VALUES('{"key1": "value1", "key2": "value2"}');
Query OK, 1 row affected (0.01 sec)
This would be especially helpful if you had a very large json object and didn't want to create a huge INSERT statement. It would also be useful if you are unsure of the exact data that will be coming in via json but you want to capture all of it. I'm currently working on a project where we occasionally add and remove items from the user input fields and this is very helpful so that I'm not constantly having to ALTER tables and edit other statements in MySQL.
More information can be found here: https://dev.mysql.com/doc/refman/5.7/en/json.html

const queryString = `insert into table_name(meta) values ('${JSON.stringify(meta)}');`;

Refreshing contents of an Express NodeJS API

I have written an API that queries a MySQL database and outputs the corresponding results visiting an url. This is the code:
//server.js
var express = require('express'),
mysql = require('mysql'),
app = express(),
connectionpool = mysql.createPool({
host : 'localhost',
user : 'root',
password : 'password',
database : 'database'
});
app.get('/:transcript', function(req,res){
var var1 = req.param('transcript');
exports.var1 = var1;
var queries = require('./queries'),
query1 = queries.query1;
//Connection to MySQL
connectionpool.getConnection(function(err, connection) {
if (err) {res.send({result: 'error connection'})}
connection.query(query1, function(err, rows) {
if (err) {res.send({result: 'error query1'})};
counter = 0; root = {};
rows.forEach(function (row) {
build_actor(row.Transcript_alias, function(exprobject1) {
counter += 1;
//Defining and filling objects
main = {};
main.Official_transcript_name = row.Transcript_name;
main.Expression = exprobject1;
root[row.Transcript_alias] = main;
if (counter == rows.length) {
res.write(JSON.stringify(root, null, '\t'));
res.end();
}
});
});
connection.release();
});
//CallBack
function build_actor(transcript, callback) {
//Other, secondary queries:
var query2 = 'SELECT * FROM expression WHERE transcript_alias = "' + transcript + '";',
connection.query(query2, function(err, rows1) {
if (err) {res.send({result: 'error query2'})}
var exprobject2 = {},
exprobject1 = {};
for (i = 0; i < rows1.length; i++) {
Conditions = rows1[i].conditions;
Tissue = rows1[i].tissue;
FPKM = rows1[i].FPKM;
exprobject2[Tissue] = FPKM;
if (Conditions in exprobject1) {
exprobject1[Conditions].push(exprobject2);
} else {
exprobject1[Conditions] = [];
exprobject1[Conditions].push(exprobject2);
}
}
callback(exprobject1);
});
}
});
});
app.listen(3000);
console.log('Listening on port 3000');
This script calls a required file where there are my queries:
//queries.js
var server = require('./server'),
query1 = 'SELECT distinct(transcript_alias)\
FROM transcript_features \
WHERE f.transcript_alias = "' + var1 + '";';
exports.query1 = query1;
I go to the contents of this script this way:
http://localhost:3000/AC149829.2_FGT004
http://localhost:3000/AC148152.3_FGT007
When I first visit http://localhost:3000/AC149829.2_FGT004, the API shows the correct results for the variable AC149829.2_FGT004. However, when changing the variable to AC148152.3_FGT007, it continues showing the information for the variable AC149829.2_FGT004. In order to see the results for AC148152.3_FGT007, I must kill the script, call it again, and visit for the first time http://localhost:3000/AC148152.3_FGT007. In conclusion, results are not refreshed.
How is that? I tried with a simple:
app.get('/:transcript', function(req,res){
var input = req.param('transcript');
res.send(input);
});
but it works well...
EDIT. I found the source of my problem. query1 is always the same. The script only calls once:
exports.var1 = var1;
var queries = require('./queries'),
query1 = queries.query1;
There's a way to overcome this limitation?

I found the solution for my problem. As
//server.js
exports.var1 = var1;
var queries = require('./queries'),
query1 = queries.query1;
is executed once and remains in the cache, I changed my code without exporting var1:
//server.js
var queries = require('./queries'),
query1 = queries.query1 + var1;
and
//queries.js
var server = require('./server'),
query1 = 'SELECT distinct(transcript_alias)\
FROM transcript_features \
WHERE f.transcript_alias = ';
exports.query1 = query1;
In other words, I import my query to server.js without any variable. The variable is assigned at server.js.

Using Node.js to retrieve data from Redis through an AJAX request

I'm going through a Node, Express, & Socket.io chat tutorial. I decided to use Redis to store the chat history and have successfully set it up so that my information is correctly posting to the database. I am now trying to access that information to use on the client-side (in this case I'm trying to access the list of users currently in the chat so I can show them to the side of the chat). I am using $.getJSON to make a GET request. Right now I have it setup so that the file it tries to access only has this JSON object : {"dog" : "2","cat":"3"} just to test it, and that is working, but I'm not sure where to go from there because anytime I try adding a function into that file, even if I specify to return a JSON object and call that function, the request stops returning the correct information.
For example I tried :
var data = function(){
return {"dog" : "2","cat":"3"}
}
data();
and that doesn't return anything ( I understand that when I make a GET request the function isn't run, but it doesn't even return that text, and if it doesn't run a function than I'm not sure how I can access redis from this file)
Here's what I'm thinking:
var redis = require('redis')
//figure out how to access the redis client that I have at localhost:6379, something like var db = redis.X
//and then call (for example) db.smembers('onlineUsers') and be returned the object which I can iterate through
Here's my relevant code:
server.js:
var jade = require('jade');
var PORT = 8080;
var redis = require('redis');
var db = redis.createClient();
var pub = redis.createClient();
var sub = redis.createClient();
var http = require('http');
var express = require('express');
var app = express();
var server = http.createServer(app);
var io = require('socket.io').listen(server);
server.listen(PORT, function(){
console.log("Now connected on localhost:" + PORT)
});
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
app.set("view options", {layout: false});
app.use(express.static(__dirname + '/public'));
app.get('/', function(req, res){
res.render('home');
});
io.sockets.on('connection', function(client){
sub.subscribe("chatting");
sub.on("message", function (channel, message) {
console.log("message received on server from publish");
client.send(message);
});
client.on("sendMessage", function(msg) {
pub.publish("chatting",msg);
});
client.on("setUsername", function(user){
pub.publish("chatting","A new user in connected:" + user);
db.sadd("onlineUsers",user);
}
);
client.on('disconnect', function () {
sub.quit();
pub.publish("chatting","User is disconnected :" + client.id);
});
});
script.js:
$(document).ready( function(){
$client = io.connect();
initialize();
});
var setUsername = function(){
var username = $("#usernameInput").val();
if (username)
{
var user = username;
$client.emit('setUsername', username);
$('#chatControls').show();
$('#usernameInput').hide();
$('#usernameSet').hide();
showCurrentUsers();
}
}
var showCurrentUsers = function(){
$('#list_of_users').empty();
$.getJSON('getusers.js', function(data){
for (var i = 0; i < data.length; i++){
$('list_of_users').append("<li>"+data[i]+"</li>")
}
})
}
var sendMessage = function(){
var msg = $('#messageInput').val();
var username = $("#usernameInput").val();
if (msg)
{
var data = {msg: msg, user: username}
$client.emit('message', data);
addMessage(data);
$('#messageInput').val('');
// populate(username,msg);
}
}
var addMessage = function(data) {
$("#chatEntries").append('<div class="message"><p>' + data.user + ' : ' + data.msg + '</p></div>');
}
// var populate = function(username,msg) {
// var data ;
// }
var initialize = function(){
$("#chatControls").hide();
$("#usernameSet").on('click', setUsername);
$("#submit").on('click',sendMessage);
showCurrentUsers();
}
and right now all that the getusers.js file has in it is:
{"dog" : "2","cat":"3"}

It looks like you're expecting your call to $.getJSON to load and execute the javascript it loads. It doesn't work this way. You need to make a node endpoint (via a route) which renders the JSON. The node endpoint would then do the data manipulation / querying redis:
Node:
In routes.js:
app.get('/chatdata', ChatController.getChatData);
In ChatController.js (manipulate, create the data as you like here)
exports.getChatData = function (req, res) {
var data = function(){
return {"dog" : "2","cat":"3"}
};
res.JSON(data);
};
Front-end
$.getJSON('getChatData', function(data){
//...
})

I think you need to setup a route to handle the GET request that $.getJSON makes, or if getusers.js is in the /public directory, then you need to modify your $.getJSON call as follows:
$.getJSON('http://localhost:8080/public/getusers.js', function(data){

Ok, it looks like it is a problem with your getusers.js file. $.getJSON seems to prefer double quotes. Try formatting it like this:
{
"dog" : "2",
"cat" : "3"
}
Also, try using this to display the data:
$.getJSON('getusers.js', function(data){
var items = [];
$.each( data, function( key, val ) {
items.push("<li id='" + key + "'>" + val +"</li>");
});
$('#list_of_users').append(items.join(""));
});

How to execute the function return the value to called function across node js files

I have 3 node js files :
mysqlconnection.js to store the database connection properties:
var mysql = require('mysql');
var cjson = require('cjson');
var yaml_config = require('node-yaml-config');
// project files
var config = yaml_config.load(__dirname + '/billingv2.yaml');
exports.execute = function(callback){
var connection = mysql.createConnection(
{
host : config.host,
user : config.user,
password : config.password,
database : config.database,
}
);
connection.connect();
return callback(null,connection);
}
subscriptionRestService.js to handle the REST api calls:
var express = require('express');
var app = express();
app.use(express.bodyParser());
var fs = require('fs');
// Project files
var mysql = require('./mysqlRestService.js');
// Get Resource Subscription data by Resourceuri
app.post('/pricingdetails', function(req, res) {
var workload = req.body;
if(workload.elements && workload.elements.length > 0)
{
var arr = [];
for(var index in workload.elements)
{
arr[index] = workload.elements[index].uri;
}
var resourceIdentifiers = arr.join(',');
}
console.log(resourceIdentifiers);
mysql.getPricingDetail(function(resourceIdentifiers,callback){
});
});
mysqlRestService.js to handle mysql queries/stored procedures:
// packages
var mysql = require('mysql');
var cjson = require('cjson');
var fs = require('fs');
var yaml_config = require('node-yaml-config');
// project files
var dbconnection = require('./mysqlconnection');
exports.getPricingDetail = function (resourceIdentifiers,callback){
console.log('entered into mysql function');
console.log(resourceIdentifiers);
var pricingDetail = {};
dbconnection.execute(function(err,response){
if(err){
throw err;
}
else
{
var selectqueryString = "call SP_ExposePricingDetailforUI('" + resourceIdentifiers + "')";
response.query(selectqueryString, function(err,pricingDetail){
if(err) {
throw err;
}
else
{
console.log(pricingDetail);
pricingDetail = pricingDetail;
}
});
}
});
//console.log('printing pricing details');
//console.log(pricingDetail);
};
problems faced
Unable to send the variable resourceIdentifiers from subscriptionRestService to mysqlRestService.js
Unable to return the pricingdetail from mysqlRestService.js to calling function in subscriptionRestService.
Any guidance greatly appreciated.

Unable to send the variable resourceIdentifiers from subscriptionRestService to mysqlRestService.js
Well, you didn't send it. It currently is a parameter of your callback function in the invocation, not an argument for the parameter of getPricingDetails. Use
mysql.getPricingDetail(resourceIdentifiers, function callback(result){
// use result here
});
Unable to return the pricingdetail from mysqlRestService.js to calling function in subscriptionRestService.
I've got no idea what pricingDetail = pricingDetail; was supposed to do. You have to call (invoke) back the callback here! Use
callback(pricingDetail);

How can I set req.session from this scope?

I'm writing a Node.js application using Express and a PostgreSQL database using node-postgres. I want to look up the current user's username and real name based on their email, and set them in req.session. However, if I set them where I am in the code below, they are undefined when we leave that block (i.e. the first console.log statements print the correct info, the second set prints undefined. How can I solve this?
var client = new pg.Client(app.conString);
var realname = "";
var username = "";
client.connect();
var query = client.query(
"SELECT * FROM users WHERE email = $1;",
[req.session.email]
);
query.on('row', function(row) {
req.session.realname = row.realname;
req.session.username = row.username;
console.log(req.session.realname);
console.log(req.session.username);
});
console.log(req.session.realname);
console.log(req.session.username);
query.on('end', function() {
client.end();
});

The second pair of console.log will execute before the query-results are available (in the row event handler).
If your code is going to be used in an Express route, you would use something like this:
app.get('/', function(req, res) {
var client = new pg.Client(app.conString);
var realname = "";
var username = "";
client.connect();
var query = client.query(
"SELECT * FROM users WHERE email = $1;",
[req.session.email]
);
query.on('row', function(row) {
req.session.realname = row.realname;
req.session.username = row.username;
});
query.on('end', function() {
client.end();
res.send(...); // <-- end the request by sending back a response
});
});
An alternative for using the EventEmitter interface for node-postgres would be to just pass a callback to query (which looks better with Express IMHO):
client.query(
"SELECT * FROM users WHERE email = $1;",
[req.session.email],
function(err, results) {
if (err)
// handle error
else
if (results.length)
{
req.session.realname = results[0].realname;
req.session.username = results[0].username;
}
res.send(...); // done
});