I want to make sure that my application will only call res.render('pages/students' ...) after all my queries finishes and I get all the necessary data. However, I completely have no clue how to do this at all and I am having trouble applying all the examples I see online onto my code. Can someone give me a hand? (There are two con.query(...) in my route. I want to only render('pages/students'...) after both of these con.query completely finishes. Therefore, I am using async to run all my queries to completion. However now for some reason my page wont event load.
app.get('/admin', function(req, res) {
var sql =
'SELECT events.id, name, description, MONTHNAME(start_time) AS month, DAY(start_time) AS day, ' +
"YEAR(start_time) AS year, DATE_FORMAT(start_time, '%h:%i%p') AS start_time, HOUR(start_time) AS start_hour, MINUTE(start_time) AS start_minute, " +
"DATE_FORMAT(end_time, '%h:%i%p') AS end_time, HOUR(end_time) AS end_hour, MINUTE(end_time) AS end_minute, location, max_capacity, hidden_sign_up, " +
'eventleaders.first_name AS eventLeader FROM events LEFT JOIN eventleaders_has_events ON events.id = eventleaders_has_events.events_id ' +
'LEFT JOIN eventleaders ON eventleaders_has_events.eventleaders_id = eventleaders.id;';
var events = {};
con.query(sql, function(err, results) {
if (err) throw err;
async.forEachOf(results, function(result, key, callback) {
var date = result.month + ' ' + result.day + ', ' + result.year;
var other = 'N/A';
if (result.other !== null) {
other = result.other;
}
if (typeof events[date] === 'undefined') {
events[date] = {};
}
var studentAttendees = {};
var sql =
'SELECT * FROM students INNER JOIN students_has_events ON students.id = students_has_events.Students_id ' +
'WHERE students_has_events.Events_id = ?';
var values = [result.id];
con.query(sql, values, function(err, results1) {
if (err) throw err;
async.forEachOf(results1, function(result1, key, callback) {
studentAttendees[result1.first_name + ' ' + result1.last_name] = {
netID: result1.netID,
phoneNumber: result1.phone,
email: result1.email
};
});
//still need to get the event leader attendees
events[date][result.name] = {
startTime: result.start_time,
location: result.location,
endTime: result.end_time,
description: result.description,
eventLeader: result.eventLeader,
numberRegistered: result.hidden_sign_up,
maxCapacity: result.max_capacity,
poster: '/images/sample.jpg',
other: other,
attendees: studentAttendees
};
});
});
});
});
Two options:
app.get('/admin', function(req, res) {
...
con.query(sql, function(err, result) {
...
con.query(sql, values, function(err, result) {
events[date][currRecord.name] = {
...
};
// put it here
console.log(events);
res.render('pages/admin', {
events: events
});
});
}
});
});
Or use the Promise version of mysql
const mysql = require('mysql2/promise');
// make this async
app.get('/admin', async function(req, res) {
try {
...
// might want to move this elsewhere inside async function
const con = await mysql.createConnection({host:'localhost', user: 'root', database: 'test'});
const result1 = await con.query(sql);
// run your logic preferably inside map
// result2 is an array of Promises
const result2 = result1.map((result, index) => {
var currRecord = result[index];
var date =
currRecord.month + ' ' + currRecord.day + ', ' + currRecord.year;
var other = 'N/A';
if (currRecord.other !== null) {
console.log('here');
other = currRecord.other;
}
if (typeof events[date] === 'undefined') {
events[date] = {};
}
//get all the student attendees of this event
var studentAttendees = {};
var sql =
'SELECT * FROM students INNER JOIN students_has_events ON students.id = students_has_events.Students_id ' +
'WHERE students_has_events.Events_id = ?';
var values = [currRecord.id];
return con.query(sql)
})
// result3 is an array of results. If your results is also an array, the structure might look lik [[student], [student]] etc.
const result3 = await Promise.all(result2);
// run your logic to get events
console.log(events);
res.render('pages/admin', {
events: events
});
} catch (e) {
// handle error
console.error(e)
}
});
Related
First of all, i'm new in JS. I have a function that possibly can use multiple requests to get the final data. How can i do this in the right way? In this example participants won't pushed to the dialogs array because it's in async call.
function getDialogs(token, callback) {
//getting user id
con.query("SELECT user_id FROM users_tokens WHERE user_token = '" + token + "'", function(error, results) {
if (error) {
throw error;
}
var userId = results[0].user_id;
//getting all conversation
con.query("SELECT cc.id as conversation_id, cc.type FROM chat_conversations cc INNER JOIN chat_participants cp ON cc.id = cp.conversation_id WHERE cp.user_id = " + userId + " GROUP BY cc.id", function (error, results) {
if (error) {
throw error;
}
var dialogs = [];
for (let i = 0; i < results.length; i++) {
var dialog = {id: results[i].conversation_id};
//getting chat participants
con.query("SELECT user_id FROM chat_participants WHERE conversation_id = " + results[i].conversation_id + " AND user_id != " + userId, function (error, results) {
var participants = [];
for (let j = 0; j< results.length; j++) {
participants.push(results[j].user_id);
}
dialogs[participants] = participants;
});
dialogs.push(dialog);
}
callback(dialogs);
});
});
}
Technically you can use a single request like this
SELECT user_id FROM chat_participants WHERE conversation_id IN (
SELECT
cc.id as conversation_id,
cc.type
FROM
chat_conversations cc
INNER JOIN chat_participants cp ON cc.id = cp.conversation_id
WHERE
cp.user_id IN (
SELECT
user_id
FROM
users_tokens
WHERE
user_token = "TOKEN"
)
GROUP BY
cc.id
)
but there are a few problems with this approach as well.
First of all, it seems like you are only using the user_id of your first row, so please use LIMIT 1 in such cases.
Second of all, it seems like user_id won't ever have a duplicate, so make it a primary key.
Third of all, don't concat your token, node mysql supports having placeholders using ? in your query, like this:
con.query("SELECT * FROM ? WHERE user_id = ?", ["table_name", "userid"])
Fourth of all, promisify your requests so you don't have a callback hell, do something like:
function promiseRequest(query, placeholders) {
return new Promise((res, rej) => {
con.query(query, placeholders, (err, data) => {
if (err) rej(err);
res(data);
});
});
}
I'm currently trying to update all user points every 1 minute (currently at 5 seconds for testing purposes), when I try to run it, it gets all the users, but then it sets their points to undefined.
setInterval(async function () {
var uPoints;
await db.each("SELECT points points, id id FROM users", function (err, row) {
if (err) {
console.log(err);
}
var u;
for (u in client.users.array()) {
uPoints = row.points + 10;
}
});
var u, user;
for (u in client.users.array()) {
user = client.users.array()[u];
tools.setPoints(uPoints, user.id.toString());
console.log('Updated ' + user.id.toString() + ' to ' + uPoints);
}
}, 5000);
});
tools.setPoints
module.exports.setPoints = function (amnt, id) {
db.run('UPDATE users SET points = ? WHERE id = ?', amnt, id);
}
The update is executed before or in parallel with the SELECT.
You can fix that if you move the UPDATE inside the callback like so:
setInterval(async function () {
var uPoints;
await db.each("SELECT points points, id id FROM users", function (err, row) {
if (err) {
console.log(err);
}
var u;
for (u in client.users.array()) {
uPoints = row.points + 10;
}
var user;
for (u in client.users.array()) {
user = client.users.array()[u];
tools.setPoints(uPoints, user.id.toString());
console.log('Updated ' + user.id.toString() + ' to ' + uPoints);
}
});
}, 5000);
});
I'm not sure what DB client you are using but generally, you don't want to mix await with callbacks.
You either want to have:
const rows = await db.each("SELECT points points, id id FROM users");
or
db.each("SELECT points points, id id FROM users", function (err, rows) {..});
You should check if your db client supports async/await or you can use promisify.
Here is the line (50) where this is happening:
var meetingId = meeting._id.toString(),
And here is the full, relevant code:
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var ObjectId = require('mongodb').ObjectID;
var config = require('./config'),
xlsx = require('./xlsx'),
utils = require('./utils'),
_ = require('lodash'),
url = config.DB_URL;
var meetings = [];
function findNumberOfNotesByMeeting(db, meeting, callback) {
var meetingId = meeting._id.toString(),
meetingName = meeting.name.displayValue,
attendees = meeting.attendees;
host = meeting.host;
var count = 1, pending = 0, accepted = 0;
console.log("==== Meeting: " + meetingName + '====');
_.each(attendees, function(item) {
console.log(count++ + ': ' + item.email + ' (' + item.invitationStatus + ')');
if (item.invitationStatus == 'pending') { pending++; }
else if (item.invitationStatus == 'accepted') { accepted++; }
});
console.log("*** " + attendees.length + ", " + pending + "," + accepted);
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, doc) {
var emails = [];
if (doc.emails) {
doc.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
var email = emails.join(', ');
if (utils.toSkipEmail(email)) {
callback();
} else {
db.collection('notes').find({ 'meetingId': meetingId }).count(function(err, count) {
if (count != 0) {
console.log(meetingName + ': ' + count + ',' + attendees.length + ' (' + email + ')');
meetings.push([ meetingName, count, email, attendees.length, pending, accepted ]);
}
callback();
});
}
});
}
function findMeetings(db, meeting, callback) {
var meetingId = meeting._id.toString(),
host = meeting.host;
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, doc) {
var emails = [];
if (!err && doc && doc.emails) {
doc.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
var email = emails.join(', ');
if (utils.toSkipEmail(email)) {
callback();
} else {
db.collection('notes').find({ 'meetingId': meetingId }).count(function(err, count) {
if (count != 0) {
var cursor = db.collection('meetings').find({
'email': {'$regex': 'agu', '$options': 'i' }
});
}
callback();
});
}
cursor.count(function(err, count) {
console.log('count: ' + count);
var cnt = 0;
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
findNumberOfNotesByMeeting(db, doc, function() {
cnt++;
if (cnt >= count) { callback(); }
});
}
});
});
});
}
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
findMeetings(db, function() {
var newMeetings = meetings.sort(function(m1, m2) { return m2[1] - m1[1]; });
newMeetings.splice(0, 0, [ 'Meeting Name', 'Number of Notes', 'Emails' ]);
xlsx.writeXLSX(newMeetings, config.xlsxFileNameMeetings);
db.close();
});
});
As you can see, the meeting variable (which I am almost 100% sure is the problem, not the _id property) is passed in just fine as a parameter to the earlier function findNumberOfNotesByMeeting. I have found some information here on SO about the fact that my new function may be asynchronous and needs a callback, but I've attempted to do this and am not sure how to get it to work, or even if this is the right fix for my code.
You're not passing the meeting object to findMeetings, which is expecting it as a second parameter. Instead of getting the meeting object, the function receives the callback function in its place, so trying to do meeting._id is undefined.
In fact, what is the purpose of the findMeetings function? It's name indicates it can either find all meetings in the database, or all meetings with a specific id. You're calling it without a meeting id indicating you might be trying to find all meetings, but its implementation takes a meeting object. You need to clear that up first.
I'm trying to switch from single mysql-queries to mysql-pool connection, so users can share one mysql-connection, but I'm not familiar with this at all (also new to nodejs/socket.io).
The following code is what I've done so far to send data every second to the socket in an array:
var
port = process.env.OPENSHIFT_NODEJS_PORT || 8000,
ip = process.env.OPENSHIFT_NODEJS_IP || '127.0.0.1',
app = require('http').createServer(handler),
fs = require('fs'),
request = require('request'),
mysql = require('mysql'),
moment = require('moment'),
tz = require('moment-timezone'),
pool = mysql.createPool({
connectionLimit: 100,
host: 'xxx',
user: 'xxx',
password: 'xxx',
database: 'xxx',
debug: false,
port: 3306}),
socketArray = [],
POLLING_INTERVAL = 1000,
pollingTimer;
moment.tz.setDefault("Europe/Berlin");
var io = require('socket.io').listen(app);
io.set('origins', '*:*');
function time()
{
output = new Date();
output = moment().format('(H:mm:ss.SS) ');
return output;
}
function handler(req,res)
{
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
res.statusCode = 200;
res.connection.setTimeout(0);
res.end();
}
app.listen(port,ip);
function pollingLoop () {
if (socketArray.length === 0) {
// no connections, wait and try again
setTimeout(pollingLoop, POLLING_INTERVAL);
return; // continue without sending mysql query
}
pool.getConnection(function(err,connection){
if (err) { console.log({"code" : 100, "status" : "connection-db error"}); return; }
console.log('connected as id ' + connection.threadId);
console.log('socketArray length: ' + socketArray.length);
var selection =
"SELECT\
a.`id`,a.`product_id` AS pid,a.`random` AS nr,a.`price`,a.`price_end` AS pe,\
TIMESTAMPDIFF(SECOND,NOW(),a.`datetime`) AS duration,\
ABS(TIMESTAMPDIFF(SECOND,NOW(),b.`date`)) AS hb\
FROM `auctions` AS a\
LEFT JOIN `auctions_bids` AS b ON b.`auction_id` = a.`id`\
WHERE TIMESTAMPDIFF(SECOND,NOW(),a.`datetime`) > '-1'\
GROUP BY a.`id`\
ORDER BY `duration` DESC,`id` DESC LIMIT 15";
var streamArray = [], lg = '';
var query = connection.query(selection, function(err, results, rows){
lg += ('id: '+results[0].id+' ('+results[0].duration+') ');
if
(
((results[0].duration < 2 || results[0].duration <= results[0].nr) && (results[0].price <= results[0].pe))
||
((results[0].duration < 2 || results[0].duration <= results[0].nr) && (results[0].hb > 0 && results[0].hb < 30))
)
{
min = 3;
max = 5;
rand = Math.floor(Math.random()*(max-min+1)+min);
price = results[0].price+0.01;
price = price.toFixed(2);
pool.query('UPDATE `auctions` SET `random` = ?,`price` = ?, `datetime` = DATE_ADD(`datetime`,INTERVAL(17-TIMESTAMPDIFF(SECOND,NOW(),`datetime`))SECOND) WHERE `id` = ?',[rand, price, results[0].id]);
console.log(time()+'UPDATED id '+results[0].id+': random ('+rand+') price ('+price+'€)');
}
streamArray.push(results[0]);
updateSockets({ streamArray: streamArray });
console.log("auctions pushed: " + streamArray);
connection.release();
setTimeout(pollingLoop, POLLING_INTERVAL);
});
console.log(time()+lg+' C: '+socketArray.length);
});
}
pollingLoop();
io.sockets.on('connection', function(socket) {
socket.on('disconnect', function() {
clearTimeout(pollingTimer);
var socketIndex = socketArray.indexOf(socket);
console.log(time()+'SOCKET-ID = %s DISCONNECTED', socketIndex);
if (~socketIndex) { socketArray.splice(socketIndex, 1); }
});
console.log(time()+'NEW SOCKET CONNECTED!');
socketArray.push(socket);
});
var updateSockets = function(data) {
socketArray.forEach(function(tmpSocket) { tmpSocket.volatile.emit('stream', data); });
};
console.log(time()+'server.js executed\n');
But this doesn't send me any data to the WebSocket. Is this approach (code-structure) even correct? Previously I used query.on('results') to get data like this:
var selection = "SELECT * FROM auctions";
var query = mysql.query(selection), auctions = [];
query.on('result', function(auction) {
console.log('id: '+auction.id+' ('+auction.duration+') ');
});
This worked fine showing data with auction.row but how to do this in my mysql pool connection?
Also after some seconds I'm getting an error that release() isn't even defined, but it's listed in the mysql-module documentation... so I think my whole logical process is somehow incorrect.
Should I use connection.end() and .release() at all? Because the
connection should never end.
Should I still use setInterval(function () { mysql.query('SELECT
1'); }, 5000); as answered in another StackOverflow question to keep
the connection alive here? (nodejs mysql Error: Connection lost The server closed the connection)
(Appreciate any tips or answers to even some of my questions! Better some answers than none, because I experienced that this topic isn't answered much at all.)
EDIT:
Updated my whole code (see above). Output looks like this now: http://s21.postimg.org/avsxa87rb/output.jpg
So the stream gets the data, but in the console.log is nothing and there's this javascript error?
You should be creating a pool, and using getConnection on that pool. Then, when you're done with the connection, release it. Additionally, you do not need to stop the pollingLoop or start it for each connection, one loop is enough.
I didn't understand the if statement with conditions, so i omitted it. It likely needs to go somewhere else.
var socketArr = [];
function handler(req, res) {
res.statusCode = 200;
res.connection.setTimeout(0);
res.end();
}
app.listen(port, ip);
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
function pollingLoop () {
if (socketArr.length === 0) {
// no connections, wait and try again
setTimeout(pollingLoop, 1000);
return; // continue without sending mysql query
}
pool.getConnection(function (err, connection) {
if (err) {
console.log({
"code": 100,
"status": "Error in connection database"
});
return;
}
console.log('connected as id ' + connection.threadId);
var selection = "SELECT * FROM auctions";
var streamArray = [],
lg = '';
var query = connection.query(selection, function (err, results, fields, rows) {
lg += ('id: ' + results[0].id + ' (' + results[0].duration + ') ');
/*if (conditions) {
var query_update = connection.query('UPDATE `auctions` SET `price` = ? WHERE `id` = ?', [price, auction.id]);
console.log(time() + 'UPDATED id ' + auction.id + ': price (' + price + '€)');
}*/
streamArray.push(results);
updateSockets({
streamArray: streamArray
});
console.log("auctions pushed: " + streamArray);
connection.release();
setTimeout(pollingLoop, 1000);
});
console.log(time() + lg + ' C: ' + socketArr.length);
});
}
// start loop
pollingLoop();
io.sockets.on('connection', function (socket) {
socket.on('disconnect', function () {
var socketIndex = socketArr.indexOf(socket);
console.log(time() + 'SOCKET-ID = %s DISCONNECTED', socketIndex);
if (~socketIndex) {
socketArr.splice(socketIndex, 1);
}
});
console.log(time() + 'NEW SOCKET CONNECTED!');
socketArr.push(socket);
});
var updateSockets = function (data) {
socketArr.forEach(function (tmpSocket) {
tmpSocket.volatile.emit('stream', data);
});
};
I used the closures in this loop. But its only print the right data on the console log, and the sql query did not work. The inserted data on MySQL is the last variable of the loop.
I thought this is because of writing speed of MySQL. But don't know how to fix it. Any idea?
Thanks
module.exports = function (callback) {
queryGetForSend = "SELECT * FROM image WHERE send_request is NULL AND post_request is NOT NULL AND year(moderate_time) = year(curdate()) AND month(moderate_time) = month(curdate()) AND (time(moderate_time) < (curtime() - 15));";
conn.query(queryGetForSend, function(err, rows, fields){
for (i in rows) {
if (rows[i].post_request == 'approve') {
resultSend = 1
} else {
resultSend = 2
}
var fileID = rows[i].img_md5;
queryString = fileID + "=" + resultSend;
// Request url: "http://im-api1.webpurify.com/image_queue/results/?key="
var d = new Date();
Date.masks.default = 'YYYY-MM-DD hh:mm:ss';
sendTime = d.format();
(function(queryString, sendTime) {
querySent = "UPDATE image SET send_request=1,result_sent='"+queryString+"',send_time='"+sendTime+"' WHERE send_request is NULL AND post_request is NOT NULL AND year(moderate_time) = year(curdate()) AND month(moderate_time) = month(curdate()) AND (time(moderate_time) < (curtime() - 15));";
conn.query(querySent, function (err, rows, fields) {
if (err) throw err;
console.log("http://google.com?key=" + key + "&" + queryString);
});
})(queryString, sendTime);
(function(queryString){
request.get("http://google.com" + key + "&" + queryString, function(err, res, body) {
});
})(queryString);
}
// callback(rows);
});
};
Two suggestions:
Avoid concatenation in SQL queries, use placeholders or prepared statements instead (if you care about security).
Use array.forEach() instead of a regular for-loop with a closure to avoid accidental use of variables set inside the for-loop:
conn.query(queryGetForSend, function(err, rows, fields) {
if (err) throw err;
rows.forEach(function(row) {
var resultSend;
if (row.post_request == 'approve') {
resultSend = 1
} else {
resultSend = 2
}
var fileID = row.img_md5;
var queryString = fileID + '=' + resultSend;
var d = new Date();
Date.masks.default = 'YYYY-MM-DD hh:mm:ss';
var sendTime = d.format();
var querySent = 'UPDATE image SET send_request=1, result_sent=?, send_time=? WHERE send_request IS NULL AND post_request IS NOT NULL AND year(moderate_time) = year(curdate()) AND month(moderate_time) = month(curdate()) AND (time(moderate_time) < (curtime() - 15));';
conn.query(querySent, [queryString, sendTime], function (err) {
if (err) throw err;
console.log('http://google.com?key=' + key + '&' + queryString);
});
request.get('http://google.com?key=' + key + '&' + queryString, function(err, res, body) {
});
});
});