I wrote a ~50 lines script to perform housekeeping on MySQL databases. I'm afraid my code exhibits anti-patterns as it rapidly escalates to an unreadable mess for the simple functions it performs.
I'd like some opinions for improving readability.
The full script is at the bottom of this post to give an idea.
Spotlight on the problem
The excessive nesting is caused by patterns like this repeated over and over: (snippet taken from script)
sql.query("show databases")
.then(function(rows) {
for (var r of rows) {
var db = r.Database;
(function(db) {
sql.query("show tables in " + db)
.then(function(rows) {
// [...]
}
})(db);
}
});
I'm nesting one promise under the other within both a for loop and a closure. The loop is needed to iterate across all results from sql.query(), and the closure is necessary to pass the value of db to the lower promise; without the closure, the loop would complete even before the nested promise executes at all, so db would always contain only the last element of the loop, preventing the nested promise from reading each value of db.
Full script
var mysql = require("promise-mysql");
var validator = require("mysql-validator"); // simple library to validate against mysql data types
var ignoreDbs = [ "information_schema" ],
multiplier = 2, // numeric records multiplier to check out-of-range proximity
exitStatus = {'ok': 0, 'nearOutOfRange': 1, 'systemError': 2};
(function() {
var sql,
mysqlHost = "localhost",
mysqlUser = "user",
mysqlPass = "";
mysql.createConnection({
host: mysqlHost,
user: mysqlUser,
password: mysqlPass
}).then(function(connection) {
sql = connection;
})
.then(function() {
sql.query("show databases")
.then(function(rows) {
for (var r of rows) {
var db = r.Database;
if (ignoreDbs.indexOf(db) != -1) continue;
(function(db) {
sql.query("show tables in " + db)
.then(function(rows) {
for (var r of rows) {
var table = r["Tables_in_" + db];
(function(table) {
sql.query("describe " + db + "." + table)
.then(function(rows) {
for (var r of rows) {
(function(r) {
var field = r.Field,
type = r.Type, // eg: decimal(10,2)
query = "select " + field + " from " + db + "." + table + " ";
if (table != "nonce") query += "order by date desc limit 1000";
sql.query(query)
.then(function(rows) {
for (var r of rows) {
var record, err;
// remove decimal part, only integer range is checked
record = Math.trunc(r[field]);
err = validator.check(record * multiplier, type);
if (err) {
console.log(err.message);
process.exit(exitStatus.nearOutOfRange);
}
}
});
})(r);
}
});
})(table);
}
});
})(db);
}
});
})
.then(function() {
// if (sql != null) sql.end(); // may not exit process here: sql connection terminates before async functions above
//process.exit(exitStatus.ok); //
});
})();
Trivia
The purpose of the script is to automatically and periodically monitor if any record stored in any row, table and database in MySQL is approaching the out-of-range limit for its specific data type. Several other processes connected to MySQL continuously insert new numeric data with increasing values and nonces; this script is a central point where to check for such numeric limits. The script would then be attached to Munin for continuous monitoring and alerting.
Update: Revised script
As suggested by #Kqcef I modularized the anonymous functions out of the promise nest, and used let to avoid the explicit nesting of an additional function to preserve variable context.
Still this is excessively verbose, previously I wrote the same script in Bash in about 40 lines, but performance was screaming for a port to nodejs.
"use strict";
var mysql = require("promise-mysql");
var validator = require("mysql-validator"); // a simple library to validate against mysql data types
var ignoreDbs = [ "information_schema" ],
multiplier = 2, // numeric records multiplier to check out-of-range proximity
exitStatus = {'ok': 0, 'nearOutOfRange': 1, 'systemError': 2};
var mysqlHost = "localhost",
mysqlUser = "btc",
mysqlPass = "";
// return array of DBs strings
function getDatabases(sql) {
return sql.query("show databases")
.then(function(rows) {
var dbs = [];
for (var r of rows)
dbs.push(r.Database);
return dbs;
});
}
// return array of tables strings
function getTables(sql, db) {
return sql.query("show tables in " + db)
.then(function(rows) {
var tables = [];
for (var r of rows)
tables.push(r["Tables_in_" + db]);
return tables;
});
}
// return array of descriptions
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(function(rows) {
var descrs = [];
for (var r of rows) {
descrs.push({ 'field': r.Field, // eg: price
'type': r.Type}); // eg: decimal(10,2)
}
return descrs;
});
}
// return err object
function validateRecord(record, type) {
var record, err;
if (typeof record != "number") {
console.log("error: record is not numeric.");
process.exit(exitStatus.systemError);
}
// remove decimal part, only integer range is checked
record = Math.trunc(record);
err = validator.check(record * multiplier, type);
return err;
}
(function() {
var sql;
mysql.createConnection({
host: mysqlHost,
user: mysqlUser,
password: mysqlPass
}).then(function(connection) {
sql = connection;
})
.then(function() {
return getDatabases(sql)
})
.then(function(dbs) {
dbs.forEach(function(db) {
if (ignoreDbs.indexOf(db) != -1) return;
getTables(sql, db)
.then(function(tables) {
tables.forEach(function(table) {
getTableDescription(sql, db, table)
.then(function(descrs) {
descrs.forEach(function(descr) {
let field = descr.field,
type = descr.type,
query = "select " + descr.field + " from " + db + "." + table + " ";
if (table != "nonce") query += "order by date desc limit 1000";
sql.query(query)
.then(function(rows) {
rows.forEach(function(row) {
let err = validateRecord(row[field], type);
if (err) {
console.log(err.message);
process.exit(exitStatus.nearOutOfRange);
}
});
});
});
});
});
});
});
});
/*
.then(function() {
//if (sql != null) sql.end();
//process.exit(exitStatus.ok);
});
*/
})();
I agree with Jaromanda in terms of using let in your for loops to block scope the values and avoid your usage of an immediately-invoked function, which, while totally fine in terms of functionality, is decidedly less readable.
In terms of best practices and avoiding anti-patterns, one of the most important things you can strive for in terms of writing 'good' code is building modularized, reusable blocks of code. As it stands, your code has 5 or 6 anonymous functions that exist nowhere but within your chain of promise callbacks. If you were to declare those as functions outside of that chain, not only does that improve the maintainability of your code (you can test each individual one), but, if their names are clearly indicative of their purposes, would make for a very readable promise chain.
(Updated based on User Question)
Rather than leaving inner functions...
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(function(rows) {
var descrs = [];
for (var r of rows) {
descrs.push({ 'field': r.Field, // eg: price
'type': r.Type}); // eg: decimal(10,2)
}
return descrs;
});
}
...you can easily strip that out so that your code is self-documenting:
function collectDescriptionsFromRows(rows) {
var descriptions = [];
for (var row of rows) {
descriptions.push({'field': row.Field, 'type': row.Type});
}
return descriptions;
}
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(collectDescriptionsFromRows);
}
Also, if you ever find yourself doing data collection from one array to another, it's extremely helpful to get used to using built-in higher order functions (map, filter, reduce). Instead of the collectDescriptionsFromRows I just listed, it could be simplified to:
function collectDescriptionsFromRows(rows) {
return rows.map(row => { 'field': row.Field, 'type': row.Type});
}
Much less verbose, much more readable. Your code and promise-chain will shrink and read more like a step-by-step list of instructions if you continue to extract those anonymous functions in the chain. Anywhere you see function(...there is more extracting to do! You can also do some damage (positively) by extracting all the data you need to begin with and use local logic to boil it down to what you need, rather than making several queries. Hope this helps.
Related
Parse-server doesn't support groupBy for queries. So instead of adapting code to work with the duplicate entries i've decided to create a Job to clean the data.
I've created a cloud function using underscore but the results are not good. It's deleting non-duplicate entries too.
I want to remove a entry if another entry exists with the same post_id and user_id
Parse.Cloud.job("removeDuplicateItems", function(request, status) {
var _ = require("underscore");
var hashTable = {};
function hashKeyForTestItem(testItem) {
var fields = ["user_id", "post_id"];
var hashKey = "";
_.each(fields, function (field) {
hashKey += testItem.get(field) + "/" ;
});
return hashKey;
}
var testItemsQuery = new Parse.Query("Post_shares");
testItemsQuery.each(function (testItem) {
var key = hashKeyForTestItem(testItem);
if (key in hashTable) { // this item was seen before, so destroy this
return testItem.destroy();
} else { // it is not in the hashTable, so keep it
hashTable[key] = 1;
}
}).then(function() {
status.success("removal completed successfully.");
}, function(error) {
status.error("Uh oh, something went wrong.");
});
});
Is there a better way of doing this?
Im' trying to build my first module in nodejs.
I have this code working perfectly :
var express = require('express');
var router = express.Router();
var sqlite3 = require('sqlite3').verbose();
var db;
var io = require('socket.io-client');
const notifier = require('node-notifier');
/* GET home page. */
router.get('/', function(req, res, next) {
createDb();
socket = io.connect('http://localhost:4000');
socket.on('connect', () => {
console.log("socket connected");
});
socket.on('message', (contenu) => {
console.log('message received');
console.log(contenu);
notifier.notify(contenu.contenu);
});
socket.emit('message', { contenu : 'test'});
res.render('index', { title: 'Accueil' });
});
/* SQLite */
function createDb() {
console.log("createDb chain");
db = new sqlite3.Database('./database_institut-villebon.db', createTable);
}
function createTable() {
console.log("createTable etudiants");
db.run("DROP TABLE IF EXISTS etudiants");
db.run("CREATE TABLE IF NOT EXISTS etudiants (Nom TEXT, NumeroGroupe INTEGER, NumeroCandidat INTEGER PRIMARY KEY, Filiere TEXT)", insertRows);
}
function insertRows() {
console.log("insertRows in etudiants");
var stmt = db.prepare("INSERT INTO etudiants VALUES (?,?,?,?)");
for (var i = 0; i < 3; i++) {
stmt.run("John Doe",i,i,"S");
}
stmt.finalize(readAllRows);
}
function readAllRows() {
console.log("readAllRows etudiants");
db.all("SELECT rowid AS id, Nom, NumeroGroupe, NumeroCandidat, Filiere FROM etudiants", function(err, rows) {
rows.forEach(function (row) {
console.log(row.id + ": " + row.NumeroCandidat +","+ row.Filiere);
});
closeDb();
});
}
function closeDb() {
console.log("closeDb");
db.close();
}
function runChain() {
createDb();
}
module.exports = router;
But when i try to put it in a module it say that the table "etudiants" doesn't exist ...
This is my module :
var sqlite3 = require('sqlite3').verbose();
"use strict";
/* SQLite */
var BddUtils = function () {
console.log("createDb chain");
this.database = new sqlite3.Database('./database_institut-villebon.db');
}
BddUtils.prototype.closeDb = function () {
console.log("closeDb");
this.database.close();
}
BddUtils.prototype.readAllRows = function() {
console.log("readAllRows etudiants");
this.database.all("SELECT rowid AS id, Nom, NumeroGroupe, NumeroCandidat, Filiere FROM etudiants", function(err, rows) {
rows.forEach(function (row) {
console.log(row.id + ": " + row.NumeroCandidat +","+ row.Filiere);
});
this.database.closeDb();
});
}
BddUtils.prototype.insertRows = function() {
console.log("insertRows in etudiants");
var stmt = this.database.prepare("INSERT INTO etudiants VALUES (?,?,?,?)");
for (var i = 0; i < 3; i++) {
stmt.run("John Doe",i,i,"S");
}
//stmt.finalize(this.readAllRows());
}
BddUtils.prototype.createTable = function () {
console.log("createTable etudiants");
this.database.run("DROP TABLE IF EXISTS etudiants");
this.database.run("CREATE TABLE IF NOT EXISTS etudiants (Nom TEXT, NumeroGroupe INTEGER, NumeroCandidat INTEGER PRIMARY KEY, Filiere TEXT)", this.insertRows());
}
BddUtils.prototype.init = function () {
this.createTable();
}
exports.BddUtils = exports = new BddUtils();
I have been looking for an issue and i found that if I don't drop the table everything works !
So i suppose that the "insertRows" function is called before the create table... but it's a callback function ....
Any help will be appreciate, thanks in advance.
EDIT : I'm maybe on something :
The context of the function (the this object inside the function) is
the statement object. Note that it is not possible to run the
statement again because it is automatically finalized after running
for the first time. Any subsequent attempts to run the statement again
will fail.
If execution was successful, the this object will contain two
properties named lastID and changes which contain the value of the
last inserted row ID and the number of rows affected by this query
respectively. Note that lastID only contains valid information when
the query was a successfully completed INSERT statement and changes
only contains valid information when the query was a successfully
completed UPDATE or DELETE statement. In all other cases, the content
of these properties is inaccurate and should not be used. The .run()
function is the only query method that sets these two values; all
other query methods such as .all() or .get() don't retrieve these
values.
So it's possible that my this.database is not in the current context anymore... don't know how to proceed..
It looks like you need to wrap your CREATE TABLE statement into a Database.serialize() function.
Database#serialize([callback])
Puts the execution mode into serialized. This means that at most one
statement object can execute a query at a time. Other statements wait
in a queue until the previous statements are executed.
This ensures the CREATE TABLE statement gets executed in isolation.
The example that comes from the documentation:
db.serialize(function() {
// These two queries will run sequentially.
db.run("CREATE TABLE foo (num)");
db.run("INSERT INTO foo VALUES (?)", 1, function() {
// These queries will run in parallel and the second query will probably
// fail because the table might not exist yet.
db.run("CREATE TABLE bar (num)");
db.run("INSERT INTO bar VALUES (?)", 1);
});
});
Solved by using db.serialized()
BddUtils.prototype.createTable = function () {
var db = this.database;
db.serialize(function() {
console.log("createTable etudiants");
db.run("DROP TABLE IF EXISTS etudiants");
db.run("CREATE TABLE IF NOT EXISTS etudiants (Nom TEXT, NumeroGroupe INTEGER, NumeroCandidat INTEGER PRIMARY KEY, Filiere TEXT)");
var stmt = db.prepare("INSERT INTO etudiants VALUES (?,?,?,?)");
for (var i = 0; i < 3; i++) {
stmt.run("John Doe",i,i,"S");
}
stmt.finalize(function(){
console.log("readAllRows etudiants");
db.all("SELECT rowid AS id, Nom, NumeroGroupe, NumeroCandidat, Filiere FROM etudiants", function(err, rows) {
rows.forEach(function (row) {
console.log(row.id + ": " + row.NumeroCandidat +","+ row.Filiere);
});
db.close();
});
});
})
}
I am trying to get the data from Firebird DB with sequentially select option. I would like to get the first 500 rows, as you see on my code. And for testing, I am increasing 'k' for each 'row' and logging 'k' and
'md5' to the console.
When I am running my code, it gives me random number of rows. But the number of rows are always more than 500.
How can I solve this problem? Any suggestions?
var Firebird = require('node-firebird');
var md5 = require('md5');
var options = {};
//options.host = '127.0.0.1';
//options.port = 3050;
options.database = '/Users/bla/mydb.FDB';
options.user = 'SYSDBA';
options.password = 'masterkey';
var pool = Firebird.pool(10, options);
var k = 0;
pool.get(function (err, db) {
if (err)
throw err;
db.sequentially('SELECT FIRST 500 SOME QUERY', function (row, index) {
k = k + 1;
console.log(k + ' => ' + md5(JSON.stringify(row)) + '\n');
}, function (err) {
db.detach();
});
});
Please check the link above:
https://github.com/hgourvest/node-firebird/issues/78
#sdnetwork sdnetwork commented an hour ago it's a bug in node-firebird, i have a fix for this problem. i will post it soon here. (try with that https://github.com/sdnetwork/node-firebird)
depending upon the version of firebird, "select first n" may give an error unless you also include an "order by" clause
I'm re-writing an existing application from WebSql to JayData. The app is built on WebSql which, being, depricated, needs to be replaced (sooner or later at least). I re factored all the WebSql into its own Adapter and am now attempting to write a parallel adapter using JayData.
What I want to know is how to gracefully handle a sql join. Here's an example
read: function (display) {
var sql = "",
args = [];
sql += "SELECT table1.table1Id, table1.name, table1Local.UpdateTime ";
sql += "FROM table1";
sql += "LEFT OUTER JOIN table1Local ON table1.table1Id = table1Local.table1Id ";
sql += "WHERE table1Local.Display = ? ";
args[0] = (display === true ? "1" : "0");
return database.read(sql, args);
},
I have two jayData entities "table1" and "table1Local" inside a context. This is my rough cut attempt but it doesn't join the data.
read: function (display) {
display = display === true ? "1" : "0";
var dfd = $.Deferred();
var context = new Table1Context({
name: config.database.type,
databaseName: config.database.name
});
context.onReady(function(){
return context.Table1
.filter(function( t){
// We need to use the Display property in the local "table"
return t.display == this.display;
}, {display: display})
.toArray()
.then(function (ts) {
var data= [];
ts.forEach( function(t) {
data.push(t);
});
dfd.resolve(data);
return views;
});
});
return dfd.promise();
}
I'm a little lost about how make this work properly.
I guess you have two entitysets in the Table1Context and two entity definitions, the two entity definitions reference each other. In this case you can change the code of the filter() to
t.table1local.display == this.display;
In this query listed below, my for loop is executing 4 values. In the first alert(id), it is alerting those 4 distinct values. However, in the nested query, the alert statement is printing out the last id value only, 4x with different max(b.id) values. I'm confused. Does anyone know what may be happening? Could a race condition be occurring?
My goal is to place an Ajax call in the nested query, which has input values based on both id and b.id. I am currently doing this, but the value in the ajax call for the "id" is the same for all 4 different calls, which messes up the return data. Thanks.
database.db.transaction(function (tx) {
tx.executeSql('SELECT id, name from programs d', [], function (tx, results) {
for (var i = 0; i < results.rows.length; i++) {
var id = results.rows.item(i)['id'];
var name = results.rows.item(i)['name'];
alert(id);
tx.executeSql('SELECT max(b.id) + 1 max from people b where b.sid = ?',
[id],
function (tx, results) {
lastRecord = results.rows.item(0)['max'];
alert(id + "last rec: " + name);
}
);
}
},
function (event) { alert(event.message); });
As per my comments, you to return a closed function to bind the parameter correctly.
A much simpler example is the following:
Running this produces 4 alerts, all showing 4:
for (i=0;i<4;i++) {
setTimeout( function() { alert(i)}, 1000);
}
Running this produces 4 alerts, showing 0/4, 1/4, 2/4, 3/4.
for (i=0;i<4;i++) {
setTimeout(function(inneri) {
return(
function() {
alert(inneri + "/" + i);
}
);
}(i), 1000);
}
where I've named inneri the value that was preserved upon closure. Note that i, itself is still referring to the outer scope, and thus is 4 (which is what is true at time of execution, since that is the value of i when it dumps out of the for loop, as we're delaying the execution using setTimeout().
The first case is a simpler version of what you're doing, whereas you want the second case.
Rewriting your js (and hoping I get all these ( and {'s in the right place :) ) gives:
database.db.transaction(function (tx) {
tx.executeSql('SELECT id, name from programs d', [], function (tx, results) {
for (var i = 0; i < results.rows.length; i++) {
var id = results.rows.item(i)['id'];
var name = results.rows.item(i)['name'];
alert(id);
tx.executeSql('SELECT max(b.id) + 1 max from people b where b.sid = ?',
[id],
function(innerId) {
return (
function (tx, results) {
lastRecord = results.rows.item(0)['max'];
alert(innerId + "last rec: " + name);
}
);
}(id) //be careful to avoid the ";" here!
);
}
},
function (event) { alert(event.message);
});
Where I have inserted:
function(innerId) {
return (
function (tx, results) {
lastRecord = results.rows.item(0)['max'];
alert(innerId + "last rec: " + name);
}
);
}(id)
in. This function is called immediately via the (id) and returns a function that takes tx and results as arguments and does the appropriate action.
I have checked braces/parenthesis, but don't have a direct way to verify that I didn't make any typos.