Develop Reference

Simplify nested promises within loops and closures

I wrote a ~50 lines script to perform housekeeping on MySQL databases. I'm afraid my code exhibits anti-patterns as it rapidly escalates to an unreadable mess for the simple functions it performs.
I'd like some opinions for improving readability.
The full script is at the bottom of this post to give an idea.
Spotlight on the problem
The excessive nesting is caused by patterns like this repeated over and over: (snippet taken from script)
sql.query("show databases")
.then(function(rows) {
for (var r of rows) {
var db = r.Database;
(function(db) {
sql.query("show tables in " + db)
.then(function(rows) {
// [...]
}
})(db);
}
});
I'm nesting one promise under the other within both a for loop and a closure. The loop is needed to iterate across all results from sql.query(), and the closure is necessary to pass the value of db to the lower promise; without the closure, the loop would complete even before the nested promise executes at all, so db would always contain only the last element of the loop, preventing the nested promise from reading each value of db.
Full script
var mysql = require("promise-mysql");
var validator = require("mysql-validator"); // simple library to validate against mysql data types
var ignoreDbs = [ "information_schema" ],
multiplier = 2, // numeric records multiplier to check out-of-range proximity
exitStatus = {'ok': 0, 'nearOutOfRange': 1, 'systemError': 2};
(function() {
var sql,
mysqlHost = "localhost",
mysqlUser = "user",
mysqlPass = "";
mysql.createConnection({
host: mysqlHost,
user: mysqlUser,
password: mysqlPass
}).then(function(connection) {
sql = connection;
})
.then(function() {
sql.query("show databases")
.then(function(rows) {
for (var r of rows) {
var db = r.Database;
if (ignoreDbs.indexOf(db) != -1) continue;
(function(db) {
sql.query("show tables in " + db)
.then(function(rows) {
for (var r of rows) {
var table = r["Tables_in_" + db];
(function(table) {
sql.query("describe " + db + "." + table)
.then(function(rows) {
for (var r of rows) {
(function(r) {
var field = r.Field,
type = r.Type, // eg: decimal(10,2)
query = "select " + field + " from " + db + "." + table + " ";
if (table != "nonce") query += "order by date desc limit 1000";
sql.query(query)
.then(function(rows) {
for (var r of rows) {
var record, err;
// remove decimal part, only integer range is checked
record = Math.trunc(r[field]);
err = validator.check(record * multiplier, type);
if (err) {
console.log(err.message);
process.exit(exitStatus.nearOutOfRange);
}
}
});
})(r);
}
});
})(table);
}
});
})(db);
}
});
})
.then(function() {
// if (sql != null) sql.end(); // may not exit process here: sql connection terminates before async functions above
//process.exit(exitStatus.ok); //
});
})();
Trivia
The purpose of the script is to automatically and periodically monitor if any record stored in any row, table and database in MySQL is approaching the out-of-range limit for its specific data type. Several other processes connected to MySQL continuously insert new numeric data with increasing values and nonces; this script is a central point where to check for such numeric limits. The script would then be attached to Munin for continuous monitoring and alerting.
Update: Revised script
As suggested by #Kqcef I modularized the anonymous functions out of the promise nest, and used let to avoid the explicit nesting of an additional function to preserve variable context.
Still this is excessively verbose, previously I wrote the same script in Bash in about 40 lines, but performance was screaming for a port to nodejs.
"use strict";
var mysql = require("promise-mysql");
var validator = require("mysql-validator"); // a simple library to validate against mysql data types
var ignoreDbs = [ "information_schema" ],
multiplier = 2, // numeric records multiplier to check out-of-range proximity
exitStatus = {'ok': 0, 'nearOutOfRange': 1, 'systemError': 2};
var mysqlHost = "localhost",
mysqlUser = "btc",
mysqlPass = "";
// return array of DBs strings
function getDatabases(sql) {
return sql.query("show databases")
.then(function(rows) {
var dbs = [];
for (var r of rows)
dbs.push(r.Database);
return dbs;
});
}
// return array of tables strings
function getTables(sql, db) {
return sql.query("show tables in " + db)
.then(function(rows) {
var tables = [];
for (var r of rows)
tables.push(r["Tables_in_" + db]);
return tables;
});
}
// return array of descriptions
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(function(rows) {
var descrs = [];
for (var r of rows) {
descrs.push({ 'field': r.Field, // eg: price
'type': r.Type}); // eg: decimal(10,2)
}
return descrs;
});
}
// return err object
function validateRecord(record, type) {
var record, err;
if (typeof record != "number") {
console.log("error: record is not numeric.");
process.exit(exitStatus.systemError);
}
// remove decimal part, only integer range is checked
record = Math.trunc(record);
err = validator.check(record * multiplier, type);
return err;
}
(function() {
var sql;
mysql.createConnection({
host: mysqlHost,
user: mysqlUser,
password: mysqlPass
}).then(function(connection) {
sql = connection;
})
.then(function() {
return getDatabases(sql)
})
.then(function(dbs) {
dbs.forEach(function(db) {
if (ignoreDbs.indexOf(db) != -1) return;
getTables(sql, db)
.then(function(tables) {
tables.forEach(function(table) {
getTableDescription(sql, db, table)
.then(function(descrs) {
descrs.forEach(function(descr) {
let field = descr.field,
type = descr.type,
query = "select " + descr.field + " from " + db + "." + table + " ";
if (table != "nonce") query += "order by date desc limit 1000";
sql.query(query)
.then(function(rows) {
rows.forEach(function(row) {
let err = validateRecord(row[field], type);
if (err) {
console.log(err.message);
process.exit(exitStatus.nearOutOfRange);
}
});
});
});
});
});
});
});
});
/*
.then(function() {
//if (sql != null) sql.end();
//process.exit(exitStatus.ok);
});
*/
})();

I agree with Jaromanda in terms of using let in your for loops to block scope the values and avoid your usage of an immediately-invoked function, which, while totally fine in terms of functionality, is decidedly less readable.
In terms of best practices and avoiding anti-patterns, one of the most important things you can strive for in terms of writing 'good' code is building modularized, reusable blocks of code. As it stands, your code has 5 or 6 anonymous functions that exist nowhere but within your chain of promise callbacks. If you were to declare those as functions outside of that chain, not only does that improve the maintainability of your code (you can test each individual one), but, if their names are clearly indicative of their purposes, would make for a very readable promise chain.
(Updated based on User Question)
Rather than leaving inner functions...
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(function(rows) {
var descrs = [];
for (var r of rows) {
descrs.push({ 'field': r.Field, // eg: price
'type': r.Type}); // eg: decimal(10,2)
}
return descrs;
});
}
...you can easily strip that out so that your code is self-documenting:
function collectDescriptionsFromRows(rows) {
var descriptions = [];
for (var row of rows) {
descriptions.push({'field': row.Field, 'type': row.Type});
}
return descriptions;
}
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(collectDescriptionsFromRows);
}
Also, if you ever find yourself doing data collection from one array to another, it's extremely helpful to get used to using built-in higher order functions (map, filter, reduce). Instead of the collectDescriptionsFromRows I just listed, it could be simplified to:
function collectDescriptionsFromRows(rows) {
return rows.map(row => { 'field': row.Field, 'type': row.Type});
}
Much less verbose, much more readable. Your code and promise-chain will shrink and read more like a step-by-step list of instructions if you continue to extract those anonymous functions in the chain. Anywhere you see function(...there is more extracting to do! You can also do some damage (positively) by extracting all the data you need to begin with and use local logic to boil it down to what you need, rather than making several queries. Hope this helps.

Meteor subscribe callback running when subscription contains previous subscribe result

I am fairly new to meteor, and I am running into a strange issue with subscribe callbacks. I have a database containing courses and reviews. I'm using a publish/subscribe model on the reviews to return reviews that are only relevant to a selected class, and I want this to change each time a new class is clicked on. I want to print all the reviews and compile some metrics about the reviews (average quality, difficulty rating). Using the following code, with a subscribe that updates the reviews sent to the client, the printed reviews (which are grabbed from a helper) return correctly, but the metrics (which are grabbed on an onReady callback to the helper) are inaccurate. When the onReady function is run, the current result of the local reviews collection contains the union of the clicked class and the previously clicked class, even though the reviews themselves print correctly.
I've also tried using autoTracker, but I got the same results. Is there a way to clear previous subscribe results before updating them?
publish:
Meteor.publish('reviews', function validReviews(courseId, visiblity) {
console.log(courseId);
console.log(visiblity);
var ret = null
//show valid reviews for this course
if (courseId != undefined && courseId != "" && visiblity == 1) {
console.log("checked reviews for a class");
ret = Reviews.find({class : courseId, visible : 1}, {limit: 700});
} else if (courseId != undefined && courseId != "" && visiblity == 0) { //invalidated reviews for a class
console.log("unchecked reviews for a class");
ret = Reviews.find({class : courseId, visible : 0},
{limit: 700});
} else if (visiblity == 0) { //all invalidated reviews
console.log("all unchecked reviews");
ret = Reviews.find({visible : 0}, {limit: 700});
} else { //no reviews
console.log("no reviews");
//will always be empty because visible is 0 or 1. allows meteor to still send the ready
//flag when a new publication is sent
ret = Reviews.find({visible : 10});
}
//console.log(ret.fetch())
return ret
});
subscribe:
this.helpers({
reviews() {
return Reviews.find({});
}
});
and subscribe call, in constructor with the helpers:
constructor($scope) {
$scope.viewModel(this);
//when a new class is selected, update the reviews that are returned by the database and update the gauges
this.subscribe('reviews', () => [(this.getReactively('selectedClass'))._id, 1], {
//callback function, should only run once the reveiws collection updates, BUT ISNT
//seems to be combining the previously clicked class's reviews into the collection
onReady: function() {
console.log("class is: ", this.selectedClass);
if (this.isClassSelected == true) { //will later need to check that the side window is open
//create initial variables
var countGrade = 0;
var countDiff = 0;
var countQual = 0;
var count = 0;
//table to translate grades from numerical value
var gradeTranslation = ["C-", "C", "C+", "B-", "B", "B-", "A-", "A", "A+"];
//get all current reviews, which will now have only this class's reviews because of the subscribe.
var allReviews = Reviews.find({});
console.log(allReviews.fetch());
console.log("len is " + allReviews.fetch().length)
if (allReviews.fetch().length != 0) {
console.log("exist")
allReviews.forEach(function(review) {
count++;
countGrade = countGrade + Number(review["grade"]);
countDiff = countDiff + review["difficulty"];
countQual = countQual + review["quality"];
});
this.qual = (countQual/count).toFixed(1);
this.diff = (countDiff/count).toFixed(1);
this.grade = gradeTranslation[Math.floor(countGrade/count) - 1];
} else {
console.log("first else");
this.qual = 0;
this.diff = 0;
this.grade = "-";
}
} else {
console.log("second else");
this.qual = 0;
this.diff = 0;
this.grade = "-";
}
}
})

When using pub-sub the minimongo database on the client will contain the union of subscriptions unless they are explicitly cleared. For that reason you want to repeat the query that's in the publication on the client side so that you filter and sort the same way. Minimongo is very fast on the client and you typically have much less data there so don't worry about performance.
In your constructor you have:
var allReviews = Reviews.find({});
instead use:
var allReviews = Reviews.find(
{
class : (this.getReactively('selectedClass'))._id,
visible : 1
},
{limit: 700}
);
Another side tip: javascript is quite clever about truthy and falsy values.
if (courseId != undefined && courseId != "" && visibility == 1)
can be simplified to:
if (courseId && visibility)
assuming you're using visibility == 1 to denote true and visibility == 0 to denote false

remember search query to use in function

I'm using Angularjs with the dirPagination plugin to connect with an Web API. This seems to work fine. I added a search function, to do a server side search:
$scope.searchChanged = function () {
if ($scope.searchFor.length == 0) {
$scope.calculatedValue = 'e';
} else {
vm.getData(vm.pageno, vm.getSearch($scope.searchFor));
}
}
vm.getSearch = function (query) {
if (query == undefined) {
query = 'tech';
} else {
query = query;
}
return query;
}
See Plnkr for the full code
If I start searching (e.g. sales) the API returns results and the paging is correct, the get request is:
/api/students/?category=sales&begin=1&pageSize=10
But if you want to go to another page number, the get request to the server is:
/api/students/?category=tech&begin=2&pageSize=10
How can the view remember the query 'sales', so that the paging and results are correct?

You are making a common mistake here: You don't need to pass in variable from the view if you are already using a scope variable.
Changing to this would be much less error prone
// change this to var getSearch or function getSearch if you don't need it on the view anymore
vm.getSearch = function () {
var query = vm.searchFor;
// you should only use vm, change ng-model to data.searchFor
if (query == undefined) {
query = 'tech';
}
return query;
}
vm.getData = function () {
vm.users = [];
$http.get("/api/students/?category=" + vm.getSearch() + "&begin=" + vm.pageno + "&pageSize=" + vm.itemsPerPage).success(function (response) {
vm.users = response.data;
vm.total_count = response.total_count;
});
};

Your request id good, you need to optimize the sql query so you can get the right results. it should look something like this:
#begin INT = 0,
#pageSize INT = 10
SELECT *
FROM [TableName]
ORDER BY id
OFFSET (#pageSize * #begin )
ROWS FETCH NEXT #pageSize ROWS ONLY;

Building OData $filter URLs with an Angular provider

I have an angular provider for querying an OData service.
Now I'm trying to build a $filter function onto that provider, so I can use it throughout my app.
The problem I'm running into, and so far haven't been able to solve, is that the query part of the URL needs to start with '$filter=' which I can handle fine when there is one filter, but multiple filters are added as ' and {query goes here}'.
A sample query would be:
http://www.example.com/odata/Restaurants/?$filter=id eq 2 and city eq 'Los Angeles' and substringof("Lagasse", chef)&$top=20
I'm sending all the filters in an array to the provider. Ideally, I would use "$filter=#{firstFilter}" for the first filter in the array and for the remaining filters use " and #{remainingFilter}" but I'm unsure how to structure this code.
My current code uses multiple if statements to check if a filter is there, but with the nature of building the url, it makes one of the filters mandatory at all times. I'd like to avoid this.
For example:
var filter = "$filter=id eq 2";
if (city) {
filter += " and city eq #{cityName}";
}
if (chef) {
filter += " and substringof(#{chefName}, chef)";
}
Now everytime a user inputs a query, they have to specify an id.
We are NOT using BreezeJS, JayData, or any other library. Strictly AngularJS and specifically $http, NOT $resource.

You can use odata-filter-builder to build $filter part for OData URL query options.
Then just use $http with config params.
Short example:
var filter = ODataFilterBuilder()
.eq('id', 2)
.eq('city', 'Los Angeles')
.and('substringof("Lagasse", chef)')
.toString();
$http
.get(resourceUrl, {params: {$filter: filter, $top: 20}})
.then(function(response) {
// Handle response
})
Full example:
angular
.module('OData', [])
.constant('ODataFilterBuilder', ODataFilterBuilder)
.factory('ODataService', function($http) {
return {
load: function(resourceUrl, queryParams) {
return $http.get(resourceUrl, {params: queryParams})
.then(function(response) {
return response.data.value;
});
}
}
});
angular
.module('app', ['OData'])
.controller('appController', function($http, ODataService, ODataFilterBuilder, $httpParamSerializer) {
// 1. inject ODataFilterBuilder
// use short name for filter builder
var f = ODataFilterBuilder;
// 2. build filter
var filter = f()
.eq('id', 2)
.eq('city', 'Los Angeles')
.and('substringof("Lagasse", chef)')
// 3. creater odata query params
var queryParams = {
$filter: filter.toString(),
$top: 20
// TODO: add other params
};
// 4. prepare odata resourse URL
var odataServiceUrl = 'http://path/to/odata/service/';
var odataResourseUrl = odataServiceUrl + 'entity';
// 5. Do http request with odataResourseUrl and queryParams
// use ODataService or angular $http service
// ODataService
// .load(odataResourseUrl, queryParams)
// .then(function(value) {
// // handle value
// });
// OR
// $http.get(odataResourseUrl, {params: queryParams})
// .then(function(respons) {
// // handle respons.data.value
// });
// Result examles:
// NOTE: $httpParamSerializer - default $http params serializer that converts objects to strings
var queryParamsSerialised = $httpParamSerializer(queryParams);
// put values to 'this' to use it in html
this.queryParams = queryParams;
this.queryParamsSerialised = queryParamsSerialised;
this.queryUrl = odataResourseUrl + '?' + queryParamsSerialised;
});
<div ng-app="app" ng-controller="appController as vm">
<pre>queryParams: {{vm.queryParams|json}}</pre>
<pre>queryParamsSerialised: {{vm.queryParamsSerialised}}</pre>
<pre>queryUrl: {{vm.queryUrl}}</pre>
</div>
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.5/angular.min.js"></script>
<script src="https://npmcdn.com/odata-filter-builder#^0.1/dist/odata-filter-builder.js"></script>

I know you said you are not using any other library, but in case anyone else stumbles up this...I suggest using joData. It's very expressive and requires no other dependencies: https://github.com/mccow002/joData
Here is your example in joData:
var query = new jo('http://www.example.com/odata/Restaurants');
query.filter(new jo.FilterClause('id').eq(2))
.andFilter(new jo.FilterClause('city').eq('Los Angeles'))
.andFilter(new jo.FilterClause('chef').substringof('Lagasse').eq(true))
.top(2);
Then,
query.toString()
produces...
"http://www.example.com/odata/Restaurants?$top=2&$filter=id eq 2 and city eq 'Los Angeles' and substringof('Lagasse',chef) eq true"
Anyway, to set this up for Angular here is what I did...
Reference joData js in your html before your angular.js reference.
Wrap it in an Angular service
angular.module('app.shared.oData', [])
.service('oDataBuilderService', function () {
this.jo = jo;
};
Now other services can use this odata service to construct odata queries:
angular.module('app.shared.video', [])
.service('videoService', function ($http, oDataBuilderService) {
this.getByVideoCatalogId = function (videoCatalogId) {
var query = new oDataBuilderService.jo('http://www.example.com/VideoCatalog/OData');
var videoCatalogIdEquals = new oDataBuilderService.jo.FilterClause("VideoCatalogId").eq(videoCatalogId);
query.andFilter(videoCatalogIdEquals);
var queryAsString = query.toString();
var promise = $http.get(queryAsString);
return promise;
}
});

You can add $filter= at the end, but then you'd have the same problem with the "and "
Easier would be to just create a simple function
addFilter(currentFilter, filter) {
if (currentFilter.length == 0) {
currentFilter = "$filter=" + filter;
} else {
currentFilter += "and " + filter;
}
return currentFilter;
}
so then just call currentFilter = addFilter(currentFilter, "what to filter")
var currentFilter = "";
if (city) {
currentFilter = addFilter(currentFilter, "city eq #{cityName}");
}
if (chef) {
currentFilter = addFilter(currentFilter, "substringof(#{chefName}, chef)");
}

I've done similar and have what I think is an elegant solution with no dependencies on other libraries. I use Array.join and concatenate the filters with an "and" between them:
var filter = [];
if (city) filter.push("City eq '" + city + "'");
if (chef) filter.push("Chef eq '" + chef + "'");
var filterQuery = filter.join(" and ");
This assumes you always want "and" between them, (not or), but that is the case in my apps and in your example. I also do this to build up the url:
var parts = [baseurl];
parts.push("$top=10");
parts.push("$skip=" + skip);
parts.push(filterQuery);
var url = parts.join("&");

How do I get the gender from a particular user when updating a different table? Azure mobile services

I have a table called Subscription and another table called Client I need the gender of the Client who owns the subscription every time I make an update. Here's my update script:
function update(item, user, request) {
var subscriptionId = item.id;
var subscriptionActivitiesTable = tables.getTable("SubscriptionActivity");
var userTable = tables.getTable("User");
var activityTable = tables.getTable("Activity");
var userGender = userTable.where({id: item.UserId}).select('Gender').take(1).read();
console.log(userGender);
activityTable.where({PlanId:item.PlanId, Difficulty: item.Difficulty}).read({
success: function(results){
var startDate = item.StartDate;
results.forEach(function(activity)
{
var testDate = new Date(startDate.getFullYear(),startDate.getMonth(), startDate.getDate());
testDate.setDate(testDate.getDate() + activity.Sequence + (activity.Week*7));
subscriptionActivitiesTable.insert({SubscriptionId: subscriptionId,
ActivityId: activity.id, ShowDate: new Date(testDate.getFullYear(),
testDate.getMonth(), testDate.getDate()), CreationDate: new Date()});
})
}
});
var planWeeks = 12;//VER DE DONDE SACAMOS ESTE NUMERO
var idealWeight = 0;
if (userGender === "Male")
{
idealWeight = (21.7 * Math.pow(parseInt(item.Height)/100,2));
}
else
{
idealWeight = (23 * Math.pow(parseInt(item.Height)/100,2));
}
var metabolismoBasal = idealWeight * 0.95 * 24;
var ADE = 0.1 * metabolismoBasal;
var activityFactor;
if (item.Difficulty === "Easy")
{
activityFactor = 1.25;
}
else if(item.Difficulty === "Medium")
{
activityFactor = 1.5;
}
else
{
activityFactor = 1.75;
}
var caloricRequirement = ((metabolismoBasal + ADE)*activityFactor);
activityTable.where(function(item, caloricRequirement){
return this.PlanId === item.PlanId && this.Type != "Sport" &&
this.CaloricRequirementMin <= caloricRequirement &&
this.CaloricRequirementMax >= caloricRequirement;}, item, caloricRequirement).read({
success: function(results)
{
var startDate = item.StartDate;
results.forEach(function(activity)
{
for (var i=0;i<planWeeks;i++)
{
var testDate = new Date(startDate.getFullYear(),startDate.getMonth(), startDate.getDate());
testDate.setDate(testDate.getDate() + activity.Sequence + (i*7));
subscriptionActivitiesTable.insert({SubscriptionId: subscriptionId,
ActivityId: activity.id, ShowDate: new Date(testDate.getFullYear(),
testDate.getMonth(), testDate.getDate()), CreationDate: new Date()});
}
})
}
})
request.execute();
}
I tried the code above and clientGender is undefined. As you can see I want to use the gender to set the idealWeight.

The read() method expects a function to be passed in on the success parameter - it doesn't return the result of the query like you'd think.
Try something like this instead:
function update(item, user, request) {
var clientTable = tables.getTable("Client");
var clientGender = 'DEFAULT';
clientTable.where({id: item.ClientId}).select('Gender').take(1).read({
success: function(clients) {
if (clients.length == 0) {
console.error('Unable to find client for id ' + item.ClientId);
} else {
var client = client[0];
clientGender = client.Gender;
// since we're inside the success function, we can continue to
// use the clientGender as it will reflect the correct value
// as retrieved from the database
console.log('INSIDE: ' + clientGender);
}
}
});
// this is going to get called while the clientTable query above is
// still running and will most likely show a value of DEFAULT
console.log('OUTSIDE: ' + clientGender);
}
In this sample, the client table query is kicked off, with a callback function provided in the success parameter. When the query is finished, the callback function is called, and the resulting data is displayed to the log. Meanwhile - while the query is still running, that is - the next statement after the where/take/select/read fluent code is run, another console.log statment is executed to show the value of the clientGender field outside the read function. This code will run while the read statement is still waiting on the database. Your output should look something like this in the WAMS log:
* INSIDE: Male
* OUTSIDE: Default
Since the log shows the oldest entries at the bottom, you can see that the OUTSIDE log entry was written sometime before the INSIDE log.
If you're not used to async or functional programming, this might look weird, but as far as I've found, this is now node works. Functions nested in functions nested in functions can get kind of scary, but if you plan ahead, it probably won't be too bad :-)