I am trying to get the data from Firebird DB with sequentially select option. I would like to get the first 500 rows, as you see on my code. And for testing, I am increasing 'k' for each 'row' and logging 'k' and
'md5' to the console.
When I am running my code, it gives me random number of rows. But the number of rows are always more than 500.
How can I solve this problem? Any suggestions?
var Firebird = require('node-firebird');
var md5 = require('md5');
var options = {};
//options.host = '127.0.0.1';
//options.port = 3050;
options.database = '/Users/bla/mydb.FDB';
options.user = 'SYSDBA';
options.password = 'masterkey';
var pool = Firebird.pool(10, options);
var k = 0;
pool.get(function (err, db) {
if (err)
throw err;
db.sequentially('SELECT FIRST 500 SOME QUERY', function (row, index) {
k = k + 1;
console.log(k + ' => ' + md5(JSON.stringify(row)) + '\n');
}, function (err) {
db.detach();
});
});
Please check the link above:
https://github.com/hgourvest/node-firebird/issues/78
#sdnetwork sdnetwork commented an hour ago it's a bug in node-firebird, i have a fix for this problem. i will post it soon here. (try with that https://github.com/sdnetwork/node-firebird)
depending upon the version of firebird, "select first n" may give an error unless you also include an "order by" clause
Related
I am using MongoDB and Node.JS, I am trying to get data out of my MongoDB and show into my html page which I have working with the below code however this just brings back ALL entries in no particular order:
server.js
// This is for getting the list of all players from my DB
app.get("/getPlayers", function(request, response) {
db.getPlayers().then(function(players){
console.log(players);
response.send(players);
});
});
leadership.html
<script>
$(function() {
$.get("http://localhost:9000/getPlayers", {}, function (res) {
let data = res;
console.log(res);
for (i = 0; i < data.length; i++) {
let name = data[i].name;
let score = data[i].score;
console.log(data[i].name);
$("#leadership").append("<tr><td class=\"name\">"
+ data[i].name + "</td><td class=\"score\">"
+ data[i].score + "</td></tr>");
}
});
});
</script>
After looking at W3 Schools I tried to alter the code to this:
db.getPlayers().sort().limit(10).then(function(players)
However my Chrome console brings back an internal server error 500. Can someone point out how I can sort by LARGEST NUMBER first, then LIMIT the results to say 10? Within the database there is a collection called players, which holds name and score
db.js
var Player = mongoose.model("Player", {name: String, score: Number});
module.exports.Player = Player;
Try something like this.
Order, sort and limit can be passed from front end or change default values after : mark.
Players is imported model, you can do it this way or use method in the model itself.
app.post('/api/players',(req,res)=>{
let order = req.body.order ? req.body.order : "-1";
let sortBy = req.body.sortBy ? req.body.sortBy : "_id";
let limit = req.body.limit ? parseInt(req.body.limit) : 100;
Players.
find().
sort([[sortBy,order]]).
limit(limit).
exec((err,players)=>{
if(err) return res.status(400).send(err);
res.status(200).json({
size: players.length,
players
})
})
})
I wrote a ~50 lines script to perform housekeeping on MySQL databases. I'm afraid my code exhibits anti-patterns as it rapidly escalates to an unreadable mess for the simple functions it performs.
I'd like some opinions for improving readability.
The full script is at the bottom of this post to give an idea.
Spotlight on the problem
The excessive nesting is caused by patterns like this repeated over and over: (snippet taken from script)
sql.query("show databases")
.then(function(rows) {
for (var r of rows) {
var db = r.Database;
(function(db) {
sql.query("show tables in " + db)
.then(function(rows) {
// [...]
}
})(db);
}
});
I'm nesting one promise under the other within both a for loop and a closure. The loop is needed to iterate across all results from sql.query(), and the closure is necessary to pass the value of db to the lower promise; without the closure, the loop would complete even before the nested promise executes at all, so db would always contain only the last element of the loop, preventing the nested promise from reading each value of db.
Full script
var mysql = require("promise-mysql");
var validator = require("mysql-validator"); // simple library to validate against mysql data types
var ignoreDbs = [ "information_schema" ],
multiplier = 2, // numeric records multiplier to check out-of-range proximity
exitStatus = {'ok': 0, 'nearOutOfRange': 1, 'systemError': 2};
(function() {
var sql,
mysqlHost = "localhost",
mysqlUser = "user",
mysqlPass = "";
mysql.createConnection({
host: mysqlHost,
user: mysqlUser,
password: mysqlPass
}).then(function(connection) {
sql = connection;
})
.then(function() {
sql.query("show databases")
.then(function(rows) {
for (var r of rows) {
var db = r.Database;
if (ignoreDbs.indexOf(db) != -1) continue;
(function(db) {
sql.query("show tables in " + db)
.then(function(rows) {
for (var r of rows) {
var table = r["Tables_in_" + db];
(function(table) {
sql.query("describe " + db + "." + table)
.then(function(rows) {
for (var r of rows) {
(function(r) {
var field = r.Field,
type = r.Type, // eg: decimal(10,2)
query = "select " + field + " from " + db + "." + table + " ";
if (table != "nonce") query += "order by date desc limit 1000";
sql.query(query)
.then(function(rows) {
for (var r of rows) {
var record, err;
// remove decimal part, only integer range is checked
record = Math.trunc(r[field]);
err = validator.check(record * multiplier, type);
if (err) {
console.log(err.message);
process.exit(exitStatus.nearOutOfRange);
}
}
});
})(r);
}
});
})(table);
}
});
})(db);
}
});
})
.then(function() {
// if (sql != null) sql.end(); // may not exit process here: sql connection terminates before async functions above
//process.exit(exitStatus.ok); //
});
})();
Trivia
The purpose of the script is to automatically and periodically monitor if any record stored in any row, table and database in MySQL is approaching the out-of-range limit for its specific data type. Several other processes connected to MySQL continuously insert new numeric data with increasing values and nonces; this script is a central point where to check for such numeric limits. The script would then be attached to Munin for continuous monitoring and alerting.
Update: Revised script
As suggested by #Kqcef I modularized the anonymous functions out of the promise nest, and used let to avoid the explicit nesting of an additional function to preserve variable context.
Still this is excessively verbose, previously I wrote the same script in Bash in about 40 lines, but performance was screaming for a port to nodejs.
"use strict";
var mysql = require("promise-mysql");
var validator = require("mysql-validator"); // a simple library to validate against mysql data types
var ignoreDbs = [ "information_schema" ],
multiplier = 2, // numeric records multiplier to check out-of-range proximity
exitStatus = {'ok': 0, 'nearOutOfRange': 1, 'systemError': 2};
var mysqlHost = "localhost",
mysqlUser = "btc",
mysqlPass = "";
// return array of DBs strings
function getDatabases(sql) {
return sql.query("show databases")
.then(function(rows) {
var dbs = [];
for (var r of rows)
dbs.push(r.Database);
return dbs;
});
}
// return array of tables strings
function getTables(sql, db) {
return sql.query("show tables in " + db)
.then(function(rows) {
var tables = [];
for (var r of rows)
tables.push(r["Tables_in_" + db]);
return tables;
});
}
// return array of descriptions
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(function(rows) {
var descrs = [];
for (var r of rows) {
descrs.push({ 'field': r.Field, // eg: price
'type': r.Type}); // eg: decimal(10,2)
}
return descrs;
});
}
// return err object
function validateRecord(record, type) {
var record, err;
if (typeof record != "number") {
console.log("error: record is not numeric.");
process.exit(exitStatus.systemError);
}
// remove decimal part, only integer range is checked
record = Math.trunc(record);
err = validator.check(record * multiplier, type);
return err;
}
(function() {
var sql;
mysql.createConnection({
host: mysqlHost,
user: mysqlUser,
password: mysqlPass
}).then(function(connection) {
sql = connection;
})
.then(function() {
return getDatabases(sql)
})
.then(function(dbs) {
dbs.forEach(function(db) {
if (ignoreDbs.indexOf(db) != -1) return;
getTables(sql, db)
.then(function(tables) {
tables.forEach(function(table) {
getTableDescription(sql, db, table)
.then(function(descrs) {
descrs.forEach(function(descr) {
let field = descr.field,
type = descr.type,
query = "select " + descr.field + " from " + db + "." + table + " ";
if (table != "nonce") query += "order by date desc limit 1000";
sql.query(query)
.then(function(rows) {
rows.forEach(function(row) {
let err = validateRecord(row[field], type);
if (err) {
console.log(err.message);
process.exit(exitStatus.nearOutOfRange);
}
});
});
});
});
});
});
});
});
/*
.then(function() {
//if (sql != null) sql.end();
//process.exit(exitStatus.ok);
});
*/
})();
I agree with Jaromanda in terms of using let in your for loops to block scope the values and avoid your usage of an immediately-invoked function, which, while totally fine in terms of functionality, is decidedly less readable.
In terms of best practices and avoiding anti-patterns, one of the most important things you can strive for in terms of writing 'good' code is building modularized, reusable blocks of code. As it stands, your code has 5 or 6 anonymous functions that exist nowhere but within your chain of promise callbacks. If you were to declare those as functions outside of that chain, not only does that improve the maintainability of your code (you can test each individual one), but, if their names are clearly indicative of their purposes, would make for a very readable promise chain.
(Updated based on User Question)
Rather than leaving inner functions...
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(function(rows) {
var descrs = [];
for (var r of rows) {
descrs.push({ 'field': r.Field, // eg: price
'type': r.Type}); // eg: decimal(10,2)
}
return descrs;
});
}
...you can easily strip that out so that your code is self-documenting:
function collectDescriptionsFromRows(rows) {
var descriptions = [];
for (var row of rows) {
descriptions.push({'field': row.Field, 'type': row.Type});
}
return descriptions;
}
function getTableDescription(sql, db, table) {
return sql.query("describe " + db + "." + table)
.then(collectDescriptionsFromRows);
}
Also, if you ever find yourself doing data collection from one array to another, it's extremely helpful to get used to using built-in higher order functions (map, filter, reduce). Instead of the collectDescriptionsFromRows I just listed, it could be simplified to:
function collectDescriptionsFromRows(rows) {
return rows.map(row => { 'field': row.Field, 'type': row.Type});
}
Much less verbose, much more readable. Your code and promise-chain will shrink and read more like a step-by-step list of instructions if you continue to extract those anonymous functions in the chain. Anywhere you see function(...there is more extracting to do! You can also do some damage (positively) by extracting all the data you need to begin with and use local logic to boil it down to what you need, rather than making several queries. Hope this helps.
So I am looking in my database right now and there is 3 articles, so the x should represent the number 3 for the for loop later. This code is supposed to add articles to the database if they are freshly scraped. In the titles[] array there is 100 items (scraped from news.google). When I execute the code, it seems like it correctly finds the index # of the titles[] list (which is the 55,68,60 repeats), but it shows up as this: (I don't even want the index # being displayed in console either, I want the title of article to be displayed. I had to shrink some of this code, too long)
55
68
60
55
68
60
55
68
60
55
68
60
55
68
60
55
68
Complete.
Successfully added article: undefinedto the database.
Successfully added article: undefinedto the database.
Successfully added article: undefinedto the database.
Successfully added article: undefinedto the database.
Successfully added article: undefinedto the database.
Successfully added article: undefinedto the database.
And here is my code:
// accessing the database
function DatabaseTime(sourcesDates, timeAdded, links, titles, descriptions) {
sourcesDates = sourcesDates;
links = links;
titles = titles;
descriptions = descriptions;
// put counter so params can access this in it's object scope, use it for the for-loop
// object operator. MEAT OF THE BURGER
var databaseOperation = function (sourcesDates, timeAdded, links, titles, descriptions) {
var scanParams = { TableName: "Rnews" }
// using code to setup for accessing the 2nd list
db.scan(scanParams, function(err, scanData) { // scanData = the 2nd list we are going to work with
//use this array later to hold the unique items
var arrayCheck = [];
for (let i = 0; i < scanData.Items.length; i++) {
arrayCheck.push(scanData.Items[i].title);
}
var index = 0;
var counter = 0;
var x;
for (var i = 0; i < titles.length; i++) {
index = 0;
x = 0;
for (var x = 0; x < arrayCheck.length; x++) {
//if (titles[i] === arrayCheck[x]) {
if (titles.indexOf(arrayCheck[x] === -1)) {
index = titles.indexOf(arrayCheck[x]);
console.log(index);
var autoParams = {
TableName: "Rnews",
Item: {
title: titles[index],
source: sourcesDates[index],
url: links[index],
description: descriptions[index],
lastAddOrUpdated: dbTimeStamp,
timePublish: timeAdded[index]
}
}
Insert(autoParams, titles);
}
}
}
function Insert(autoParams) {
db.put(autoParams, function(err, data) {
if (err) throw err;
console.log("Successfully added article: " + titles[i] + "to the database.");
});
}
console.log("Complete.");
});
};
databaseOperation(sourcesDates, timeAdded, links, titles, descriptions);
}
//// END
DatabaseTime(sourcesDates, timeAdded, links, titles, descriptions);
(printing the index):
You're doing console.log(index); in the first if statement, which prints the numbers.
(undefined problem):
you're trying to access titles[i]. the problem is that neither items nor i are defined here. What do you see in your console?
You're also trying to pass items to the Insert function as the second argument, but the function declaration doesn't include a second argument.
if you want to print the item's name, you could either pass more information to your function like this:
function Insert(autoParams, currentItemsName) {
db.put(autoParams, function(err, data) {
if (err) throw err;
console.log("Successfully added article: " + currentItemsName + " to the database.");
});
and then call it:
Insert(autoParams, titles[i]);
instead of:
Insert(autoParams, titles);
Or you could use the data that is already available to you inside the function and print that, which is the better option in my opinion.
function Insert(autoParams) {
var currentItemsName = autoParams.Item.title
db.put(autoParams, function(err, data) {
if (err) throw err;
console.log("Successfully added article: " + currentItemsName + " to the database.");
});
var prompt = ...
var connection = ...
prompt.start();
var property = {
name: 'yesno',
message: 'approve this screencast?',
validator: /y[es]*|n[o]?/,
warning: 'Must respond yes or no',
default: 'no'
};
connection.queryAsync('SELECT * FROM screencasts WHERE approved = 0')
.spread(function(screencasts) {
screencasts.forEach(function(screencast) {
console.log('Title: "' + screencast.title + '".');
prompt.get(property, function(err, res) {
console.log('Command-line input received:');
console.log(' result: ' + res.yesno);
});
});
});
Aim: To enumerate the screencasts and prompt the user to approve or deny them, interactively.
Problem: I think the problem is that, the loop does not block at all, resulting in the next element being processed too quickly:
How do I wait for the user to input a value before "processing" the next element in the sequence?
Someone mentioned mapSeries, and that got me thinking about the example that shows how to avoid mapSeries and instead use bluebird's map.
What I ended up with is this:
var promise = ...
var prompt = ...
var connection = ...
promise.promisifyAll(prompt);
...
connection.queryAsync('SELECT * FROM screencasts WHERE approved = 0')
.spread(function(screencasts) {
var current = promise.resolve();
promise.map(screencasts, function(screencast) {
current = current.then(function() {
console.log('Title: "' + screencast.title + '".');
return prompt.getAsync(property);
}).then(function(res) {
console.log(res);
});
return current;
});
});
It seems to work well. I am curious, though. Is map the most appropiate function for this?
I'm trying to get my WebSQL database to popluate using a JSON array (Object is called myJSONObject, array is called costcodes).
The function runs on click, the database successfully creates, but the data is not inserted into the database. It doesn't even throw and error, it just doesn't do anything.
My initial thought was that the data isn't escaped properly, but I don't know exactly where/how to escape it. So I'm stumped.
localDB = null;
function initDB()
{
if (localDB==null)
{
var shortName = 'Costcode';
var version = '1.0';
var displayName = 'Costcode';
var maxSize = 217802; // in bytes
localDB = window.openDatabase(shortName, version, displayName, maxSize);
}
}
function buildTable()
{
var sQuery = 'CREATE TABLE IF NOT EXISTS Costcode ('+
'id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,' +
'cost_code_no VARCHAR NULL,' +
'row_unique_no VARCHAR NULL,' +
'cost_class_no VARCHAR NULL,' +
'Table_Version VARCHAR DEFAULT "1.0");';
try
{
initDB();
localDB.transaction(function(transaction)
{
transaction.executeSql(sQuery, []);
console.log('sucess');
});
}
catch (e)
{
alert("Error: Unable to create table 'x" + "' " + e + ".");
return;
}
}
function exeSQLFast()
{
initDB();
localDB.transaction(function(transaction)
{
for (var x = 0; x <myJSONObject.costcodes.length; x++)
{
var costcodeno = myJSONObject.costcodes[x].cost_code_no;
var rowuniqueid = myJSONObject.costcodes[x].row_unique_id;
var costclassno = myJSONObject.costcodes[x].cost_class_no;
console.log(costcodeno);
console.log(rowuniqueid);
console.log(costclassno);
transaction.executeSql('INSERT INTO Costcode (cost_code_no, row_unique_id, cost_class_no) VALUES (? , ? , ?)',
[costcodeno,
rowuniqueid,
costclassno]
, function(transaction, results)
{
console.log(costcodeno);
console.log('hooray');
}
)};
}
)}
</script>
</head>
<body onLoad="buildTable();">
<input type="button" onClick="exeSQLFast();" value='button'>
</body>
</html>
The console log shows that the variables are all being properly defined, but it's not running the insert statement. Any ideas?
Here's an example of myJSONObject.costcodes[2]
cost_class_no: " 3"
cost_code_no: " 1000"
row_unique_id: 335
That looks like a problem doesn't it...
The problem was that I called the row_unique_no column row_unique_id in the insert statement. Now I feel stupid.
But if anyone is curious how to populate a WebSQL database properly, this is how you do it. Just don't mistype the column names.