I have a table in my mysql db with columns id and drivers_license_number. Now data in drivers_license_number column is stored in plaintext, and I want to encrypt it.
This table contains about 400000 records. I have read about problems with LIMIT and OFFSET issue and try to use in my query late row lookups.
So first of all I need to get data from db, then encrypt the right field and update db with this encrypted field. I don't understand, how to organize my code to send parameters with limit and offset to db. I need to work in loop in this situation?
function getUpdatedEncryptedField(db_table_name, field_name) {
return getDataFromDb(db_table_name, field_name).then(function(result){
return encryptData(result, field_name).then(function (result){
var promises = result.map(function(item) {
var data = [item[field_name], item.id];
return updateFieldNameData(db_table_name, field_name, data);
});
return q.all(promises);
});
});
}
After the first pass I will get for example first 1000 records and how to move forward to get another 1000 rows?
function getDataFromDb(db_table_name, field_name, limit, offset) {
var sql = 'SELECT ds.id, ' + field_name + ' FROM ( SELECT id FROM ' + db_table_name + 'WHERE ' + field_name +
' IS NOT NULL ORDER BY id LIMIT ' + limit + ', ' + offset + ') d JOIN ' + db_table_name + ' ds ON ds.id = d.id ORDER BY ds.id;'
return db.modio.executeSql(sql);
}
You can use a carring/high order function to update the chunks by e.g 1000 and get the next 1000 records until the 400000, like so:
function getUpdatedEncryptedField(db_table_name, field_name) {
var idFrom = 0;
var idTo = 1;
return function getDataFromDB(db_table_name, field_name){
idFrom = idTo;
idTo += 1000;
console.log(id range is ${idFrom}-${idTo}`);
// call your db to get the recods
}
}
// assign high order func to updateChunks var, which is again a func
var updateChunks = getUpdatedEncryptedField('studentsDB','creditCard')
// so each time you execute updateChunks() the values idFrom & idTo are updated
// updateChunks()
// id range is 1-1001
// updateChunks()
// id range is 1001-2001
// and so on
// create a loop until you get all the table records, e.g.
// pay attention to define 'i' with let and not with var because of async loop
for(let i=0; i <= 400000/1000; i++){
// call the updateChunk func here
// at eaach itteration it will ask for the next chunk of records
updateChunks()
}
`
Of course you should update a little bit your mysql query, to get the range of Ids from - to, this is out of the question scope.
Hope helps, good luck.
Related
I'm new to Firebase, and I'm going to build a list of item collected from server with following criterias:
max 10 items
order by item value (eg. 'count')
this is my code
FBRef.orderByChild("count").limitToLast(10).on("child_added", function(snapshot){}
and it worked fine, but when a new record inserted, is there a Firebase method that can check and auto update the list if the new item is in top 10 ? Or I need to to it myself ?
var FBRef = new Firebase('yours.firebaseio.com');
FBRef.orderByChild("count").limitToLast(5).on("child_added", function(snapshot){
var data = snapshot.val();
var html = "<li>value: " + data.value + " - count: " + data.count + "</li>";
$("#result").prepend(html);
});
$(document).ready(function () {
$("#inputaddbtn").click(add);
});
function add(){
var value = $("#inputvalue").val();
var count = $("#inputcount").val();
$("#inputcount").val("");
$("#inputvalue").val("");
FBRef.push({
value : value,
count : count
})
}
You'll need to monitor more events and handle more arguments to keep the list limited.
As a first step: Firebase will fire a child_removed event when an element has been removed from the location or (in your case) falls outside of the query.
You can handle that event to remove the child from the list:
var FBRef = new Firebase('yours.firebaseio.com');
var query = FBRef.orderByChild("count").limitToLast(5);
query.on("child_added", function(snapshot){
var data = snapshot.val();
var html = "<li id='key_"+snapshot.key()+"'>value: " + data.value + " - count: " + data.count + "</li>";
$("#result").prepend(html);
});
query.on("child_removed", function(snapshot){
$('#key_'+snapshot.key()).remove();
});
You'll note that I give the li an id when the child is added and then remove the li based on that id when Firebase tells me to.
This covers simple adding/removing items. For a complete app that handles all cases, you should also handle child_changed and child_moved. And you'll probably also want to handle the second argument to the callbacks, which indicates the key of the item that the child needs to go after. But maybe your use-case doesn't require handling these cases, in which case the above is enough.
I'm trying to write one row of data into a Google Cloud SQL database, using a prepared statement as described here:
https://developers.google.com/apps-script/guides/jdbc
I have the data stored in an object called valuesByTitle, which looks like this:
{NAME: 'fun event', LOCATION: '123 Main St.'}
The data is inserted into the SQL table if I write everything out like this:
var conn = getConnection(); // connects to db
var stmt = conn.prepareStatement('INSERT INTO events '
+ '(NAME, LOCATION) values (?, ?)');
stmt.setString(1, valuesByTitle['NAME']);
stmt.setString(2, valuesByTitle['LOCATION']);
stmt.execute();
}
However, I would like to automate the process, so I don't have to change the code each time the variables change.
I have two new variables, one for the column names and one for a set of place holders (e.g. "?") for the insert statement.
var columns = Object.keys(valuesByTitle);
var placeholders = columns.map(function(column) {
return "?";
};
Using these, this prepared statement should automatically insert whatever is in the valuesByTitle object (assuming all the values are string values):
var conn = getConnection();
var stmt = conn.prepareStatement("INSERT INTO events (" + columns.join(",") + ") VALUES (" + placeholders.join(",") + ")");
for (i = 0; i < columns.length; i++) {
stmt.setString(i+1, valuesByTitle[columns[i]]);
}
stmt.execute();
For some reason, it's not inserting the data. Surprised not to find any examples either. Is the logic off or is it just not possible to do?
Thanks.
All your code is OK, you simply forgot to close the map() bracket i.e.
var columns = Object.keys(valuesByTitle);
var placeholders = columns.map(function(column) {
return "?";
};
should be
var columns = Object.keys(valuesByTitle);
var placeholders = columns.map(function(column) {
return "?";
});
(note the ending bracket after return "?";})
I've one Restful service developed using node.js which takes some values from a user. According to the values given it reformats a query string. But, Whenever I want to pass this query to findone() of mongoDB and calls that service it showing "query selector must be an object" message on browser.
var query = "{" ;
if( orderNo == " "){
// Don't append anything
}else{
query = query + "'orderNo' : " + orderNo ;
}
.
.
.
query = query + "}";
And, I also tried like below :
var query = {};
query[orderNo] = orderNo;
db.collection('*****').findOne(query, function(err, item) {
console.log(item);
res.jsonp(item);
});
In both cases I'm getting the same result in browser.
Anyone guide me how I can query these kind of queries in MongoDB.
You were almost there on your second attempt. When you do this
var query = {};
query[orderNo] = orderNo;
you create an object with key name the same as the value! For example, if the value of orderNo is 12345, then the above statement will produce the object
query = {
"12345": 12345
}
You could use the same bracket notation to create a proper query object property
query = {
"orderNo": 12345
}
as follows:
var query = {};
if(orderNo != " ") query["orderNo"] = orderNo;
db.collection('*****').findOne(query, function(err, item) {
console.log(item);
res.jsonp(item);
});
This question already has answers here:
JavaScript closure inside loops – simple practical example
(44 answers)
Closed 8 years ago.
I'm quite frustrated by a seemingly simple sqlite problem. I have a table with 3 records on the client. After collecting these records up and making a trip to the server to insert them, I have a cleanup function that hunts down new records on the server and runs deletes on the client in preparation to add them back in with server-specific ids. I don't necessarily like deleting, when updating would be preferable, however, the web-based app I've written may be accessed from multiple iPad devices and users, and we need a way to keep the client data in sync with the server. I have tried various methods that became problematic once offline possibilities were factored in.
So, to the code. I collect ids and perform a delete (that multiple_records_deleted is a boolean used to indicate to a setTimeout when the delete statement has completed).
ids = ids.substring(0,ids.length-1);
db.transaction(function (tx) {
tx.executeSql("DELETE FROM " + table + " WHERE id IN (" + ids + ")", [], function (tx, results) {
console.log("DELETE FROM " + table + " WHERE id IN (" + ids + ")");
multiple_records_deleted = true;
});
});
Then, the goal is to insert new records that may or may not be identical (there are keys and timestamp fields and other complex relationships crucial to the app that only the server has.)
sqlInserts[0] = "sql to insert first record";
sqlInserts[1] = "sql to insert second record";
sqlInserts[2] = "sql to insert third record";
function insertMultipleRecords() {
for (var x = 0; x < sqlInserts.length - 1; x++) {
db.transaction(function (tx) {
tx.executeSql(sqlInserts[x],[]);
}, badCB, goodCB);
}
}
The long and short of my problem, is that every time I run this code, only the last item in the array seems to execute. I have tried just about every variation of db.transaction I can think of. used timers. used other callbacks. but to no avail. The weird thing is that I run a similar code block in another part of the app and it works fine.
Any ideas why it is only ever the last item in the array that successfully runs, while the rest seem to go nowhere?
Thanks..
Robin
EDIT:
Here is a follow up snip of code.
for (var rec in response.table.records.record) {
db.transaction(function (tx) {
console.log("DELETE FROM " + table + " WHERE id = " + response.table.records.record[rec].f[record_idnumber_fld_ctr]);
tx.executeSql("DELETE FROM " + table + " WHERE id = " + response.table.records.record[rec].f[record_idnumber_fld_ctr], [], function (tx, results) {
// then insert
fld_ctr = 0;
setTableColumns("table_columns",table); // ensure we are using the right table
sql = "INSERT INTO " + table + " (";
for (x = 0; x < table_columns.length; x++) {
sql += table_columns[x] + ",";
}
sql = sql.substring(0,sql.length-1) + ") ";
sql += "SELECT ";
for (var fld in response.table.fields.field) {
var obj = response.table.fields.field[fld];
for (var prop in obj) {
if (prop === "#attributes") { // cast the data
sql += formatData(obj[prop].field_type,response.table.records.record[rec].f[fld_ctr]) + ",";
}
}
fld_ctr++;
}
// special cases for certain tables
if (table == "orders") {
if (response.table.records.record[rec].f[43] != "") sql += formatData("string","YES") + ","; // siggy
else sql += formatData("string","") + ",";
}
if (table == "pictures") {
sql += formatData("string","") + ","; // datauri
}
sql += "'SYNCED',";
sql += "'NO',";
sql += formatData("integer",response.table.records.record[rec].f[record_idnumber_fld_ctr]) + ",";
sql += "'VALID',";
sql += "'NO ERRORS'";
console.log(sql);
db.transaction(insertRecords(sql), errorHandler);
});
});
}
I run through a resultset of records from the server. And for each one, I attempt to delete from the table where a key is the same. Both console.logs display the same verbiage all 3 times... matching the last one in the array. I'm sure this is a closure problem, just not sure how to attack it.
It is the classic issue where the variable is just a reference and not a snapshot in time. So by the time the callback is executed, the value of x has been changed. So you need to either pull out the function into a separate function or use a closure.
function runTrans (x) {
db.transaction(function (tx) {
tx.executeSql(sqlInserts[x],[]);
}, badCB, goodCB);
}
function insertMultipleRecords() {
for (var x = 0; x < sqlInserts.length - 1; x++) {
runTrans(x);
}
}
or
function insertMultipleRecords() {
for (var x = 0; x < sqlInserts.length - 1; x++) {
(function(x) {
db.transaction(function (tx) {
tx.executeSql(sqlInserts[x],[]);
}, badCB, goodCB);
}(x));
}
}
I am trying to create a simple page that takes a binary file and inserts the values into a Web SQL database.
This is the function I am using to insert the data:
function bin2dbfunc()
{
var result, n, aByte, byteStr;
var i=0;
var sql = new Array();
result = fr.result; //Input file
for (n = 0; n < result.length; ++n)
{
aByte = result.charCodeAt(n);
byteStr = aByte.toString(16);
if (byteStr.length < 2)
{
byteStr = "0" + byteStr;
} //Format to add leading 0 for hex values
//Looping through taking each byte read from file and adding to array
//sql[i] = aByte; //Value
sql[i] = byteStr; //String
//When completed one row of database run single SQL insert statement with array contents
if(i==15)
{
i=0; //Clear counter for next row
db.transaction(function (tx)
{
tx.executeSql('INSERT INTO binary_data VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [sql[0], sql[1], sql[2], sql[3], sql[4], sql[5], sql[6], sql[7], sql[8], sql[9], sql[10], sql[11], sql[12], sql[13], sql[14], sql[15]]);
}, function (tx, err) {
document.getElementById("result3").innerHTML += 'ERROR '; //Display error message if SQL not run successfully
});
}
else
{
i++; //Otherwise increment counter
}
}
}
I have stripped back the code to remove all my debug messages but essentially the code appears to run. I am using a binary file with 6 rows worth of data, the code however inserts the last row of data 6 times into the database.
Can anyone spot where I am going wrong?
Variables are captured by reference and not by value in JavaScript closures (this is true for objects such as arrays but for primitive values as well); this means that the functions passed to db.transaction will use the current value of sql when they run. It seems that they run asynchronously in your case, thus using the value of sql once bin2dbfunc has returned (which explains why you end up with the last row of data).
You have to copy the value of sql in order to make sure that the right one is used:
function transactionCallback(sql_) {
var sql = sql_.slice(0); // copy
return function(tx) {
tx.executeSql(…, sql);
};
}
db.transaction(transactionCallback(sql), …);
or
function transactionCallback(sql) {
return function(tx) {
tx.executeSql(…, sql);
};
}
db.transaction(transactionCallback(sql), …);
sql = [];
as you don't need sql's values afterwards in bin2dbfunc; this second solution would also enable you to get rid of i and use a more elegant solution based on push and length.