i have a csv that I want to save into my mySQL Table. My parser works well and it also save the created json Array to my table. My problem is that he insert row for row in the background and don't response it.
My code looks like that:
var file= './mytable.csv';
connection.connect(function (err) {});
var csv = require('csvtojson');
csv({ delimiter:","})
.fromFile(file)
.on('end_parsed', function(jsonArray){
for(var i = 0; i < jsonArray.length; i++){
var post = jsonArray[i]
conn.query('INSERT INTO mytable SET ?', post, function(err, results) {
if (err) throw err;
console.log(result.insertId);
});
}
res.end("done");
})
.on('done', function(error){
console.log('end')
})
My Goal is that my api send: its "done" with (res.json("Done")) when the complete query is done. What should I change?
Greetings
edit: my csv is realy large, with almost 500k rows!
EDIT:
I inserted async into my parser like that:
csv({ delimiter:";"})
.fromFile(file)
.on('end_parsed', function(jsonArray) {
async.forEach(jsonArray, function (jsonArrays, callback) {
conn.query('INSERT INTO mytable SET ?', jsonArrays, callback);
}, function (err) {
if (err) return next(err);
res.json("done");
console.log("done")
});
});
But he don't responde with "done" (in Terminal he write it, but postman give me only "Could not get any response")
Your call to res.end()/ res.json() doesn't wait for all inserts to be finished.
And if you start your inserts within a for-loop you start them all more or less in parallel. You should take look at something like the async library (http://caolan.github.io/async). There you find a eachLimit() function that lets you run async operations on a collection/array. With this function you can limit how many operations can run in parallel. And you get a callback that is called when an error happens or all async calls have finished. Within this callback you can call the res.json(...) function to send your response.
Sample:
var async = require('async');
//...
function save_row_to_db (post, callback) {
conn.query('INSERT INTO mytable SET ?', post, callback);
}
function finished(err) {
if (err) throw err;
res.end("done");
}
async.eachLimit(csvRows, 20, save_row_to_db, finished);
Related
In the code
var stuff_i_want = '';
stuff_i_want = get_info(parm);
And the function get_info:
get_info(data){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
console.log(stuff_i_want); // Yep. Value assigned..
}
in the larger scope
stuff_i_want = null
What am i missing regarding returning mysql data and assigning it to a variable?
============ New code per Alex suggestion
var parent_id = '';
get_info(data, cb){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
return cb(results[0].objid); // Scope is larger than function
}
==== New Code in Use
get_data(parent_recording, function(result){
parent_id = result;
console.log("Parent ID: " + parent_id); // Data is delivered
});
However
console.log("Parent ID: " + parent_id);
In the scope outside the function parent_id is null
You're going to need to get your head around asynchronous calls and callbacks with javascript, this isn't C#, PHP, etc...
Here's an example using your code:
function get_info(data, callback){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
return callback(results[0].objid);
})
}
//usage
var stuff_i_want = '';
get_info(parm, function(result){
stuff_i_want = result;
//rest of your code goes in here
});
When you call get_info this, in turn, calls connection.query, which takes a callback (that's what function(err, results) is
The scope is then passed to this callback, and so on.
Welcome to javascript callback hell...
It's easy when you get the hang of it, just takes a bit of getting used to, coming from something like C#
I guess what you really want to do here is returning a Promise object with the results. This way you can deal with the async operation of retrieving data from the DBMS: when you have the results, you make use of the Promise resolve function to somehow "return the value" / "resolve the promise".
Here's an example:
getEmployeeNames = function(){
return new Promise(function(resolve, reject){
connection.query(
"SELECT Name, Surname FROM Employee",
function(err, rows){
if(rows === undefined){
reject(new Error("Error rows is undefined"));
}else{
resolve(rows);
}
}
)}
)}
On the caller side, you use the then function to manage fulfillment, and the catch function to manage rejection.
Here's an example that makes use of the code above:
getEmployeeNames()
.then(function(results){
render(results)
})
.catch(function(err){
console.log("Promise rejection error: "+err);
})
At this point you can set up the view for your results (which are indeed returned as an array of objects):
render = function(results){ for (var i in results) console.log(results[i].Name) }
Edit
I'm adding a basic example on how to return HTML content with the results, which is a more typical scenario for Node. Just use the then function of the promise to set the HTTP response, and open your browser at http://localhost:3001
require('http').createServer( function(req, res){
if(req.method == 'GET'){
if(req.url == '/'){
res.setHeader('Content-type', 'text/html');
getEmployeeNames()
.then(function(results){
html = "<h2>"+results.length+" employees found</h2>"
html += "<ul>"
for (var i in results) html += "<li>" + results[i].Name + " " +results[i].Surname + "</li>";
html += "</ul>"
res.end(html);
})
.catch(function(err){
console.log("Promise rejection error: "+err);
res.end("<h1>ERROR</h1>")
})
}
}
}).listen(3001)
Five years later, I understand asynchronous operations much better.
Also with the new syntax of async/await in ES6 I refactored this particular piece of code:
const mysql = require('mysql2') // built-in promise functionality
const DB = process.env.DATABASE
const conn = mysql.createConnection(DB)
async function getInfo(data){
var sql = "SELECT a from b where info = data"
const results = await conn.promise().query(sql)
return results[0]
}
module.exports = {
getInfo
}
Then, where ever I need this data, I would wrap it in an async function, invoke getInfo(data) and use the results as needed.
This was a situation where I was inserting new records to a child table and needed the prent record key, based only on a name.
This was a good example of understanding the asynchronous nature of node.
I needed to wrap the all the code affecting the child records inside the call to find the parent record id.
I was approaching this from a sequential (PHP, JAVA) perspective, which was all wrong.
Easier if you send in a promise to be resolved
e.g
function get_info(data, promise){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
promise.resolve(results[0].objid);
}
}
This way Node.js will stay fast because it's busy doing other things while your promise is waiting to be resolved
I've been working on this goal since few weeks, without any result, and I finally found a way to assign in a variable the result of any mysql query using await/async and promises.
You don't need to understand promises in order to use it, eh, I don't know how to use promises neither anyway
I'm doing it using a Model class for my database like this :
class DB {
constructor(db) {
this.db = db;
}
async getUsers() {
let query = "SELECT * FROM asimov_users";
return this.doQuery(query)
}
async getUserById(array) {
let query = "SELECT * FROM asimov_users WHERE id = ?";
return this.doQueryParams(query, array);
}
// CORE FUNCTIONS DON'T TOUCH
async doQuery(queryToDo) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
async doQueryParams(queryToDo, array) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, array, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
}
Then, you need to instantiate your class by passing in parameter to constructor the connection variable given by mysql. After this, all you need to do is calling one of your class methods with an await before. With this, you can chain queries without worrying of scopes.
Example :
connection.connect(function(err) {
if (err) throw err;
let DBModel = new DB(connection);
(async function() {
let oneUser = await DBModel.getUserById([1]);
let allUsers = await DBModel.getUsers();
res.render("index.ejs", {oneUser : oneUser, allUsers : allUsers});
})();
});
Notes :
if you need to do another query, you just have to write a new method in your class and calling it in your code with an await inside an async function, just copy/paste a method and modify it
there are two "core functions" in the class, doQuery and doQueryParams, the first one only takes a string as a parameter which basically is your mysql query. The second one is used for parameters in your query, it takes an array of values.
it's relevant to notice that the return value of your methods will always be an array of objects, it means that you'll have to do var[0] if you do a query which returns only one row. In case of multiple rows, just loop on it.
I have the following code that reads a CSV file and then pulls a document from the database:
fs.readFile process.env.PWD + '/data/errorports.csv', 'utf8', (err, data) ->
if err
console.log "Error reading csv", err
return
rows = data.split('\n')
for row in rows
columns = row.split(',')
airportCode = columns[0]
airport = Airports.findOne({_id: airportCode})
console.log 'airport:', airport
But when I call Airports.findOne({_id: airportCode}) it throws the error:
/Users/abemiessler/.meteor/packages/meteor-tool/.1.3.4.19lp8gr++os.osx.x86_64+web.browser+web.cordova/mt-os.osx.x86_64/dev_bundle/server-lib/node_modules/fibers/future.js:159
throw new Error('Can\'t wait without a fiber');
^
Error: Can't wait without a fiber
Can anyone see why I would be getting this error? Any suggestions on how to get around it?
You probably need to make use of Meteor.wrapAsync() to do this (to make your async function run inside a Fider...which will allow you to execute your meteor code inside). Here is an example.
var syncReadFile = Meteor.wrapAsync(fs.readFile);
syncReadFile(process.env.PWD + '/data/errorports.csv', 'utf8', function(err, data) {
if (err) {
console.log "Error reading csv", err;
return;
}
rows = data.split('\n');
for (row in rows) {
columns = row.split(',');
airportCode = columns[0];
airport = Airports.findOne({_id: airportCode});
console.log 'airport:', airport;
}
});
Or ... you can try to wrap your callback into
Meteor.bindEnvironment()
Would be something like
fs.readFile process.env.PWD + '/data/errorports.csv', 'utf8',
Meteor.bindEnvironment(function(err, data) {
[..your code..]
});
I created a Node.js script that creates a large array of randomly generated test data and I want to write it to a Redis DB. I am using the redis client library and the async library. Initially, I tried executing a redisClient.hset(...) command within the for loop that generates my test data, but after some Googling, I learned the Redis method is asynchronous while the for loop is synchronous. After seeing some questions on StackOverflow, I can't get it to work the way I want.
I can write to Redis without a problem with a small array or larger, such as one with 100,000 items. However, it does not work well when I have an array of 5,000,000 items. I end up not having enough memory because the redis commands seem to be queueing up, but aren't executed until after async.each(...) is complete and the node process does not exit. How do I get the Redis client to actually execute the commands, as I call redisClient.hset(...)?
Here a fragment of the code I am working with.
var redis = require('redis');
var async = require('async');
var redisClient = redis.createClient(6379, '192.168.1.150');
var testData = generateTestData();
async.each(testData, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
});
callback();
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
You could call eachLimit to ensure you are not executing too many redisClient.hset calls at the same time.
To avoid overflowing the call stack you could do setTimeout(callback, 0); instead of calling the callback directly.
edit:
Forget what I said about setTimeout. All you need to do is call the callback at the right place. Like so:
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
callback();
});
You may still want to use eachLimit and try out which limit works best.
By the way - async.each is supposed to be used only on code that schedules the invocation of the callback in the javascript event queue (e.g. timer, network, etc) . Never use it on code that calls the callback immediately as was the case in your original code.
edit:
You can implement your own eachLimit function that instead of an array takes a generator as it's first argument. Then you write a generator function to create the test data. For that to work, node needs to be run with "node --harmony code.js".
function eachLimit(generator, limit, iterator, callback) {
var isError = false, j;
function startNextSetOfActions() {
var elems = [];
for(var i = 0; i < limit; i++) {
j = generator.next();
if(j.done) break;
elems.push(j.value);
}
var activeActions = elems.length;
if(activeActions === 0) {
callback(null);
}
elems.forEach(function(elem) {
iterator(elem, function(err) {
if(isError) return;
else if(err) {
callback(err);
isError = true;
return;
}
activeActions--;
if(activeActions === 0) startNextSetOfActions();
});
});
}
startNextSetOfActions();
}
function* testData() {
while(...) {
yield new Data(...);
}
}
eachLimit(testData(), 10, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
if(err) callback(err);
else {
console.log("Item was persisted. Result: " +reply);
callback();
}
});
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
I got a file newuser.js (node.js environment featuring a mongodb database managed via mongoose) containing the following code:
//newuser.js
//basically creates new user documents in the database and takes a GET parameter and an externally generated random code (see randomcode.js)
[...]
var randomCode = require ('randomcode');
var newTempUser = new tempUser({name: req.body.name, vericode: randomCode.randomveriCode(parameter)
});
newTempUser.save(function (err){
//some output
});
//randomcode.js
//creates a random sequence of characters (=vericode), checks if code already exists in DB and restarts function if so or returns generated code
exports.randomveriCode = function randomveriCode(parameter){
[...]
var TempUser = conn.model('TempUser', TempUserSchema);
TempUser.count({vericode: generatedcode}, function(err, counter){
if (counter=='0'){
return generatedcode;
}else{
randomveriCode(parameter);
}
});
};
Problem is, that newuser.js throws an error as variable vericode is 'undefined' (thus mongoose model validations fails). The error does not occur if I skip the database query and instantly return the generated code (which in fact has got a value as verified by several console.log instructions). It occurs to me that the db query takes to long and empty or null value returned before query is complete? I thought about introducing promises unless you got any other suggestions or hints what may cause this behaviour?
Kind regards
Igor
Since querying the database is a non-blocking operation, you cannot expect the function call to return the value from the database immediately. Try passing in a callback instead:
// newuser.js
var randomCode = require('randomcode');
randomCode.randomveriCode(parameter, function(err, code) {
if (err) throw err; // TODO: handle better
var newTempUser = new tempUser({name: req.body.name, vericode: code});
newTempUser.save(function (err){
//some output
});
});
// randomcode.js
exports.randomveriCode = function randomveriCode(parameter, cb) {
var TempUser = conn.model('TempUser', TempUserSchema);
TempUser.count({vericode: generatedcode}, function(err, counter) {
if (err) return cb(err);
if (counter == '0') {
cb(null, generatedcode);
} else {
randomveriCode(parameter, cb);
}
});
};
your randomveriCode function contains calls to an asynchronous function and therefore, your function really needs to provide a callback argument like this:
exports.randomveriCode = function randomveriCode(parameter, callback){
[...]
var TempUser = conn.model('TempUser', TempUserSchema);
TempUser.count({vericode: generatedcode}, function(err, counter){
if(err) return callback(err);
if (counter=='0'){
return callback(null, generatedcode);
}else{
randomveriCode(parameter, callback);
}
});
};
You'd then call it like so:
var randomCode = require ('randomcode');
randomCode(function(err, vericode){
if(err) throw err;
var newTempUser = new tempUser({name: req.body.name, vericode: vericode});
newTempUser.save(function(err,newUser){
//do something here
});
});
Btw - you could also use a synchronous function to create a GUID. See https://www.npmjs.org/package/node-uuid.
I have frustrating problem with learning to work with callback style of programming in Node.js. I have a query to a MongoDB database. If I pass in a function to execute on the result it works but I'd rather flatten it out and have it return the value. Any help or direction on how to do this correctly is appreciated. Here's my code:
var getLots = function(response){
db.open(function(err, db){
db.collection('lots', function(err, collection){
collection.find(function(err, cursor){
cursor.toArray(function(err, items){
response(items);
})
})
})
})
}
I want something more like this:
lots = function(){
console.log("Getting lots")
return db.open(openCollection(err, db));
}
openCollection = function(err, db){
console.log("Connected to lots");
return (db.collection('lots',findLots(err, collection))
);
}
findLots = function(err, collection){
console.log("querying 2");
return collection.find(getLots(err, cursor));
}
getLots = function(err, cursor) {
console.log("Getting lots");
return cursor.toArray();
}
Where the final set of data would bubble back up through the function calls.
The problem is that I get an error from Node.js saying that err is not defined or that the collection is not defined. For some reason when I nest the callbacks the correct object is getting passed down. When I try going to this flattened style it complains that things are not defined. I don't know how to get it to pass the necessary objects.
What you need is one of the many control flow libraries available for node via npm and catalogued on the Node.js wiki. My specific recommendation is caolan/async, and you would use the async.waterfall function to accomplish this type of flow where each async operation must be executed in order and each requires the results from the previous operation.
Pseudocode example:
function getLots(db, callback) {
db.collection("lots", callback);
}
function findLots(collection, callback) {
collection.find(callback);
}
function toArray(cursor, callback) {
cursor.toArray(callback);
}
async.waterfall([db.open, getLots, find, toArray], function (err, items) {
//items is the array of results
//Do whatever you need here
response(items);
});
async is a good flow control library. Frame.js offers some specific advantages like better debugging, and better arrangement for synchronous function execution. (though it is not currently in npm like async is)
Here is what it would look like in Frame:
Frame(function(next){
db.open(next);
});
Frame(function(next, err, db){
db.collection('lots', next);
});
Frame(function(next, err, collection){
collection.find(next);
});
Frame(function(next, err, cursor){
cursor.toArray(next);
});
Frame(function(next, err, items){
response(items);
next();
});
Frame.init();