So i have a csv file containing my information, i need to do a mass add/update
exports.add_questions_from_file = function (file_path, surveyid, callback)
{
var U = [{}];
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
console.log(data);
d = data.split(/\r\n|\n/);
for (x=0;x <d.length;x++)
{
line = d[x].split(',');
if (line[0] == "") {return};
RQuestion.add_by_line (line,function (err, question)
{
U.push({id:question.id});
console.log(U);
});
}
}
});
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":U}},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
};
But even though im using callback functions the update seems to happen with the same object always, even the console.log here
U.push({id:question.id});
console.log(U);
returns the same object (even that all the other were created)
Im doing something wrong?
I see a few issues.
First for:
if (line[0] == "") {return};
Don't you mean to use a break or continue instead? Otherwise the entire function will quit if there is a blank line anywhere in the file. This is very important because Survey.update won't get called either.
Second: I assumed that RQuestion.add_by_line and Survey.update are doing something async like updating a database. Your code needs to be restructured to wait for those async items to complete before moving on to the next step. I'd recommend an npm package named async for that.
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
d = data.split(/\r\n|\n/);
async.map(d, function(line, callback) {
//this function is called for each line
add_by_line (line,function (err, question)
{
callback(err,{id:question.id});
});
}, function(err, results) {
//this function is called when all of the items are done
console.log("done with async");
console.dir(results);
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":results},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
});
}
});
Related
I have my text data like this and I mark * charset to check unfinished job.
And here's my part of code.
I want to make it when txt lines are start with * charset, it should be processed and after that processed string's * charset should be removed.
fs.readFile('data.txt', async function (err, data) {
if (err) throw err;
let array = data.toString().split("\n");
for (i in array) {
if (array[i].charAt(0) === '*') {
console.log(`Now Processing : ${array[i]} | ${array.length - i -1} items left`);
//
// SOME JOBS
//
let newValue = array[i].replace('*', '');
fs.writeFile('data.txt', newValue, 'utf-8', function (err, data) {
if (err) throw err;
console.log('Done!');
})
} else {
console.log(`${array[i]} Already Captured`)
}
From what I understand you are trying to read through a file, find lines containing "*" character, doing some work, then removing the * from the affected lines.
Firstly, the call to fs.writeFile is happening inside a loop, so every iteration of that loop calls the writeFile function. From the nodejs docs this method will "... asynchronously write data to the file, replacing the file if it already exists." You are replacing the file with every iteration. What you want to do is use fs.appendFile or better yet pass the 'append' system flag to writeFile. System flags can be seen here. Take a look at the 'a' flag and pass it in the options object of writeFile.
Your usage of async in the readFile callback is incomplete also as you don't call await within that callback.
fs.readFile('data.txt', async function (err, data) {
if (err) throw err;
let array = data.toString().split("\n");
for (i in array) {
if (array[i].charAt(0) === '*') {
console.log(`Now Processing : ${array[i]} | ${array.length - i -1} items left`);
//
// SOME JOBS
//
let newValue = array[i].replace('*', '');
newValue = newValue + '\n';
await fs.appendFile('data.txt', newValue, 'utf-8', function (err, data) {
if (err) throw err;
console.log('Done!');
})
} else {
console.log(`${array[i]} Already Captured`)
}
}
});
I am new to NodeJs and I'm finding the Non Blocking and Asynchronous nature of JS extremely difficult to understand and handle,
I have a piece of code which is supposed to Iterate an array
and for every iteration, I'm supposed to make a DB update.
Can someone provide the correct implementation of Async library functions and help fix my code?
Code example -
function updateFunction(conn, requestBody, callback) {
let arr = [];
async.each(requestBody.arr, function(item, callback) {
let sqlData = []
let columns = "";
if(item.columnData != null){
sqlData.push(item.columnData);
columns += "`columnName` = ?,";
}
if(columns != ''){
columns = columns.substring(0,columns.length-1);
let sqlQuery = 'UPDATE someTable SET '+columns
+' WHERE id = "' + item.id + '"';
conn.query(sqlQuery, sqlData, function (err, result) {
if (err) {
return callback(err, false);
}
})
}
else{
return callback(null, false);
}
columns = "";
sqlData = [];
},
function(err, results) {
//Code never reaches here, don't know why
if (err) {
return callback(err, false);
}
else{
return callback(null, true);
}
});
} // END
During your database query call, on a successful query your callback is not called, therefore causing your code to never reach the final callback.
You will want to add another return statement after your if (err) { return callback(err); } to let async know your database query is finished.
And another thing, according to the docs, the async each method's final callback does not invoke with results in its callback.
A callback which is called when all iteratee functions have finished, or an error occurs. Invoked with (err).
Therefore, it is not required for you to pass a value into the callback statement within your iteratee function.
Modify your code to do this and it will work.
conn.query(sqlQuery, sqlData, function (err, result) {
if (err) {
return callback(err);
}
return callback(null);
})
Hope this helps.
conn.query(sqlQuery, sqlData, async function (err, result) {
if (err) {
return await callback(err, false);
}
})
Something like this.. so the function callback is async here and we gave await which actually waits until the return call is finished..
this is a follow up question to Asynchron Errorhandling inside $.each. As mentioned in the comments there, i want to handle data after the last async job from a $.each loop.
So for instance:
var errors = 0;
var started = 0;
var successful = 0;
$.each(..., function(){
started++;
connection.query('INSERT INTO tableName SET ?', post, function(err, result)
{
if (err) {
if (err.code === 'ER_DUP_ENTRY')
{ errors++; }
else
{ throw err; }
} else { successful++;}
if (started == successful + errors) {
// all done
console.log(errors + " errors occurred");
}
});
});
In this case everything logs out properly when the // all done comment is reached. But what if i want to use this data later on instead of just logging it out.
Is there a way to wait for this data outside of the $.each scope? Or do i always have to handle everything in the nested function?
You can use promises instead
var promises = [];
$.each(..., function() {
var promise = new Promise(function(resolve, reject) {;
connection.query('INSERT INTO tableName SET ?', post, function(err, result) {
if (err) {
resolve(err.code);
} else {
resolve(result);
}
});
});
promises.push(promise);
});
var result = Promise.all(promises);
And then when you want to use the data, you do
result.then(function(data) {
// use data array looking like ["result data", "result data", "ER_DUP_ENTRY" .. etc]
})
I have updated the post with the actual code.
The problem is that the node app hangs and does not exit unless I comment out the query in addArticle. I am wonder what I'm doing wrong here (in regards to the hanging problem).
function addArticle(title, text, date, link) {
connection.query("SELECT * FROM articles WHERE link LIKE "+connection.escape(link), function(error, rows, fields) {
if(rows.length == 0) {
console.log("article not in database");
console.log(connection.escape(title));
var values = [connection.escape(title), connection.escape(text), date, connection.escape(link), '{}'];
connection.query("INSERT INTO articles (title, text, date, link, topics) VALUES ?", [[values]], function(err) {
if(err) throw err;
});
}
});
}
function scrapeReuters() {
var url = 'http://www.reuters.com/news/archive/technologyNews?date=10092013';
request(url, function(err, resp, body){
$ = cheerio.load(body);
links = $('a');
$(links).each(function(i, link){
var addr = $(link).attr('href');
if(addr != undefined && addr.indexOf('article') != -1 && addr.indexOf('http') == -1 ) {
var full_link = "http://www.reuters.com"+addr;
var title = $(link).text();
request(full_link, function(err, resp, body){
$ = cheerio.load(body);
para = $('p').text();
addArticle(title, para,new Date().getTime(), full_link);
});
}
});
});
}
You probably need to close the connection after all the queries have finished. You can try using the https://github.com/caolan/async library to run the queries in sequence and then in a master callback, close the connection.
Its a little tricky, but first you need to define an array of functions to execute. Then you run async.sequence(arrayOfFns,masterCallback). The master callback gets errs and results (notice plural, its from all the functions). In that master callback, terminate the mysql connection/and or end the process.
To do this, I would rewrite the addArticle query to just return the query string. Then before your $(links).each loop, I would make an array called toInsert
In each loop I would say
toInsert.push(function(callback) {
connection.query(addArticle(...),function(err) {
if(err) callback(err);
else callback(null,true);
});
});
Then after the loop run
async.sequence(toInsert,function(errs,results) {
connection.close() //not sure if correct
process.exit(); //maybe, if needed?
});
I am just starting out with mongodb, but I am running into a problem when trying to use .find() on a collection.
I've created a DataAccessObject which opens a specific databate and then lets your perform operations on it. Here is the code:
The constructor:
var DataAccessObject = function(db_name, host, port){
this.db = new Db(db_name, new Server(host, port, {auto_reconnect: true}, {}));
this.db.open(function(){});
}
A getCollection function:
DataAccessObject.prototype.getCollection = function(collection_name, callback) {
this.db.collection(collection_name, function(error, collection) {
if(error) callback(error);
else callback(null, collection);
});
};
A save function:
DataAccessObject.prototype.save = function(collection_name, data, callback){
this.getCollection(collection_name, function(error, collection){
if(error) callback(error);
else{
//in case it's just one article and not an array of articles
if(typeof (data.length) === 'undefined'){
data = [data];
}
//insert to collection
collection.insert(data, function(){
callback(null, data);
});
}
});
}
And what seems to be the problematic one - a findAll function:
DataAccessObject.prototype.findAll = function(collection_name, callback) {
this.getCollection(collection_name, function(error, collection) {
if(error) callback(error)
else {
collection.find().toArray(function(error, results){
if(error) callback(error);
else callback(null, results);
});
}
});
};
Whenever I try to dao.findAll(error, callback), the callback never gets called.
I've narrowed the problem down to the following part of the code:
collection.find().toArray(function(error, result){
//... whatever is in here never gets executed
});
I've looked at how other people do it. In fact, I'm following this tutorial very closely. No one else seems to have this problem with colelction.find().toArray(), and it doesn't come up in my searches.
Thanks,
Xaan.
You are not using the open callback so if you are trying to make the findall request right after creating the dao then it won't be ready.
If your code is like this, it will not work.
var dao = new DataAccessObject("my_dbase", "localhost", 27017);
dao.findAll("my_collection",function() {console.log(arguments);});
I tested it and it doesn't find records, and it also gives no error. I think it should give an error.
But if you change it so that you give a callback to the constructor, then it should work.
var DataAccessObject = function(db_name, host, port, callback){
this.db = new Db(db_name, new Server(host, port, {auto_reconnect: true}, {}));
this.db.open(callback);
}
And make your code like this.
var dao = new DataAccessObject("my_dbase", "localhost", 27017, function() {
dao.findAll("my_collection",function() {console.log(arguments);});
});