Develop Reference

Node.JS RemoteExec call not firing properly

Querying a database for a list of servers to perform a command on. The array is populated properly and echos out as planned, but none of the connections occur. I tried both passing the array directly into rexec and looping through a forEachAsync. Neither process the server list properly. Am I referencing the array elements improperly?
Mind the syntax errors at the end, I was just trying to include both methods I tried.
#!
var mysql = require('mysql');
var resultset = require('node-array');
var rexec = require('remote-exec');
var fs = require('fs');
var _ = require('lodash');
//var streamBuffers = require('stream-buffers');
var moment = require('moment');
var util = require('util');
var now = moment().format('YYYYMMDD_HHmmss');
var logStdout = process.stdout;
var errStderr = process.stderr;
console.log = function () {
logStdout.write(util.format.apply(null, arguments) + '\n');
}
console.error = function () {
errStderr.write(util.format.apply(null, arguments) + '\n');
}
var connection = mysql.createConnection({
host : 'abc',
user : 'user',
password : '******',
database : 'db'
});
var ssh_options = {
port: 22,
username: 'e109gh',
privateKey: fs.readFileSync('R:/nodeJS/sshkey.priv'),
stdout: fs.createWriteStream('./out.txt'),
stderr: fs.createWriteStream('./err.txt')
}
var my_conn_options = _.clone(ssh_options);
var cmds = ['hostname -i'];
connection.query('SELECT name FROM server', function(err, rows) {
rows.forEachAsync(function(element, index, array) {
console.log(element.name);
rexec(element.name,cmds,my_conn_options,function(err){
if (err) {
now = moment().format('YYYY-MM-DD HH:mm:ss');
console.error(err);
} else {
console.log("it worked for "+element.name);
}
});
});
});
// var buffer = new streamBuffers.WritableStreamBuffer();
connection.end(function(err) {});
// my_conn_options.stdout = buffer;
//
// rexec(rows,cmds,my_conn_options,function(err){
// if (err) {
// now = moment().format('YYYY-MM-DD HH:mm:ss');
// console.error(err);
// } else {
// console.log()
// }
// });
//
//});

Querying embedded mongodb with node js

I try to fetch a value from my age collection and check by applying the condition if(age===5) then it will fetch data again from other collection question and send the data to the browser. But it will show me the error
Error While Saving the result TypeError: Cannot read property 'age_group_5' of undefined
Code:
1). Node js
var agegroupQuiz = require('../models/agegroupSchema.js');
var questionQuiz = require('../models/question.js');
exports.agegroupController = function(req,res,next){
try{
//var age=5;
var queryObj = {};
var projection1 = '-_id, age_group.age_group_5.max_age';
var projection2 = '-_id question';
//agegropup Schema
var a = agegroupQuiz.findOne(queryObj,projection1);
console.log(a.age_group.age_group_5.max_age);
var age = a.age_group.age_group_5.max_age;
if(age===5){
//question Schema
questionQuiz.find(queryObj,projection2,function(err, data){
if(err){
console.log('getQuestions : Error while getting Questions ' + err);
return next(err);
}
//console.log(question);
res.send(data);
});
}else{console.log("error");}
}catch(err){
console.log('Error While Saving the reuslt ' +err);
return next(err);
}
}
/* exports.agemethod = function(req, res, next){
}*/
2). Mongodb Schema
a). var mongoose = require('mongoose');
module.exports = (function question () {
var Schema = mongoose.Schema;
var question = new Schema({
question:{type:Array,
_id:{type:Number},
title:{type:String},
options:{type:Array},
result:{type:Array},
feedback:{type:String}
},
metadata:{
type:String,
category:String,
age_group:String,
location:String
}
});
var results = mongoose.model('userquestion', question);
return results;
})();
b). var mongoose = require('mongoose');
module.exports = (function agegroup () {
var Schema = mongoose.Schema;
var agegroup = new Schema({
age_group : {
age_group_5: {
_id:{type:String},
max_age:{type:Number}
}
}
});
var results = mongoose.model('age', agegroup);
return results;
})();

I would recommend you try this code and hopefully it should work.
var agegroupQuiz = require('../models/agegroupSchema.js');
var questionQuiz = require('../models/question.js');
exports.agegroupController = function(req,res,next){
try{
//var age=5;
var queryObj1 = {}; //for max age 5
var queryObj2 = {}; //for question
queryObj1 = {
_id: //id,
maxAge: 5 //please put the key in accordance to your schema
};
queryObj2 = {
_id: //id,
question: //question
}
//agegropup Schema
return agegroupQuiz.findOne(queryObj1)
.then(function(maxAge){
var age = maxAge;
if(age == 5){
questionQuiz.find(queryObj,projection2,function(err, data){
if(err){
console.log('getQuestions : Error while getting Questions ' + err);
}
//console.log(question);
res.send(data);
});
}else{
console.log("Error");
}
}).catch(function(error){
console.log('Error While Saving the reuslt ' +err);
});
}
}
I may have missed some curly braces, please verify that, and ignore the indentation.
make sure you use the correct keys of your model.

NodeJs need to wrap function in a callback

I am still fairly new to NodeJS. I realize I need to put the following code in a callback so I can wait for the database call to finish, but I can't see how to get it to work.
I need to return my result to the parent function where marked at the bottom of this code:
exports.gtoken = function(req, sfinfo) {
var outcome = {};
req.app.db.models.Account.findById(req.user.roles.account.id, 'isVerified extraemail search memberid name').exec(function(err, account) {
if (err) {
return console.error(err + 'issue');
}
outcome.account = account;
var memberid = JSON.stringify(outcome.account);
var g = (new Date().getTime())/1000;
var n = (g + 86400);
var memberid = outcome.account.memberid;
var secondaryemails = outcome.account.extraemail;
var isVerified = outcome.account.isVerified;
var IDName = outcome.account.name;
var payload = {
"email": req.user.email ,
"exp": n,
"iss": req.app.locals.issid ,
"userid": req.user.id ,
"signedin": 'true' ,
"memberid": memberid ,
"secondaryemails": secondaryemails ,
"isVerified": isVerified ,
"IDName": IDName ,
"sfinfo": sfinfo,
};
var token = jwt.encode( payload, req.app.config.cryptoKey, 'HS512' );
console.log(token);
return token;
});
// I need to return my result here to the parent function
};

Without more details, it's difficult to guess what this method is doing and what value do you expect to be returned.
I believe, however, you're trying to get the token with this method. So here's what you might be looking for:
exports.gtoken = function(req, sfinfo, callback) {
req.app.db.models.Account.findById(req.user.roles.account.id, 'isVerified extraemail search memberid name').exec(function(err, account) {
if (err) {
callback(err, null);
} else {
var outcome = {};
outcome.account = account;
var memberid = JSON.stringify(outcome.account);
var g = (new Date().getTime()) / 1000;
var n = (g + 86400);
var memberid = outcome.account.memberid;
var secondaryemails = outcome.account.extraemail;
var isVerified = outcome.account.isVerified;
var IDName = outcome.account.name;
var payload = {
"email": req.user.email,
"exp": n,
"iss": req.app.locals.issid,
"userid": req.user.id,
"signedin": 'true',
"memberid": memberid,
"secondaryemails": secondaryemails,
"isVerified": isVerified,
"IDName": IDName,
"sfinfo": sfinfo,
};
var token = jwt.encode(payload, req.app.config.cryptoKey, 'HS512');
callback(null, token);
}
});
};
And here's how you could use the method:
gtoken(req, sfinfo, function(err, token) {
if (err) {
console.error(err + 'issue');
} else {
console.log(token);
}
});

Reading a file line by line, parse them and insert them in mongo in node js

I have a file which is tab separated. It has thousands of data. How can I use nodeJs to read the file, line by line, parse them and create an object and insert them in a mongo DB.
I am just learning node and mongo. I come from different background. So how can this be done.
Finally the Mongo DB has to be populated with proper data.
I searched in net but I could not find the complete solution.
Thanks.

I had an issue with the answer by Juvenik. My problem was that the database would not be populated by the time readline had completed. The lines were being read synchronously, but the DB insertion was asynchronous.
Instead, I found a simpler solution with the line-reader package. It reads the lines and waits for a callback before continuing.
var MongoClient = require('mongodb').MongoClient
var dbName = 'yourDbName'
var url = 'mongodb://localhost:27017/' + dbName
var collectionName = 'yourCollectionName'
var filename = 'yourFileName.txt'
var printLine = 1000
MongoClient.connect(url, function(err, db) {
if (err) {
console.error('Problem connecting to database')
} else {
console.log('Connected correctly to server.')
var lineReader = require('line-reader')
var collection = db.collection(collectionName)
var lineNum = -1
var headers = []
lineReader.eachLine(filename, function(line, last, cb) {
lineNum++
try {
var split = line.split('\t')
var object = {}
if (lineNum > 0) {
for (var i = 0; i < split.length; i += 1) {
object[headers[i]] = split[i]
}
collection.insert(object, function (insertErr, insertObj) {
if (insertErr) console.error(insertErr)
if (lineNum % printLine === 0) console.log('Line ' + lineNum)
if (last) {
console.log('Done with ' + filename + ' (' + lineNum + ' records)')
process.exit(0)
} else {
cb()
}
})
} else {
headers = line.split('\t')
cb()
}
} catch (lineError) {
console.error(lineError)
}
})
}
})

I came across similar problem. This approach worked for me.
Have a look, it might be helpful.
var mongoDb = require('mongodb');
var mongoClient = mongoDb.MongoClient;
var dbname = 'YOUR_DB_NAME';
var collectionName = 'YOUR_COLLECTION_NAME';
var url = 'mongodb://localhost:27017/'+dbname;
var filename = 'FIle_Name.txt';
console.log('***************Process started');
mongoClient.connect(url,function(err,db){
if(err){
console.log('error on connection '+err);
}
else{
console.log('***************Successfully connected to mongodb');
var collection = db.collection(collectionName);
var fs = require('fs');
var readline = require('readline');
var stream = require('stream');
var instream = fs.createReadStream(filename);
var outstream = new stream;
var rl = readline.createInterface(instream,outstream);
console.log('***************Parsing, please wait ...');
rl.on('line',function(line){
try{
var arr = line.split('\t');
var object = {};
//Parse them here
//Example
object['name'] = arr[0]; //Just an example
var res = collection.insert(object);
}
catch (err){
console.log(err);
}
});
rl.on('close',function(){
db.close();
console.log('***************completed');
});
}
});
I am a learner too. If someone can make it better, it will be good.

Here is a more performant (inserting batches of objects) and updated version (using async and latest mongo driver) of frank-0's answer
const lineReader = require('line-reader');
async function readFileAndInsertInMongo(file) {
let total = 0;
return new Promise((resolve, reject) => {
let buffer = [];
lineReader.eachLine(file, (line, last, cb) => {
// prepare your object based on the line content
let insertObject = {'some_content': 'some_value'};
if (total % 10000 === 0 || last) {
collection.insertMany(buffer, function(err, res){
if (last) {
if (err) {
reject(err);
} else {
resolve(res);
}
} else {
buffer = [];
return cb();
}
});
} else {
buffer.push(insertObject);
return cb();
}
});
});
}
This really is the best solution I have found to parse huge files and insert them in the database without exploding Node's memory. Hope this can help ;)

Node.js loading Postgres SQL data into JS FIle

I'm attempting to load the Postgres data into node.js and I'm having a lot of difficulty with it. I know the constring works because I used it to send the data from the a textfile into the database.
var request = require('request');
var pg = require('pg');
var squel=require('squel');
var client = new pg.Client(conString);
var conString="postgres://postgres:Password#localhost:5433/postgres"
client.connect(function (err,data){
if(err) console.log("Error connecting to PG", err);
else{
var query = squel.select().from('"ASNTable"')
console.log(query.toString());
client.query(query, function(res){
var outputJSON = [];
for (row in res){
outputJSON.push(row);
}
return outputJSON
console.log(outputJSON)
});
}
});
I keep getting "SELECT * FROM "ASNTable" so its like the client.query part never does anything?

#aembke I figured it out.
The var conString should go before client.
var request = require('request');
var pg = require('pg');
var squel=require('squel');
var conString="postgres://postgres:Vcu3student#localhost:5433/postgres"
var client = new pg.Client(conString);
var outputJSON = [];
client.connect(function (err,data){
if(err) console.log("'Error connecting to PG'", err);
else{
var query = squel.select().field("asnumber").from('"asntable"');
client.query(query.toString(), function (err,res){
if (err) throw err;
var x = JSON.stringify(res);
var y = x.split(",")
// console.log(y[7])
console.log(y.length);
for (var i=0; i<y.length; i++){
// console.log(y[i]);
if (y[i].indexOf('"asnumber":') > 0)
{
console.log(y[i])
outputJSON.push(y[i]);
}
};
// console.log("result "+JSON.stringify(res))
});
}
});
console.log(outputJSON);