number variables incrementation - unexpected numbers when running - javascript

given the following node\JavaScript code, for some reason the counters variables (failedCounter, successMatchedCounter, successUnmatchedCounter) are counted in an unexpectedly manner when running. I think that it's a matter of scopes and asynchronously but still can't spot the cause.
UPDATE: I think that I should be notified when all calls to connection.query() have finished and only then to log the counters..
See the line of "//BUG: counted numbers are not logged as expected":
var MongoClient = require('mongodb').MongoClient;
var mysql = require('mysql2');
var fs = require('fs');
var dir = './logs';
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
var testMode = false;
var mongoUrl = 'mongodb://xxx:27017/yy';
var mySqlconnString = {
host: 'xxxxx',
user: 'xxxx',
password: 'xxxxx',
database: 'xxxxx'
};
var connection = mysql.createConnection(mySqlconnString);
connection.connect(function(err) {
if (err) {
console.log('Error connecting to MySql DB');
return;
}
console.log('Connection established to MySql DB');
MongoClient.connect(mongoUrl, function(err, db) {
if (err) {
console.log('Error connecting to MongoDB');
return;
}
console.log("Connection established to MongoDB");
markSitesAsDeleted(db, function() {
console.log('closing DBs connections..');
connection.end(function(err) {});
db.close();
});
});
});
var failedCounter = 0;
var successMatchedCounter = 0;
var successUnmatchedCounter = 0;
var totalCounter = 0;
var markSitesAsDeleted = function(db, closeConnectionsCallback) {
console.log(`\nMigration process is starting..`);
var cursor = db.collection('someCollection').find({
"isDeleted": true
});
console.log(`Migrating data..\r\n`);
cursor.each(function(err, siteDoc) {
if (siteDoc != null) {
var siteID = Math.trunc(siteDoc._id)
if (testMode === false) {
connection.query(`CALL MarkSiteAsDeleted_3(${siteID})`, function(error, rows) {
if (error) {
//TODO: Print error
failedCounter++;
fs.appendFileSync('logs/log.txt', `Error occured when calling MarkSiteAsDeleted_3 SP for SiteId=${siteID}. see error: ${JSON.stringify(error)}\n`);
} else {
if (rows.affectedRows === 1) { // Has match
successMatchedCounter++;
} else {
successUnmatchedCounter++;
}
}
});
}
totalCounter++;
} else {
//BUG: counted numbers are not logged as expected
fs.appendFileSync('logs/log.txt', `Total: ${totalCounter}, Success and Matched: ${successMatchedCounter}, Success but Unmatched: ${successUnmatchedCounter}, Failed: ${failedCounter}\r\n`);
closeConnectionsCallback();
}
});
};

It sure looks like an async problem. Inside your cursor.each you have async callbacks to handle connection.query. So as the each iterates it makes queries. but all the queries are running asynchronously.
Once the each is finished setting up all the async queries it then hits your else which uses a synchronous write. That is where the problem occurs. Some of your async query handlers have not finished when that write occurs.

Related

JQuery Custom Validation Rule Mysql function always showing error

I am trying to add a custom validation rule to check if username exists or not. My code:
jQuery.validator.addMethod("checkexist", function(myvalue, element) {
checkifexist(function(result) {
if (result == true)
{
return true;
}
else
{
return false;
}
});
}, 'Does not exists!');
and
async function checkifexist(callback) {
const secret = keytar.getPassword('Userdata', 'MysqlPassword');
secret.then((result) => {
var mysql = require("mysql");
var connection = mysql.createConnection({
host: store.get('ip'),
port: store.get('port'),
user: store.get('username'),
password: result,
database: "database"
});
var querystring = 'SELECT * FROM `testdb`';
try
{
connection.query(querystring, (err, rows, fields) => {
if(err){
return callback("error with query", err);
}
var found;
for (var i in rows)
{
if (some check...)
{
return callback(true);
}
}
return callback(false);
});
connection.end(() => {
});
}
catch { }
});
However, if I check the valditation it correctly outputs true or false on every keypress. I still get only validation errors, though it should be a valid input.

ssh2 connect to multiple server and get the output nodejs

I am using ssh2 nodejs module to connect to a UNIX application and run a script and it is successful.
Now i want to connect to multiple servers one by one and get the output and store it.
When i try using a for loop to pass the servers one by one from a json as input to the ssh2 the for loop completes much faster than the block which is supposed to get the output from the server.
This is also causing me handshake error.
Kindly help
Here is the code
inc_cron.schedule("*/20 * * * * *", function(id) {
});
//inc_cron.js
var cronFunction = function(inputStr) {
if(appNames['applications'].length>0){
for (i = 0; i < appNames["applications"].length; i++) {
getDataFromServer(appNames["applications"][i].Name,appNames["applications"][i].hostname, appNames["applications"][i].username, appNames["applications"][i].password, appNames["applications"][i].log_path, function(err, data) {
if(err) {
logger.info("Error is in cronFunction = ", err);
} else if(data) {
output_data +=data;
} ssh.close_second();
});
}
}
}
var getDataFromServer = function(Name,hostname, username, password, log_path, cb) {
ssh.close_second();
ssh.connect_second({
host: hostname,
username: username,
password: password
}, function(err) {
if(err) {
logger.error('Err: ', err);
} else {
ssh.sftp("",'grep -o "ERROR" '+log_path+'.log.'+yr+'-'+mnth+'-* | wc -l', function(err, data) {
cb(err, data);
}); } }); }
//connect.js
SSHConnection.prototype.sftp = function(type, path, cb) {
var self = this;
var log_data = '';
self.connection2.exec(path +' ' + type, { pty: true }, function(err, stream) {
if (err) {
logger.log('SECOND :: exec error: ' + err);
}
stream.on('end', function() {
self.connection2.end(); // close parent (and this) connection
}).on('data', function(data) {
logger.info(data.toString());
});
});
};
Without watch your code, be sure to handle correctly the async issue with ssh2... use a promise factory.
One way to do this is to use es7 async await. For this you have to rewrite your getDataFromServer function to return a promise:
var getDataFromServer = function(Name,hostname, username, password, log_path, cb) {
return new Promise(function(resolve,reject){
ssh.close_second();
sh.connect_second({
host: hostname,
username: username,
password: password
},function(err) {
if(err){
reject(err)
}else{
ssh.sftp("",'grep -o "ERROR" '+log_path+'.log.'+yr+'-'+mnth+'-* | wc -l', function(err, data) {
if(err){
reject(err)
}else{
resolve(data)
}
})
}
})
})
}
now you can rewrite your cron function to be an async function.
var cronFunction = async function(inputStr) {
if(appNames['applications'].length>0){
for (i = 0; i < appNames["applications"].length; i++) {
try{
output_data + = await getDataFromServer(appNames["applications"][i].Name,appNames["applications"][i].hostname, appNames["applications"][i].username, appNames["applications"][i].password, appNames["applications"][i].log_path)
}catch(err){
logger.info("Error is in cronFunction = ", err);
}
ssh.close_second();
}
}
}
async await enables you to write async code in syncronous coding style.
However async await is currently (node 7.*) hidden behind a flag (--harmony-async-await). this feature will be enable by default in the upcomming node release (8.0.0) in April 2017.
so to start your app you currently have to use
node --harmony-async-await yourapp.js
P.S.: This code is currently untested and most probably contains bugs .. but you get the idea.

Exporting Mysql Connection in nodeJS

In my database.js I have
var Mysql = require('Mysql');
var Jwt = require('jsonwebtoken');
var bcrypt = require('bcrypt');
var supersecretkey = 'JMDub_Super_Secret_key';
var config = require('./config');
var signupErrors = require('./Signuperrors.js');
var sucessMsg = require('./SucessMessages.js');
var App_errors = require('./error.js');
var query = require('./queryDB.js');
var connection = Mysql.createConnection({
"host": "******",
"user": "****",
"password": "***",
"database": "***"
});
connection.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
console.log('connected as id ' + connection.threadId);
});
//Sign Up Methods
var createUser = function createwithCredentails(post,callback) {
bcrypt.hash(post.password, 10, function(err, hash){
//console.log('Cache Hash : +',hash);
var createUserQuery = connection.query('INSERT INTO users SET ?',{"email":post.email,"password":hash,"username":post.username},function(err,result){
if (err) {
if (err.code == 'ER_DUP_ENTRY') {
//console.log(err.code);
callback(signupErrors.error_5000);
}
else callback(App_errors.error_1003);
}
if (result) {
callback(sucessMsg.success_signup);
}
});
});
}
//connection.query('SELECT * FROM Users Where Username = '' AND Password = ''');
var validateUser = function ValidateUserWithUserNameAndPassword(post,callback) {
var UserCheckQuery = connection.query('SELECT * FROM users WHERE email="'+post.email+'"',function(err, results, fields) {
if (err){
console.log(err);
callback(App_errors.error_1000);
}
if (results.length == 1) {
//console.log(results[0].password,post.password);
var givenPassword = post.password;
var DBhash = results[0].password;
bcrypt.compare(givenPassword, DBhash,function(err, res) {
if (res) {
console.log('Password matched');
var token = Jwt.sign({"email":post.email,"username":post.username},supersecretkey, {
expiresIn: 60*60*5 // expires in 5 hours
});
callback({
message:{
"success":1,
"description":"sucessfully logged in - please cache the token for any queries in future",
"environment":"test",
"errorCode":null
},
"token":token
});
}
if (!res) {
console.log('password doesnt match');
callback(signupErrors.error_6000);
}
if (err) {
console.log('Error Comparing Passwords');
callback(App_errors.error_1004);
}
});
}
else{
callback(signupErrors.error_6000);
}
});
};
var isauthenticate = function isauthenticated(post,route,callback) {
if (post.headers.token) {
Jwt.verify(post.headers.token, supersecretkey, function(err, decoded) {
if (decoded) {
//console.log(decoded);
//From this part the user is Sucessully Authenticated and autherization params can be extracted from token if required
//Write Business Logic in future as per the requirement
//Operation 1 - Update Profile
//Profile Details consists of {1.first name 2.last name 3. profile pictur(base 64 encoded) 4.further settings in future that can be added to DB if required}
if (route == '/update-profile') {
query.updateProfile(connection,decoded.email,post.body,function(response) {
callback(response);
});
}
//callback({"message":"is a valid token"});
}
if (decoded == null) {
console.log('is not a valid token');
//callback(App_errors.error_1000);
}
if (err) {
console.log('error verifying token');
callback(App_errors.error_1000);
}
});
}
else{
callback(App_errors.error_1001);
}
};
module.exports = {
validateUser:validateUser,
createUser:createUser,
isauthenticate:isauthenticate,
connection:connection
}
I am exporting connection object to queryDB.js file. But when I try to log the exported connection object I get undefined object. Why is this happening?
When I pass connection object as function argument, everything works fine. Not sure why?
below is queryDB.js file
var errors = require('./error.js')
var Dbconnection = require('./Database.js').connection;
var updateProfile = function profiledata(connection,email,data,callback) {
console.log(Dbconnection);
if ((!data)|| (Object.keys(data).length < 1)) {
//console.log(data);
callback(errors.error_1001);
}
else{
callback({"message":"update Sucesss"});
//console.log(connection);
//var updateData = mapProfileDataTomodel(data);
//console.log(updateData);
connection.query('SELECT * FROM users WHERE email = "'+email+'"',function(err, result,feilds) {
if (err) throw err;
if (result) {
console.log(result);
}
});
}
}
var mapProfileDataTomodel = function mapProfileDataTomodel(data) {
var profileDataModel = {};
for (var key in data) {
//console.log('looping and mapping data');
if (data.firstname) {
profileDataModel.firstname = data.firstname;
}
if (data.lastname) {
profileDataModel.lastname = data.lastname;
}
if (data.profilepic) {
profileDataModel.profilepic = data.profilepic;
}
}
return profileDataModel;
}
module.exports = {
updateProfile:updateProfile
}
I have commented out connection object log via function arguments.
So, Why I am unable to get the connection object that is exported? But I used the same exported connection object in my app.js file. It did work fine there.

Node js custom callback function error

I'm trying to make a simple authentication with node js. Because I read user data from a database, I have to make it asynchronous. Here's my function, which checks if authentication is ok:
function auth(req, callback) {
var header = req.headers['authorization'];
console.log(cb.type);
console.log("Authorization Header is: ", header);
if(!header) {
callback(false);
}
else if(header) {
var tmp = header.split(' ');
var buf = new Buffer(tmp[1], 'base64');
var plain_auth = buf.toString();
console.log("Decoded Authorization ", plain_auth);
var creds = plain_auth.split(':');
var name = creds[0];
var password = creds[1];
User.findOne({name:name, password:password}, function(err, user) {
if (user){
callback(true);
}else {
callback(false);
}
});
}
}
And here I call the function:
auth (req, function (success){
if (!success){
res.setHeader('WWW-Authenticate', 'Basic realm="myRealm');
res.status(401).send("Unauthorized");
}else{
if(user!==req.user) {
res.status(403).send("Unauthorized");
}else{
User.findOneAndUpdate({user:userid}, {user:req.body.user, name:req.body.name, email:req.user.email, password:User.generateHash(req.body.password)},
{upsert:true}, function(err, user) {
if(!err) {
res.status(200).send("OK");
}else{
res.status(400).send("Error");
}
});
}
}
});
This gives me error "TypeError: object is not a function", pointing at "callback(false)". I have no idea what could cause this error, as I pass a function as a parameter, and the first log message prints "[function]". Any help would be appreciated.

Using Node.js to connect to a REST API

Is it sensible to use Node.js to write a stand alone app that will connect two REST API's?
One end will be a POS - Point of sale - system
The other will be a hosted eCommerce platform
There will be a minimal interface for configuration of the service. nothing more.
Yes, Node.js is perfectly suited to making calls to external APIs. Just like everything in Node, however, the functions for making these calls are based around events, which means doing things like buffering response data as opposed to receiving a single completed response.
For example:
// get walking directions from central park to the empire state building
var http = require("http");
url = "http://maps.googleapis.com/maps/api/directions/json?origin=Central Park&destination=Empire State Building&sensor=false&mode=walking";
// get is a simple wrapper for request()
// which sets the http method to GET
var request = http.get(url, function (response) {
// data is streamed in chunks from the server
// so we have to handle the "data" event
var buffer = "",
data,
route;
response.on("data", function (chunk) {
buffer += chunk;
});
response.on("end", function (err) {
// finished transferring data
// dump the raw data
console.log(buffer);
console.log("\n");
data = JSON.parse(buffer);
route = data.routes[0];
// extract the distance and time
console.log("Walking Distance: " + route.legs[0].distance.text);
console.log("Time: " + route.legs[0].duration.text);
});
});
It may make sense to find a simple wrapper library (or write your own) if you are going to be making a lot of these calls.
Sure. The node.js API contains methods to make HTTP requests:
http.request
http.get
I assume the app you're writing is a web app. You might want to use a framework like Express to remove some of the grunt work (see also this question on node.js web frameworks).
/*Below logics covered in below sample GET API
-DB connection created in class
-common function to execute the query
-logging through bunyan library*/
const { APIResponse} = require('./../commonFun/utils');
const createlog = require('./../lib/createlog');
var obj = new DB();
//Test API
routes.get('/testapi', (req, res) => {
res.status(201).json({ message: 'API microservices test' });
});
dbObj = new DB();
routes.get('/getStore', (req, res) => {
try {
//create DB instance
const store_id = req.body.storeID;
const promiseReturnwithResult = selectQueryData('tablename', whereField, dbObj.conn);
(promiseReturnwithResult).then((result) => {
APIResponse(200, 'Data fetched successfully', result).then((result) => {
res.send(result);
});
}).catch((err) => { console.log(err); throw err; })
} catch (err) {
console.log('Exception caught in getuser API', err);
const e = new Error();
if (err.errors && err.errors.length > 0) {
e.Error = 'Exception caught in getuser API';
e.message = err.errors[0].message;
e.code = 500;
res.status(404).send(APIResponse(e.code, e.message, e.Error));
createlog.writeErrorInLog(err);
}
}
});
//create connection
"use strict"
const mysql = require("mysql");
class DB {
constructor() {
this.conn = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'pass',
database: 'db_name'
});
}
connect() {
this.conn.connect(function (err) {
if (err) {
console.error("error connecting: " + err.stack);
return;
}
console.log("connected to DBB");
});
}
//End class
}
module.exports = DB
//queryTransaction.js File
selectQueryData= (table,where,db_conn)=>{
return new Promise(function(resolve,reject){
try{
db_conn.query(`SELECT * FROM ${table} WHERE id = ${where}`,function(err,result){
if(err){
reject(err);
}else{
resolve(result);
}
});
}catch(err){
console.log(err);
}
});
}
module.exports= {selectQueryData};
//utils.js file
APIResponse = async (status, msg, data = '',error=null) => {
try {
if (status) {
return { statusCode: status, message: msg, PayLoad: data,error:error }
}
} catch (err) {
console.log('Exception caught in getuser API', err);
}
}
module.exports={
logsSetting: {
name: "USER-API",
streams: [
{
level: 'error',
path: '' // log ERROR and above to a file
}
],
},APIResponse
}
//createlogs.js File
var bunyan = require('bunyan');
const dateFormat = require('dateformat');
const {logsSetting} = require('./../commonFun/utils');
module.exports.writeErrorInLog = (customError) => {
let logConfig = {...logsSetting};
console.log('reached in writeErrorInLog',customError)
const currentDate = dateFormat(new Date(), 'yyyy-mm-dd');
const path = logConfig.streams[0].path = `${__dirname}/../log/${currentDate}error.log`;
const log = bunyan.createLogger(logConfig);
log.error(customError);
}
A more easy and useful tool is just using an API like Unirest; URest is a package in NPM that is just too easy to use jus like
app.get('/any-route', function(req, res){
unirest.get("https://rest.url.to.consume/param1/paramN")
.header("Any-Key", "XXXXXXXXXXXXXXXXXX")
.header("Accept", "text/plain")
.end(function (result) {
res.render('name-of-the-page-according-to-your-engine', {
layout: 'some-layout-if-you-want',
markup: result.body.any-property,
});
});

Categories

Resources