Develop Reference

Read files from directory and save to array of object

I have a directory of the tax file of employees. Each file has a filename as employee code. I am reading each file and extract some components and save to an array of employee objects.
const readline = require('readline');
let empArr = [];
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
for await (file of filenames) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone =true;
empArr.push(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
})
}
readFiles(directoryPath);
I am not able to get empArr. After getting the array, I need to save to excel. That part I will try after getting the array of employee objects.

I got it working after reading several articles on closure and promises. The below code works for me and sends me array of employees that are processed.
const directoryPath = './tax/';
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
let promiseArr = filenames.map( file=> {
return new Promise((resolve)=>{
processFile(file, resolve)
})
});
Promise.all(promiseArr).then((ret)=>console.log(ret));
})
}
function processFile(file, callback) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone = true;
return callback(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
readFiles(directoryPath);
Surely, the code can be improved further.

IBM MQ change userID when an app is started on remote MQ

When I deploy my app to remote IBM MQ. Then I see that userID is changed to a user of my pc. I set userID = prod, but see in logs (get logs from a remote MQ), that userID = ps (ps - the user of my pc).
But, if the app is started locally, I don't see this problem.
I use ubuntu, docker, Kubernetes, node.js.
I put userID in code, but might I should config it through Docker?
Or how should the conf be changed to fix this problem?
("use strict");
const mq = require("ibmmq");
const fs = require("fs");
const logger = require("../config/logerConfig");
const MQC = mq.MQC;
const StringDecoder = require("string_decoder").StringDecoder;
const decoder = new StringDecoder("utf8");
function ToMQ() {
const qMgr = "queueManagerName";
const qName = "queueName";
const connName = "somehost";
let queueHandle;
const cno = new mq.MQCNO();
const sco = new mq.MQSCO();
const csp = new mq.MQCSP();
const cd = new mq.MQCD();
csp.UserId = "prod";
csp.Password = "";
cno.SecurityParms = csp;
cno.Options |= MQC.MQCNO_CLIENT_BINDING;
cd.ConnectionName = connName;
cd.ChannelName = "channelName";
//cd.SSLCipherSpec = "TLS_RSA_WITH_AES_128_CBC_SHA256";
cd.SSLClientAuth = MQC.MQSCA_OPTIONAL;
cno.ClientConn = cd;
cno.SSLConfig = sco;
mq.setTuningParameters({
syncMQICompat: true
});
mq.Connx(qMgr, cno, function(err, hConn) {
if (err) {
logger.errorLogger().error("Failed to connect to MQ!");
} else {
logger.serverLogger().info(`Connection successful`);
const od = new mq.MQOD();
od.ObjectName = qName;
od.ObjectType = MQC.MQOT_Q;
const openOptions = MQC.MQOO_BROWSE;
mq.Open(hConn, od, openOptions, function(err, hObj) {
queueHandle = hObj;
if (err) {
logger.errorLogger().error(err.message);
} else {
getMessages();
}
});
}
});
}
function formatErr(err) {
if (err) {
ok = false;
return "MQ call failed at " + err.message;
} else {
return "MQ call successful";
}
}
function getMessages() {
const md = new mq.MQMD();
const gmo = new mq.MQGMO();
gmo.Options =
MQC.MQGMO_NO_SYNCPOINT |
MQC.MQGMO_MQWI_UNLIMITED |
MQC.MQGMO_CONVERT |
MQC.MQGMO_FAIL_IF_QUIESCING;
gmo.Options |= MQC.MQGMO_BROWSE_FIRST;
gmo.MatchOptions = MQC.MQMO_NONE;
mq.setTuningParameters({
getLoopPollTimeMs: 500
});
mq.Get(queueHandle, md, gmo, getCB);
}
function getCB(err, hObj, gmo, md, buf, hConn) {
if (err) {
if (err.mqrc == MQC.MQRC_NO_MSG_AVAILABLE) {
logger.serverLogger().info("No more messages available.");
} else {
logger.errorLogger().error(formatErr(err.message));
exitCode = 1;
}
ok = false;
mq.GetDone(hObj);
} else {
if (md.Format == "MQSTR") {
const message = decoder.write(buf);
const metaJSON = getMetaJson(message);
try {
fs.writeFileSync(
.... process
);
logger.serverLogger().info(message);
} catch (e) {
logger.errorLogger().error("Cannot write file ", e.message);
}
} else {
logger.serverLogger().info("binary message: " + buf);
}
gmo.Options &= ~MQC.MQGMO_BROWSE_FIRST;
gmo.Options |= MQC.MQGMO_BROWSE_NEXT;
}
}
function getMetaJson(message) {
// parse JSON
}

Nested function calls do not work after wrapping them with module.exports() in express.js

The nested functions calls do not work after putting all the functions inside the module.exports() like below. It used to work fine before exporting them.
First, isValidCreds is called, which calls hasValidated, which then calls validateJWT.
However, when I run the server I only get "isValidCreds is called".
What is the problem?
module.exports = {
isValidCreds: function (req, res, next) {
console.log("isValidCreds is called");
if(hasValidated(req)){
next();
}
res.status(401);
res.end("Unauthorized");
},
hasValidated: function (req) {
console.log("hasValidated is called");
var authz = req.headers["authorize"];
if(typeof authz !== 'undefined'){
var authzParts = authz.split(" ");
if(authzParts[0] === "Bearer"){
if( validateJWT(authzParts[1]) ){
console.log("Authorized");
return true;
}
}
}
return false;
},
validateJWT: function (jwtstring) {
console.log("validateJWT is called");
var jwtParts = jwtstring.split(".");
var header = JSON.parse(Buffer.from(jwtParts[0], 'base64').toString());
var payload = JSON.parse(Buffer.from(jwtParts[1], 'base64').toString());
var signature = jwtParts[2];
var encodedHeader = Buffer.from(JSON.stringify(header)).toString('base64');
var encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64');
var username = payload.sub;
currentUser = username;
console.log("Verifying user:",username);
var secret = secrets[username].secret;
console.log("secret:",secret);
var salt = payload.salt;
secrets[username].salt = salt;
var veriSign = CryptoJS.enc.Base64.stringify(
CryptoJS.enc.Utf8.parse(
CryptoJS.HmacSHA256(encodedHeader + "." + encodedPayload,salt + secret).toString(CryptoJS.enc.Hex)
)
);
veriSign = veriSign.replace('/','_');
veriSign = veriSign.replace('+','-');
return veriSign === signature;
}
};

Because there is no function or variable with the name hasValidated anymore, there it only an object that has the property hasValidated to which an anonymous function is assigned.
What you could do is to create regular function and export those. That way those functions don't depend on an object:
function isValidCreds(req, res, next) {
console.log("isValidCreds is called");
if (hasValidated(req)) {
next();
}
res.status(401);
res.end("Unauthorized");
}
function hasValidated(req) {
console.log("hasValidated is called");
var authz = req.headers["authorize"];
if (typeof authz !== 'undefined') {
var authzParts = authz.split(" ");
if (authzParts[0] === "Bearer") {
if (validateJWT(authzParts[1])) {
console.log("Authorized");
return true;
}
}
}
return false;
}
function validateJWT(jwtstring) {
console.log("validateJWT is called");
var jwtParts = jwtstring.split(".");
var header = JSON.parse(Buffer.from(jwtParts[0], 'base64').toString());
var payload = JSON.parse(Buffer.from(jwtParts[1], 'base64').toString());
var signature = jwtParts[2];
var encodedHeader = Buffer.from(JSON.stringify(header)).toString('base64');
var encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64');
var username = payload.sub;
currentUser = username;
console.log("Verifying user:", username);
var secret = secrets[username].secret;
console.log("secret:", secret);
var salt = payload.salt;
secrets[username].salt = salt;
var veriSign = CryptoJS.enc.Base64.stringify(
CryptoJS.enc.Utf8.parse(
CryptoJS.HmacSHA256(encodedHeader + "." + encodedPayload, salt + secret).toString(CryptoJS.enc.Hex)
)
);
veriSign = veriSign.replace('/', '_');
veriSign = veriSign.replace('+', '-');
return veriSign === signature;
}
module.exports = {
isValidCreds,
validateJWT,
hasValidated
};
Or reference those functions using the exports object exports.hasValidated:
module.exports = {
isValidCreds: function (req, res, next) {
console.log("isValidCreds is called");
if(exports.hasValidated(req)){ // <===============
next();
}
res.status(401);
res.end("Unauthorized");
},
hasValidated: function (req) {
console.log("hasValidated is called");
var authz = req.headers["authorize"];
if(typeof authz !== 'undefined'){
var authzParts = authz.split(" ");
if(authzParts[0] === "Bearer"){
if( exports.validateJWT(authzParts[1]) ){ // <===============
console.log("Authorized");
return true;
}
}
}
return false;
},
validateJWT: function (jwtstring) {
console.log("validateJWT is called");
var jwtParts = jwtstring.split(".");
var header = JSON.parse(Buffer.from(jwtParts[0], 'base64').toString());
var payload = JSON.parse(Buffer.from(jwtParts[1], 'base64').toString());
var signature = jwtParts[2];
var encodedHeader = Buffer.from(JSON.stringify(header)).toString('base64');
var encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64');
var username = payload.sub;
currentUser = username;
console.log("Verifying user:",username);
var secret = secrets[username].secret;
console.log("secret:",secret);
var salt = payload.salt;
secrets[username].salt = salt;
var veriSign = CryptoJS.enc.Base64.stringify(
CryptoJS.enc.Utf8.parse(
CryptoJS.HmacSHA256(encodedHeader + "." + encodedPayload,salt + secret).toString(CryptoJS.enc.Hex)
)
);
veriSign = veriSign.replace('/','_');
veriSign = veriSign.replace('+','-');
return veriSign === signature;
}
};
this.hasValidated would also work, but is a bad idea, because it is not obvious to the reader that the functions exported by the module depend would need to be called on the exports of that module.
So something like:
let {isValidCreds} = require('your module')
//....
app.use(isValidCreds)
would fail if you use this.hasValidated

Try to use this.methodName
var test = {
prop: 42,
func: function() {
return this.prop;
},
};
// expected output: 42
console.log(test.func());

Firebase Cloud Function error, Function returned undefined, expected Promise or value

I have a cloud function which triggers on certain database write(onCreate), it works as expected but it also throws an error "Function returned undefined, expected Promise or value" though I am returning a promise.
Attaching the code snippet below. there are nested promises in it. is there a better way to handle nested promises, I already checked many posts for nested promises but not able to figure out a proper solution.
Thanks in Advance
exports.calculateAnswer = function(snap, context, dbPath,bucket) {
const answerKey = snap.val();
const incidentId = context.params.incidentId;
const matchId = context.params.match;
var globalIncidentPath = globalIncidentRootPath.replace('${match}', matchId);
globalIncidentPath = globalIncidentPath + incidentId + '/'
var pdPath = pdRootPath.replace('${match}', matchId);
pdPath = pdPath + incidentId
pdPath = pdPath + "/" + bucket
var incidentsPath = incidentsRootPath.replace('${match}', matchId);
var earningsNodePath = earningsNodeRootPath.replace('${match}', matchId);
let app = admin.app();
var globalData = null;
var globalData = null;
const globalPromise = app.database(dbPath).ref(globalIncidentPath).once('value').then(function(snapshot) {
globalData = snapshot.val();
console.log("globalData ",globalIncidentPath, "data ",globalData);
if(globalData) {
console.log("fetching pddata")
return app.database(dbPath).ref(pdPath).once('value')
}
else{
console.log("No global data found");
return true
}
}).then(function(pdSnashot){
const pdData = pdSnashot.val()
if(pdData) {
var promises = []
pdSnashot.forEach(function(childSnap){
console.log('key ',childSnap.key)
console.log('users count ',childSnap.numChildren())
childSnap.forEach(function(usersSnap){
const userId = usersSnap.key
const incidentProcessed = incidentsPath + userId + '/processed/' + incidentId
if (childSnap.key === answerKey) {
const earningUserIdEPath = earningsNodePath + userId
//const earningEPath = earningUserIdEPath + '/e/'
let gocashValue = globalData['v'];
const earningFetchPromise = app.database(dbPath).ref(earningUserIdEPath).once('value').then(function(snapshot1){
let snapDict = snapshot1.val();
var newGoCash = gocashValue
var newPDGoCash = gocashValue
if (snapDict){
let currentGoCash =snapDict['e'];
let currentPDCash = snapDict['pd']
if(currentGoCash) {
newGoCash = currentGoCash + gocashValue;
}
if(currentPDCash) {
newPDGoCash = currentPDCash + gocashValue;
}
}
const obj = Object()
obj["e"] = newGoCash
obj["pd"] = newPDGoCash
const earningPromise = app.database(dbPath).ref(earningUserIdEPath).update(obj)
const tempGlobal = globalData
tempGlobal["skip"] = false;
const processedPromise = app.database(dbPath).ref(incidentProcessed).set(tempGlobal)
return Promise.all([earningPromise,processedPromise])
});
promises.push(earningFetchPromise)
}
else{
const tempGlobal = globalData
tempGlobal["skip"] = true;
const processIncidentPromise = app.database(dbPath).ref(incidentProcessed).set(tempGlobal);
promises.push(processIncidentPromise)
}
})
})
return Promise.all(promises).then(value => {
console.log("Pd promises completed",value);
return true
})
}
else{
console.log("No Pd Data Found");
return true
}
})
.catch(function(error){
console.log('error in promise resolve',error)
})
console.log('global promise',globalPromise)
return Promise.all([globalPromise])
})

I would modify your code as follows. See the comments within the code.
var globalData = null;
const globalPromise = app
.database(dbPath)
.ref(globalIncidentPath)
.once('value')
.then(function(snapshot) {
globalData = snapshot.val();
console.log('globalData ', globalIncidentPath, 'data ', globalData);
if (globalData) {
console.log('fetching pddata');
return app
.database(dbPath)
.ref(pdPath)
.once('value');
} else {
console.log('No global data found');
// return true; Better to throw an error here
throw new Error('No global data found');
}
})
//The following 3 lines don't bring anything
//Moreover they are most probably the reason of your error as you don't return anything in this then()
//.then(function(pdSnashot){
// console.log("");
//})
.catch(function(error) {
console.log('error in promise resolve', error);
return true;
//Note that if you don't need the console.log you may ommit the entire catch since the platform will handle the error itself.
});
console.log('global promise', globalPromise);
//return Promise.all([globalPromise]); // There is no reason to use Promise.all() here since there is only one promise chain returned in globalPromise
return globalPromise;

Right way to cipher / uncipher message with Node.JS?

I use this to Cipher a message:
const CIPHER_ALGORITHM = 'aes256';
var cipherMessage = function(data, key) {
try {
var cipher = crypto.createCipher(CIPHER_ALGORITHM, key);
var cipheredData = cipher.update(data, "binary", "hex");
cipheredData += cipher.final("hex");
return cipheredData;
} catch(e) {
return null;
}
}
And this to uncipher a message:
var decipherMessage = function(msg, key) {
var ret = {};
try {
var decipher = crypto.createDecipher(CIPHER_ALGORITHM, key);
var decipheredMessage = decipher.update(msg, 'hex', 'binary');
decipheredMessage += decipher.final("binary");
ret = JSON.parse(decipheredMessage);
} catch(e) {
return null;
}
return ret;
}
I have to call them multiple time (every second), it it the right way to do it ?
Is there any possibility of memory leak with that ?