I have a directory of the tax file of employees. Each file has a filename as employee code. I am reading each file and extract some components and save to an array of employee objects.
const readline = require('readline');
let empArr = [];
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
for await (file of filenames) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone =true;
empArr.push(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
})
}
readFiles(directoryPath);
I am not able to get empArr. After getting the array, I need to save to excel. That part I will try after getting the array of employee objects.
I got it working after reading several articles on closure and promises. The below code works for me and sends me array of employees that are processed.
const directoryPath = './tax/';
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
let promiseArr = filenames.map( file=> {
return new Promise((resolve)=>{
processFile(file, resolve)
})
});
Promise.all(promiseArr).then((ret)=>console.log(ret));
})
}
function processFile(file, callback) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone = true;
return callback(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
readFiles(directoryPath);
Surely, the code can be improved further.
Related
When I deploy my app to remote IBM MQ. Then I see that userID is changed to a user of my pc. I set userID = prod, but see in logs (get logs from a remote MQ), that userID = ps (ps - the user of my pc).
But, if the app is started locally, I don't see this problem.
I use ubuntu, docker, Kubernetes, node.js.
I put userID in code, but might I should config it through Docker?
Or how should the conf be changed to fix this problem?
("use strict");
const mq = require("ibmmq");
const fs = require("fs");
const logger = require("../config/logerConfig");
const MQC = mq.MQC;
const StringDecoder = require("string_decoder").StringDecoder;
const decoder = new StringDecoder("utf8");
function ToMQ() {
const qMgr = "queueManagerName";
const qName = "queueName";
const connName = "somehost";
let queueHandle;
const cno = new mq.MQCNO();
const sco = new mq.MQSCO();
const csp = new mq.MQCSP();
const cd = new mq.MQCD();
csp.UserId = "prod";
csp.Password = "";
cno.SecurityParms = csp;
cno.Options |= MQC.MQCNO_CLIENT_BINDING;
cd.ConnectionName = connName;
cd.ChannelName = "channelName";
//cd.SSLCipherSpec = "TLS_RSA_WITH_AES_128_CBC_SHA256";
cd.SSLClientAuth = MQC.MQSCA_OPTIONAL;
cno.ClientConn = cd;
cno.SSLConfig = sco;
mq.setTuningParameters({
syncMQICompat: true
});
mq.Connx(qMgr, cno, function(err, hConn) {
if (err) {
logger.errorLogger().error("Failed to connect to MQ!");
} else {
logger.serverLogger().info(`Connection successful`);
const od = new mq.MQOD();
od.ObjectName = qName;
od.ObjectType = MQC.MQOT_Q;
const openOptions = MQC.MQOO_BROWSE;
mq.Open(hConn, od, openOptions, function(err, hObj) {
queueHandle = hObj;
if (err) {
logger.errorLogger().error(err.message);
} else {
getMessages();
}
});
}
});
}
function formatErr(err) {
if (err) {
ok = false;
return "MQ call failed at " + err.message;
} else {
return "MQ call successful";
}
}
function getMessages() {
const md = new mq.MQMD();
const gmo = new mq.MQGMO();
gmo.Options =
MQC.MQGMO_NO_SYNCPOINT |
MQC.MQGMO_MQWI_UNLIMITED |
MQC.MQGMO_CONVERT |
MQC.MQGMO_FAIL_IF_QUIESCING;
gmo.Options |= MQC.MQGMO_BROWSE_FIRST;
gmo.MatchOptions = MQC.MQMO_NONE;
mq.setTuningParameters({
getLoopPollTimeMs: 500
});
mq.Get(queueHandle, md, gmo, getCB);
}
function getCB(err, hObj, gmo, md, buf, hConn) {
if (err) {
if (err.mqrc == MQC.MQRC_NO_MSG_AVAILABLE) {
logger.serverLogger().info("No more messages available.");
} else {
logger.errorLogger().error(formatErr(err.message));
exitCode = 1;
}
ok = false;
mq.GetDone(hObj);
} else {
if (md.Format == "MQSTR") {
const message = decoder.write(buf);
const metaJSON = getMetaJson(message);
try {
fs.writeFileSync(
.... process
);
logger.serverLogger().info(message);
} catch (e) {
logger.errorLogger().error("Cannot write file ", e.message);
}
} else {
logger.serverLogger().info("binary message: " + buf);
}
gmo.Options &= ~MQC.MQGMO_BROWSE_FIRST;
gmo.Options |= MQC.MQGMO_BROWSE_NEXT;
}
}
function getMetaJson(message) {
// parse JSON
}
I am trying to read some data from a file and store it in a database.
This is part of a larger transaction and I need the returned ids for further steps.
async parseHeaders(mysqlCon, ghID, csv) {
var self = this;
var hIDs = [];
var skip = true;
var idx = 0;
console.log("Parsing headers");
return new Promise(async function(resolve, reject) {
try {
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(csv)
});
await lineReader.on('close', async function () {
console.log("done: ", JSON.stringify(hIDs));
resolve(hIDs);
});
await lineReader.on('line', async function (line) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx++] = await self.getParamID(mysqlCon, ghID, v, c, data);//, function(hID,sidx) { //add data in case the parameter is not in DB, yet.
}
});
} catch(e) {
console.log(JSON.stringify(e));
reject("some error occured: " + e);
}
});
}
async getParamID(mysqlCon,ghID,variable,category,data) {
return new Promise(function(resolve, reject) {
var sql = "SELECT ID FROM Parameter WHERE GreenHouseID="+ghID+" AND Variable = '" + variable + "' AND Category='" + category + "'";
mysqlCon.query(sql, function (err, result, fields) {
if(result.length === 0 || err) { //apparently not in DB, yet ... add it (Acronym and Machine need to be set manually).
sql = "INSERT INTO Parameter (GreenHouseID,Variable,Category,Control) VALUES ("+ghID+",'"+variable+"','"+category+"','"+data[3]+"')";
mysqlCon.query(sql, function (err, result) {
if(err) {
console.log(result,err,this.sql);
reject(err);
} else {
console.log("Inserting ",variable," into DB: ",JSON.stringify(result));
resolve(result.insertId); //added, return generated ID.
}
});
} else {
resolve(result[0].ID); //found in DB .. return ID.
}
});
});
}
The functions above are in the base class and called by the following code:
let headerIDs = await self.parseHeaders(mysqlCon, ghID, filePath);
console.log("headers:",JSON.stringify(headerIDs));
The sequence of events is that everything in parseHeaders completes except for the call to self.getParamID and control returns to the calling function which prints an empty array for headerIDs.
The console.log statements in self.getParamID are then printed afterward.
What am I missing?
Thank you
As you want to execute an asynchronous action for every line we could define a handler to do right that:
const once = (target, evt) => new Promise(res => target.on(evt, res));
function mapLines(reader, action) {
const results = [];
let index = 0;
reader.on("line", line => results.push(action(line, index++)));
return once(reader, "close").then(() => Promise.all(results));
}
So now you can solve that easily:
let skip = false;
const hIDs = [];
await mapLines(lineReader, async function (line, idx) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx] = await self.getParamID(mysqlCon, ghID, v, c, data);
}
});
How can I handle request fails in this example of axios.all requests. I.e. if all servers are responde with JSON all is okay and I have JSON file at end of a cycle. But if one of this servers not responde with JSON or not responde at all I do have nothing in "/data.json" file, even all other servers are working perfectly. How can I catch a server fail and skip it?
var fs = require("fs");
var axios = require('axios');
var util = require('util');
var round = 0;
var tmp = {};
var streem = fs.createWriteStream(__dirname + '/data.json', {flags : 'w'});
toFile = function(d) { //
streem.write(util.format(d));
};
start();
setInterval(start, 27000);
function start(){
streem = fs.createWriteStream(__dirname + '/data.json', {flags : 'w'});
monitor();
}
function monitor(){
axios.all([
axios.get('server1:api'),
axios.get('server2:api'),
axios.get('server3:api'),
axios.get('server4:api'),
]).then(axios.spread((response1, response2, response3, response4) => {
tmp.servers = {};
tmp.servers.server1 = {};
tmp.servers.server1 = response1.data;
tmp.servers.server2 = {};
tmp.servers.server2 = response2.data;
tmp.servers.server3 = {};
tmp.servers.server3 = response3.data;
tmp.servers.server4 = {};
tmp.servers.server4 = response4.data;
toFile(JSON.stringify(tmp));
round++;
streem.end();
streem.on('finish', () => {
console.error('Round: ' + round);
});
})).catch(error => {
console.log(error);
});
}
The most standard way to approach this would be a recursive function like below.
let promises = [
axios.get('server1:api'),
axios.get('server2:api'),
axios.get('server3:api'),
axios.get('server4:api'),
];
async function monitor() {
const responses = (arguments.length === 1 ? [arguments[0]] : Array.apply(null, arguments))[0];
const nextPromise = promises.shift();
if (nextPromise) {
try {
const response = await getSentenceFragment(offset);
responses.push(response);
}
catch (error) {
responses.push({});
}
return responses.concat(await monitor(responses));
} else {
return responses;
}
}
monitor([]).then(([response1, response2, response3, response4]) => {
tmp.servers = {};
tmp.servers.server1 = {};
tmp.servers.server1 = response1.data;
tmp.servers.server2 = {};
tmp.servers.server2 = response2.data;
tmp.servers.server3 = {};
tmp.servers.server3 = response3.data;
tmp.servers.server4 = {};
tmp.servers.server4 = response4.data;
toFile(JSON.stringify(tmp));
round++;
streem.end();
streem.on('finish', () => {
console.error('Round: ' + round);
});
});
I want to save a private variable(secret) when require a file/module. The secret shall be saved in the "object" of file sec_test.js and it shall not be readable or writable just execution-able. Is this a correct way?
Question 1:
Is it possible to get the secret somehow during execution from the testing_sec_test.js?
Question 2:
is it possible to have a constructor-ish function in sec_test.js ?
file: sec_test.js
module.exports = function (string) {
var module = {};
let secret = null;
module.get_secret_length = function (callback) {
generate_secret();
if(secret == null){
const json_err = {
"Success":false,
"error":"generating secret failed"
};
callback(json_err,null);
}else{
const json_err = {
"Success":true,
"result":"secret has been generated",
"secret_length":get_secret_length()
};
callback(json_err,null);
}
}
function generate_secret(){
if(secret == null){
secret = getRandomString()+string+getRandomString();
}
}
function get_secret_length(){
return secret.length;
}
function getRandomString(){
const length = Math.floor(Math.random() * Math.floor(200));
const characters_allowed = '#1#2$3&/=?:.;,+_-><~*^|4567890'+
'qwertyuioplkjhgfdsazxcvbnmQWERTYUIOPLKJHGFDSAZXCVBNM';
let random_string = "";
for(let i =0;i<length;i++){
let random_nbr = Math.floor(Math.random() * Math.floor(characters_allowed.length));
random_string += characters_allowed.charAt(random_nbr);
}
return random_string;
}
return module;
};
file: testing_sec_test.js
const sec_test = require('./sec_test')("IS THIS SECRET A PRIVATE VARIABLE");
console.log(sec_test.get_secret_length.toString());
sec_test.get_secret_length(function(err,result){
if(err){
console.log(err);
}else{
console.log(result);
}
});
---------------------------------------------------------------
I Guess i have to formulate my question a little better,, sorry
Question 1: Is it possible to get the key or ivKey AFTER the object has been required and the parameter has been inputed. Or is this object not safe to use becase its key or ivKey is public accessable?
file: testing_sec_test.js
//lets pretend that these keys is written in from the terminal to the object and are NOT hardcoded in the code!.
let sec_Obj = {
"key": '1234zr3p67VC61jmV54rIYu1545x4TlY',
"ivKey": "123460iP0h6vJoEa",
"salt": "1kg8kfjfd2js93zg7sdg485sd74g63d2",
"key_iterations": 87923
}
const sec_test = require('./sec_test')(sec_Obj);
sec_Obj = null;
console.log(sec_test);
let plain_text = "This is a national secret";
console.log("plain_text == "+plain_text);
sec_test.encrypt(plain_text,function(err,encrypted){
if(err){
console.log(err);
}else{
console.log("encrypted == "+encrypted);
sec_test.decrypt(encrypted,function(err,decrypted){
if(err){
console.log(err);
}else{
console.log("decrypted == "+decrypted);
}
});
}
});
file: sec_test.js
const crypto = require('crypto');
module.exports = function (keysObj) {
//is the parameter keysObj private??
var module = {};
module.encrypt = function (clearData,callback) {
let str_encoding = "utf8";
let encoding = "base64";
try {
let encipher = crypto.createCipheriv('aes-256-ctr', getPrivateKey(), getPrivateIvKey());
let result = encipher.update(clearData, str_encoding, encoding);
result += encipher.final(encoding);
callback(null,result);
} catch (error) {
callback({"success":false,"error":error},null);
}
}
module.decrypt = function(encrypted,callback) {
let str_encoding = "utf8";
let encoding = "base64";
try {
let decipher = crypto.createDecipheriv('aes-256-ctr',getPrivateKey(), getPrivateIvKey());
let result = decipher.update(encrypted, encoding, str_encoding);
result += decipher.final(str_encoding);
callback(null,result);
} catch (error) {
callback({"success":false,"error":error},null);
}
}
//is this a private function
function getPrivateKey(){
return crypto.pbkdf2Sync(keysObj['key'], keysObj['salt'], keysObj['key_iterations'], 32, 'sha512');
}
//is this a private function
function getPrivateIvKey(){
return new Buffer(keysObj['ivKey']);
}
return module;
};
Simple example
var privateVar = 'private';
module.exports = {
test:function(){
console.log('I am '+privateVar);
}
}
var test = require('./test.js');
//logs i am private
test.test()
//logs undefined
test.privateVar
I'm trying to make something for a little community I'm in. But I'm not very well versed in JavaScript and NodeJS yet. The script I'm making, is supposed to first find all possible combinations of a list of users in groups of 6.
Then I need to take each of those groups skill rating average, and compare so that I can find the two that matches the closest. This way we get two teams that can play against each other, and be somewhat balanced at least.
But my first issue is that I seem to be unable to even print out anything from the array that I'm making, and I don't understand why.
var filePath = 'data.txt';
function readFile() {
var data = [];
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(filePath)
});
lineReader.on('line', function(line) {
var splitString = line.split(',');
var arr = {
sr: splitString[0],
role: splitString[1],
discord: splitString[3]
};
data.push(arr);
console.log(arr);
});
lineReader.on('close', () => {
return data;
});
}
function balance() {
var data = readFile();
for(var i = 0; i < data.length; i++) {
console.log(data[i]);
}
}
balance();
The output is always undefined. What am I doing wrong?
What you can do is not to change the nature of the reading function, instead change your approach.
I will give you two options.
Using callback:
var filePath = 'data.txt';
function readFile(callback) {
var data = [];
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(filePath)
});
lineReader.on('line', function (line) {
var splitString = line.split(',');
var arr = {
sr: splitString[0],
role: splitString[1],
discord: splitString[3]
};
data.push(arr);
console.log(arr);
});
lineReader.on('close', () => {
callback(data);
});
}
function balance() {
readFile(function (data) {
for (var i = 0; i < data.length; i++) {
console.log(data[i]);
}
});
}
balance();
Using Promise:
var filePath = 'data.txt';
function readFile() {
return new Promise(function (resolve, reject) {
var data = [];
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(filePath)
});
lineReader.on('line', function (line) {
var splitString = line.split(',');
var arr = {
sr: splitString[0],
role: splitString[1],
discord: splitString[3]
};
data.push(arr);
console.log(arr);
});
lineReader.on('close', () => {
resolve(data);
});
});
}
function balance() {
readFile().then(function (data) {
for (var i = 0; i < data.length; i++) {
console.log(data[i]);
}
}, function (error) {
});
}
balance();
Using async await
const filePath = './data.txt';
async function readFile() {
return new Promise((resolve) => {
const data = [];
const lineReader = require('readline').createInterface({
input: require('fs').createReadStream(filePath),
});
lineReader.on('line', (line) => {
const [sr, role, discord] = line.split(',');
const arr = {
sr,
role,
discord,
};
data.push(arr);
});
lineReader.on('close', () => {
resolve(data);
});
});
}
async function balance() {
const data = await readFile();
for (let i = 0; i < data.length; i += 1) {
console.log(data[i]);
}
return 1;
}
balance().then(() => { });
Using observer pattern
const { EventEmitter } = require('events');
const fs = require('fs');
class FileParser extends EventEmitter {
constructor(file) {
super();
this.file = file;
}
parse() {
const self = this;
const data = [];
const lineReader = require('readline').createInterface({
input: require('fs').createReadStream(self.file),
});
lineReader.on('line', (line) => {
const [sr, role, discord] = line.split(',');
const arr = {
sr,
role,
discord,
};
data.push(arr);
});
lineReader.on('close', () => {
self.emit('done', data);
});
return this;
}
}
const fileParser = new FileParser('./data.txt');
fileParser
.parse()
.on('done', (data) => {
for (let i = 0; i < data.length; i += 1) {
console.log(data[i]);
}
});
Refactored a bit to use the latest ES6 syntax;