Develop Reference

Read files from directory and save to array of object

I have a directory of the tax file of employees. Each file has a filename as employee code. I am reading each file and extract some components and save to an array of employee objects.
const readline = require('readline');
let empArr = [];
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
for await (file of filenames) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone =true;
empArr.push(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
})
}
readFiles(directoryPath);
I am not able to get empArr. After getting the array, I need to save to excel. That part I will try after getting the array of employee objects.

I got it working after reading several articles on closure and promises. The below code works for me and sends me array of employees that are processed.
const directoryPath = './tax/';
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
let promiseArr = filenames.map( file=> {
return new Promise((resolve)=>{
processFile(file, resolve)
})
});
Promise.all(promiseArr).then((ret)=>console.log(ret));
})
}
function processFile(file, callback) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone = true;
return callback(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
readFiles(directoryPath);
Surely, the code can be improved further.

js share array between thread

I try to make threads is js, 1 thread will read a file and push words into an array, 2nd thread will read the array and console.log the word.
I didn't find any solutions except using worker_threads.
This is my main file:
var data = [];
var i = 0;
var finish = false;
function runWorker() {
const worker = new Worker('./worker.js', { workerData: { i, data } });
const worker2 = new Worker('./worker2.js', { workerData: { i, data, finish } });
worker.on('message', function(x) {
data.push[x.data]
if(x.finish != undefined){
finish = true
}
});
worker.on('error', function(error) {
console.log(error);
});
worker2.on('message', function(x) {
console.log(x)
});
worker2.on('error', function(error) {
console.log(error);
});
worker.on('exit', code => {
if (code !== 0) console.log(new Error(`Worker stopped with exit code ${code}`));
});
}
runWorker();
this is my worker.js file :
const { workerData, parentPort } = require('worker_threads');
var fs = require('fs');
const chalk = require('chalk');
fs.readFile('./input.txt', async function read(err, data) {
if (err) {
console.log(' |-> ' + chalk.bold.red("impossible a lire"))
}
else{
console.log(' |-> ' + chalk.blue("début lecture du fichier"))
data = data.toString().split(" ");
for(var i = 0; i<data.length; i++){
parentPort.postMessage({data:data[i]})
}
parentPort.postMessage({finish:true})
}
})
Here I read a file, and send each word to the main programme to put it into an array
this is my worker2.js file :
const { workerData, parentPort } = require('worker_threads');
let finish = false
while(!finish){
let data = workerData.data;
let i = workerData.i;
finish = workerData.finish;
console.log(data[i]);
}
Here I have an infinite loop, and if something is on the array, print it.

How to handle axios.all request fails

How can I handle request fails in this example of axios.all requests. I.e. if all servers are responde with JSON all is okay and I have JSON file at end of a cycle. But if one of this servers not responde with JSON or not responde at all I do have nothing in "/data.json" file, even all other servers are working perfectly. How can I catch a server fail and skip it?
var fs = require("fs");
var axios = require('axios');
var util = require('util');
var round = 0;
var tmp = {};
var streem = fs.createWriteStream(__dirname + '/data.json', {flags : 'w'});
toFile = function(d) { //
streem.write(util.format(d));
};
start();
setInterval(start, 27000);
function start(){
streem = fs.createWriteStream(__dirname + '/data.json', {flags : 'w'});
monitor();
}
function monitor(){
axios.all([
axios.get('server1:api'),
axios.get('server2:api'),
axios.get('server3:api'),
axios.get('server4:api'),
]).then(axios.spread((response1, response2, response3, response4) => {
tmp.servers = {};
tmp.servers.server1 = {};
tmp.servers.server1 = response1.data;
tmp.servers.server2 = {};
tmp.servers.server2 = response2.data;
tmp.servers.server3 = {};
tmp.servers.server3 = response3.data;
tmp.servers.server4 = {};
tmp.servers.server4 = response4.data;
toFile(JSON.stringify(tmp));
round++;
streem.end();
streem.on('finish', () => {
console.error('Round: ' + round);
});
})).catch(error => {
console.log(error);
});
}

The most standard way to approach this would be a recursive function like below.
let promises = [
axios.get('server1:api'),
axios.get('server2:api'),
axios.get('server3:api'),
axios.get('server4:api'),
];
async function monitor() {
const responses = (arguments.length === 1 ? [arguments[0]] : Array.apply(null, arguments))[0];
const nextPromise = promises.shift();
if (nextPromise) {
try {
const response = await getSentenceFragment(offset);
responses.push(response);
}
catch (error) {
responses.push({});
}
return responses.concat(await monitor(responses));
} else {
return responses;
}
}
monitor([]).then(([response1, response2, response3, response4]) => {
tmp.servers = {};
tmp.servers.server1 = {};
tmp.servers.server1 = response1.data;
tmp.servers.server2 = {};
tmp.servers.server2 = response2.data;
tmp.servers.server3 = {};
tmp.servers.server3 = response3.data;
tmp.servers.server4 = {};
tmp.servers.server4 = response4.data;
toFile(JSON.stringify(tmp));
round++;
streem.end();
streem.on('finish', () => {
console.error('Round: ' + round);
});
});

How to make private varaible in JS

I want to save a private variable(secret) when require a file/module. The secret shall be saved in the "object" of file sec_test.js and it shall not be readable or writable just execution-able. Is this a correct way?
Question 1:
Is it possible to get the secret somehow during execution from the testing_sec_test.js?
Question 2:
is it possible to have a constructor-ish function in sec_test.js ?
file: sec_test.js
module.exports = function (string) {
var module = {};
let secret = null;
module.get_secret_length = function (callback) {
generate_secret();
if(secret == null){
const json_err = {
"Success":false,
"error":"generating secret failed"
};
callback(json_err,null);
}else{
const json_err = {
"Success":true,
"result":"secret has been generated",
"secret_length":get_secret_length()
};
callback(json_err,null);
}
}
function generate_secret(){
if(secret == null){
secret = getRandomString()+string+getRandomString();
}
}
function get_secret_length(){
return secret.length;
}
function getRandomString(){
const length = Math.floor(Math.random() * Math.floor(200));
const characters_allowed = '#1#2$3&/=?:.;,+_-><~*^|4567890'+
'qwertyuioplkjhgfdsazxcvbnmQWERTYUIOPLKJHGFDSAZXCVBNM';
let random_string = "";
for(let i =0;i<length;i++){
let random_nbr = Math.floor(Math.random() * Math.floor(characters_allowed.length));
random_string += characters_allowed.charAt(random_nbr);
}
return random_string;
}
return module;
};
file: testing_sec_test.js
const sec_test = require('./sec_test')("IS THIS SECRET A PRIVATE VARIABLE");
console.log(sec_test.get_secret_length.toString());
sec_test.get_secret_length(function(err,result){
if(err){
console.log(err);
}else{
console.log(result);
}
});
---------------------------------------------------------------
I Guess i have to formulate my question a little better,, sorry
Question 1: Is it possible to get the key or ivKey AFTER the object has been required and the parameter has been inputed. Or is this object not safe to use becase its key or ivKey is public accessable?
file: testing_sec_test.js
//lets pretend that these keys is written in from the terminal to the object and are NOT hardcoded in the code!.
let sec_Obj = {
"key": '1234zr3p67VC61jmV54rIYu1545x4TlY',
"ivKey": "123460iP0h6vJoEa",
"salt": "1kg8kfjfd2js93zg7sdg485sd74g63d2",
"key_iterations": 87923
}
const sec_test = require('./sec_test')(sec_Obj);
sec_Obj = null;
console.log(sec_test);
let plain_text = "This is a national secret";
console.log("plain_text == "+plain_text);
sec_test.encrypt(plain_text,function(err,encrypted){
if(err){
console.log(err);
}else{
console.log("encrypted == "+encrypted);
sec_test.decrypt(encrypted,function(err,decrypted){
if(err){
console.log(err);
}else{
console.log("decrypted == "+decrypted);
}
});
}
});
file: sec_test.js
const crypto = require('crypto');
module.exports = function (keysObj) {
//is the parameter keysObj private??
var module = {};
module.encrypt = function (clearData,callback) {
let str_encoding = "utf8";
let encoding = "base64";
try {
let encipher = crypto.createCipheriv('aes-256-ctr', getPrivateKey(), getPrivateIvKey());
let result = encipher.update(clearData, str_encoding, encoding);
result += encipher.final(encoding);
callback(null,result);
} catch (error) {
callback({"success":false,"error":error},null);
}
}
module.decrypt = function(encrypted,callback) {
let str_encoding = "utf8";
let encoding = "base64";
try {
let decipher = crypto.createDecipheriv('aes-256-ctr',getPrivateKey(), getPrivateIvKey());
let result = decipher.update(encrypted, encoding, str_encoding);
result += decipher.final(str_encoding);
callback(null,result);
} catch (error) {
callback({"success":false,"error":error},null);
}
}
//is this a private function
function getPrivateKey(){
return crypto.pbkdf2Sync(keysObj['key'], keysObj['salt'], keysObj['key_iterations'], 32, 'sha512');
}
//is this a private function
function getPrivateIvKey(){
return new Buffer(keysObj['ivKey']);
}
return module;
};

Simple example
var privateVar = 'private';
module.exports = {
test:function(){
console.log('I am '+privateVar);
}
}
var test = require('./test.js');
//logs i am private
test.test()
//logs undefined
test.privateVar

Need Help Making Java Script Discord Music Bot

I have tried to use this code to make a discord music bot but i am getting error telling me i need ffmpeg, but how would I implement it into this code?
index.js file
const Discord = require('discord.js');
const bot = new Discord.Client();
const config = require("./config.json");
const ytdl = require('ytdl-core');
var youtube = require('./youtube.js');
var ytAudioQueue = [];
var dispatcher = null;
bot.on('message', function(message) {
var messageParts = message.content.split(' ');
var command = messageParts[0].toLowerCase();
var parameters = messageParts.splice(1, messageParts.length);
switch (command) {
case "-join" :
message.reply("Attempting to join channel " + parameters[0]);
JoinCommand(parameters[0], message);
break;
case "-play" :
PlayCommand(parameters.join(" "), message);
break;
case "-playqueue":
PlayQueueCommand(message);
break;
}
});
function PlayCommand(searchTerm) {
if(bot.voiceConnections.array().length == 0) {
var defaultVoiceChannel = bot.channels.find(val => val.type === 'voice').name;
JoinCommand(defaultVoiceChannel);
}
youtube.search(searchTerm, QueueYtAudioStream);
}
function PlayQueueCommand(message) {
var queueString = "";
for(var x = 0; x < ytAudioQueue.length; x++) {
queueString += ytAudioQueue[x].videoName + ", ";
}
queueString = queueString.substring(0, queueString.length - 2);
message.reply(queueString);
}
function JoinCommand(ChannelName) {
var voiceChannel = GetChannelByName(ChannelName);
if (voiceChannel) {
voiceChannel.join();
console.log("Joined " + voiceChannel.name);
}
return voiceChannel;
}
/* Helper Methods */
function GetChannelByName(name) {
var channel = bot.channels.find(val => val.name === name);
return channel;
}
function QueueYtAudioStream(videoId, videoName) {
var streamUrl = youtube.watchVideoUrl + videoId;
if (!ytAudioQueue.length) {
ytAudioQueue.push(
{
'streamUrl' : streamUrl,
'videoName' : videoName
}
);
console.log('Queued audio ' + videoName);
PlayStream(ytAudioQueue[0].streamUrl);
}
else {
ytAudioQueue.push(
{
'streamUrl' : streamUrl,
'videoName' : videoName
}
);
}
console.log("Queued audio " + videoName);
}
function PlayStream(streamUrl) {
const streamOptions = {seek: 0, volume: 1};
if (streamUrl) {
const stream = ytdl(streamUrl, {filter: 'audioonly'});
if (dispatcher == null) {
var voiceConnection = bot.voiceConnections.first();
if(voiceConnection) {
console.log("Now Playing " + ytAudioQueue[0].videoname);
dispatcher = bot.voiceConnections.first().playStream(stream, streamOptions);
dispatcher.on('end', () => {
dispatcher = null;
PlayNextStreamInQueue();
});
dispatcher.on('error', (err) => {
console.log(err);
});
}
} else {
dispatcher = bot.voiceConnections.first().playStream(stream, streamOptions);
}
}
}
function PlayNextStreamInQueue() {
ytAudioQueue.splice(0, 1);
if (ytAudioQueue.length != 0) {
console.log("now Playing " + ytAudioQueue[0].videoName);
PlayStream(ytAudioQueue[0].streamUrl);
}
}
bot.login(config.token);
youtube.js file
var request = require('superagent');
const API_KEY = "My API KEY";
const WATCH_VIDEO_URL = "https://www.youtube.com/watch?v=";
exports.watchVideoUrl = WATCH_VIDEO_URL;
exports.search = function search(searchKeywords, callback) {
var requestUrl = 'https://www.googleapis.com/youtube/v3/search' + '?part=snippet&q=' + escape(searchKeywords) + '&key=' + API_KEY;
request(requestUrl, (error, response) => {
if (!error && response.statusCode == 200) {
var body = response.body;
if (body.items.length == 0) {
console.log("I Could Not Find Anything!");
return;
}
for (var item of body.items) {
if (item.id.kind == 'youtube#video') {
callback(item.id.videoId, item.snippet.title);
return;
}
}
} else {
console.log("Unexpected error!");
return;
}
});
return;
};
Error I am getting when I try to run code :
Joined General
C:\Discord Bot\node_modules\discord.js\src\client\voice\pcm\FfmpegConverterEngine.
js:80
throw new Error(
^
Error: FFMPEG was not found on your system, so audio cannot be played. Please make
sure FFMPEG is installed and in your PATH.

You need to download/extract FFMPEG and make it as your PATH/System variable[environment variable]
Make sure that you have it as your System variable like this :
The System Variable should named as 'FFMPEG' and should be directed to where the execution(.exe) file is. It should be in a folder called 'bin' in your FFMPEG folder.
You can google/search online on how to add a new System/PATH variable for your specific OS.