I am trying to export database properties stored in properties file from Javascript module. By the time I read database properties file, Javascript file is already exported and data properties appear undefined wherever I use in other modules.
const Pool = require('pg').Pool;
const fs = require('fs')
const path = require('path');
class DbConfig {
constructor(dbData) {
this.pool = new Pool({
user: dbData['user'],
host: dbData['host'],
database: dbData['database'],
password: dbData['password'],
max: 20,
port: 5432
});
}
}
function getdbconf() {
const dbData = {};
fs.readFile("../../db_properties.txt"), 'utf8', (err, data) => {
if (err) {
console.error(err)
return
}
// dbData = {"user":"postgres", "password": "1234"...};
return dbData;
});
}
let db = new DbConfig(getdbconf());
let dbPool = db.pool;
console.log("dbpool : -> : ",dbPool); // username and password appear undefined
module.exports = { dbPool };
Is there a way to read data before exporting data from Javascript module?
Usually database config or any other sensitive info is read from a .env file using dotenv .
Or
you could also provide env from command line itself like
DB_HOST=127.0.0.1 node index.js
inside your index.js
console.log(process.env.DB_HOST)
Please create a new file (connection-pool.js) and paste this code:
const { Pool } = require('pg');
const poolConnection = new Pool({
user: 'postgresUserName',
host: 'yourHost',
database: 'someNameDataBase',
password: 'postgresUserPassword',
port: 5432,
});
console.log('connectionOptions', poolConnection.options);
module.exports = poolConnection;
For use it, create a new file (demo-connection.js) and paste this code:
const pool = require('./connection-pool');
pool.query('SELECT NOW();', (err, res) => {
if (err) {
// throw err;
console.log('connection error');
return;
}
if (res) {
console.log(res.rows);
pool.end();
}
});
This is an alternative option 🙂
Exporting the result of async calls
To export values which have been obtained asynchronously, export a Promise.
const fs = require('fs/promises'); // `/promise` means no callbacks, Promise returned
const dbDataPromise = fs.readFile('fileToRead')); //`readFile` returns Promise now
module.exports = dbDataPromise;
Importing
When you need to use the value,
const dbDataPromise = require('./dbdata');
async init() {
const dbData = await dbDataPromise;
}
//or without async, using Promise callbacks
init() {
dbDataPromise
.then(dbData => the rest of your code that depends on dbData here);
}
Current code broken
Please note that your current code, as pasted above, is broken:
function getdbconf() {
const dbData = {};
fs.readFile("../../db_properties.txt"), 'utf8', (err, data) => {
//[...] snipped for brevity
return dbData;
});
}
fs.readFile "returns" dbData, but there is nothing to return to, since you are in a callback which you did not call yourself. Function getdbconf returns nothing.
The line that says let db = new DbConfig(getdbconf()); will NOT work. It needs to be inside the callback.
The only way to avoid putting all of your code inside the callback (and "flatten" it) is to use await, or to use readFileSync
Avoiding the issue
Using environment variables
Suhas Nama's suggestion is a good one, and is common practice. Try putting the values you need in environment variables.
Using synchronous readFile
While using synchronous calls does block the event loop, it's ok to do during initialization, before your app is up and running.
This avoids the problem of having everything in a callback or having to export Promises, and is often the best solution.
Related
im trying to make a command that connects to the database, i created a little CLI script that loops through files in specific folders to get command class modules
my problem is that in one of my commands, i'm trying to connect to sequelize, and it just doesn't seem to be doing anything. i get no output to the console, nor does it even seem to try to connect
this is probably because i'm still kind of struggling to figure out how to properly do sync / async / await stuff...notice how i use glob.sync cause i want to loop through the files sychronously, but then in my command i need to connect to the database using await
cli.js:
#! /usr/bin/env node
const patterns = [
'./node_modules/unity/src/commands/**/*.js',
'./src/commands/**/*.js',
]
const glob = require('glob')
const path = require('path')
const yargs = require('yargs')
const signature = yargs.argv._[0]
const process = require('process')
patterns.forEach(pattern => {
glob.sync(pattern).forEach(file => {
const commandPath = path.resolve(file)
const command = require(commandPath)
if (command.signature == signature) {
command.argv = yargs.argv
command.run()
process.exit()
}
})
})
console.log('Command not found.')
here is an example command in one of the commands folders:
const { Sequelize } = require('sequelize')
class MigrateCommand {
static signature = 'migrate'
static argv = {}
static run() {
const sequelize = new Sequelize({
dialect: 'mysql',
host: 'localhost',
port: 3306,
database: 'dio_unity2',
username: 'root',
password: '',
})
const connect = async () => {
try {
await sequelize.authenticate()
console.log('Connection successful.')
}
catch (error) {
console.error('Unable to connect.', error)
}
}
connect()
console.log('migrate run complete')
}
}
module.exports = MigrateCommand
i've set npx up so i can just run npx unity migrate and it will call this command based on the migrate signature
now my console should say connection successful or unable to connect and then migrate run complete, but all i see in the console is migrate run complete. it's like it isn't even trying to connect at all...
i have no idea what i'm doing wrong here.
You can't make an asynchronous process synchronous without spawning a new thread and synchronously waiting on it. (There's an npm package that does that, via execSync.)
But the good news here is there's no need to in your code. You want to do things in series, but doing things in series isn't quite the same as doing them synchronously. Here's how:
First, run can't make the async process it's starting via authenticate synchronous. So instead, run should just be async (and we don't need connect):
static async run() {
const sequelize = new Sequelize({
dialect: 'mysql',
host: 'localhost',
port: 3306,
database: 'dio_unity2',
username: 'root',
password: '',
})
try {
await sequelize.authenticate()
console.log('Connection successful.')
} catch (error) {
console.error('Unable to connect.', error)
}
console.log('migrate run complete')
}
Next, in the script, we loop through your patterns and files using an async function:
(async () => {
for (const pattern of patterns) {
for (const file of glob.sync(pattern)) {
const commandPath = path.resolve(file)
const command = require(commandPath)
if (command.signature == signature) {
command.argv = yargs.argv
await command.run() // *** Note the `await`
process.exit()
}
}
}
console.log("Command not found.")
})();
That does the work asynchronously, but one after another (in series).
If it were me, I wouldn't use process.exit() to terminate the process, it's a very aggressive form of process termination which may prevent some things from finishing (details here), and may not be as clear to people doing code maintenance later as alternatives.. Now that we've put the code in a wrapper function, I'd just return out of the function to break the loops:
await command.run();
return;
What is different between module.exports = testMethod ; and module.exports = { testMethod } ; Because when I am using module.exports = testMethod ; it is throwing error as below.
Error: Route.get() requires a callback function but got a [object Undefined]
But I am okay with module.exports = { testMethod } ;
Whole codes are
const testMethod = asyncErrorWrapper(async (req, res, next) => {
const information = req.body;
const question = await Question.create({
title: information.title,
content: information.content,
user: req.user.id,
});
res.status(200).json({
success: true,
data: question,
});
});
module.exports = { testMethod };
From VSCode, change between the ES5 or ES6 version to Js can take you on a bad way.
So, be carreful, i have the same problem recently, and after refactoring by use on ES6 module.exports = router to the end of some Js file (Node project using express) it was done.
Strange for me, on Cloud9 on Aws i have no problems.
Both are worked to export your module to outside function. But when you are using any callback function with
module.exports = somectrl
then it will fail but
module.exports = { somectrl }
because when you create an object, it actually instanciate it but when you pass a ref function/ const function name then it will behave as a existing function which does not work right.
you can do something like this to work,
module.exports = somectrl()
or
module.exports = new somectrl()
Thank you in advance for your help.
I am unsure how to do the following. I have a module to send emails, and the Config is injected into the module using summon.js dependency injection, but I need to use the sendMail method and pass it the parameter mailOptions. Here is the code example:
'use strict';
const nodemailer = require('nodemailer');
const ejs = require('ejs');
const fs = require('fs');
module.exports = function(Configs) {
// create reusable transporter object using the default SMTP transport
let transporter = nodemailer.createTransport({
host: Configs.email.host,
port: Configs.email.port,
auth: {
user: Configs.email.user,
pass: Configs.email.pass
}
});
this.sendMail = function(mailOptions) {
mailOptions.to = Configs.mockEmail || mailOptions.to
mailOptions.from = Configs.email.user
return new Promise((resolve, reject) => {
if (mailOptions.template) {
ejs.renderFile('/../templates/' + mailOptions.template +
'.ejs', mailOptions.data, null, (err, html) => {
if (err) {
return reject(err)
}
resolve(html)
})
return
}
resolve()
}).then(html => {
mailOptions.html = html || mailOptions.html
return new Promise((resolve, reject) => {
// send mail with defined transport object
transporter.sendMail(mailOptions, (error, info) => {
if (error) {
return reject(error)
}
resolve(info)
})
})
})
}
return this
}
Then, I want to make use of this module:
const EmailUtil = require('email')
async function foo() {
// Do something async with await.
const mailOptions = {...}
EmailUtils.sendMail(mailOption);
}
However, it gives me the error:
TypeError: EmailUtils.sendMail is not a function
Note: I can remove the module.export = function(Configs) but them that will not be good since i would need to hard code the path of my config file and I have multiple configuration files for each environment. Then, I want to be able to keep Summon.js dependency injection while calling sendMail from another module. Thanks
Any ideas??
Thank you!
Since you're exporting a function you need to actually call it after requiring the module:
const EmailUtil = require('email')
async function foo() {
// Do something async with await.
const mailOptions = {...}
EmailUtil().sendMail(mailOption);
}
The use of this suggests that a function is supposed to be used as a constructor. There will be desired this object only if a function is called with new or bound to some context.
There is a convention in JavaScript to use pascal-cased names for constructors, so they could be identified unambiguously in the code.
For given EmailUtil, it should be:
const EmailUtil = require('email');
const emailUtil = new EmailUtil(config);
...
emailUtil.sendMail(mailOption);
I answer this by including EmailUtils in the depend.json file which takes care of defining the dependencies for summonjs. In this way, I was able to pass the Configs to EmailUtils and call sendMail in such way.
EmailUtils.sendMail(mailOptions);
There was no need to use the keyword new, which is a great answer. I was not aware a module could be instantiated in this way.
I am working on post method in the server side to retrieve all files inside the requested directory (not recursive), and below is my code.
I am having difficulty sending the response back (res.json(pathContent);) with the updated pathContent without using the setTimeout.
I understand that this is due to the asynchronous behavior of the file system methods used (readdir and stat) and need to use some sort of callback, async, or promise technique.
I tried to use the async.waterfall with the entire body of readdir as one function and the res.json(pathContent) as the other, but it didn't send the updated array to the client side.
I know that there have been thousands of questions regarding this asynchronous operation but could not figure out how to solve my case after reading number of posts.
Any comments would be appreciated. Thanks.
const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');
var pathName = '';
const pathContent = [];
app.post('/api/files', (req, res) => {
const newPath = req.body.path;
fs.readdir(newPath, (err, files) => {
if (err) {
res.status(422).json({ message: `${err}` });
return;
}
// set the pathName and empty pathContent
pathName = newPath;
pathContent.length = 0;
// iterate each file
const absPath = path.resolve(pathName);
files.forEach(file => {
// get file info and store in pathContent
fs.stat(absPath + '/' + file, (err, stats) => {
if (err) {
console.log(`${err}`);
return;
}
if (stats.isFile()) {
pathContent.push({
path: pathName,
name: file.substring(0, file.lastIndexOf('.')),
type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
})
} else if (stats.isDirectory()) {
pathContent.push({
path: pathName,
name: file,
type: 'Directory',
});
}
});
});
});
setTimeout(() => { res.json(pathContent); }, 100);
});
The easiest and cleanest way would be use await/async, that way you can make use of promises and the code will almost look like synchronous code.
You therefor need a promisified version of readdir and stat that can be create by the promisify of the utils core lib.
const { promisify } = require('util')
const readdir = promisify(require('fs').readdir)
const stat = promisify(require('fs').stat)
async function getPathContent(newPath) {
// move pathContent otherwise can have conflicts with concurrent requests
const pathContent = [];
let files = await readdir(newPath)
let pathName = newPath;
// pathContent.length = 0; // not needed anymore because pathContent is new for each request
const absPath = path.resolve(pathName);
// iterate each file
// replace forEach with (for ... of) because this makes it easier
// to work with "async"
// otherwise you would need to use files.map and Promise.all
for (let file of files) {
// get file info and store in pathContent
try {
let stats = await stat(absPath + '/' + file)
if (stats.isFile()) {
pathContent.push({
path: pathName,
name: file.substring(0, file.lastIndexOf('.')),
type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
})
} else if (stats.isDirectory()) {
pathContent.push({
path: pathName,
name: file,
type: 'Directory',
});
}
} catch (err) {
console.log(`${err}`);
}
}
return pathContent;
}
app.post('/api/files', (req, res, next) => {
const newPath = req.body.path;
getPathContent(newPath).then((pathContent) => {
res.json(pathContent);
}, (err) => {
res.status(422).json({
message: `${err}`
});
})
})
And you should not concatenated paths using + (absPath + '/' + file), use path.join(absPath, file) or path.resolve(absPath, file) instead.
And you never should write your code in a way that the code executed for the request, relays on global variables like var pathName = ''; and const pathContent = [];. This might work in your testing environment, but will for sure lead to problems in production. Where two request work on the variable at the "same time"
Based on the initial comment I received and the reference, I used readdirSync and statSync instead and was able to make it work. I will review other answers as well and learn about other ways to implement this.
Thank you all for your kind inputs.
Here is my solution.
const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');
var pathName = '';
const pathContent = [];
app.post('/api/files', (req, res) => {
const newPath = req.body.path;
// validate path
let files;
try {
files = fs.readdirSync(newPath);
} catch (err) {
res.status(422).json({ message: `${err}` });
return;
}
// set the pathName and empty pathContent
pathName = newPath;
pathContent.length = 0;
// iterate each file
let absPath = path.resolve(pathName);
files.forEach(file => {
// get file info and store in pathContent
let fileStat = fs.statSync(absPath + '/' + file);
if (fileStat.isFile()) {
pathContent.push({
path: pathName,
name: file.substring(0, file.lastIndexOf('.')),
type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
})
} else if (fileStat.isDirectory()) {
pathContent.push({
path: pathName,
name: file,
type: 'Directory',
});
}
});
res.json(pathContent);
});
There is different way to do it :
You can first promisify the function with using new Promise() then second, use async/await or .then()
You can use the function ProsifyAll() of the Bluebird package (https://www.npmjs.com/package/bluebird)
You can use the synchrone version of the fs functions
https://nodejs.org/api/fs.html#fs_fs_readdirsync_path_options
https://nodejs.org/api/fs.html#fs_fs_statsync_path_options
Here's some options:
Use the synchronous file methods (check the docs, but they usually end with Sync). Slower, but a fairly simple code change, and very easy to understand.
Use promises (or util.promisify) to create a promise for each stat, and Promise.all to wait for all the stats to complete. After that, you can use async functions and await as well for easier to read code and simpler error handling. (Probably the largest code change, but it will make the async code easier to follow)
Keep a counter of the number of stats you have done, and if that number is the size you expect, then call res.json form inside the stat callback (smallest code change, but very error prone)
I'm having difficulty decrypting multiple environment variables in nodejs for an AWS lambda. I've looked at the code sample supplied in the console and the following two related questions:
Question 1,
Question 2
I have been able to successfully decrypt a single environment variable through their code sample, however, when I try to apply a cleaner approach through the use of promises (methods outlined in the questions above), I get this error when testing the lambda function in the console:
TypeError: First argument must be a string, Buffer, ArrayBuffer,
Array, or array-like object.
I was wondering if anyone has had this issue before and how I could go about resolving it?
Edit:
I've added some samples from my code below
const AWS = require('aws-sdk');
const mysql = require('mysql');
let connection;
const encrypted = {
username: process.env.username,
password: process.env.password,
database: process.env.database,
host: process.env.host
};
let decrypted = {};
const encryptedEnvVars = [process.env.username, process.env.password, process.env.database, process.env.host ];
exports.handler = (event, context, callback) => {
if (isEnvVarsDecrypted()) {
processEvent(event, context);
} else {
Promise.all(encryptedEnvVars.map(decryptKMS))
.then(decryptEnvVars)
.catch(console.log);
}
};
function decryptKMS(key) {
return new Promise((resolve, reject) => {
const kms = new AWS.KMS()
kms.decrypt({ CiphertextBlob: new Buffer(key, 'base64') }, (err, data) => {
if(err) { reject(err); }
else { resolve(data.Plaintext.toString('ascii')); }
});
});
}
var decryptEnvVars = data => {
return new Promise((resolve, reject) => {
console.log(data);
decrypted.username = data[0].Plaintext.toString('ascii');
decrypted.password = data[1].Plaintext.toString('ascii');
decrypted.database = data[2].Plaintext.toString('ascii');
decrypted.host = data[3].Plaintext.toString('ascii');
resolve();
});
};
var isEnvVarsDecrypted = () => {
return decrypted.username && decrypted.password && decrypted.database && decrypted.host;
}
If key is null, then new Buffer(key, 'base64') will fail with the error you describe.
When I ran your code myself:
If any environment variable was missing, the error occurred
When all environment variables were declared, the error ceased
So, you should confirm that the environment variables you reference are actually defined.
A couple of other pointers:
Make sure you are always calling the lambda callback, regardless of success/failure; this is how you signal to the lambda environment that execution has ended.
After calling decryptEnvVars, you should call your processEvent function