I'm having difficulty decrypting multiple environment variables in nodejs for an AWS lambda. I've looked at the code sample supplied in the console and the following two related questions:
Question 1,
Question 2
I have been able to successfully decrypt a single environment variable through their code sample, however, when I try to apply a cleaner approach through the use of promises (methods outlined in the questions above), I get this error when testing the lambda function in the console:
TypeError: First argument must be a string, Buffer, ArrayBuffer,
Array, or array-like object.
I was wondering if anyone has had this issue before and how I could go about resolving it?
Edit:
I've added some samples from my code below
const AWS = require('aws-sdk');
const mysql = require('mysql');
let connection;
const encrypted = {
username: process.env.username,
password: process.env.password,
database: process.env.database,
host: process.env.host
};
let decrypted = {};
const encryptedEnvVars = [process.env.username, process.env.password, process.env.database, process.env.host ];
exports.handler = (event, context, callback) => {
if (isEnvVarsDecrypted()) {
processEvent(event, context);
} else {
Promise.all(encryptedEnvVars.map(decryptKMS))
.then(decryptEnvVars)
.catch(console.log);
}
};
function decryptKMS(key) {
return new Promise((resolve, reject) => {
const kms = new AWS.KMS()
kms.decrypt({ CiphertextBlob: new Buffer(key, 'base64') }, (err, data) => {
if(err) { reject(err); }
else { resolve(data.Plaintext.toString('ascii')); }
});
});
}
var decryptEnvVars = data => {
return new Promise((resolve, reject) => {
console.log(data);
decrypted.username = data[0].Plaintext.toString('ascii');
decrypted.password = data[1].Plaintext.toString('ascii');
decrypted.database = data[2].Plaintext.toString('ascii');
decrypted.host = data[3].Plaintext.toString('ascii');
resolve();
});
};
var isEnvVarsDecrypted = () => {
return decrypted.username && decrypted.password && decrypted.database && decrypted.host;
}
If key is null, then new Buffer(key, 'base64') will fail with the error you describe.
When I ran your code myself:
If any environment variable was missing, the error occurred
When all environment variables were declared, the error ceased
So, you should confirm that the environment variables you reference are actually defined.
A couple of other pointers:
Make sure you are always calling the lambda callback, regardless of success/failure; this is how you signal to the lambda environment that execution has ended.
After calling decryptEnvVars, you should call your processEvent function
Related
I am trying to export database properties stored in properties file from Javascript module. By the time I read database properties file, Javascript file is already exported and data properties appear undefined wherever I use in other modules.
const Pool = require('pg').Pool;
const fs = require('fs')
const path = require('path');
class DbConfig {
constructor(dbData) {
this.pool = new Pool({
user: dbData['user'],
host: dbData['host'],
database: dbData['database'],
password: dbData['password'],
max: 20,
port: 5432
});
}
}
function getdbconf() {
const dbData = {};
fs.readFile("../../db_properties.txt"), 'utf8', (err, data) => {
if (err) {
console.error(err)
return
}
// dbData = {"user":"postgres", "password": "1234"...};
return dbData;
});
}
let db = new DbConfig(getdbconf());
let dbPool = db.pool;
console.log("dbpool : -> : ",dbPool); // username and password appear undefined
module.exports = { dbPool };
Is there a way to read data before exporting data from Javascript module?
Usually database config or any other sensitive info is read from a .env file using dotenv .
Or
you could also provide env from command line itself like
DB_HOST=127.0.0.1 node index.js
inside your index.js
console.log(process.env.DB_HOST)
Please create a new file (connection-pool.js) and paste this code:
const { Pool } = require('pg');
const poolConnection = new Pool({
user: 'postgresUserName',
host: 'yourHost',
database: 'someNameDataBase',
password: 'postgresUserPassword',
port: 5432,
});
console.log('connectionOptions', poolConnection.options);
module.exports = poolConnection;
For use it, create a new file (demo-connection.js) and paste this code:
const pool = require('./connection-pool');
pool.query('SELECT NOW();', (err, res) => {
if (err) {
// throw err;
console.log('connection error');
return;
}
if (res) {
console.log(res.rows);
pool.end();
}
});
This is an alternative option 🙂
Exporting the result of async calls
To export values which have been obtained asynchronously, export a Promise.
const fs = require('fs/promises'); // `/promise` means no callbacks, Promise returned
const dbDataPromise = fs.readFile('fileToRead')); //`readFile` returns Promise now
module.exports = dbDataPromise;
Importing
When you need to use the value,
const dbDataPromise = require('./dbdata');
async init() {
const dbData = await dbDataPromise;
}
//or without async, using Promise callbacks
init() {
dbDataPromise
.then(dbData => the rest of your code that depends on dbData here);
}
Current code broken
Please note that your current code, as pasted above, is broken:
function getdbconf() {
const dbData = {};
fs.readFile("../../db_properties.txt"), 'utf8', (err, data) => {
//[...] snipped for brevity
return dbData;
});
}
fs.readFile "returns" dbData, but there is nothing to return to, since you are in a callback which you did not call yourself. Function getdbconf returns nothing.
The line that says let db = new DbConfig(getdbconf()); will NOT work. It needs to be inside the callback.
The only way to avoid putting all of your code inside the callback (and "flatten" it) is to use await, or to use readFileSync
Avoiding the issue
Using environment variables
Suhas Nama's suggestion is a good one, and is common practice. Try putting the values you need in environment variables.
Using synchronous readFile
While using synchronous calls does block the event loop, it's ok to do during initialization, before your app is up and running.
This avoids the problem of having everything in a callback or having to export Promises, and is often the best solution.
Sometimes I'm getting duplicated documents from a callable function that looks like this:
const { default: Big } = require('big.js');
const { firestore } = require('firebase-admin');
const functions = require('firebase-functions');
const { createLog } = require('./utils/createLog');
const { payCart } = require('./utils/payCart');
const { unlockCart } = require('./utils/unlockCart');
exports.completeRechargedTransaction = functions.https.onCall(
async (data, context) => {
try {
if (!context.auth) {
throw new functions.https.HttpsError(
'unauthenticated',
'unauthenticated'
);
}
const requiredProperties = [
'foo',
'bar',
'etc'
];
const isDataValid = requiredProperties.every(prop => {
return Object.keys(data).includes(prop);
});
if (!isDataValid) {
throw new functions.https.HttpsError(
'failed-precondition',
'failed-precondition'
);
}
const transactionRef = firestore()
.collection('transactions')
.doc(data.transactionID);
const userRef = firestore().collection('users').doc(data.paidBy.userID);
let currentTransaction = null;
await firestore().runTransaction(async transaction => {
try {
const transactionSnap = await transaction.get(transactionRef);
if (!transactionSnap.exists) {
throw new functions.https.HttpsError(
'not-found',
'not-found'
);
}
const transactionData = transactionSnap.data();
if (transactionData.status !== 'recharged') {
throw new functions.https.HttpsError(
'invalid-argument',
'invalid-argument'
);
}
if (transactionData.type !== 'recharge') {
throw new functions.https.HttpsError(
'invalid-argument',
'invalid-argument'
);
}
if (transactionData.paidBy === null) {
throw new functions.https.HttpsError(
'invalid-argument',
'invalid-argument',
);
}
const userSnap = await transaction.get(userRef);
if (!userSnap.exists) {
throw new functions.https.HttpsError(
'not-found',
'not-found',
);
}
const userData = userSnap.data();
const newUserPoints = new Big(userData.points).plus(data.points);
if (!data.isGoldUser) {
transaction.update(userRef, {
points: parseFloat(newUserPoints.toFixed(2))
});
}
currentTransaction = {
...data,
remainingBalance: parseFloat(newUserPoints.toFixed(2)),
status: 'completed'
};
transaction.update(transactionRef, currentTransaction);
} catch (error) {
console.error(error);
throw error;
}
});
const { paymentMethod } = data.rechargeDetails;
let cashAmount = 0;
if (paymentMethod && paymentMethod.paymentMethod === 'cash') {
cashAmount = data.points;
}
let cartResponse = null;
if (
data.rechargeDetails.isProcessingCart &&
Boolean(data.paidBy.userID) &&
!data.isGoldUser
) {
cartResponse = await payCart(context, data.paidBy.userID, cashAmount);
// This is the function that does all the writes and for some reason it is getting
// called twice or thrice in some rare cases, and I'm pretty much sure that
// The Angular Client is only calling this function "completeRechargedTransaction " once.
}
await createLog({
message: 'Success',
createdAt: new Date(),
type: 'activity',
collectionName: 'transactions',
callerID: context.auth.uid || null,
docID: transactionRef.id
});
return {
code: 200,
message: 'Success',
transaction: currentTransaction,
cartResponse
};
} catch (error) {
console.error(error);
await unlockCart(data.paidBy.userID);
await createLog({
message: error.message,
createdAt: new Date(),
type: 'error',
collectionName: 'transactions',
callerID: context.auth.uid || null,
docID: data.transactionID,
errorSource:
'completeRechargedTransaction'
});
throw error;
}
}
);
I'm reading a lot of firebase documentation, but I can't find a solution to implement idempotency on my callable functions, the context parameter in callable function is very different from background functions and triggers, the callable context looks like this:
https://firebase.google.com/docs/reference/functions/providers_https_.callablecontext
I did find a helpful blogpost to implement idempotency with firebase triggers:
Cloud Functions pro tips: Building idempotent functions
But I don't fully understand this approach because I think it's assuming that the document writes are made on the client aka the front end application, and I don't really think that's a good approach because is it too reliant on the client and I'm afraid of security issues as well.
So yeah, I would like to know is there's a way to implement Idempotency on Callable Functions, I need something like an EventID but for callable functions to safely implement payments on my app and third party apis, such as stripe.
I will appreciate any help or hint you can give me.
The use of idempotent functions mainly applies to the automatically triggered Cloud Functions that respond to events such as a file uploaded to Cloud Storage or document added to Firestore. In these cases, the event triggers the function to be executed, and if the function succeeds, all is well. However, if the function fails, it will get retried automatically which leads to the problems discussed in the blog post you linked.
In the case of user-triggered cloud functions (a HTTPS Event or Callable cloud function), these are not retried automatically. It is left up to the caller of these functions to choose to handle any errors and whether they are retried by the client calling the function again.
As these user-triggered functions are only executed by your client code, you should check to make sure that completeRechargedTransaction() isn't being called more than once. A method of testing this is to supply your own value for Event ID prior to calling the function like so:
// using a firebase push ID as a UUID
// could also use someFirestoreCollectionReference.doc().id or uuid()
const eventId = firebase.database.ref().push().key;
completeRechargedTransaction({
eventId,
/* ... other data ... */
})
.then(console.log.bind(null, "Successfully completed recharged transaction:"))
.catch(console.error.bind(null, "Failed to complete recharged transaction:"));
Note: One of the most common ways functions will get called twice by the client is because of rerenders where you've updated the state to show a "loading" message and then your call to the function gets made a second time. As an example for React, you would make sure your database call is wrapped in it's own useEffect() call.
I am having problem with my Discord bot trying to access the Google Sheets API v4.0. I want it to read, write, and update data in the sheet. My code work fine for Node.js application but when I put my code in a Discord bot, it gives me this error:
UnhandledPromiseRejectionWarning: Error: The request is missing a valid API key
I don't understand why I am getting this error. I even made a service account for my Discord bot.
Here is the Discord bot's code.
require('dotenv').config();
const sheet = require('./sheet');
const { Client } = require('discord.js');
const { sheets } = require('googleapis/build/src/apis/sheets');
const client = new Client();
Sheet file code:
const {google} = require('googleapis');
const keys = require('./keys.json');
const client = new google.auth.JWT(
keys.client_email,
keys.private_key_id,
keys.private_key,
['https://www.googleapis.com/auth/spreadsheets']
);
client.authorize(function(err,tokens){
if(err){
console.log(err);
return;
}
// else{
// console.log('Connected');
// gsrun(client);
// }
});
const getdata = async function gsrun(cl){
const gsapi = google.sheets({version:'v4',auth:cl});
const opt = {
spreadsheetId:'1LzdhD4rb2FdElTyQCAkgb5oyeGeu0d9rT2abS4n_4i8',
range: 'A2:B5'
};
let data = await gsapi.spreadsheets.values.get(opt);
console.log(data.data.values);
}
exports.getdata = getdata;
I have given access to bot email id to sheet which is discordbotapi#nodejs-api-testing-discordbot.iam.gserviceaccount.com. I have also enabled the Google Sheets API. Where am I making error?
When you tried it in Node.js without Discord.js, you didn't export anything and you simply called gsrun(client) after the authorisation was successful, so it worked fine. The problem is that now you try to use getData without any authorisation. Although you have client.authorize in your code, you never use it.
To solve this, I would make at least two different functions here; one for generating the client and one for the get request itself and export them both.
To generate a client I’d wrap this in a promise. This way I could use async/await later. This function will create a client with the JWT, perform the authorisation, and either resolve with the client or reject the promise depending on the results of client.authorize().
function connect() {
return new Promise((resolve, reject) => {
const scope = ['https://www.googleapis.com/auth/spreadsheets'];
const { client_email, private_key, private_key_id } = keys;
const client = new google.auth.JWT(
client_email,
private_key_id,
private_key,
scope,
);
client.authorize((err) => {
if (err) {
reject(err);
} else {
resolve(client);
}
});
});
}
Now you can simply connect and get the client by using const client = await connect().
The second part is to get some data from the spreadsheet. Again, I'd wrap it in a promise. This function will accept the client (we’ve just created above) and the options with the spreadsheetId and range. Inside the promise you just call the API endpoint with the options:
function getData(client, options) {
return new Promise((resolve, reject) => {
const endpoint = google.sheets({ version: 'v4', auth: client });
endpoint.spreadsheets.values.get(options, (err, data) => {
if (err) {
reject(err);
} else {
// or resolve with data.data.values if you only want the values
resolve(data.data);
}
});
});
}
You can export both of these in an object. Here is the full sheet.js file:
const { google } = require('googleapis');
const keys = require('./keys.json');
function connect() {
return new Promise((resolve, reject) => {
const scope = ['https://www.googleapis.com/auth/spreadsheets'];
const { client_email, private_key, private_key_id } = keys;
const client = new google.auth.JWT(
client_email,
private_key_id,
private_key,
scope,
);
client.authorize((err) => {
if (err) {
reject(err);
} else {
resolve(client);
}
});
});
}
function getData(client, options) {
return new Promise((resolve, reject) => {
const endpoint = google.sheets({ version: 'v4', auth: client });
endpoint.spreadsheets.values.get(options, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data.data);
}
});
});
}
module.exports = { connect, getData };
Then, you can import it in your bot's file and use it in there to connect first and then get the values:
const { Client } = require('discord.js');
const { connect, getData } = require('./sheet');
const client = new Client();
client.on('message', async (message) => {
if (message.author.bot) return;
const auth = await connect();
const options = {
spreadsheetId: '1LzdhD4rb2FdElTyQCAkgb5oyeGeu0d9rT2abS4n_4i8',
range: 'A2:B5',
};
const { values } = await getData(auth, options);
message.channel.send(values);
});
I'm new the Node.js and I've been working with a sample project by a third party provider and I'm trying to use Azure Key Vault to store configuration values.
I'm having trouble getting a process to wait before executing the rest. I'll try to detail as much as I know.
The sample project has a file named agent.js which is the start page/file. On line 16 (agent_config = require('./config/config.js')[process.env.LP_ACCOUNT][process.env.LP_USER]) it calls a config file with values. I'm trying to set these value using Key Vault. I've tried many combinations of calling functions, and even implementing async / await but the value for agent_config always contains a [Promise] object and not the data returned by Key Vault.
If I'm right, this is because the Key Vault itself uses async / await too and the config file returns before the Key Vault values are returned.
How can Key Vault be added/implemented in a situation like this?
Here's what I've tried:
First updated agent.js to
let agent_config = {};
try {
agent_config = require('./config/config.js')['123']['accountName'];
} catch (ex) {
log.warn(`[agent.js] Error loading config: ${ex}`)
}
console.log(agent_config);
Test 1
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
function getValue(secretName, secretVersion) {
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' }).then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
module.exports = {
'123': {
'accountName': {
accountId: getValue('mySecretName', '')
}
}
};
Results
{ accountsId: undefined }
Test 2
Made getValue an async function and wrapped it around another function (tried without the wrapping and didn't work either)
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
async function getValue(secretName, secretVersion) {
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' }).then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
async function config() {
module.exports = {
'123': {
'accountName': {
accountId: await getValue('mySecretName', '')
}
}
};
}
config();
Results
{}
Test 3
Made getValue an async function and wrapped it around another function (tried without the wrapping and didn't work either)
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
async function getValue(secretName, secretVersion) {
return msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
module.exports = {
'123': {
'accountName': {
accountId: getValue('mySecretName', '')
}
}
};
config();
Results
{ accountId: { <pending> } }
Other
I've tried many others ways like module.exports = async (value) =< {...} (found through other questions/solutions without success.
I'm starting to think I need to do some "waiting" on agent.js but I haven't found good info on this.
Any help would be great!
One issue is that your getValue function is not returning anything as your returns need to be explicit.
(and without the promise being returned, there's nothing to await on)
async function getValue(secretName, secretVersion) {
return msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
You could also get away with less explicit returns using arrow functions..
const getValue = async (secretName, secretVersion) =>
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then(credentials => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion)
.then(response => response.Value);
});
Introducing the Azure Key Vault read, which is async, means your whole config read is async. There' nothing you can do to get around that. This will mean that the code that uses the config will need to handle it appropriately. You start by exporting an async function that will return the config..
async function getConfig() {
return {
'123': {
'accountName': {
accountId: await getValue('mySecretName', '')
}
}
};
}
module.exports = getConfig;
In your agent code you call that function. This will mean that your agent code will need to be wrapped in a function too, so maybe something like this..
const Bot = require('./bot/bot.js');
const getConfig = require('./config/config.js');
getConfig().then(agentConfig => {
const agent = new Bot(agentConfig);
agent.on(Bot.const.CONNECTED, data => {
log.info(`[agent.js] CONNECTED ${JSON.stringify(data)}`);
});
});
The package azure-keyvault has been deprecated in favor of the new packages to deal with Keyvault keys, secrets and certificates separately. For your scenario, you can use the new #azure/keyvault-secrets package to talk to Key Vault and the new #azure/identity package to create the credential.
const { SecretClient } = require("#azure/keyvault-secrets");
const { DefaultAzureCredential } = require("#azure/identity");
async function getValue(secretName, secretVersion) {
const credential = new DefaultAzureCredential();
const client = new SecretClient(KEY_VAULT_URI, credential);
const secret = await client.getSecret(secretName);
return secret.value;
}
The DefaultAzureCredential assumes that you have set the below env variables
AZURE_TENANT_ID: The tenant ID in Azure Active Directory
AZURE_CLIENT_ID: The application (client) ID registered in the AAD tenant
AZURE_CLIENT_SECRET: The client secret for the registered application
To try other credentials, see the readme for #azure/identity
If you are moving from the older azure-keyvault package, checkout the migration guide to understand the major changes
Thank you in advance for your help.
I am unsure how to do the following. I have a module to send emails, and the Config is injected into the module using summon.js dependency injection, but I need to use the sendMail method and pass it the parameter mailOptions. Here is the code example:
'use strict';
const nodemailer = require('nodemailer');
const ejs = require('ejs');
const fs = require('fs');
module.exports = function(Configs) {
// create reusable transporter object using the default SMTP transport
let transporter = nodemailer.createTransport({
host: Configs.email.host,
port: Configs.email.port,
auth: {
user: Configs.email.user,
pass: Configs.email.pass
}
});
this.sendMail = function(mailOptions) {
mailOptions.to = Configs.mockEmail || mailOptions.to
mailOptions.from = Configs.email.user
return new Promise((resolve, reject) => {
if (mailOptions.template) {
ejs.renderFile('/../templates/' + mailOptions.template +
'.ejs', mailOptions.data, null, (err, html) => {
if (err) {
return reject(err)
}
resolve(html)
})
return
}
resolve()
}).then(html => {
mailOptions.html = html || mailOptions.html
return new Promise((resolve, reject) => {
// send mail with defined transport object
transporter.sendMail(mailOptions, (error, info) => {
if (error) {
return reject(error)
}
resolve(info)
})
})
})
}
return this
}
Then, I want to make use of this module:
const EmailUtil = require('email')
async function foo() {
// Do something async with await.
const mailOptions = {...}
EmailUtils.sendMail(mailOption);
}
However, it gives me the error:
TypeError: EmailUtils.sendMail is not a function
Note: I can remove the module.export = function(Configs) but them that will not be good since i would need to hard code the path of my config file and I have multiple configuration files for each environment. Then, I want to be able to keep Summon.js dependency injection while calling sendMail from another module. Thanks
Any ideas??
Thank you!
Since you're exporting a function you need to actually call it after requiring the module:
const EmailUtil = require('email')
async function foo() {
// Do something async with await.
const mailOptions = {...}
EmailUtil().sendMail(mailOption);
}
The use of this suggests that a function is supposed to be used as a constructor. There will be desired this object only if a function is called with new or bound to some context.
There is a convention in JavaScript to use pascal-cased names for constructors, so they could be identified unambiguously in the code.
For given EmailUtil, it should be:
const EmailUtil = require('email');
const emailUtil = new EmailUtil(config);
...
emailUtil.sendMail(mailOption);
I answer this by including EmailUtils in the depend.json file which takes care of defining the dependencies for summonjs. In this way, I was able to pass the Configs to EmailUtils and call sendMail in such way.
EmailUtils.sendMail(mailOptions);
There was no need to use the keyword new, which is a great answer. I was not aware a module could be instantiated in this way.