We are overriding console.log in our productive ExpressJS application to add certain fields (e.g. timestamp, sessionid, requestid) as well as send logs to a syslog server.
We are doing this by adding console.requestId and console.sessionId properties to the global console object and then overriding console.log() to output these values as well as send logs to syslog. This works fine:
let originalConsoleLog = console.log;
console.log = function() {
let args = Array.from(arguments);
args.unshift(this.sessionId);
args.unshift(this.requestId);
originalConsoleLog.log.apply(console, args);
syslog.log(arguments);
}
app.get('/some-endpoint', async (req, res) => {
console.requestId = req.header('X-Request-Id');
console.sessionId = req.session = req.cookies['SESSIONID'];
let response = await someAsyncProcess();
console.log('foo');
req.json(response);
});
The problem comes when 2 requests are processed concurrently:
Request 1 sets requestId/sessionId REQ1/SESSA
Request 1 waits for someAsyncProcess()
Request 2 sets requestId/sessionId REQ2/SESSB
Request 1 logs REQ2/SESSB foo when it should log REQ1/SESSA foo
So globals (like console) are out.
Locals are also out since we do not want to pass some logger instance to each and every module.
What we need is a "pseudo global" variable which is scoped to a single request in ExpressJS. Something like closure but to external modules as well:
let logger = new Logger(sessionId, requestId);
function() {
// I can see logger
logger.log('hello');
// This module can't
let foo = require('foo');
}
The idea is that we can set a request and session id on this logger variable and all functions/modules/methods called from here on will console.log with this session/request id. The variable "belongs" to all code called from the request responder.
We don't want to inject this logger instance into every submodule we use but we want to capure console.log requests dynamically and inject the relevant local sessionId and requestId.
You'll probably need to use async hooks for this, something like
const asyncHooks = require('async_hooks');
const store = new Map();
const asyncHook = asyncHooks.createHook({
init: (asyncId, _, triggerAsyncId) => {
if (store.has(triggerAsyncId)) {
store.set(asyncId, store.get(triggerAsyncId))
}
},
destroy: (asyncId) => {
if (store.has(asyncId)) {
store.delete(asyncId);
}
}
});
asyncHook.enable();
const createRequestContext = (sessionId, requestId) => {
const context = { sessionId, requestId };
store.set(asyncHooks.executionAsyncId(), context);
return context;
};
const getRequestContext = () => {
return store.get(asyncHooks.executionAsyncId());
};
let originalConsoleLog = console.log;
console.log = function() {
let args = Array.from(arguments);
const { sessionId, requestId } = getRequestContext();
args.unshift(sessionId);
args.unshift(requestId);
originalConsoleLog.log.apply(console, args);
syslog.log(arguments);
}
app.get('/some-endpoint', async (req, res) => {
const requestId = req.header('X-Request-Id');
const sessionId = req.session = req.cookies['SESSIONID'];
createRequestContext(requestId, sesscionId);
let response = await someAsyncProcess();
console.log('foo');
req.json(response);
});
Related
I don't understand why my spy is not being used. I have used this code elsewhere and it has worked fine.
Here is my test:
const {DocumentEngine} = require('../documentEngine')
const fileUtils = require('../utils/fileUtils')
const request = {...}
const fieldConfig = {...}
test('If the Carbone addons file is not found, context is built with the carboneAddons property as an empty object', async () => {
const expectedResult = {
carboneAddons: {},
}
const fileExistSpy = jest
.spyOn(fileUtils, 'checkFileExists')
.mockResolvedValue(false)
const result = await docEngine.buildContext(request, fieldConfig)
expect(fileExistSpy).toHaveBeenCalledTimes(1)
})
Here is the code that it is being tested:
async function buildContextForLocalResources(request, fieldConfig) {
/* other code */
const addonFormatters = await getCarboneAddonFormatters()
const context = {
sourceJson,
addonFormatters,
documentFormat,
documentTemplateId,
documentTemplateFile,
responseType,
jsonTransformContext
}
return context
}
async function getCarboneAddonFormatters() {
const addOnPath = path.resolve(
docConfig.DOC_GEN_RESOURCE_LOCATION,
'library/addon-formatters.js'
)
if (await checkFileExists(addOnPath)) {
logger.info('Formatters found and are being used')
const {formatters} = require(addOnPath)
return formatters
}
logger.info('No formatters were found')
return {}
}
This is the code from my fileUtils file:
const fs = require('fs/promises')
async function checkFileExists(filePath) {
try {
await fs.stat(filePath)
return true
} catch (e) {
return false
}
}
My DocumentEngine class calls the buildContext function which in turn calls the its method getCarboneAddonFormatters. The fileUtils is outside of DocumentEngine class in a utilities folder. The original code I had this working on was TypeScript as opposed to this which is just NodeJS Javascript. The config files for both are the same. When I try to step through the code (VSCode debugger), as soon as I hit the line with await fs.stat(filePath) in the checkFileExists function, it kicks me out of the test and moves on to the next test - no error messages or warnings.
I've spent most of the day trying to figure this out. I don't think I need to do an instance wrapper for the documentEngine, because checkFileExists is not a class member, and that looks like a React thing...
Any help in getting this to work would be appreciated.
I have three files
BotJobActions.js
TestDate.js
CreateCron.js
The BotJobActions file creates a function called getUser that returns the user connected to a specific job, then exports the getUser along with a bunch of other functions.
const getUser = async (jobId) =>{
await mongoConnect(process.env.DB_PWORD)
try {
const user = await User.findOne({pendingJobs:jobId})
return user
} catch (err) {
console.log(err)
}
}
module.exports = { newJob, getUserJobs, getUser, updateUserJob, destroyUserPendingJob, destroyUserCompletedJob, activateJob, deactivateJob, endJob }
TestDate defines a function called runBot which runs a bot Job. In runBot it also calls the getUser function, so I can make changes to a specific user. Then exports the function because it will be used in other files.
const { getUser } = require("../bot/botJobActions");
const runBot = async (todayJobs) =>{
// await mongoConnect(process.env.DB_PWORD)
for(const job of todayJobs){
const clubPassword = decryptToken(job.clubPassword.token, job.clubPassword.iv)
const user = await getUser(job.id)
if(job.proxy){
const proxyConfig = await getProxyConfig(user)
if(proxyConfig.status === "no proxy") console.log("[-] Proxy Config Retrival Error/Running Without Proxy")
// await startBot(member=job.member?job.member : null, proxy=proxyConfig.status === 'success'?proxyConfig:null, job.clubUsername, clubPassword, job.startTime, job.endTime, job.courseList, job.id)
await console.log(member=job.member?job.member : null, proxy=proxyConfig.status === 'success'?proxyConfig:null, job.clubUsername, clubPassword, job.startTime, job.endTime, job.courseList, job.id)
}else{
// await startBot(member=job.member?job.member : null, proxy=null, job.clubUsername, clubPassword, job.startTime, job.endTime, job.courseList, job.id)
await console.log(member=job.member?job.member : null, proxy=null, job.clubUsername, clubPassword, job.startTime, job.endTime, job.courseList, job.id)
}
}
return
}
module.exports = { runBot, getJobs }
CreateCron is a function that runs whenever a job is created with a specific start time. This function will create a cron job for that specified time to run the bot.
const schedule = require('node-schedule');
const { runBot } = require('./testDate');
const createCron = (job) =>{
const startDate = new Date(job.botStartDate)
const startTime = new Date(`09/19/2000 ${job.botStartTime}`)
startDate.setHours(startTime.getHours())
startDate.setMinutes(startTime.getMinutes())
console.log(startDate.toUTCString())
schedule.scheduleJob(startDate, async function(){
console.log('run job')
await runBot([job])
})
}
My problem thought is that whenever I run the createCron function, I get an error saying that the getUser is not a function. Even thought it is.
Any help is appreciated!!
I was able to fix the problem. All I had to do was use the absolute path to the function instead of the relative path. Then the functions worked. Hope this can help somebody!
I'm new the Node.js and I've been working with a sample project by a third party provider and I'm trying to use Azure Key Vault to store configuration values.
I'm having trouble getting a process to wait before executing the rest. I'll try to detail as much as I know.
The sample project has a file named agent.js which is the start page/file. On line 16 (agent_config = require('./config/config.js')[process.env.LP_ACCOUNT][process.env.LP_USER]) it calls a config file with values. I'm trying to set these value using Key Vault. I've tried many combinations of calling functions, and even implementing async / await but the value for agent_config always contains a [Promise] object and not the data returned by Key Vault.
If I'm right, this is because the Key Vault itself uses async / await too and the config file returns before the Key Vault values are returned.
How can Key Vault be added/implemented in a situation like this?
Here's what I've tried:
First updated agent.js to
let agent_config = {};
try {
agent_config = require('./config/config.js')['123']['accountName'];
} catch (ex) {
log.warn(`[agent.js] Error loading config: ${ex}`)
}
console.log(agent_config);
Test 1
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
function getValue(secretName, secretVersion) {
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' }).then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
module.exports = {
'123': {
'accountName': {
accountId: getValue('mySecretName', '')
}
}
};
Results
{ accountsId: undefined }
Test 2
Made getValue an async function and wrapped it around another function (tried without the wrapping and didn't work either)
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
async function getValue(secretName, secretVersion) {
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' }).then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
async function config() {
module.exports = {
'123': {
'accountName': {
accountId: await getValue('mySecretName', '')
}
}
};
}
config();
Results
{}
Test 3
Made getValue an async function and wrapped it around another function (tried without the wrapping and didn't work either)
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
async function getValue(secretName, secretVersion) {
return msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
module.exports = {
'123': {
'accountName': {
accountId: getValue('mySecretName', '')
}
}
};
config();
Results
{ accountId: { <pending> } }
Other
I've tried many others ways like module.exports = async (value) =< {...} (found through other questions/solutions without success.
I'm starting to think I need to do some "waiting" on agent.js but I haven't found good info on this.
Any help would be great!
One issue is that your getValue function is not returning anything as your returns need to be explicit.
(and without the promise being returned, there's nothing to await on)
async function getValue(secretName, secretVersion) {
return msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
You could also get away with less explicit returns using arrow functions..
const getValue = async (secretName, secretVersion) =>
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then(credentials => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion)
.then(response => response.Value);
});
Introducing the Azure Key Vault read, which is async, means your whole config read is async. There' nothing you can do to get around that. This will mean that the code that uses the config will need to handle it appropriately. You start by exporting an async function that will return the config..
async function getConfig() {
return {
'123': {
'accountName': {
accountId: await getValue('mySecretName', '')
}
}
};
}
module.exports = getConfig;
In your agent code you call that function. This will mean that your agent code will need to be wrapped in a function too, so maybe something like this..
const Bot = require('./bot/bot.js');
const getConfig = require('./config/config.js');
getConfig().then(agentConfig => {
const agent = new Bot(agentConfig);
agent.on(Bot.const.CONNECTED, data => {
log.info(`[agent.js] CONNECTED ${JSON.stringify(data)}`);
});
});
The package azure-keyvault has been deprecated in favor of the new packages to deal with Keyvault keys, secrets and certificates separately. For your scenario, you can use the new #azure/keyvault-secrets package to talk to Key Vault and the new #azure/identity package to create the credential.
const { SecretClient } = require("#azure/keyvault-secrets");
const { DefaultAzureCredential } = require("#azure/identity");
async function getValue(secretName, secretVersion) {
const credential = new DefaultAzureCredential();
const client = new SecretClient(KEY_VAULT_URI, credential);
const secret = await client.getSecret(secretName);
return secret.value;
}
The DefaultAzureCredential assumes that you have set the below env variables
AZURE_TENANT_ID: The tenant ID in Azure Active Directory
AZURE_CLIENT_ID: The application (client) ID registered in the AAD tenant
AZURE_CLIENT_SECRET: The client secret for the registered application
To try other credentials, see the readme for #azure/identity
If you are moving from the older azure-keyvault package, checkout the migration guide to understand the major changes
I am trying to modify a Node.js function called 'splunk-logger'. The problem is that when the SNS Message comes into the function, the events from the Anti-Virus (Trend Micro DeepSecurity) console are grouped together. I already contacted their support and they said this is just the way events are sent and they can't help.
Example: {Message {Event_1} {Event_2} {Event_3}}
Now the JavaScript function works great and the events are forwarded to Splunk. However, since they are grouped together BEFORE they even hit the Lambda function, Splunk sees them as 1 single event instead of 3.
My thought is to take the 'event' variable (since it contains the sns 'message') and parse through that to separate each event (probably using regex or something). Then, I can either create another function to send each event immediately or simply call the "logger.flushAsync" function to send them.
Link to splunk-dev explaining the funciton: http://dev.splunk.com/view/event-collector/SP-CAAAE6Y#create.
Here is the code from the index.js:
const loggerConfig = {
url: process.env.SPLUNK_HEC_URL,
token: process.env.SPLUNK_HEC_TOKEN,
};
const SplunkLogger = require('./lib/mysplunklogger');
const logger = new SplunkLogger(loggerConfig);
exports.handler = (event, context, callback) => {
console.log('Received event:', JSON.stringify(event, null, 2));
// Log JSON objects to Splunk
logger.log(event);
// Send all the events in a single batch to Splunk
logger.flushAsync((error, response) => {
if (error) {
callback(error);
} else {
console.log(`Response from Splunk:\n${response}`);
callback(null, event.key1); // Echo back the first key value
}
});
};
Here is the code from the mysplunklogger.js file.
'use strict';
const url = require('url');
const Logger = function Logger(config) {
this.url = config.url;
this.token = config.token;
this.addMetadata = true;
this.setSource = true;
this.parsedUrl = url.parse(this.url);
// eslint-disable-next-line import/no-dynamic-require
this.requester = require(this.parsedUrl.protocol.substring(0, this.parsedUrl.protocol.length - 1));
// Initialize request options which can be overridden & extended by consumer as needed
this.requestOptions = {
hostname: this.parsedUrl.hostname,
path: this.parsedUrl.path,
port: this.parsedUrl.port,
method: 'POST',
headers: {
Authorization: `Splunk ${this.token}`,
},
rejectUnauthorized: false,
};
this.payloads = [];
};
// Simple logging API for Lambda functions
Logger.prototype.log = function log(message, context) {
this.logWithTime(Date.now(), message, context);
};
Logger.prototype.logWithTime = function logWithTime(time, message, context) {
const payload = {};
if (Object.prototype.toString.call(message) === '[object Array]') {
throw new Error('message argument must be a string or a JSON object.');
}
payload.event = message;
// Add Lambda metadata
if (typeof context !== 'undefined') {
if (this.addMetadata) {
// Enrich event only if it is an object
if (message === Object(message)) {
payload.event = JSON.parse(JSON.stringify(message)); // deep copy
payload.event.awsRequestId = context.awsRequestId;
}
}
if (this.setSource) {
payload.source = `lambda:${context.functionName}`;
}
}
payload.time = new Date(time).getTime() / 1000;
this.logEvent(payload);
};
Logger.prototype.logEvent = function logEvent(payload) {
this.payloads.push(JSON.stringify(payload));
};
Logger.prototype.flushAsync = function flushAsync(callback) {
callback = callback || (() => {}); // eslint-disable-line no-param-reassign
console.log('Sending event(s)');
const req = this.requester.request(this.requestOptions, (res) => {
res.setEncoding('utf8');
console.log('Response received');
res.on('data', (data) => {
let error = null;
if (res.statusCode !== 200) {
error = new Error(`error: statusCode=${res.statusCode}\n\n${data}`);
console.error(error);
}
this.payloads.length = 0;
callback(error, data);
});
});
req.on('error', (error) => {
callback(error);
});
req.end(this.payloads.join(''), 'utf8');
};
module.exports = Logger;
import requests
import re
import json
import os
def lambda_handler(event, context):
data = json.dumps(event)
EventIds = re.findall(r'{\\\".+?\\\"}', data)
EventLength = len(EventIds)
headers = {'Authorization': 'Splunk ' + os.environ['SPLUNK_HEC_TOKEN']}
i = 0
while i < EventLength:
response = requests.post(os.environ['SPLUNK_HEC_URL'], headers=headers, json={"event":EventIds[i]}, verify=True)
i+=1
Arrays are the data type used when Deep Security 10.0 or newer sends events to Amazon SNS. But Splunk wants one event per message. So don't send the array directly.
Instead, use the Splunk logger or Lambda to iterate through the array, sending each item as an individual message. You can modify this sample Lambda script for Node.js:
https://github.com/deep-security/amazon-sns/blob/master/lambda-save-ds-event-to-s3.js
It sends events to S3 individually (which is what you need). Just change it to send to Splunk instead.
Disclosure: I work for Trend Micro.
I have this module and the corresponding test with proxyquire.
// sqlQuery.js
const sql = require('mssql');
module.exports = function sqlFn() {
const request = new sql.Request();
}
// sqlQueryTest.js
const proxyquire = require('proxyquire');
const sql = require('mssql');
let sqlStub = sinon.stub(sql);
describe('Sql wrapper', function() {
let sqlQuery = proxyquire('../public/sqlQuery', {
'sql': sqlStub
})
sqlQuery()
it('Should call sql once', function() {
expect(sqlStub.Request.callCount).to.deep.equal(1) // passes
})
})
If in the sqlQuery I want to use the request object's functions, the test won't run:
TypeError: request.query is not a function
I tried rewire as well to set require to a stub:
let stub1 = sinon.stub()
let sqlQuery = rewire('../public/sqlQuery')
sqlQuery.__set__('request', {
input: stub1
})
How can I test if request is called with given methods in the code?