I am trying to modify a Node.js function called 'splunk-logger'. The problem is that when the SNS Message comes into the function, the events from the Anti-Virus (Trend Micro DeepSecurity) console are grouped together. I already contacted their support and they said this is just the way events are sent and they can't help.
Example: {Message {Event_1} {Event_2} {Event_3}}
Now the JavaScript function works great and the events are forwarded to Splunk. However, since they are grouped together BEFORE they even hit the Lambda function, Splunk sees them as 1 single event instead of 3.
My thought is to take the 'event' variable (since it contains the sns 'message') and parse through that to separate each event (probably using regex or something). Then, I can either create another function to send each event immediately or simply call the "logger.flushAsync" function to send them.
Link to splunk-dev explaining the funciton: http://dev.splunk.com/view/event-collector/SP-CAAAE6Y#create.
Here is the code from the index.js:
const loggerConfig = {
url: process.env.SPLUNK_HEC_URL,
token: process.env.SPLUNK_HEC_TOKEN,
};
const SplunkLogger = require('./lib/mysplunklogger');
const logger = new SplunkLogger(loggerConfig);
exports.handler = (event, context, callback) => {
console.log('Received event:', JSON.stringify(event, null, 2));
// Log JSON objects to Splunk
logger.log(event);
// Send all the events in a single batch to Splunk
logger.flushAsync((error, response) => {
if (error) {
callback(error);
} else {
console.log(`Response from Splunk:\n${response}`);
callback(null, event.key1); // Echo back the first key value
}
});
};
Here is the code from the mysplunklogger.js file.
'use strict';
const url = require('url');
const Logger = function Logger(config) {
this.url = config.url;
this.token = config.token;
this.addMetadata = true;
this.setSource = true;
this.parsedUrl = url.parse(this.url);
// eslint-disable-next-line import/no-dynamic-require
this.requester = require(this.parsedUrl.protocol.substring(0, this.parsedUrl.protocol.length - 1));
// Initialize request options which can be overridden & extended by consumer as needed
this.requestOptions = {
hostname: this.parsedUrl.hostname,
path: this.parsedUrl.path,
port: this.parsedUrl.port,
method: 'POST',
headers: {
Authorization: `Splunk ${this.token}`,
},
rejectUnauthorized: false,
};
this.payloads = [];
};
// Simple logging API for Lambda functions
Logger.prototype.log = function log(message, context) {
this.logWithTime(Date.now(), message, context);
};
Logger.prototype.logWithTime = function logWithTime(time, message, context) {
const payload = {};
if (Object.prototype.toString.call(message) === '[object Array]') {
throw new Error('message argument must be a string or a JSON object.');
}
payload.event = message;
// Add Lambda metadata
if (typeof context !== 'undefined') {
if (this.addMetadata) {
// Enrich event only if it is an object
if (message === Object(message)) {
payload.event = JSON.parse(JSON.stringify(message)); // deep copy
payload.event.awsRequestId = context.awsRequestId;
}
}
if (this.setSource) {
payload.source = `lambda:${context.functionName}`;
}
}
payload.time = new Date(time).getTime() / 1000;
this.logEvent(payload);
};
Logger.prototype.logEvent = function logEvent(payload) {
this.payloads.push(JSON.stringify(payload));
};
Logger.prototype.flushAsync = function flushAsync(callback) {
callback = callback || (() => {}); // eslint-disable-line no-param-reassign
console.log('Sending event(s)');
const req = this.requester.request(this.requestOptions, (res) => {
res.setEncoding('utf8');
console.log('Response received');
res.on('data', (data) => {
let error = null;
if (res.statusCode !== 200) {
error = new Error(`error: statusCode=${res.statusCode}\n\n${data}`);
console.error(error);
}
this.payloads.length = 0;
callback(error, data);
});
});
req.on('error', (error) => {
callback(error);
});
req.end(this.payloads.join(''), 'utf8');
};
module.exports = Logger;
import requests
import re
import json
import os
def lambda_handler(event, context):
data = json.dumps(event)
EventIds = re.findall(r'{\\\".+?\\\"}', data)
EventLength = len(EventIds)
headers = {'Authorization': 'Splunk ' + os.environ['SPLUNK_HEC_TOKEN']}
i = 0
while i < EventLength:
response = requests.post(os.environ['SPLUNK_HEC_URL'], headers=headers, json={"event":EventIds[i]}, verify=True)
i+=1
Arrays are the data type used when Deep Security 10.0 or newer sends events to Amazon SNS. But Splunk wants one event per message. So don't send the array directly.
Instead, use the Splunk logger or Lambda to iterate through the array, sending each item as an individual message. You can modify this sample Lambda script for Node.js:
https://github.com/deep-security/amazon-sns/blob/master/lambda-save-ds-event-to-s3.js
It sends events to S3 individually (which is what you need). Just change it to send to Splunk instead.
Disclosure: I work for Trend Micro.
Related
I need one solution. I am using axios module to fetch response from other server and after that I need to export value to other file using Node.js. I am explaining my code below.
const axios = require('axios');
let protocol = '';
let opticalIp = '';
axios.get(`www.mydomain.com/api`).then(function(response) {
//console.log('succ', response);
if(response.data && response.data.data) {
protocol = response.data.data['scheme'];
opticalIp = response.data.data['optical_ip'];
}
}).catch(function(error) {
console.log('error', error);
});
console.log('data', opticalIp);
const env = {
API_URL: `${protocol}://${opticalIp}/`,
configHeaders: ''
};
module.exports = env;
Here I am fetching the data from other server and also here I am returning some env value to other file. In my case before response the env object is returning so that the API_URL value is going blank. Here I need after the success response the API_URL will be prepared and then this env object should return to other file where ever it is called. Please help me to get the proper solution.
In this case, you'll want to just export a function that you can call from the other file. This will avoid quite a bit of headache later on. Since axios returns a promise (hence the .then()) call after the request, you can just return the axios request call and modify whichever value you'd like to return from within the .then() call.
Here's an overview of promise chaining provided by javascript.info.
// CHILD MODULE
const axios = require('axios');
function fetchOptical () {
let protocol = '';
let opticalIp = '';
return axios.get(`https://10.0.12.17:9201/api/v1.0/settings/custom-form/Optical`)
.then(function(response) {
//console.log('succ', response);
if(response.data && response.data.data) {
protocol = response.data.data['scheme'];
opticalIp = response.data.data['optical_ip'];
const env = {
API_URL: `${protocol}://${opticalIp}/`,
configHeaders: ''
};
console.log('data', opticalIp);
return env
}
}).catch(function(error) {
console.log('error', error);
return error
});
})
}
module.exports = fetchOptical;
// PARENT MODULE
import child from './child'
child().then(env => {
})
Modifying protocol and opticalIp will not change API_URL. Instead, in the then, try:
if(response.data && response.data.data) {
const protocol = response.data.data['scheme'];
const opticalIp = response.data.data['optical_ip'];
env.API_URL = `${protocol}://${opticalIp}/`;
}
// Then when defining env:
const env = {
API_URL: null, // null or '' or whatever you want the default to be.
configHeaders: ''
};
I have a small web application listening for incoming messages from a Websocket server. I receive them like so
const webSocket = new WebSocket("wss://echo.websocket.org");
webSocket.onopen = event => webSocket.send("test");
webSocket.onmessage = event => console.log(event.data);
but the sending server is more complex. There are multiple types of messages that could come e.g. "UserConnected", "TaskDeleted", "ChannelMoved"
How to detect which type of message was sent? For now I modified the code to
const webSocket = new WebSocket("wss://echo.websocket.org");
webSocket.onopen = event => {
const objectToSend = JSON.stringify({
message: "test-message",
data: "test"
});
webSocket.send(objectToSend);
};
webSocket.onmessage = event => {
const objectToRead = JSON.parse(event.data);
if (objectToRead.message === "test-message") {
console.log(objectToRead.data);
}
};
So do I have to send an object from the server containing the "method name" / "message type" e.g. "TaskDeleted" to identify the correct method to execute at the client? That would result in a big switch case statement, no?
Are there any better ways?
You can avoid the big switch-case statement by mapping the methods directly:
// List of white-listed methods to avoid any funny business
let allowedMethods = ["test", "taskDeleted"];
function methodHandlers(){
this.test = function(data)
{
console.log('test was called', data);
}
this.taskDeleted = function(data)
{
console.log('taskDeleted was called', data);
}
}
webSocket.onmessage = event => {
const objectToRead = JSON.parse(event.data);
let methodName = objectToRead.message;
if (allowerMethods.indexOf(methodName)>=0)
{
let handler = new methodHandlers();
handler[methodName](data);
}
else
{
console.error("Method not allowed: ", methodName)
}
};
As you have requested in one of your comments to have a fluent interface for the websockets like socket.io.
You can make it fluent by using a simple PubSub (Publish Subscribe) design pattern so you can subscribe to specific message types. Node offers the EventEmitter class so you can inherit the on and emit events, however, in this example is a quick mockup using a similar API.
In a production environment I would suggest using the native EventEmitter in a node.js environment, and a browser compatible npm package in the front end.
Check the comments for a description of each piece.
The subscribers are saved in a simple object with a Set of callbacks, you can add unsubscribe if you need it.
note: if you are using node.js you can just extend EventEmitter
// This uses a similar API to node's EventEmitter, you could get it from a node or a number of browser compatible npm packages.
class EventEmitter {
// { [event: string]: Set<(data: any) => void> }
__subscribers = {}
// subscribe to specific message types
on(type, cb) {
if (!this.__subscribers[type]) {
this.__subscribers[type] = new Set
}
this.__subscribers[type].add(cb)
}
// emit a subscribed callback
emit(type, data) {
if (typeof this.__subscribers[type] !== 'undefined') {
const callbacks = [...this.__subscribers[type]]
callbacks.forEach(cb => cb(data))
}
}
}
class SocketYO extends EventEmitter {
constructor({ host }) {
super()
// initialize the socket
this.webSocket = new WebSocket(host);
this.webSocket.onopen = () => {
this.connected = true
this.emit('connect', this)
}
this.webSocket.onerror = console.error.bind(console, 'SockyError')
this.webSocket.onmessage = this.__onmessage
}
// send a json message to the socket
send(type, data) {
this.webSocket.send(JSON.stringify({
type,
data
}))
}
on(type, cb) {
// if the socket is already connected immediately call the callback
if (type === 'connect' && this.connected) {
return cb(this)
}
// proxy EventEmitters `on` method
return super.on(type, cb)
}
// catch any message from the socket and call the appropriate callback
__onmessage = e => {
const { type, data } = JSON.parse(e.data)
this.emit(type, data)
}
}
// create your SocketYO instance
const socket = new SocketYO({
host: 'wss://echo.websocket.org'
})
socket.on('connect', (socket) => {
// you can only send messages once the socket has been connected
socket.send('myEvent', {
message: 'hello'
})
})
// you can subscribe without the socket being connected
socket.on('myEvent', (data) => {
console.log('myEvent', data)
})
I am trying to learn Google Assistant integration using DialogFlow, I have written the following code which works like charm when I test in dialogFlow but fails when I test the same on Google Assistant (i.e. Context passed from Intent getBillingInfo becomes null when accessed in payBill intent). Kindly help me understand where I am going wrong.
Code:
var https = require ('https');
const functions = require('firebase-functions');
const {WebhookClient} = require('dialogflow-fulfillment');
const {Card, Suggestion} = require('dialogflow-fulfillment');
process.env.DEBUG = 'dialogflow:debug'; // enables lib debugging statements
exports.dialogflowFirebaseFulfillment = functions.https.onRequest((request, response) => {
const agent = new WebhookClient({ request, response });
console.log('Dialogflow Request headers: ' + JSON.stringify(request.headers));
console.log('Dialogflow Request body: ' + JSON.stringify(request.body));
function welcome(agent) {
agent.add(`Welcome to my agent!`);
}
function fallback(agent) {
agent.add(`I didn't understand`);
agent.add(`I'm sorry, can you try again?`);
}
// // below to get this function to be run when a Dialogflow intent is matched
function getBillingInfoHandler(agent) {
const parameters = request.body.queryResult.parameters;
var phoneNumber = parameters['phone-number'];
console.log("Phone Number: "+ phoneNumber);
let url = "https://testapi.io/api/shwej//getBillingInfo";
return new Promise((resolve, reject) => {
https.get(url, (res) => {
let body = ''; // var to store the response chunks
res.on('data', (chunk) => {body += chunk; });
res.on('end', () => {
// After all the data has been received, parse the JSON for desired data
let response = JSON.parse(body);
let output = response.billing_amount;
// Resolve the promise with the output text
console.log(body);
agent.add("Your bill for " + phoneNumber + " is " + output + " ₹ ");
agent.add(new Suggestion(`Click to Pay`));
//agent.setContext('billing_context');
//const context = {'phoneNumber': phoneNumber, 'billAmount': output};
//agent.setContext(context);
agent.setContext({
'name':'billing-context',
'lifespan': 50,
'parameters':{
'phoneNumber': phoneNumber,
'billAmount': output
}
});
resolve();
});
res.on('error', (error) => {
agent.add("Error occurred while calling API.");
console.log(`Error calling the API: ${error}`);
reject();
});
});
});
}
function payBillHandler(agent) {
let billingContext = agent.getContext('billing-context');
if (typeof billingContext === 'undefined' || billingContext === null){
agent.add("Some error with passing context!!!");
}else{
agent.add("Your payment is successful! ");
agent.add(" Phone Number : " + billingContext.parameters.phoneNumber);
agent.add(" Amount paid : " + billingContext.parameters.billAmount + " ₹ ");
}
}
// Run the proper function handler based on the matched Dialogflow intent name
let intentMap = new Map();
intentMap.set('Default Welcome Intent', welcome);
intentMap.set('Default Fallback Intent', fallback);
intentMap.set('getBillingInfo', getBillingInfoHandler);
intentMap.set('payBill', payBillHandler);
agent.handleRequest(intentMap);
});
It doesn't look like you're importing the Actions on Google client library. Try replacing the first few lines of your fulfillment with:
// Import the Dialogflow module and response creation dependencies
// from the Actions on Google client library.
const {
dialogflow,
BasicCard,
Permission,
Suggestions,
} = require('actions-on-google');
// Import the firebase-functions package for deployment.
const functions = require('firebase-functions');
// Instantiate the Dialogflow client.
const app = dialogflow({debug: true});
You can also try out the Actions on Google codelabs (level 1, level 2) or review the Actions on Google github repos.
I'm new the Node.js and I've been working with a sample project by a third party provider and I'm trying to use Azure Key Vault to store configuration values.
I'm having trouble getting a process to wait before executing the rest. I'll try to detail as much as I know.
The sample project has a file named agent.js which is the start page/file. On line 16 (agent_config = require('./config/config.js')[process.env.LP_ACCOUNT][process.env.LP_USER]) it calls a config file with values. I'm trying to set these value using Key Vault. I've tried many combinations of calling functions, and even implementing async / await but the value for agent_config always contains a [Promise] object and not the data returned by Key Vault.
If I'm right, this is because the Key Vault itself uses async / await too and the config file returns before the Key Vault values are returned.
How can Key Vault be added/implemented in a situation like this?
Here's what I've tried:
First updated agent.js to
let agent_config = {};
try {
agent_config = require('./config/config.js')['123']['accountName'];
} catch (ex) {
log.warn(`[agent.js] Error loading config: ${ex}`)
}
console.log(agent_config);
Test 1
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
function getValue(secretName, secretVersion) {
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' }).then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
module.exports = {
'123': {
'accountName': {
accountId: getValue('mySecretName', '')
}
}
};
Results
{ accountsId: undefined }
Test 2
Made getValue an async function and wrapped it around another function (tried without the wrapping and didn't work either)
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
async function getValue(secretName, secretVersion) {
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' }).then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
async function config() {
module.exports = {
'123': {
'accountName': {
accountId: await getValue('mySecretName', '')
}
}
};
}
config();
Results
{}
Test 3
Made getValue an async function and wrapped it around another function (tried without the wrapping and didn't work either)
./config/config.js
const KeyVault = require('azure-keyvault');
const msRestAzure = require('ms-rest-azure');
const KEY_VAULT_URI = 'https://' + '{my vault}' + '.vault.azure.net/' || process.env['KEY_VAULT_URI'];
async function getValue(secretName, secretVersion) {
return msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
module.exports = {
'123': {
'accountName': {
accountId: getValue('mySecretName', '')
}
}
};
config();
Results
{ accountId: { <pending> } }
Other
I've tried many others ways like module.exports = async (value) =< {...} (found through other questions/solutions without success.
I'm starting to think I need to do some "waiting" on agent.js but I haven't found good info on this.
Any help would be great!
One issue is that your getValue function is not returning anything as your returns need to be explicit.
(and without the promise being returned, there's nothing to await on)
async function getValue(secretName, secretVersion) {
return msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then((credentials) => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion).then(
function (response) {
return response.Value;
});
});
}
You could also get away with less explicit returns using arrow functions..
const getValue = async (secretName, secretVersion) =>
msRestAzure.loginWithAppServiceMSI({ resource: 'https://vault.azure.net' })
.then(credentials => {
const client = new KeyVault.KeyVaultClient(credentials);
return client.getSecret(KEY_VAULT_URI, secretName, secretVersion)
.then(response => response.Value);
});
Introducing the Azure Key Vault read, which is async, means your whole config read is async. There' nothing you can do to get around that. This will mean that the code that uses the config will need to handle it appropriately. You start by exporting an async function that will return the config..
async function getConfig() {
return {
'123': {
'accountName': {
accountId: await getValue('mySecretName', '')
}
}
};
}
module.exports = getConfig;
In your agent code you call that function. This will mean that your agent code will need to be wrapped in a function too, so maybe something like this..
const Bot = require('./bot/bot.js');
const getConfig = require('./config/config.js');
getConfig().then(agentConfig => {
const agent = new Bot(agentConfig);
agent.on(Bot.const.CONNECTED, data => {
log.info(`[agent.js] CONNECTED ${JSON.stringify(data)}`);
});
});
The package azure-keyvault has been deprecated in favor of the new packages to deal with Keyvault keys, secrets and certificates separately. For your scenario, you can use the new #azure/keyvault-secrets package to talk to Key Vault and the new #azure/identity package to create the credential.
const { SecretClient } = require("#azure/keyvault-secrets");
const { DefaultAzureCredential } = require("#azure/identity");
async function getValue(secretName, secretVersion) {
const credential = new DefaultAzureCredential();
const client = new SecretClient(KEY_VAULT_URI, credential);
const secret = await client.getSecret(secretName);
return secret.value;
}
The DefaultAzureCredential assumes that you have set the below env variables
AZURE_TENANT_ID: The tenant ID in Azure Active Directory
AZURE_CLIENT_ID: The application (client) ID registered in the AAD tenant
AZURE_CLIENT_SECRET: The client secret for the registered application
To try other credentials, see the readme for #azure/identity
If you are moving from the older azure-keyvault package, checkout the migration guide to understand the major changes
I've read through the Firebase Cloud Functions reference, guides and sample code to try and determine why my function is triggered twice, but am yet to find a successful resolution. I've also trialed Firebase-Queue as a work-around, however its latest update suggests Cloud Functions is the way to go.
In short, I'm retrieving notices from an external API using request-promise, checking those notices against ones I already have in my database, and when a new notice is identified, posting it to said database. The corresponding venue is then updated with a reference to the new notice. Code is as follows:
'use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const request = require('request');
const rp = require('request-promise');
admin.initializeApp(functions.config().firebase);
const db = admin.database();
const venues = db.ref("/venues/");
exports.getNotices = functions.https.onRequest((req, res) => {
var options = {
uri: 'https://xxxxx.xxxxx',
qs: {
format: 'json',
type: 'venue',
...
},
json: true
};
rp(options).then(data => {
processNotices(data);
console.log(`venues received: ${data.length}`);
res.status(200).send('OK');
})
.catch(error => {
console.log(`Caught Error: ${error}`);
res.status(`${error.statusCode}`).send(`Error: ${error.statusCode}`);
});
});
function processNotices(data) {
venues.once("value").then(snapshot => {
snapshot.forEach(childSnapshot => {
var existingKey = childSnapshot.val().key;
for (var i = 0; i < data.length; i++) {
var notice = data[i];
var noticeKey = notice.key;
if (noticeKey !== existingKey) {
console.log(`New notice identified: ${noticeKey}`)
postNotice(notice);
}
}
return true;
});
});
}
function postNotice(notice) {
var ref = venues.push();
var key = ref.key;
var loc = notice.location;
return ref.set(notice).then(() => {
console.log('notice posted...');
updateVenue(key, loc);
});
}
function updateVenue(key, location) {
var updates = {};
updates[key] = "true";
var venueNoticesRef = db.ref("/venues/" + location + "/notices/");
return venueNoticesRef.update(updates).then(() => {
console.log(`${location} successfully updated with ${key}`);
});
}
Any suggestions as to how to rectify the double-triggering would be greatly appreciated. Thanks in advance!
Problem solved - some misinformation from the Firebase Console Logs (repeating entries), coupled with nested for loops in the wrong order were responsible for the apparent double triggering.