I am streaming data to Amazon Kinesis, and I use Amazon Lambda to handle data and write it to DynamoDB.
My Lambda code:
var doc = require('dynamodb-doc');
var dynamo = new doc.DynamoDB();
exports.handler = function(event, context) {
//console.log('Received event:', JSON.stringify(event, null, 2));
event.Records.forEach(function(record) {
// Kinesis data is base64 encoded so decode here
var payload = new Buffer(record.kinesis.data, 'base64').toString('ascii');
console.log('Decoded payload:', payload);
var tableName = "_events";
var datetime = new Date().getTime().toString();
dynamo.putItem({
"TableName": tableName,
"Item" : {
"eventID" : record["eventID"],
"eventName" : payload
}
}, function(err, data) {
if (err) {
console.log("dynamodb error: " + err);
context.done('error putting item into dynamodb failed: '+err);
}
else {
console.log('great success: '+JSON.stringify(data, null, ' '));
context.succeed('K THX BY');
}
});
});
// context.succeed("Successfully processed " + event.Records.length + " records.");
};
When I run test, data is successfully saved to DynamoDB. But when I stream the real data, it doesn't happen, while logs show that data was received by lambda function.
Also console.log() function doesn't work in putItem() block, so I have no idea how to debug this problem.
The problems were:
1. I didn't set the correct permissions
2. I just did't wait enough so the data could be processed by the lambda function.
Related
I have an AWS Lambda function that sends an email through SNS, with a Node.js 10.x runtime. The event object contains a variable called client_id that corresponds to a Raspberry Pi device id in IoT Core service. I see the following in the Cloudwatch logs for the Lambda function:
console.log("EVENT: \n" + JSON.stringify(event, null, 2));
2020-09-05T00:21:35.587Z 229b8110-852a-4abe-8b9c-4d06a121d63f INFO EVENT:
{
"client_id": "RaspberryPi001"
}
I need to extract the client_id from the event and include it in the Message text for the email, in order to specify which device is issuing an email alert. How do I extract the client_id and include it in the email Message text?
'use strict';
var AWS = require('aws-sdk');
AWS.config.update({region: 'us-west-2'});
exports.handler = async (event, context, callback) => {
// Create publish parameters
var params = {
Message: `The air quality in your building has triggered a High Risk Alert.`,
Subject: `Air Quality - High Risk Alert`,
TopicArn: `arn:aws:sns:us-west-2:xyz`,
};
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS({apiVersion: '2010-03-31'}).publish(params).promise();
// Handle promise's fulfilled/rejected states
publishTextPromise.then(
function(data) {
console.log("Message: " + params.Message);
console.log("Sent to the topic: " + params.TopicArn);
console.log("MessageID is " + data.MessageId);
console.log("EVENT: \n" + JSON.stringify(event, null, 2));
}).catch(
function(err) {
console.error(err, err.stack);
});
const data = await publishTextPromise;
return data;
};
Thanks!
I am node.js learner.I have written a local file which will connect to dynamo db service from my local m/c and do a query.I have exported the function which does the query.
Now I have created a http local server in node.js and called the exported function which will call the dynamo db query.
I have used promise in while doing the query to dynamo db but getting empty object.
when I am executing the dynamodb query function standalone means just calling the function from same file
it is returning values as expected but when calling the function from http server then I am facing the issue of empty object.
var AWS = require("aws-sdk");
var awsConfig = {
"region": "us-east-1",
"endpoint": "http://dynamodb.us-east-1.amazonaws.com"
};
if (typeof Promise === 'undefined') {
AWS.config.setPromisesDependency(require('bluebird'));
}
const loadCarDetailsWithImage = async function (pKeyValueModelNo,sKeyValueCarName) {
AWS.config.update(awsConfig);
var docClient = new AWS.DynamoDB.DocumentClient();
console.log(pKeyValueModelNo);
console.log(sKeyValueCarName);
var params = {
TableName: "ImageDetails",
ProjectionExpression:"ModelNo,CarName,ImageUrl",
KeyConditionExpression: "ModelNo = :mdNo and CarName = :carName",
ExpressionAttributeValues: {
":mdNo": pKeyValueModelNo,
":carName": sKeyValueCarName
},
};
//1.Tried like this as option 1
//but data is empty
await docClient.query(params).promise().then(function(data) {
console.log(JSON.stringify(data, null, 2));
}).catch(function(err) {
console.log(JSON.stringify(err, null, 2));
});
//2.Tried alternative way as option 2
//but data is empty
return new Promise(function(resolve,reject){
docClient.query(params, function (err, data) {
if (err) {
console.log("error - " + JSON.stringify(err, null, 2));
reject(err);
}
else {
console.log("success - " + JSON.stringify(data, null, 2));
resolve(JSON.stringify(data, null, 2));
}
})
})
}
module.exports = loadCarDetailsWithImage;
The called function which is called when a req is received in http server is like
var car = require('./loadCarDetailsWithImage'); // this I have written at beginning
function callBusinessProcess(id,carName,res){
console.log("id: "+id+" and name: "+carName);
var initializePromise = car(id,carName);
initializePromise.then(function(result) {
console.log(result);
res = result;
},function(err){
console.log(err);
}).then(function(result){
console.log(result);
})
}
OutPut is coming like:
success - {
"Items": [],
"Count": 0,
"ScannedCount": 0
}
{
"Items": [],
"Count": 0,
"ScannedCount": 0
}
called the function like
if(req.method.toLowerCase() == 'get'
&& req.url.substring(0,9) == '/content/'){
var q = url.parse(req.url,true).query;
console.log("Model No :"+q.model);
console.log("Car Name :"+q.name);
callBusinessProcess(q.model,q.name,res);
}
also having the variables values in output like below.These values are available in db
Model No :bmwx1
Car Name :BMW
pKeyValueModelNo :bmwx1
sKeyValueCarName :BMW
I am very confused that though I have used promise in my function still I am receiving empty object
I am new to javascript and I am trying to write a lambda function that would get triggered by PUT event in the bucket, the function would write the file name and some metadata field that's on the s3 object to the dynamodb table. I have most of it worked out but I'm stuck at grabbing the x-amz-meta header info and pass the variable to dynamo.put parameter. Can anyone tell me what I am doing wrong in my code? thanks!
var AWS = require('aws-sdk');
var dynamo = new AWS.DynamoDB.DocumentClient({region: 'us-east-1'});
var s3 = new AWS.S3();
//specify the parameters from event to write to specified db table
exports.handler = function(event, context, callback) {
var srcKey = unescape(event.Records[0].s3.object.key);
var srcEtag = unescape(event.Records[0].s3.object.eTag);
var scrUploadTime = unescape(event.Records[0].eventTime);
var bucket= unescape(event.Records[0].s3.bucket.name);
var checksum =
s3.headObject(
{
Bucket: bucket,
Key: srcKey
},
function(err, data)
{
if (err)
{
console.log(err);
context.done('Error', 'Error getting s3 object: ' + err);
}
else
{
return console.log(this.httpResponse.headers['x-amz-meta-checksum']);
}
});
var params = {
Item: {
filename: srcKey,
uploadtime: scrUploadTime,
client_checksum : checksum
},
TableName: 'S3_log'
};
//write to dynammodb
dynamo.put(params, function(err, data){
if (err) {
callback(err, null);
}else{
callback(null, data);
}
});
};
It looks like you want this:
console.log(data.Metadata['x-amz-meta-checksum']);
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
But also, note that your code doesn't appear to be structured correctly. s3.headObject is asynchronous so your code continues execution at var params = ... before s3.headObject returns. The next actions should probably be inside the callback or handled in another way (waterfall, promises, etc.) to delay the next action until this one completes.
I have written a node.js lambda function that triggers based on a dynamodb stream when new records are inserted into a particular table.
The function receives only new events, filters for inserted records, and then for each record, uses a couple of fields to retrieve data from other tables. Using this combined data a message is composed and sent via SNS to specific target ARN.
The function performs correctly. All the relevant data is retrieved, and a push notification is sent out.
However, for some reason the function appears to be called several times for the same stream, and processes the newly inserted records several times. The result is the target device receiving the same push notification several times.
Should I be placing the callback in a different place, or am I not calling on the context correctly?
This is the function:
'use strict';
var AWS = require("aws-sdk");
var dynamodb = new AWS.DynamoDB();
var sns = new AWS.SNS();
console.log('Loading function');
exports.handler = (event, context, callback) => {
console.log('Received event:', JSON.stringify(event, null, 2));
event.Records.forEach((record) => {
console.log(record.eventID);
console.log(record.eventName);
console.log('DynamoDB Record: %j', record.dynamodb);
if (record.eventName == 'INSERT') {
var matchId = record.dynamodb.NewImage.eventId.S;
var match_params = {
Key: {
"eventId": {
S: matchId
}
},
TableName: "xxxxxxxxxxx-mobilehub-xxxxxxx-Event"
};
//retrieve the match information from Event table
dynamodb.getItem(match_params, function(err, data) {
var match_description = "";
if (err) {
console.log(err, err.stack);
context.fail('No match event record found in Event table');
} else {
match_description = data.Item.description.S;
var uId = record.dynamodb.NewImage.participantUserId.S; //participantUserId
var user_params = {
Key: {
"userId": {
S: uId
}
},
TableName: "xxxxxxxxxxx-mobilehub-xxxxxxxxx-User"
};
//retrieve the user record from User table
dynamodb.getItem(user_params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
context.fail('Error occurred. See log.');
} else {
console.log(data); // successful response
if (data.length === 0) {
console.log("No User Record Found.");
context.fail('No user found for participantUserId.');
} else {
var deviceARN = data.Item.device_arn.S;
if (deviceARN <= 1) {
console.log("User has not registered their device for push notifications.");
context.fail('User has not registered for notifications');
} else {
var json_message = JSON.stringify({
APNS_SANDBOX: JSON.stringify({
aps: {
alert: "You are playing in an upcoming match " + match_description,
badge: 1,
sound: 'default'
}
})
});
var snsparams = {
Message: json_message,
MessageStructure: 'json',
TargetArn: deviceARN
};
sns.publish(snsparams, function(err, data) {
if (err) {
console.log(err); // an error occurred
context.fail('SNS send failed. See log.');
} else {
console.log(data); // successful response
context.success('Push notification sent to user.');
}
});
}
}
}
});
}
});
}
});
callback(null, `Successfully processed ${event.Records.length} records.`);
};
In my case, I added the same event source multiple times.
Quote from the conversation with an AWS support engineer:
Using my internal tools, I noticed that the Lambda function xxxxxx has
the event source:
arn:aws:events:my_region:my_acct_id:rule/my_event_target
configured twice as push event source. This means that this might be the cause
why you are seeing two invokes at every minute. Would you please
confirm on your side if this event is configured twice for the $LATEST
version of your lambda and also confirm if it's intended?
I hope this could save someelse :)
In your lambda page at the bottom, try tweaking "Concurrency" Unreserved account concurrency to 1 and "Asynchronous invocation" Retry attempts to 0 . As a test try these and observe the behaviour. Might help.
This is the code I have for publishing to the topic (I have changed the target and topic arn for security reasons):
var AWS = require("aws-sdk");
var sns = new AWS.SNS();
var targetArn = 'arn:aws:sns:us-east-1:4363657289:endpoint/GCM/APP_NAME/3185sfdnfe283925sgSeaa0e';
var topicArn = 'arn:aws:s-s:us-east-1:4363657289436:TOPIC_NAME';
var payload = {
GCM: {
data: {
message: "test"
}
}
};
payload.GCM = JSON.stringify(payload.GCM);
payload = JSON.stringify(payload);
var params= {
TopicArn: topicArn,
TargetArn: targetArn,
Message: payload,
MessageStructure: 'json'
};
var responsefromSNS = sns.publish(params , function(error, data) {
if (error) {
console.log("ERROR: " + error.stack);
}
else {
console.log("SENT DATA: " + JSON.stringify(data));
context.done(null, data);
}
});
console.log(responsefromSNS);
My issue is that I never see log statements from either the if or else block and the push notification never reaches the mobile app. I have consulted both the AWS JavaScript SDK Documentation and countless stack overflow posts about this and nothing that I have tried works. And, I have given the lambda function permission to publish to the topic.
---UPDATE-----
I have changed my code a bit and now it looks like this:
var AWS = require("aws-sdk");
AWS.config.update({region:'us-east-1'});
var topicarn = 'arn:aws:s-s:us-east-1:927579412028:alexapushdemo';
var targetarn = 'arn:aws:sns:us-east-1:927579412028:endpoint/GCM/automation.home.visa.com.homeautomation/3af761b2-1955-34d8-b66a-85e232e0aa0e';
var payload = {
default: "test",
GCM: {
data: {
message: "test"
}
}
};
payload.GCM = JSON.stringify(payload.GCM);
payload = JSON.stringify(payload);
var sns = new AWS.SNS();
console.log('start of sns function')
sns.publish({
TargetArn: targetarn,
Message: payload,
MessageStructure: 'json'
}, function(err, data) {
if (err) {
console.log(err.stack);
// Notify Lambda that we are finished, but with errors
context.done(err, 'sns function finished with errors!');
return;
}
console.log('push sent');
console.log(data);
// Notify Lambda that we are finished
context.done(null, 'sns function finished!');
});
console.log('end of sns functions');
The error I get is:
ConfigError: Missing region in config\\n
at Request.VALIDATE_REGION (/node_modules/aws-sdk/lib/event_listeners.js:81:45)\\n
at Request.callListeners (/node_modules/aws-sdk/lib/sequential_executor.js:105:20)\\n
at callNextListener (/node_modules/aws-sdk/lib/sequential_executor.js:95:12)\\n
at /node_modules/aws-sdk/lib/event_listeners.js:75:9\\n
at finish (/node_modules/aws-sdk/lib/config.js:228:7)\\n
at /node_modules/aws-sdk/lib/config.js:268:9\\n
at resolveNext (/node_modules/aws-sdk/lib/credentials/credential_provider_chain.js:84:9)\\n
at /node_modules/aws-sdk/lib/credentials/credential_provider_chain.js:97:11\\n
at /node_modules/aws-sdk/lib/credentials.js:123:23\\n
at /node_modules/aws-sdk/lib/credentials/ec2_metadata_credentials.js:66:7\\"\",\"ip\":\"127.0.0.1\"}",
Why am I getting this even though I'm calling AWS.config.update.
iram,
If I take your exact code and paste it into a Lambda Node.js 4.3 function and execute a test from the Lambda Console, this is the result:
ERROR: InvalidParameter: Invalid parameter: TopicArn Reason: Both TopicArn and TargetArn specified. Use only one or the other
This means that in your params, you need to comment out either TopicArn or TargetArn or put in some logic to determine if the incoming payload contains an Arn that is a target endpoint or a topic endpoint.
You could still have permissions issues with Lambda execution role to SNS or to CW Logs, however, regardless if you have permission to publish or send logs to CloudWatch from your Lambda function, running a test from the console will always spit out some logging of what's going on.
Good luck.
const AWS = require('aws-sdk');
AWS.config.region = 'us-east-1';
specify the regeion to be used by the aws sdk like this