I am new to javascript and I am trying to write a lambda function that would get triggered by PUT event in the bucket, the function would write the file name and some metadata field that's on the s3 object to the dynamodb table. I have most of it worked out but I'm stuck at grabbing the x-amz-meta header info and pass the variable to dynamo.put parameter. Can anyone tell me what I am doing wrong in my code? thanks!
var AWS = require('aws-sdk');
var dynamo = new AWS.DynamoDB.DocumentClient({region: 'us-east-1'});
var s3 = new AWS.S3();
//specify the parameters from event to write to specified db table
exports.handler = function(event, context, callback) {
var srcKey = unescape(event.Records[0].s3.object.key);
var srcEtag = unescape(event.Records[0].s3.object.eTag);
var scrUploadTime = unescape(event.Records[0].eventTime);
var bucket= unescape(event.Records[0].s3.bucket.name);
var checksum =
s3.headObject(
{
Bucket: bucket,
Key: srcKey
},
function(err, data)
{
if (err)
{
console.log(err);
context.done('Error', 'Error getting s3 object: ' + err);
}
else
{
return console.log(this.httpResponse.headers['x-amz-meta-checksum']);
}
});
var params = {
Item: {
filename: srcKey,
uploadtime: scrUploadTime,
client_checksum : checksum
},
TableName: 'S3_log'
};
//write to dynammodb
dynamo.put(params, function(err, data){
if (err) {
callback(err, null);
}else{
callback(null, data);
}
});
};
It looks like you want this:
console.log(data.Metadata['x-amz-meta-checksum']);
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
But also, note that your code doesn't appear to be structured correctly. s3.headObject is asynchronous so your code continues execution at var params = ... before s3.headObject returns. The next actions should probably be inside the callback or handled in another way (waterfall, promises, etc.) to delay the next action until this one completes.
Related
I have an AWS Lambda function. It goes through a list of the array which is an array of URLs and saves their screenshot and puts them in s3. How do I return the output of this function which is screenshotLinks array that has all the links to files saved in s3? I used the callback function in the end, but it just returns null! I want callback function to output all the s3 file links saved inside screenshotLinks array.
exports.handler = (event, context, callback) => {
desktopLinks.forEach(function (url, index) {
https.request(url, function(res) {
var data = new Stream();
res.on('data', function(chunk) {
// Agregates chunks
data.push(chunk);
});
res.on('end', function() {
var body = data.read();
// Once you received all chunks, send to S3
var currentLink = links[index];
var linkAddress = encodeURIComponent(currentLink);
var slashPosition = getPosition(currentLink, '/', 3)+1;
var linkName = currentLink.substr(slashPosition, currentLink.length)
var params = {
Bucket: bucket,
Key: completeDate + '/screenshots/' + linkName + '.png',
Body: body
};
s3.putObject(params, function(err, data, callback) {
if (err) {
console.error(err, err.stack);
} else {
bunch = params.Bucket + '/' + params.Key;
screenshotLinks.push(bunch);
}
});
});
}).end();
})
callback(null, screenshotLinks)
};
Your code is event driven / asynchronous which means you are calling the callback before screenshotLinks has been populated.
The node http.ClientRequest.end() method finishes sending a request, but that doesn't mean that the response has been received and handled, as that is done by an asynchronous event handler. However, the callback is executed immediately after the call to request.end(), which is just after the request has been fired off, therefore screenshotLinks is empty.
You need to execute your callback from the callback you pass to s3.putObject. I suggest you pass your callback a response/result object that indicates whether the putObject succeeded and contains the url it relates to and either an error message or a screenshotLink, e.g. something like this:
s3.putObject(params, function(err, data, callback) {
var s3Response;
s3Response.url = url;
if (err) {
s3Response.success = false;
s3Response.error = err;
console.error(err, err.stack);
} else {
bunch = params.Bucket + '/' + params.Key;
s3Response.success = true;
s3Response.screenshotLink = bunch;
}
callback(null, s3Response);
});
I would like to suggest you use an 8.10 node runtime.
ref: https://aws.amazon.com/blogs/compute/node-js-8-10-runtime-now-available-in-aws-lambda/
Then your entry point should be:
export async function <function_name>(event) {}
Then:
let s3 = new AWS.S3({ region: process.env.AWS_REGION, apiVersion: '2006-03-01' });
let params=
{
Bucket: /* a path to bucket (string) */,
Key: name /* string */,
Body: /* supported types (Buffer, Typed Array, Blob, String, ReadableStream) */,
ACL: 'public-read',
ContentType: 'image/png'
};
try
{
let s3Response = await s3.upload(params).promise();
// if succceed
console.log(`File uploaded to S3 at ${s3Response.Bucket} bucket. File location: ${s3Response.Location}`);
}
catch (ex) // if error occured
{
console.error(ex);
}
we'e writing a serverless function in AWS Lambda that does the following:
Fetches JSON files from AWS S3 (Multiple files).
Merges JSON files into one single file.
Uploads the new file from (2) back to S3.
To handle this flow, we're trying to use the async library in Nodejs within our Lambda. Here's the code we have so far:
'use-strict';
const AWS = require('aws-sdk');
const async = require('async');
const BUCKET = 'sample-bucket';
const s3path = 'path/to/directory';
exports.handler = (event, context, callback) => {
let filepaths = []; // Stores filepaths
let all_data = []; // Array to store all data retrieved
let s3Client = new AWS.S3({ params: { Bucket: BUCKET } }); // S3 bucket config
// Runs functions in the series provided to ensure each function runs one after the other
async.series([
// Read file IDs from the event and saves in filepaths array
function(callbackSeries) {
console.log('Starting function...');
event.forEach(id => {
filepaths.push(`${id}.json`);
});
console.log('All filenames pushed.');
callbackSeries();
},
// Fetches each file from S3 using filepaths
// Then pushes objects from each file to the all_data array
function(callbackSeries) {
async.each(filepaths, (file, callback) => {
let params = {
'Bucket': BUCKET,
'Key': s3path + file
}
s3Client.getObject(params, (err, data) => {
if(err) {
throw err;
} else {
if(data.Body.toString()) {
console.log('Fetching data...');
all_data.push(JSON.parse(data.Body.toString()));
callback();
console.log('Fetched.');
}
}
});
},
(err) => {
if (err) throw err;
else callbackSeries();
});
},
// NOTHING IS BEING EXECUTED AFTER THIS
// Puts object back into bucket as one merged file
function(callbackSeries) {
console.log('Final function.')
// Concatenates multiple arrays in all_data into a single array
all_data = Array.prototype.concat(...all_data);
let params = {
'Bucket': BUCKET,
'Body': JSON.stringify(all_data),
'Key': 'obj.json'
};
s3Client.putObject(params, (err, data) => {
if(err) throw err;
else console.log('Success!');
})
}
], (err) => {
if(err) throw err;
else console.log('End.');
})
}
The first two functions in our async.series are running normally and all the console.log statements are executed. But our third function i.e.:
// Puts object back into bucket as one merged file
function(callbackSeries) {
console.log('Final function.')
// Concatenates multiple arrays in all_data into a single array
all_data = Array.prototype.concat(...all_data);
let params = {
'Bucket': BUCKET,
'Body': JSON.stringify(all_data),
'Key': 'obj.json'
};
s3Client.putObject(params, (err, data) => {
if(err) throw err;
else console.log('Success!');
})
}
is not being executed at all. We added a console.log statements but it doesn't seem to be executing at all.
I've consulted both AWS Support as well as async's documentation but can't seem to figure out the problem.
Any assistance would be highly appreciated.
Many thanks.
I am streaming data to Amazon Kinesis, and I use Amazon Lambda to handle data and write it to DynamoDB.
My Lambda code:
var doc = require('dynamodb-doc');
var dynamo = new doc.DynamoDB();
exports.handler = function(event, context) {
//console.log('Received event:', JSON.stringify(event, null, 2));
event.Records.forEach(function(record) {
// Kinesis data is base64 encoded so decode here
var payload = new Buffer(record.kinesis.data, 'base64').toString('ascii');
console.log('Decoded payload:', payload);
var tableName = "_events";
var datetime = new Date().getTime().toString();
dynamo.putItem({
"TableName": tableName,
"Item" : {
"eventID" : record["eventID"],
"eventName" : payload
}
}, function(err, data) {
if (err) {
console.log("dynamodb error: " + err);
context.done('error putting item into dynamodb failed: '+err);
}
else {
console.log('great success: '+JSON.stringify(data, null, ' '));
context.succeed('K THX BY');
}
});
});
// context.succeed("Successfully processed " + event.Records.length + " records.");
};
When I run test, data is successfully saved to DynamoDB. But when I stream the real data, it doesn't happen, while logs show that data was received by lambda function.
Also console.log() function doesn't work in putItem() block, so I have no idea how to debug this problem.
The problems were:
1. I didn't set the correct permissions
2. I just did't wait enough so the data could be processed by the lambda function.
I want to mock Amazon AWS S3 getObject
The code I want to test is the following one: Its in helper.js
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
exports.get_data_and_callback = function(callback, extra){
s3.getObject( {Bucket: SRC_BUCKET, Key: SRC_KEY},
function (err, data) {
if (err != null) {
console.log("Couldn't retrieve object: " + err);
}else{
console.log("Loaded " + data.ContentLength + " bytes");
callback(data, extra);
}
});
}
In test/helper_test.js I wrote a test that should mock the module AWS
var assert = require('assert');
var mockery = require('mockery');
describe("helper", function() {
it('loads and returns data from S3 to a callback', function(){
mockery.enable();
var fakeaws = {
S3: function(){
return {
getObject: function(params, callback){
callback(null, "hello")
}
}
}
}
mockery.registerSubstitute('aws-sdk', fakeaws);
function replace_function(err, data){
console.log(data);
}
require('../helper.js').get_data_and_callback(replace_function, null);
});
});
When I require AWS in the Test-File test/helper_test.js like this:
aws = require('aws-sdk');
s3 = new aws.S3;
s3.getObject(replace_function)
Then my code works, it prints out hello.
BUT the execution of require('../helper.js').get_data_and_callback(replace_function, null);
Doesn't work like expected, AWS stays the same its not replaced with my fakeaws. What do I wrong? Do you maybe have other solutions to replace S3 Thanks
We have created an aws-sdk-mock npm module which mocks out all the AWS SDK services and methods. https://github.com/dwyl/aws-sdk-mock
It's really easy to use. Just call AWS.mock with the service, method and a stub function.
AWS.mock('S3', 'getObject', function(params, callback) {
callback(null, 'success');
});
Then restore the methods after your tests by calling:
AWS.restore('S3', 'getObject');
This works for every service and method in the as-sdk.
I'm over my head at the moment.
I'm new to node and writing a passportjs module for Freshbooks. There's a Passport function I'm trying to implement that get's a user's profile.
This code uses Passport's OAuth foo to make a request.
this._oauth.post(url, token, tokenSecret, post_body, post_content_type, function (err, body, res) {
if (err) { return done(new InternalOAuthError('failed to fetch user profile', err)); }
try {
var parser = require('xml2json');
var json = parser.toJson(body); //returns a string containing the JSON structure by default
var util = require('util');
console.log(util.inspect(json));
var profile = { provider: 'freshbooks' };
profile.id = json.response.staff.staff_id;
profile.displayName = json.response.staff.first_name + ' ' + json.response.staff.last_name;
profile.name = { familyName: json.response.staff.last_name,
givenName: json.response.staff.first_name };
if (json.response.staff.email) { profile.emails = [{ value: json.response.staff.email }]; }
profile._raw = body;
profile._json = json;
console.log(util.inspect(json));
done(null, profile);
} catch(e) {
done(e);
}
});
I get a response. It's xml. I'm converting it to JSON, but I don't want that actually. I want a plain-old javascript object.
I looked at https://github.com/Leonidas-from-XIV/node-xml2js but the examples don't show how to get the result out.
var parseString = require('xml2js').parseString;
var xml = "<root>Hello xml2js!</root>"
parseString(xml, function (err, result) {
console.dir(result);
});
What do I do to block around this code till the call is complete and get result out? I'm not sure how to merge these two callbacks together.
you can ask xml2json to return object:
var json = parser.toJson(body, {object: true});
if you decide to use async parser then just put your done callback inside json result handler. There is no need to "block" async function:
var parseString = require('xml2js').parseString;
parseString(body, function(err, json) {
// handle error: return done(err)
// do your logic if no error
// ...
// profile._json = json;
// ...
//
// 'return' result
done(null, profile);
});