I want to mock Amazon AWS S3 getObject
The code I want to test is the following one: Its in helper.js
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
exports.get_data_and_callback = function(callback, extra){
s3.getObject( {Bucket: SRC_BUCKET, Key: SRC_KEY},
function (err, data) {
if (err != null) {
console.log("Couldn't retrieve object: " + err);
}else{
console.log("Loaded " + data.ContentLength + " bytes");
callback(data, extra);
}
});
}
In test/helper_test.js I wrote a test that should mock the module AWS
var assert = require('assert');
var mockery = require('mockery');
describe("helper", function() {
it('loads and returns data from S3 to a callback', function(){
mockery.enable();
var fakeaws = {
S3: function(){
return {
getObject: function(params, callback){
callback(null, "hello")
}
}
}
}
mockery.registerSubstitute('aws-sdk', fakeaws);
function replace_function(err, data){
console.log(data);
}
require('../helper.js').get_data_and_callback(replace_function, null);
});
});
When I require AWS in the Test-File test/helper_test.js like this:
aws = require('aws-sdk');
s3 = new aws.S3;
s3.getObject(replace_function)
Then my code works, it prints out hello.
BUT the execution of require('../helper.js').get_data_and_callback(replace_function, null);
Doesn't work like expected, AWS stays the same its not replaced with my fakeaws. What do I wrong? Do you maybe have other solutions to replace S3 Thanks
We have created an aws-sdk-mock npm module which mocks out all the AWS SDK services and methods. https://github.com/dwyl/aws-sdk-mock
It's really easy to use. Just call AWS.mock with the service, method and a stub function.
AWS.mock('S3', 'getObject', function(params, callback) {
callback(null, 'success');
});
Then restore the methods after your tests by calling:
AWS.restore('S3', 'getObject');
This works for every service and method in the as-sdk.
Related
I'm totally new in azure and I would like to create azure function, which will read the content from azure storage container file.json.
Folder structure :
Storage account name: storageaccounttest
Container name: test
File name: file.json
File.json:
[
{
"name":"Kate",
"age":"28"
},
{
"name":"John",
"age":"30"
}
]
Cors on storage account: get enabled.
Environemnts variable added: process.env.AZURE_STORAGE_NAME and process.env.AZURE_STORAGE_KEY and process.env.AZURE_CONNECTION_STRING
I'm using VisualStudioCode to deploy the function.
I installed locally the dependencies:
"dependencies": {
"azure-storage": "^2.10.3",
"dotenv": "^8.1.0"
}
I choose the javascript -> HttpTrigger fn-> anonymus options
I'm using getBlobToText fn.
My index.js:
var storage = require('azure-storage');
var blobService = storage.createBlobService();
var containerName = 'test';
var blobName = 'file.json';
module.exports = blobService.getBlobToText(
containerName,
blobName,
function(err, blobContent) {
if (err) {
console.error("Couldn't download blob");
console.error(err);
} else {
console.log("Sucessfully downloaded blob");
console.log(blobContent);
}
});
Fn is deployed successfully, but I'm not able to see results.
After start, fn is finished with status 500, Internal Server Errror, Console: No new trace in the past 1 min(s).
What I made wrong?
Just summarized for helping others who get the same issue.
I think you were using context.binding.response to pass the blobContent value to the output response as the offical document Azure Functions JavaScript developer guide said.
Here is my sample code with Promise feature to solve it.
var azure = require('azure-storage');
var blobService = azure.createBlobService();
var containerName = 'test';
var blobName = 'file.json';
async function getBlobContent(containerName, blobName) {
return new Promise((resolve, reject) => {
blobService.getBlobToText(containerName, blobName, function(err, blobContent) {
if (err) {
reject(err);
} else {
resolve(blobContent);
}
});
});
}
module.exports = async function (context, req) {
await getBlobContent(containerName, blobName).then(
function(content) {
context.res = {
headers: {"Content-Type": "application/json"},
body: content
}
}, function(error) {
context.res = {
status: 400,
body: error
}
}
);
};
It works as the figure below.
I am trying to use SNS located in São Paulo (sa-east-1) from a lambda function (Node.js 8.10) on Ohio (us-east-2). This is the first time I try to use a AWS service located in another region. So far, this is what I am doing:
//init aws resources
const AWS = require('aws-sdk');
const sns = new AWS.SNS({apiVersion: '2010-03-31', region: 'sa-east-1'});
//promisefy AWS.SNS.createPlatformEndpoint method
snsCreatePlatformEndpoint = params => new Promise(
(resolve, reject)=>{
sns.createPlatformEndpoint(params, function(error, data){
if (error) { reject(error); }
else { resolve(data); }
});
}
);
exports.handler = (awsEvent, context, callback) => {
//parse stuff in here
...
HandleToken(token, callback);
};
async function HandleToken(token, callback){
try{
let params = {
PlatformApplicationArn: process.env.PlatAppArn,
Token: token,
};
console.log('params:', params); // this prints as expected
let {EndpointArn} = await snsCreatePlatformEndpoint(params);
console.log('It should pass through here'); // it is not printed
//returns a success response
...
} catch (error) {
//returns an error response
...
}
}
I have set a really high timeout for my lambda function: 5mins.
I also have tested the same code on a lambda function located in São Paulo(sa-east-1), and it works.
I have been receiving the following error on my client:
"Request failed with status code 504"
"Endpoint request timed out"
Question: How can I use SNS in another AWS region correctly?
You shouldn't need to do any special setup beyond setting the region.
E.g., I use the following pattern to send notifications from us-east-1 to Tokyo (ap-northeast-1):
// this lambda runs in us-east-1
let AWS = require("aws-sdk");
AWS.config.update({ region: "ap-northeast-1" }); // asia-pacific region
exports.handler = async (event, context) => {
var params = {
Message: 'my payload',
TopicArn: 'arn:aws:sns:ap-northeast-1:xxxxxx:tokyoSNS'
};
let SNS = new AWS.SNS({apiVersion: '2010-03-31'});
var data = await SNS.publish(params).promise();
// check if successful then return
}
No endpoints, etc., was setup. Are you required to run your lambda in a VPC? That's the only complication I can think of at the moment.
I have an AWS Lambda function. It goes through a list of the array which is an array of URLs and saves their screenshot and puts them in s3. How do I return the output of this function which is screenshotLinks array that has all the links to files saved in s3? I used the callback function in the end, but it just returns null! I want callback function to output all the s3 file links saved inside screenshotLinks array.
exports.handler = (event, context, callback) => {
desktopLinks.forEach(function (url, index) {
https.request(url, function(res) {
var data = new Stream();
res.on('data', function(chunk) {
// Agregates chunks
data.push(chunk);
});
res.on('end', function() {
var body = data.read();
// Once you received all chunks, send to S3
var currentLink = links[index];
var linkAddress = encodeURIComponent(currentLink);
var slashPosition = getPosition(currentLink, '/', 3)+1;
var linkName = currentLink.substr(slashPosition, currentLink.length)
var params = {
Bucket: bucket,
Key: completeDate + '/screenshots/' + linkName + '.png',
Body: body
};
s3.putObject(params, function(err, data, callback) {
if (err) {
console.error(err, err.stack);
} else {
bunch = params.Bucket + '/' + params.Key;
screenshotLinks.push(bunch);
}
});
});
}).end();
})
callback(null, screenshotLinks)
};
Your code is event driven / asynchronous which means you are calling the callback before screenshotLinks has been populated.
The node http.ClientRequest.end() method finishes sending a request, but that doesn't mean that the response has been received and handled, as that is done by an asynchronous event handler. However, the callback is executed immediately after the call to request.end(), which is just after the request has been fired off, therefore screenshotLinks is empty.
You need to execute your callback from the callback you pass to s3.putObject. I suggest you pass your callback a response/result object that indicates whether the putObject succeeded and contains the url it relates to and either an error message or a screenshotLink, e.g. something like this:
s3.putObject(params, function(err, data, callback) {
var s3Response;
s3Response.url = url;
if (err) {
s3Response.success = false;
s3Response.error = err;
console.error(err, err.stack);
} else {
bunch = params.Bucket + '/' + params.Key;
s3Response.success = true;
s3Response.screenshotLink = bunch;
}
callback(null, s3Response);
});
I would like to suggest you use an 8.10 node runtime.
ref: https://aws.amazon.com/blogs/compute/node-js-8-10-runtime-now-available-in-aws-lambda/
Then your entry point should be:
export async function <function_name>(event) {}
Then:
let s3 = new AWS.S3({ region: process.env.AWS_REGION, apiVersion: '2006-03-01' });
let params=
{
Bucket: /* a path to bucket (string) */,
Key: name /* string */,
Body: /* supported types (Buffer, Typed Array, Blob, String, ReadableStream) */,
ACL: 'public-read',
ContentType: 'image/png'
};
try
{
let s3Response = await s3.upload(params).promise();
// if succceed
console.log(`File uploaded to S3 at ${s3Response.Bucket} bucket. File location: ${s3Response.Location}`);
}
catch (ex) // if error occured
{
console.error(ex);
}
I am new to javascript and I am trying to write a lambda function that would get triggered by PUT event in the bucket, the function would write the file name and some metadata field that's on the s3 object to the dynamodb table. I have most of it worked out but I'm stuck at grabbing the x-amz-meta header info and pass the variable to dynamo.put parameter. Can anyone tell me what I am doing wrong in my code? thanks!
var AWS = require('aws-sdk');
var dynamo = new AWS.DynamoDB.DocumentClient({region: 'us-east-1'});
var s3 = new AWS.S3();
//specify the parameters from event to write to specified db table
exports.handler = function(event, context, callback) {
var srcKey = unescape(event.Records[0].s3.object.key);
var srcEtag = unescape(event.Records[0].s3.object.eTag);
var scrUploadTime = unescape(event.Records[0].eventTime);
var bucket= unescape(event.Records[0].s3.bucket.name);
var checksum =
s3.headObject(
{
Bucket: bucket,
Key: srcKey
},
function(err, data)
{
if (err)
{
console.log(err);
context.done('Error', 'Error getting s3 object: ' + err);
}
else
{
return console.log(this.httpResponse.headers['x-amz-meta-checksum']);
}
});
var params = {
Item: {
filename: srcKey,
uploadtime: scrUploadTime,
client_checksum : checksum
},
TableName: 'S3_log'
};
//write to dynammodb
dynamo.put(params, function(err, data){
if (err) {
callback(err, null);
}else{
callback(null, data);
}
});
};
It looks like you want this:
console.log(data.Metadata['x-amz-meta-checksum']);
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
But also, note that your code doesn't appear to be structured correctly. s3.headObject is asynchronous so your code continues execution at var params = ... before s3.headObject returns. The next actions should probably be inside the callback or handled in another way (waterfall, promises, etc.) to delay the next action until this one completes.
This is the code I have for publishing to the topic (I have changed the target and topic arn for security reasons):
var AWS = require("aws-sdk");
var sns = new AWS.SNS();
var targetArn = 'arn:aws:sns:us-east-1:4363657289:endpoint/GCM/APP_NAME/3185sfdnfe283925sgSeaa0e';
var topicArn = 'arn:aws:s-s:us-east-1:4363657289436:TOPIC_NAME';
var payload = {
GCM: {
data: {
message: "test"
}
}
};
payload.GCM = JSON.stringify(payload.GCM);
payload = JSON.stringify(payload);
var params= {
TopicArn: topicArn,
TargetArn: targetArn,
Message: payload,
MessageStructure: 'json'
};
var responsefromSNS = sns.publish(params , function(error, data) {
if (error) {
console.log("ERROR: " + error.stack);
}
else {
console.log("SENT DATA: " + JSON.stringify(data));
context.done(null, data);
}
});
console.log(responsefromSNS);
My issue is that I never see log statements from either the if or else block and the push notification never reaches the mobile app. I have consulted both the AWS JavaScript SDK Documentation and countless stack overflow posts about this and nothing that I have tried works. And, I have given the lambda function permission to publish to the topic.
---UPDATE-----
I have changed my code a bit and now it looks like this:
var AWS = require("aws-sdk");
AWS.config.update({region:'us-east-1'});
var topicarn = 'arn:aws:s-s:us-east-1:927579412028:alexapushdemo';
var targetarn = 'arn:aws:sns:us-east-1:927579412028:endpoint/GCM/automation.home.visa.com.homeautomation/3af761b2-1955-34d8-b66a-85e232e0aa0e';
var payload = {
default: "test",
GCM: {
data: {
message: "test"
}
}
};
payload.GCM = JSON.stringify(payload.GCM);
payload = JSON.stringify(payload);
var sns = new AWS.SNS();
console.log('start of sns function')
sns.publish({
TargetArn: targetarn,
Message: payload,
MessageStructure: 'json'
}, function(err, data) {
if (err) {
console.log(err.stack);
// Notify Lambda that we are finished, but with errors
context.done(err, 'sns function finished with errors!');
return;
}
console.log('push sent');
console.log(data);
// Notify Lambda that we are finished
context.done(null, 'sns function finished!');
});
console.log('end of sns functions');
The error I get is:
ConfigError: Missing region in config\\n
at Request.VALIDATE_REGION (/node_modules/aws-sdk/lib/event_listeners.js:81:45)\\n
at Request.callListeners (/node_modules/aws-sdk/lib/sequential_executor.js:105:20)\\n
at callNextListener (/node_modules/aws-sdk/lib/sequential_executor.js:95:12)\\n
at /node_modules/aws-sdk/lib/event_listeners.js:75:9\\n
at finish (/node_modules/aws-sdk/lib/config.js:228:7)\\n
at /node_modules/aws-sdk/lib/config.js:268:9\\n
at resolveNext (/node_modules/aws-sdk/lib/credentials/credential_provider_chain.js:84:9)\\n
at /node_modules/aws-sdk/lib/credentials/credential_provider_chain.js:97:11\\n
at /node_modules/aws-sdk/lib/credentials.js:123:23\\n
at /node_modules/aws-sdk/lib/credentials/ec2_metadata_credentials.js:66:7\\"\",\"ip\":\"127.0.0.1\"}",
Why am I getting this even though I'm calling AWS.config.update.
iram,
If I take your exact code and paste it into a Lambda Node.js 4.3 function and execute a test from the Lambda Console, this is the result:
ERROR: InvalidParameter: Invalid parameter: TopicArn Reason: Both TopicArn and TargetArn specified. Use only one or the other
This means that in your params, you need to comment out either TopicArn or TargetArn or put in some logic to determine if the incoming payload contains an Arn that is a target endpoint or a topic endpoint.
You could still have permissions issues with Lambda execution role to SNS or to CW Logs, however, regardless if you have permission to publish or send logs to CloudWatch from your Lambda function, running a test from the console will always spit out some logging of what's going on.
Good luck.
const AWS = require('aws-sdk');
AWS.config.region = 'us-east-1';
specify the regeion to be used by the aws sdk like this