Develop Reference

Cannot seem to access S3 bucket contents with key

I am using the aws javascript sdk and for some reason I can access the entire buckets contents, but when I add a prefix I get null returned rather than a subset of those contents. For example, the following returns all bucket contents:
AWS.config.accessKeyId = this.s3.config["accessKeyId"];
AWS.config.secretAccessKey = this.s3.config["secretAccessKey"];
AWS.config.region = 'us-east-2';
var aws = new AWS.S3();
var all_params = {Bucket: 'bucket-name'};
new Promise(resolve => {
aws.listObjectsV2(all_params, function (err, url) {
console.log(url)
resolve(url)
});
})
The object returned contains 1000 records, most of them in the format Key: "clients/after_fronts/000...". However when I run the following, I get a null object:
AWS.config.accessKeyId = this.s3.config["accessKeyId"];
AWS.config.secretAccessKey = this.s3.config["secretAccessKey"];
AWS.config.region = 'us-east-2';
var key = "clients"
var aws = new AWS.S3();
var params = {Bucket: 'bucket-name', prefix: key};
return new Promise(resolve => {
aws.listObjectsV2(params, function (err, url) {
console.log(url)
resolve(url)
});
})
I thought it might be a permissions issue but I'm not sure why it returns data without a prefix and then no data with the prefix. What else could be going on?

Well, after staring at this for an hour I realized the docs call for Prefix not prefix and that capitalization made all the difference.

AWS Lambda thumbnail generator not working on larger file sizes

So I followed this tutorial by amazon: https://docs.aws.amazon.com/lambda/latest/dg/with-s3-example.html to make a thumbnail generator for every image that I upload to my S3 bucket. It works fine on small images around 500KB, and it correctly puts them in the thumbnail bucket, but I attempted to upload a 300MB image file to my S3 bucket, and my lambda function doesn't seem to work correctly.
I looked through other forums and I have attempted to fiddle with some timeout and memory size settings in AWS Lambda becuase I thought that the function maybe needed more memory, but that wasn't the case and I personally don't know what else I have left to go off of.
Here is the copy of the lambda function straight from the link I used, the error occurs at line 57 when the MAX_HEIGHT and MAX_WIDTH are being set. It seems that size in the case of large files seems to always be undefined.
// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var gm = require('gm')
.subClass({ imageMagick: true }); // Enable ImageMagick integration.
var util = require('util');
// constants
var MAX_WIDTH = 100;
var MAX_HEIGHT = 100;
// get reference to S3 client
var s3 = new AWS.S3();
exports.handler = function(event, context, callback) {
// Read options from the event.
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
var srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
var srcKey =
decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
var dstBucket = srcBucket + "resized";
var dstKey = "resized-" + srcKey;
// Sanity check: validate that source and destination are different buckets.
if (srcBucket == dstBucket) {
callback("Source and destination buckets are the same.");
return;
}
// Infer the image type.
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
callback("Could not determine the image type.");
return;
}
var imageType = typeMatch[1];
if (imageType != "jpg" && imageType != "png") {
callback('Unsupported image type: ${imageType}');
return;
}
// Download the image from S3, transform, and upload to a different S3 bucket.
async.waterfall([
function download(next) {
// Download the image from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transform(response, next) {
gm(response.Body).size(function(err, size) {
// Infer the scaling factor to avoid stretching the image unnaturally.
var scalingFactor = Math.min(
MAX_WIDTH / size.width,
MAX_HEIGHT / size.height
);
var width = scalingFactor * size.width;
var height = scalingFactor * size.height;
// Transform the image buffer in memory.
this.resize(width, height)
.toBuffer(imageType, function(err, buffer) {
if (err) {
next(err);
} else {
next(null, response.ContentType, buffer);
}
});
});
},
function upload(contentType, data, next) {
// Stream the transformed image to a different S3 bucket.
s3.putObject({
Bucket: dstBucket,
Key: dstKey,
Body: data,
ContentType: contentType
},
next);
}
], function (err) {
if (err) {
console.error(
'Unable to resize ' + srcBucket + '/' + srcKey +
' and upload to ' + dstBucket + '/' + dstKey +
' due to an error: ' + err
);
} else {
console.log(
'Successfully resized ' + srcBucket + '/' + srcKey +
' and uploaded to ' + dstBucket + '/' + dstKey
);
}
callback(null, "message");
}
);
};
Here is the error message directly from AAWS Cloud Watch logs:
2019-05-14T22:31:28.731Z b5fccf54-e55f-49f2-9206-462fa5769149 TypeError: Cannot read property 'width' of undefined
at gm.<anonymous> (/var/task/index.js:57:38)
at emitMany (events.js:147:13)
at gm.emit (events.js:224:7)
at gm.<anonymous> (/var/task/node_modules/gm/lib/getters.js:70:16)
at cb (/var/task/node_modules/gm/lib/command.js:322:16)
at gm._spawn (/var/task/node_modules/gm/lib/command.js:226:14)
at gm._exec (/var/task/node_modules/gm/lib/command.js:190:17)
at gm.proto.(anonymous function) [as size] (/var/task/node_modules/gm/lib/getters.js:68:12)
at transform (/var/task/index.js:54:31)
at nextTask (/var/task/node_modules/async/dist/async.js:5324:14)
EDIT: In addition I also seem to get this error which alternated between the two:
2019-05-14T22:55:25.923Z 7a7f1ec2-cd78-4fa5-a296-fee58033aea6 Unable to resize MYBUCKET/image.png and upload to MYBUCKETRESIZE/resize-image.png due to an error: Error: Stream yields empty buffer
EDIT: Added the report line:
REPORT RequestId: a67e1e79-ebec-4b17-9832-4049ff31bd89 Duration: 7164.64 ms Billed Duration: 7200 ms Memory Size: 1024 MB Max Memory Used: 810 MB

Your two errors are for slightly different reasons.
Stream yields empty buffer almost definitely means you're running out of memory.
The other error might be a specific imagemagick error, which I don't think you're handling. Though it is also probably related to the memory issue. Because it's in a callback, you need to check the err variable:
function transform(response, next) {
gm(response.Body).size(function(err, size) {
if (err) throw err;
The imagemamgick module might be writing something to /tmp. A snippet for checking/clearing disk usage from a project i worked on
const fs = require('fs')
const folder = '/tmp/'
// Deleting any files in the /tmp folder (lambda will reuse the same container, we will run out of space)
const files = fs.readdirSync(folder)
for (const file of files) {
console.log('Deleting file from previous invocation: ' + folder + file)
fs.unlinkSync(folder + file)
}

how can I get the url after all images are saved in S3 using nodejs

The code here intend to save the images uploaded in multipart/form-data (files have been successfully parsed in multiparty module) to Amazon S3, then get all the image bucket URLs and save it to imageUrl array field in the mongodb. However, the imageUrl is always empty.
I found out inside the loopwithcb function the imageUrl has correct URLs.
In the callback function afterSave, the imageUrl is empty. I think it is asynchronous problem, but still cannot figure it out. Any thought would be helpful.
//save images to S3
var i=-1;
var imageUrl =[];
function loopwithcb(afterSave){
Object.keys(files.files).forEach(function(){
i=i+1;
var myFile = files.files[i];
var fileName = Date.now()+myFile.originalFilename;
s3Client.upload({
Bucket: bucket,
Key: fileName,
ACL: 'public-read',
Body: fs.createReadStream(myFile.path)
//ContentLength: part.byteCount
}, function (err, data) {
if (err) {
//handle error
} else {
//handle upload complete
var s3url = "https://s3.amazonaws.com/" + bucket + '/' + fileName;
imageUrl.push(s3url);
console.log('imageurl:'+ imageUrl);
//delete the temp file
fs.unlink(myFile.path);
console.log('url:' + imageUrl);
}
});
});
afterSave();
}
function afterSave(){
console.log('i:'+ i);
console.log('outside print imageurl:'+ imageUrl);
Listing.create({
title : fields.ltitle,
type : 'newlist',
description : fields.lbody,
image : imageUrl
}, function (err, small) {
if (err) return handleError(err);
// saved!
console.log(small);
console.log('listingId:' + ObjectId(small._id).valueOf());
//res.json(small);
});
}
loopwithcb(afterSave); //call back

Changed some part of code, now it can print out the correct imageUrl. However, the new code will NOT upload files to AWS S3. Found this solution (async for loop in node.js) is exactly what I want.
function loopwithcb(afterSave){
Object.keys(files.files).forEach(function(){
i=i+1;
var myFile = files.files[i];
var fileName = Date.now()+myFile.originalFilename;
var saved = s3Client.upload({
Bucket: bucket,
Key: fileName,
ACL: 'public-read',
Body: fs.createReadStream(myFile.path)
//ContentLength: part.byteCount
});
if (saved.err) {
//handle error
} else {
//handle upload complete
var s3url = "https://s3.amazonaws.com/" + bucket + '/' + fileName;
imageUrl.push(s3url);
console.log('imageurl:'+ imageUrl);
//delete the temp file
fs.unlink(myFile.path);
console.log('url:' + imageUrl);
}
});
afterSave();
}

How to get contents of a text file from AWS s3 using a lambda function?

I was wondering if I could set up a lambda function for AWS, triggered whenever a new text file is uploaded into an s3 bucket. In the function, I would like to get the contents of the text file and process it somehow. I was wondering if this was possible...?
For example, if I upload foo.txt, with contents foobarbaz, I would like to somehow get foobarbaz in my lambda function so I can do stuff with it. I know I can get metadata from getObject, or a similar method.
Thanks!

The S3 object key and bucket name are passed into your Lambda function via the event parameter. You can then get the object from S3 and read its contents.
Basic code to retrieve bucket and object key from the Lambda event is as follows:
exports.handler = function(event, context, callback) {
const bkt = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
};
Once you have the bucket and key, you can call getObject to retrieve the object:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
exports.handler = function(event, context, callback) {
// Retrieve the bucket & key for the uploaded S3 object that
// caused this Lambda function to be triggered
const Bucket = event.Records[0].s3.bucket.name;
const Key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
// Retrieve the object
s3.getObject({ Bucket, Key }, function(err, data) {
if (err) {
console.log(err, err.stack);
callback(err);
} else {
console.log("Raw text:\n" + data.Body.toString('ascii'));
callback(null, null);
}
});
};
Here's an updated JavaScript example using ES6-style code and promises, minus error-handling:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
exports.handler = async (event, context) => {
const Bucket = event.Records[0].s3.bucket.name;
const Key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const data = await s3.getObject({ Bucket, Key }).promise();
console.log("Raw text:\n" + data.Body.toString('ascii'));
};
A number of posters have asked for the equivalent in Java, so here's an example:
package example;
import java.net.URLDecoder;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.S3Event;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord;
public class S3GetTextBody implements RequestHandler<S3Event, String> {
public String handleRequest(S3Event s3event, Context context) {
try {
S3EventNotificationRecord record = s3event.getRecords().get(0);
// Retrieve the bucket & key for the uploaded S3 object that
// caused this Lambda function to be triggered
String bkt = record.getS3().getBucket().getName();
String key = record.getS3().getObject().getKey().replace('+', ' ');
key = URLDecoder.decode(key, "UTF-8");
// Read the source file as text
AmazonS3 s3Client = new AmazonS3Client();
String body = s3Client.getObjectAsString(bkt, key);
System.out.println("Body: " + body);
return "ok";
} catch (Exception e) {
System.err.println("Exception: " + e);
return "error";
}
}
}

I am using lambda function with a python 3.6 environment.
The code below will read the contents of a file main.txt inside bucket my_s3_bucket. Make sure to replace name of bucket and file name according to your needs.
def lambda_handler(event, context):
# TODO implement
import boto3
s3 = boto3.client('s3')
data = s3.get_object(Bucket='my_s3_bucket', Key='main.txt')
contents = data['Body'].read()
print(contents)

You can use data.Body.toString('ascii') to get the contents of the text file, assuming that the text file was encoded used ascii format. You can also pass other encoding types to the function. Check out Node-Buffer for further details.

The new AWS SDK v3 means that the files are read back as a readable stream. You'll need to take that into consideration from now on as well.
https://carova.io/snippets/read-data-from-aws-s3-with-nodejs

GridFS put() with Mongoose dies in write concern code?

I have the following code (trimmed, assume all the closing stuff is there), which dies deep down inside GridFS:
var Grid = require('mongodb').Grid;
var mongoose = require('mongoose');
var db = mongoose.connect('mongodb://localhost:27017/ksnap');
router.route('/').post(function(req, res) {
var post = new Post();
var busboy = new Busboy({ headers: req.headers });
req.pipe(busboy);
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
if (fieldname != 'img') { return; }
var bufs = [];
file.on('data', function(data) {
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
bufs.push(data);
}); // busboy file on data
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
var buf = Buffer.concat(bufs);
var grid = new Grid(db, 'fs');
grid.put(buf, {metadata:{category:'image'}, content_type: 'image'}, function(err, result) {
if (err) { console.log(err); } else { console.log(result); }
});
Stack trace:
/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/gridstore.js:1552
} else if(self.safe.w != null || typeof self.safe.j == 'boolean' || typeof s
^
TypeError: Cannot read property 'w' of undefined
at _getWriteConcern (/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/gridstore.js:1552:22)
at Stream.GridStore (/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/gridstore.js:100:23)
at Grid.put (/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/grid.js:52:19)
at FileStream.<anonymous> (/opt/ksnap-server/server.js:83:13)
at FileStream.emit (events.js:117:20)
at _stream_readable.js:943:16
at process._tickCallback (node.js:419:13)
Busboy returns a stream which I put into a buffer, so far so good. This works fine, I've tested it. But when I try to grid.put() the buffer, it dies as above. I've tried to trace it, but I'm having trouble. As far as I can tell, the all the options get eaten in grid.js, so by the time they get passed down to gridstore.js it's just an empty object. Mongoose just doesn't set this, I guess.

I was able to get past this error by manually setting db.safe = {w: 1}; after opening the connection, however when I did the grid.put() it just stuck there. Swapping out mongoose for a regular mongodb connection worked, so I guess currently mongoose just doesn't work with GridFS.
I was finally able to get everything (apparently) working by adding the streamifier and gridfs-stream modules, and the following mongo setup:
var streamifier = require('streamifier');
var Grid = require('gridfs-stream');
mongoose.connect('mongodb://localhost:27017/ksnap');
Then later, when I'm ready to save the file to GridFS:
var gfs = new Grid(mongoose.connection.db, mongoose.mongo);
var writestream = gfs.createWriteStream({
mode: 'w',
filename: post.id,
content_type: 'image/jpeg'
});
streamifier.createReadStream(buffer).pipe(writestream);
writestream.on('close', function (file) {
console.log("saved 300px as "+file.filename);
});
And save the post document itself to MongoDB:
post.save(function(err) {
if (err) { res.send(err); }
console.log('saved post '+post.id);
res.send(post);
});
This was the combination of options that worked for me. One of the keys was using mongoose.connect(), not mongoose.createConnection(), which would let me save the files, but not the documents.

I know this has been a while - I saw the same issue - make sure your mongoose session is connected to the DB - ie
mongoose.connection.once("connected", function () {...} has been called, then load the require files and files. This ensures the db object in the connection is bound to an existing mongo session. If you find the mongoose.connection.db is null and mongoose.connection is NOT null then you will have initialized your grid stream with an uninitialized mongodb connection.