I have the following code (trimmed, assume all the closing stuff is there), which dies deep down inside GridFS:
var Grid = require('mongodb').Grid;
var mongoose = require('mongoose');
var db = mongoose.connect('mongodb://localhost:27017/ksnap');
router.route('/').post(function(req, res) {
var post = new Post();
var busboy = new Busboy({ headers: req.headers });
req.pipe(busboy);
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
if (fieldname != 'img') { return; }
var bufs = [];
file.on('data', function(data) {
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
bufs.push(data);
}); // busboy file on data
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
var buf = Buffer.concat(bufs);
var grid = new Grid(db, 'fs');
grid.put(buf, {metadata:{category:'image'}, content_type: 'image'}, function(err, result) {
if (err) { console.log(err); } else { console.log(result); }
});
Stack trace:
/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/gridstore.js:1552
} else if(self.safe.w != null || typeof self.safe.j == 'boolean' || typeof s
^
TypeError: Cannot read property 'w' of undefined
at _getWriteConcern (/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/gridstore.js:1552:22)
at Stream.GridStore (/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/gridstore.js:100:23)
at Grid.put (/opt/ksnap-server/node_modules/mongodb/lib/mongodb/gridfs/grid.js:52:19)
at FileStream.<anonymous> (/opt/ksnap-server/server.js:83:13)
at FileStream.emit (events.js:117:20)
at _stream_readable.js:943:16
at process._tickCallback (node.js:419:13)
Busboy returns a stream which I put into a buffer, so far so good. This works fine, I've tested it. But when I try to grid.put() the buffer, it dies as above. I've tried to trace it, but I'm having trouble. As far as I can tell, the all the options get eaten in grid.js, so by the time they get passed down to gridstore.js it's just an empty object. Mongoose just doesn't set this, I guess.
I was able to get past this error by manually setting db.safe = {w: 1}; after opening the connection, however when I did the grid.put() it just stuck there. Swapping out mongoose for a regular mongodb connection worked, so I guess currently mongoose just doesn't work with GridFS.
I was finally able to get everything (apparently) working by adding the streamifier and gridfs-stream modules, and the following mongo setup:
var streamifier = require('streamifier');
var Grid = require('gridfs-stream');
mongoose.connect('mongodb://localhost:27017/ksnap');
Then later, when I'm ready to save the file to GridFS:
var gfs = new Grid(mongoose.connection.db, mongoose.mongo);
var writestream = gfs.createWriteStream({
mode: 'w',
filename: post.id,
content_type: 'image/jpeg'
});
streamifier.createReadStream(buffer).pipe(writestream);
writestream.on('close', function (file) {
console.log("saved 300px as "+file.filename);
});
And save the post document itself to MongoDB:
post.save(function(err) {
if (err) { res.send(err); }
console.log('saved post '+post.id);
res.send(post);
});
This was the combination of options that worked for me. One of the keys was using mongoose.connect(), not mongoose.createConnection(), which would let me save the files, but not the documents.
I know this has been a while - I saw the same issue - make sure your mongoose session is connected to the DB - ie
mongoose.connection.once("connected", function () {...} has been called, then load the require files and files. This ensures the db object in the connection is bound to an existing mongo session. If you find the mongoose.connection.db is null and mongoose.connection is NOT null then you will have initialized your grid stream with an uninitialized mongodb connection.
Related
So I followed this tutorial by amazon: https://docs.aws.amazon.com/lambda/latest/dg/with-s3-example.html to make a thumbnail generator for every image that I upload to my S3 bucket. It works fine on small images around 500KB, and it correctly puts them in the thumbnail bucket, but I attempted to upload a 300MB image file to my S3 bucket, and my lambda function doesn't seem to work correctly.
I looked through other forums and I have attempted to fiddle with some timeout and memory size settings in AWS Lambda becuase I thought that the function maybe needed more memory, but that wasn't the case and I personally don't know what else I have left to go off of.
Here is the copy of the lambda function straight from the link I used, the error occurs at line 57 when the MAX_HEIGHT and MAX_WIDTH are being set. It seems that size in the case of large files seems to always be undefined.
// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var gm = require('gm')
.subClass({ imageMagick: true }); // Enable ImageMagick integration.
var util = require('util');
// constants
var MAX_WIDTH = 100;
var MAX_HEIGHT = 100;
// get reference to S3 client
var s3 = new AWS.S3();
exports.handler = function(event, context, callback) {
// Read options from the event.
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
var srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
var srcKey =
decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
var dstBucket = srcBucket + "resized";
var dstKey = "resized-" + srcKey;
// Sanity check: validate that source and destination are different buckets.
if (srcBucket == dstBucket) {
callback("Source and destination buckets are the same.");
return;
}
// Infer the image type.
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
callback("Could not determine the image type.");
return;
}
var imageType = typeMatch[1];
if (imageType != "jpg" && imageType != "png") {
callback('Unsupported image type: ${imageType}');
return;
}
// Download the image from S3, transform, and upload to a different S3 bucket.
async.waterfall([
function download(next) {
// Download the image from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transform(response, next) {
gm(response.Body).size(function(err, size) {
// Infer the scaling factor to avoid stretching the image unnaturally.
var scalingFactor = Math.min(
MAX_WIDTH / size.width,
MAX_HEIGHT / size.height
);
var width = scalingFactor * size.width;
var height = scalingFactor * size.height;
// Transform the image buffer in memory.
this.resize(width, height)
.toBuffer(imageType, function(err, buffer) {
if (err) {
next(err);
} else {
next(null, response.ContentType, buffer);
}
});
});
},
function upload(contentType, data, next) {
// Stream the transformed image to a different S3 bucket.
s3.putObject({
Bucket: dstBucket,
Key: dstKey,
Body: data,
ContentType: contentType
},
next);
}
], function (err) {
if (err) {
console.error(
'Unable to resize ' + srcBucket + '/' + srcKey +
' and upload to ' + dstBucket + '/' + dstKey +
' due to an error: ' + err
);
} else {
console.log(
'Successfully resized ' + srcBucket + '/' + srcKey +
' and uploaded to ' + dstBucket + '/' + dstKey
);
}
callback(null, "message");
}
);
};
Here is the error message directly from AAWS Cloud Watch logs:
2019-05-14T22:31:28.731Z b5fccf54-e55f-49f2-9206-462fa5769149 TypeError: Cannot read property 'width' of undefined
at gm.<anonymous> (/var/task/index.js:57:38)
at emitMany (events.js:147:13)
at gm.emit (events.js:224:7)
at gm.<anonymous> (/var/task/node_modules/gm/lib/getters.js:70:16)
at cb (/var/task/node_modules/gm/lib/command.js:322:16)
at gm._spawn (/var/task/node_modules/gm/lib/command.js:226:14)
at gm._exec (/var/task/node_modules/gm/lib/command.js:190:17)
at gm.proto.(anonymous function) [as size] (/var/task/node_modules/gm/lib/getters.js:68:12)
at transform (/var/task/index.js:54:31)
at nextTask (/var/task/node_modules/async/dist/async.js:5324:14)
EDIT: In addition I also seem to get this error which alternated between the two:
2019-05-14T22:55:25.923Z 7a7f1ec2-cd78-4fa5-a296-fee58033aea6 Unable to resize MYBUCKET/image.png and upload to MYBUCKETRESIZE/resize-image.png due to an error: Error: Stream yields empty buffer
EDIT: Added the report line:
REPORT RequestId: a67e1e79-ebec-4b17-9832-4049ff31bd89 Duration: 7164.64 ms Billed Duration: 7200 ms Memory Size: 1024 MB Max Memory Used: 810 MB
Your two errors are for slightly different reasons.
Stream yields empty buffer almost definitely means you're running out of memory.
The other error might be a specific imagemagick error, which I don't think you're handling. Though it is also probably related to the memory issue. Because it's in a callback, you need to check the err variable:
function transform(response, next) {
gm(response.Body).size(function(err, size) {
if (err) throw err;
The imagemamgick module might be writing something to /tmp. A snippet for checking/clearing disk usage from a project i worked on
const fs = require('fs')
const folder = '/tmp/'
// Deleting any files in the /tmp folder (lambda will reuse the same container, we will run out of space)
const files = fs.readdirSync(folder)
for (const file of files) {
console.log('Deleting file from previous invocation: ' + folder + file)
fs.unlinkSync(folder + file)
}
I am working on a lambda function that would be invoked by a S3 PUT event and would display the metadata field of the s3 object. I tried to set the key and bucket as variable but when I run it I get a { BadRequest: null error. below is my code in javascript. when I hardcode the key and bucket it would work but not with variable passed in, can someone explain what am I doing wrong? thanks!
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
exports.handler = function(event, context) {
var srcbucket = ("\'" + (event.Records[0].s3.object.key).toString() + "\'");
var srcKey = ("\'" + (event.Records[0].s3.bucket.name).toString() + "\'");
console.log (srcKey);
s3.headObject(
{
Bucket : srcbucket,
Key: srcKey
},
function(err, data)
{
if (err)
{
console.log(err);
context.done('Error', 'Error getting s3 object: ' + err);
}
else
{
var data = JSON.stringify(this.httpResponse.headers['x-amz-meta-checksum']).replace(/\"/g, "");
console.log (data.replace(/\"/g, ""));
}
First, your variable is mixed up. The srcbucket is pointer to object's Key, and vice versa.
Secondly, you may want remove the additional quote ' that was applied to the variable.
var srcbucket = event.Records[0].s3.bucket.name;
var srcKey = event.Records[0].s3.object.key;
I'm certain I'm missing something obvious, but the gist of the problem is I'm receiving a PNG from a Mapbox call with the intent of writing it to the file system and serving it to the client. I've successfully relayed the call, received a response of raw data and written a file. The problem is that my file ends up truncated no matter what path I take, and I've exhausted the answers I've found skirting the subject. I've dumped the raw response to the log, and it's robust, but any file I make tends to be about a chunk's worth of unreadable data.
Here's the code I've got at present for the file making. I tried this buffer move as a last ditch after several failed and comparably fruitless iterations. Any help would be greatly appreciated.
module.exports = function(req, res, cb) {
var cartography = function() {
return https.get({
hostname: 'api.mapbox.com',
path: '/v4/mapbox.wheatpaste/' + req.body[0] + ',' + req.body[1] + ',6/750x350.png?access_token=' + process.env.MAPBOX_API
}, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var mapPath = 'map' + req.body[0] + req.body[1] + '.png';
var map = new Buffer(body, 'base64');
fs.writeFile(__dirname + '/client/images/maps/' + mapPath, map, 'base64', function(err) {
if (err) throw err;
cb(mapPath);
})
})
});
};
cartography();
};
It is possible to rewrite your code in more compact subroutine:
const fs = require('fs');
const https = require('https');
https.get(url, (response)=> { //request itself
if(response) {
let imageName = 'image.png'; // for this purpose I usually use crypto
response.pipe( //pipe response to a write stream (file)
fs.createWriteStream( //create write stream
'./public/' + imageName //create a file with name image.png
)
);
return imageName; //if public folder is set as default in app.js
} else {
return false;
}
})
You could get original name and extension from url, but it safer to generate a new name with crypto and get file extension like i said from url or with read-chunk and file-type modules.
I have this piece of code in order to download files from Google Drive:
function downloadDrive(fileId, callback) {
var fileExt = fileId.split(".");
var file = Date.now() + "." + fileExt[fileExt.length - 1];
var dest = fs.createWriteStream("./files/"+file);
service.files.get({
auth: oauth2Client,
fileId: fileExt[0],
alt: "media"
})
.on("finish", function() {
callback(file);
})
.on("error", function(err) {
console.log("Error during download", err);
})
.pipe(dest);
}
It works very well on small files ~500Mb. However, when trying to download a quite a big gzip file ~3Gb, it throws the following error.
buffer.js:23
const ui8 = new Uint8Array(size);
^
RangeError: Invalid typed array length
at new Uint8Array (native)
at createBuffer (buffer.js:23:15)
at allocate (buffer.js:98:12)
at new Buffer (buffer.js:53:12)
at Function.Buffer.concat (buffer.js:225:16)
at BufferList.copy (/Synology/server_Metagenomics/server/node_modules/googleapis/node_modules/google-auth-library/node_modules/request/node_modules/bl/bl.js:124:21)
at BufferList.slice (/Synology/server_Metagenomics/server/node_modules/googleapis/node_modules/google-auth-library/node_modules/request/node_modules/bl/bl.js:99:15)
at BufferList.toString (/Synology/server_Metagenomics/server/node_modules/googleapis/node_modules/google-auth-library/node_modules/request/node_modules/bl/bl.js:166:15)
at Request.<anonymous> (/Synology/server_Metagenomics/server/node_modules/googleapis/node_modules/google-auth-library/node_modules/request/request.js:1035:36)
at emitOne (events.js:82:20)
at Request.emit (events.js:169:7)
I didn't find a lot of information about it. What is going on?
You can try following to download file. The error seems to be because you are downloading large file.
var filename = 'zzz.txt';
var proxyUrl = "http://" + user + ":" + password + "#" + host + ":" + port;
var token = 'YOUR_TOKEN';
var req = request.get('https://www.googleapis.com/drive/v3/files/YOUR_FILE_ID?alt=media', {
'auth': {
'bearer': token
},
'proxy': proxyUrl
}).on('response', function(res) {
// create file write stream
var fws = fs.createWriteStream(filename);
// setup piping
res.pipe(fws);
res.on('err', function() {
console.log("error occured.....");
});
res.on('end', function() {
console.log('Done');
// go on with processing
});
});
Trying to allow users to upload image files to the Node.js server in a MEAN Stack application. I am using ng-file-upload for the client side angular directive. That seems to be working good enough. I run into an error when I pass the image to the server.
I use an API route to handle the work on the server side. The server will be responsible for saving the file to disk with node-multiparty module. It seems to hit route but when it tries to emit a close event I get the error. throw new Error('"name" and "value" are required for setHeader().'
The file I want is in my temp folder but it doesn't get saved to the target directory on my server plus I get the header error after the file should have been saved. So I need to stop the error and save the file with fs.rename() to the target image directory.
Here is the code that is breaking.
file api.js
// router to save images
router.route('/img/upload')
.post(function (req, res) {
console.log("image upload hits the router")
var options = {};
var count = 0;
var form = new multiparty.Form(options);
//save file to disk
form.on('file', function (name, file) {
var uploadDirectory = 'img/user/profile/';
var oldPath = file.path;
var newPath = uploadDirectory + file.originalFilename;
fs.rename(oldPath, newPath, function (err) {
if (err) throw err;
console.log('renamed complete');
});
});
// Close emitted after form parsed
form.on('close', function () {
console.log('Upload completed!');
res.setHeader('text/plain'); // Here is the line that gives an error.
res.end('Received ' + count + ' files');
});
// Parse req
form.parse(req);
});
So this is what I got to work for me
The actual line that gave me an error was setHeaders. It appears I needed to put the name and value as strings separated by a comma. This works perfectly for me now. I hope it saves everyone time coding.
// post
.post(function (req, res) {
var options = {};
var count = 0;
var form = new multiparty.Form(options);
form.on('error', function (err) {
console.log('Error parsing form: ' + err.stack);
});
//save file to disk
form.on('file', function (name, file) {
var uploadDirectory = '/img/user/profile/';
var oldPath = file.path;
var newPath = uploadDirectory + file.originalFilename;
fs.rename(oldPath, newPath, function (err) {
if (err) throw err;
console.log('renamed complete');
});
});
// Close emitted after form parsed
form.on('close', function () {
console.log('Upload completed!');
res.setHeader('Content-Type', 'text/plain');
res.end('Received ' + count + ' files');
});
// Parse req
form.parse(req);
});