I am trying to setup a file API in my node.js application. My goal is to be able to write the file stream directly to gridfs, without needing to store the file to disk initially. It seems like my create code is working. I am able to save a file upload to gridfs. The problem is reading the file. When I try to download a saved file via a web browser window, I see that the file contents are wrapped with something like the following:
------WebKitFormBoundarye38W9pfG1wiA100l
Content-Disposition: form-data; name="file"; filename="myfile.txt"
Content-Type: text/javascript
***File contents here***
------WebKitFormBoundarye38W9pfG1wiA100l--
So my question is what do I need to do to strip the boundary information from the file stream before saving it to gridfs? Here's the code i'm working with:
'use strict';
var mongoose = require('mongoose');
var _ = require('lodash');
var Grid = require('gridfs-stream');
Grid.mongo = mongoose.mongo;
var gfs = new Grid(mongoose.connection.db);
// I think this works. I see the file record in fs.files
exports.create = function(req, res) {
var fileId = new mongoose.Types.ObjectId();
var writeStream = gfs.createWriteStream({
_id: fileId,
filename: req.query.name,
mode: 'w',
content_type: req.query.type,
metadata: {
uploadedBy: req.user._id,
}
});
writeStream.on('finish', function() {
return res.status(200).send({
message: fileId.toString()
});
});
req.pipe(writeStream);
};
// File data is returned, but it's wrapped with
// WebKitFormBoundary and has headers.
exports.read = function(req, res) {
gfs.findOne({ _id: req.params.id }, function (err, file) {
if (err) return res.status(400).send(err);
// With this commented out, my browser will prompt
// me to download the raw file where I can see the
// webkit boundary and request headers
//res.writeHead(200, { 'Content-Type': file.contentType });
var readstream = gfs.createReadStream({
_id: req.params.id
// I also tried this way:
//_id: file._id
});
readstream.pipe(res);
});
};
By the way, i'm not currently using any middleware for these routes, but am open to doing so. I just didn't want the file to hit the disk prior to being sent to gridfs.
Edit:
Per #fardjad, I added the node-multiparty module for multipart/form-data parsing and it kind of worked. But when I download an uploaded file and compare with an original (as text), there are lots of differences in the encoding, and the downloaded file won't open. Here's my latest attempt.
'use strict';
var mongoose = require('mongoose');
var _ = require('lodash');
var multiparty = require('multiparty');
var Grid = require('gridfs-stream');
Grid.mongo = mongoose.mongo;
var gfs = new Grid(mongoose.connection.db);
exports.create = function(req, res) {
var form = new multiparty.Form();
var fileId = new mongoose.Types.ObjectId();
form.on('error', function(err) {
console.log('Error parsing form: ' + err.stack);
});
form.on('part', function(part) {
if (part.filename) {
var writeStream = gfs.createWriteStream({
_id: fileId,
filename: part.filename,
mode: 'w',
content_type: part.headers['content-type'],
metadata: {
uploadedBy: req.user._id,
}
})
part.pipe(writeStream);
}
});
// Close emitted after form parsed
form.on('close', function() {
return res.status(200).send({
message: fileId.toString()
});
});
// Parse req
form.parse(req);
};
exports.read = function(req, res) {
gfs.findOne({ _id: req.params.id }, function (err, file) {
if (err) return res.status(400).send(err);
res.writeHead(200, { 'Content-Type': file.contentType });
var readstream = gfs.createReadStream({
_id: req.params.id
});
readstream.pipe(res);
});
};
Final Edit:
Here's a simple implementation that I copied from another developer and modified. This is working for me: (I'm still trying to figure out why it won't work in my original express app. Something seems to be interfering)
https://gist.github.com/pos1tron/094ac862c9d116096572
var Busboy = require('busboy'); // 0.2.9
var express = require('express'); // 4.12.3
var mongo = require('mongodb'); // 2.0.31
var Grid = require('gridfs-stream'); // 1.1.1"
var app = express();
var server = app.listen(9002);
var db = new mongo.Db('test', new mongo.Server('127.0.0.1', 27017));
var gfs;
db.open(function(err, db) {
if (err) throw err;
gfs = Grid(db, mongo);
});
app.post('/file', function(req, res) {
var busboy = new Busboy({ headers : req.headers });
var fileId = new mongo.ObjectId();
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('got file', filename, mimetype, encoding);
var writeStream = gfs.createWriteStream({
_id: fileId,
filename: filename,
mode: 'w',
content_type: mimetype,
});
file.pipe(writeStream);
}).on('finish', function() {
// show a link to the uploaded file
res.writeHead(200, {'content-type': 'text/html'});
res.end('download file');
});
req.pipe(busboy);
});
app.get('/', function(req, res) {
// show a file upload form
res.writeHead(200, {'content-type': 'text/html'});
res.end(
'<form action="/file" enctype="multipart/form-data" method="post">'+
'<input type="file" name="file"><br>'+
'<input type="submit" value="Upload">'+
'</form>'
);
});
app.get('/file/:id', function(req, res) {
gfs.findOne({ _id: req.params.id }, function (err, file) {
if (err) return res.status(400).send(err);
if (!file) return res.status(404).send('');
res.set('Content-Type', file.contentType);
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
var readstream = gfs.createReadStream({
_id: file._id
});
readstream.on("error", function(err) {
console.log("Got error while processing stream " + err.message);
res.end();
});
readstream.pipe(res);
});
});
See my comment on the issue you created on github. I had the same problem but I managed to debug the issue. I narrowed it down to where i was confident that the problem was a piece of express middleware modified the request. I disabled my middleware one by one until i found the unlikely culprit: connect-livereload
I commented out app.use(require('connect-livereload')()); and the problem went away.
I believe it was injecting the livereload script into the response (a binary image file).
Looks like the file has been uploaded through an HTML form, in that case you need to decode the multipart/form-data encoded data, re-assemble the parts if needed and save the file to GridFS. For parsing, you can use something like node-multiparty.
Related
Currently, I am using multer library to save files on File system. This application is using Node and Express.
I can save the file first on server and then encrypt it. Later on delete the unencrypted file. However, I do not want to add unencrypted file on server directly. So, I am looking for a way to encrypt the incoming file from the front end and then save it to disk.
const defaultFolder = 'upload';
const filePath = resolveHome(process.env.FILE_STORAGE_LOCATION || defaultFolder);
const key = 'test';
const cipher = crypto.createCipher('aes-256-cbc', key);
const decipher = crypto.createDecipher('aes-256-cbc', key);
const upload = multer({
storage: multer.diskStorage({
destination: filePath,
filename: (req, file, cb) => {
const parts = file.originalname.split('.');
const ext = parts.pop();
const name = parts.join('.');
cb(null, name + '-' + Date.now() + '.' + ext);
},
}),
}).single('file');
app.post('/upload', (req, res) => {
upload(req, res, err => {
if (err) {
return res.status(400).send({
error: 'The request was invalid',
fileName: req.file.originalname,
});
}
return res.status(200).send({
fileName: req.file.filename,
});
});
});
I tried to use crypto library to encrypt the file but it's not working. I believe the req.cipher is invalid as I would normally use req.file to get reference to the file.
app.post('/upload', (req, res) => {
upload(req, res, err => {
output = fs.createWriteStream(filePath + '/' + req.file.originalname);
req.pipe(cipher).pipe(output).on('finish', () => console.log('Encrypted file written on disk'));
if (err) {
return res.status(400).send({
error: 'The request was invalid',
fileName: req.file.originalname,
});
}
return res.status(200).send({
fileName: req.file.originalname,
});
});
});
I had tried to just write file without using cipher and file was empty. Adding this information in case it helps.
req.pipe(output).on('finish', () => console.log('Encrypted file written on disk'));
Can you try this
app.post('/upload', function(req, res) {
upload(req, res, function(err) {
var fileName = req.file.destination +"\\"+ req.file.filename
var input = fs.createReadStream(fileName);
var output = fs.createWriteStream(fileName + ".enc");
input.pipe(cipher).pipe(output);
output.on('finish', function() {
fs.unlink(fileName, (err) => {
if (err) throw err;
console.log('Encrypted file written to disk!');
res.end('Encrypted file written to disk!')
});
});
})
})
I created a node.js server that uses busboy to take requests, and pipe the files to Imgur for upload. However, I keep getting an "Uploading file too fast!" response from Imgur, and I'm not sure exactly what the problem is. Here is the code snippet involving busboy:
var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();
router.post('/u', function(req, res, next) {
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if(fieldname == 'image') {
var options = {
uri: 'https://api.imgur.com/3/image',
method: 'POST',
headers: {
'Authorization': 'Client-ID ' + clientID // put client id here
},
form: {
image: file,
type: 'file'
}
};
request(options)
.then(function(parsedBody) {
console.log(parsedBody);
})
.catch(function(err) {
console.log(err);
});
}
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('field');
});
busboy.on('finish', function() {
res.status(200).end();
});
req.pipe(busboy);
});
As you can see I'm piping the request file directly into my request for imgur. Providing a ReadStream by simply saving the file to disc and then using fs.createReadStream() works perfectly, so I'm not really sure why trying to pipe directly from request to request gives me the error. The exact response I'm getting from Imgur is:
StatusCodeError: 400 - {"data":{"error":"Uploading file too fast!","request":"\/3\/image","method":"POST"},"success":false,"status":400}
If anyone has encountered this before, it would be helpful...
The first issue is that you should be using formData instead of form for file uploads. Otherwise, the request library won't send the correct HTTP request.
The second issue is that the stream object won't have the correct content length until it's fully processed. We can buffer the data ourselves and pass it after the initial file stream from busboy has processed.*
This gives us something that looks like
var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();
router.post('/u', function(req, res, next) {
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if(fieldname == 'image') {
// the buffer
file.fileRead = [];
file.on('data', function(data) {
// add to the buffer as data comes in
this.fileRead.push(data);
});
file.on('end', function() {
// create a new stream with our buffered data
var finalBuffer = Buffer.concat(this.fileRead);
var options = {
uri: 'https://api.imgur.com/3/image',
method: 'POST',
headers: {
'Authorization': 'Client-ID ' + clientID // put client id here
},
formData: {
image: finalBuffer,
type: 'file'
}
};
request(options)
.then(function(parsedBody) {
console.log(parsedBody);
})
.catch(function(err) {
console.log(err);
});
});
}
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('field');
});
busboy.on('finish', function() {
res.status(200).end();
});
req.pipe(busboy);
});
Code for the buffering is from http://thau.me/2014/02/nodejs-streaming-files-to-amazons3/
Lastly, you may want to consider using the request library, as the request-promise library discourages the use of streams. See the github repo for more details: https://github.com/request/request-promise
I have been able to successfully upload images to mongoDB using GridFs.
Below are images from my database:
fs.files:
fs.chunks:
Below is the code I used to upload images:
var Grid = require('gridfs-stream');
var mongoose = require("mongoose");
Grid.mongo = mongoose.mongo;
var gfs = new Grid(mongoose.connection.db);
app.post('/picture', function(req, res) {
var part = req.files.filefield;
var writeStream = gfs.createWriteStream({
filename: part.name,
mode: 'w',
content_type:part.mimetype
});
writeStream.on('close', function() {
return res.status(200).send({
message: 'Success'
});
});
writeStream.write(part.name);
writeStream.end();
});
The Issue:
I can't seem to figure out how to read this image from mongoDB and display it on the front end in a HTML <img> tag.
I have tried this so far but it only displays the name of the file:
app.get('/picture', function(req, res) {
gfs.files.find({ filename: 'trooper.jpeg' }).toArray(function (err, files) {
if(files.length===0){
return res.status(400).send({
message: 'File not found'
});
}
res.writeHead(200, {'Content-Type': files[0].contentType});
var readstream = gfs.createReadStream({
filename: files[0].filename
});
readstream.on('data', function(chunk) {
res.write(chunk);
});
readstream.on('end', function() {
res.end();
});
readstream.on('error', function (err) {
console.log('An error occurred!', err);
throw err;
});
});
});
I have taken the code above from here
Can anyone please help. Ive been stuck on this for DAYS!!!!
It should really be
writeStream.write(part.data);
not
writeStream.write(part.name);
I am now using angular-file-upload packages to upload files. After I press item.upload(), it claims to be successfully uploaded the file, but I see the req.body is empty. Please Help!
Here is the angular code to handle it:
var uploader = $scope.uploader = $fileUploader.create({
scope: $scope, // to automatically update the html. Default: $rootScope
url: '/api/teams/upload',
formData: [
{ key: 'value' }
],
filters: [
function (item) { // first user filter
$scope.previewImage(item);
return true;
}
]
});
And here is the way to trigger the upload:
uploader.bind('afteraddingfile', function (event, item) {
// console.info(item.file);
console.info('After adding a file', item);
// console.log('item.upload();');
item.upload();
});
And finally here is the express js code:
exports.upload = function(req, res) {
// console.log('req.headers');
// console.log(req.headers);
console.log('req.body');
console.log(req.body);
What wrong's with it?
First make sure your POST is encoded as enctype="multipart/form-data"....
In Express 4 you need to set the body parser in your server:
var bodyParser = require('dy-parser');
//...
var app = express();
//...
app.use(bodyParser()); // pull information from html in POST
var busboy = require('connect-busboy');
app.use(busboy());
In earlier version of Express you only needed to add the body parser from the framework itself and files will be store on the configured location:
app.use(express.bodyParser({limit: '10mb', uploadDir: __dirname + '/public/uploads' })); // pull information from html in POST
Since version 4 removed support for connect now you need to add your custom support for multipart/form data to parser multi/part POSTs, so you will have to to do something like:
var fs = require('fs');
var busboy = require('connect-busboy');
//...
app.use(busboy());
//...
app.post('/api/teams/upload', function(req, res) {
var fstream;
req.pipe(req.busboy);
req.busboy.on('file', function (fieldname, file, filename) {
console.log("Uploading: " + filename);
fstream = fs.createWriteStream(__dirname + '/files/' + filename);
file.pipe(fstream);
fstream.on('close', function () {
res.redirect('back');
});
});
});
On the client side you need to call the $upload.upload To start the upload
I am trying to figure out how I can post an image directly to GridFS without storing it anywhere on the server as a temporary file first.
I am using Postman (chrome ext.) to post a file, and I manage to store this post as a file using:
req.pipe(fs.createWriteStream('./test.png'));
I am also able to store directly to GridFS from a readStream when the readStream is created from a file on the server. (see code)
I have the following files, saveFromReq.js which listens for the POST and basically just passes this on to the savePic.js.
saveFromReq.js:
var express = require('express');
var app = express();
var savePic = require('./savePic');
var fs = require('fs');
var GridStore = require('mongodb').GridStore;
var pic = './square.png';
var picID;
//When the following
//var pic = fs.createReadStream('./square.png', {autoClose: true});
//is not commented out, and 'req' is replaced with 'pic' in the savePic function,
//the file square.png is stored correctly to GridFS
app.post('/picture', function(req, res){
savePic(req, function(id){});
res.writeHead(200, {'Content-Type': 'text' });
res.end("Sucsess!\n");
});
app.listen(process.env.PORT || 3413);
savePic.js:
var savePic = function(req, callback){
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
BSON = require('mongodb').pure().BSON,
assert = require('assert');
fs = require('fs');
//When the following
//req.pipe(fs.createWriteStream('./test.png'));
//is not commented out, the correct image is stored to test.png, and
//the sequence after req.on("data"... starts
//(That sequence does not start at all when this is commented out..)
var fileId = new ObjectID();
var db = new Db('testDB', new Server('localhost', 27017));
// Establish connection to db
db.open(function(err, db) {
var gridStore = new GridStore(db, 'test', 'w');
//open
gridStore.open(function(err, gridStore) {
console.log("opened");
req.on("data", function (data) {
console.log("data recieved");
gridStore.write(data, function (err, gridStore) {
if (err) {
console.log("error writing file");
}
});
});
req.on("end", function () {
gridStore.close(function (err, gridStore) {
if (!err) {
console.log("The file has been stored to database.");
db.close();
}
});
});
req.pipe(gridStore);
});
});
callback(fileId);
};
module.exports = savePic;
Any help would be greatly appreciated!
gridfs-stream makes that pretty easy:
// `gfs` is a gridfs-stream instance
app.post('/picture', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}));
res.send("Success!");
});
while #robertklep's answer is correct, I would like to add something to his answer. This code shows how you can send back the stored file's metadata.
app.post('/picture', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}).on('close', function(savedFile){
console.log('file saved', savedFile);
return res.json({file: savedFile});
}));
})
This worked for me with mongoose:
var gfs = Grid(mongoose.connection.db, mongoose.mongo);
var writeStream = gfs.createWriteStream({
filename: name,
mode: 'w',
content_type: 'video/mp4'
});
writeStream.on('close', function() {
console.log('close event');
});
fs.createReadStream('uploads/' + name + '/' + name + '.mp4').pipe(writeStream);
console.log('stream.write: ' + name + '/' + name + '.mp4');
I am struggling a couple of days with getting the video on client side browser. That is what I tried so far:
var readstream = gfs.createReadStream({
filename: file.filename
});
readstream.on('data', function(data) {
res.write(data);
console.log(data);
});
readstream.on('end', function() {
res.end();
});
readstream.on('error', function (err) {
console.log('An error occurred!', err);
throw err;
});
My Data on MongoDB side looks like:
db.fs.chunks.find()
{ "_id" : ObjectId("5757e76df14741bf0391aaca"), "files_id" : ObjectId("5757e76df14741bf0391aac8"), "n" : 0, "data" : BinData(0,"AAAAIGZ0eXBpc29....
And the contentType is 'video/mp4':
logging on browser side prints this:
Object { 0: "�", 1: "�", 2: "�", 3: " ", 4: "f", 5: "t", 6: "y", 7: "p", 8: "i", 9: "s", 85003 more… }
Could someone please save my live? I hope you do not see my post as not convenient in this place.
Complete code to insert the txtfile in mongodb using gridfs in nodejs.This works well `
var mongoose=require("mongoose");
var gridfsstream=require("gridfs-stream");
var fs=require("fs");
mongoose.connect("mongodb://localhost:27017/testimage");
var conn=mongoose.connection;
gridfsstream.mongo=mongoose.mongo;
conn.once("open",function()
{
console.log("database connected successfully");
var gfs=gridfsstream(conn.db);
var writestream=gfs.createWriteStream({
filename:"danger.txt"
});
fs.createReadStream("sivakasi.txt").pipe(writestream);
writestream.on("close",function(file)
{
console.log(file.filename +"stored successfully into mongodb using gridfs");
});
writestream.on("error",function(file)
{
console.log(file.filename +"not stored into mongodb using gridfs");
});
});
conn.on("error",function()
{
console.log("database not connected try again!!!");
});
`
complete code to post the image from html to nodejs store that image in mongodb using gridfs system and display that image in server.This code works well.
var express=require("express");
var bodyparser=require("body-parser");
var multer=require("multer");
var app=express();
var upload = multer({ dest: '/tmp/'});
app.use(bodyparser.urlencoded({extended:false}));
app.post("/uploadimage",upload.single("file"),function(request,response)
{
var mongoose=require("mongoose");
var gridfsstream=require("gridfs-stream");
var fs=require("fs");
mongoose.connect("mongodb://localhost:27017/testimage");
var con=mongoose.connection;
gridfsstream.mongo=mongoose.mongo;
con.once("open",function()
{
console.log("test image database connected successfully");
var gfs=gridfsstream(con.db);
var readstream=fs.createReadStream(request.file.originalname);
var writestream=gfs.createWriteStream({
filename:"mentorpicthree.jpg"
});
readstream.pipe(writestream);
writestream.on("close",function()
{
console.log("image stored in mongodb database successfully");
fs.readFile(request.file.originalname,function(err,data)
{
if(err)
{
response.writeHead(404,{"Content-Type":"text/plain"});
console.log("error");
}
else
{
response.writeHead(200,{"Content-Type":"image/jpg"});
response.end(data);
}
});
});
writestream.on("error",function()
{
console.log("image not stored in mongodb database");
});
});
con.on("error",function()
{
console.log("database not connected try again!!!");
});
});
app.listen(8086,function()
{
console.log("server running on port 8086");
});
<html>
<head>
<title>FILE UPLOAD</title>
</head>
<body>
<p>Ryan Dhal</p>
<form action="http://127.0.0.1:8086/uploadimage" method="POST" enctype="multipart/form-data">
<input type="file" name="file">
<br>
<input type="submit" value="UPLOAD">
</form>
</body>
</html>