Related
I am trying to upload a file to Cloud Functions, using Express to handle requests there, but i am not succeeding. I created a version that works locally:
serverside js
const express = require('express');
const cors = require('cors');
const fileUpload = require('express-fileupload');
const app = express();
app.use(fileUpload());
app.use(cors());
app.post('/upload', (req, res) => {
res.send('files: ' + Object.keys(req.files).join(', '));
});
clientside js
const formData = new FormData();
Array.from(this.$refs.fileSelect.files).forEach((file, index) => {
formData.append('sample' + index, file, 'sample');
});
axios.post(
url,
formData,
{
headers: { 'Content-Type': 'multipart/form-data' },
}
);
This exact same code seems to break when deployed to Cloud Functions, where req.files is undefined. Does anyone have any idea what is happening here?
EDIT
I also had a go at using multer, which worked fine locally, but once uploaded to Cloud Functions, this got me an empty array (same clientside code):
const app = express();
const upload = multer();
app.use(cors());
app.post('/upload', upload.any(), (req, res) => {
res.send(JSON.stringify(req.files));
});
There was indeed a breaking change in the Cloud Functions setup that triggered this issue. It has to do with the way the middleware works that gets applied to all Express apps (including the default app) used to serve HTTPS functions. Basically, Cloud Functions will parse the body of the request and decide what to do with it, leaving the raw contents of the body in a Buffer in req.rawBody. You can use this to directly parse your multipart content, but you can't do it with middleware (like multer).
Instead, you can use a module called busboy to deal with the raw body content directly. It can accept the rawBody buffer and will call you back with the files it found. Here is some sample code that will iterate all the uploaded content, save them as files, then delete them. You'll obviously want to do something more useful.
const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');
exports.upload = functions.https.onRequest((req, res) => {
if (req.method === 'POST') {
const busboy = new Busboy({ headers: req.headers });
// This object will accumulate all the uploaded files, keyed by their name
const uploads = {}
// This callback will be invoked for each file uploaded
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`);
// Note that os.tmpdir() is an in-memory file system, so should only
// be used for files small enough to fit in memory.
const filepath = path.join(os.tmpdir(), fieldname);
uploads[fieldname] = { file: filepath }
console.log(`Saving '${fieldname}' to ${filepath}`);
file.pipe(fs.createWriteStream(filepath));
});
// This callback will be invoked after all uploaded files are saved.
busboy.on('finish', () => {
for (const name in uploads) {
const upload = uploads[name];
const file = upload.file;
res.write(`${file}\n`);
fs.unlinkSync(file);
}
res.end();
});
// The raw bytes of the upload will be in req.rawBody. Send it to busboy, and get
// a callback when it's finished.
busboy.end(req.rawBody);
} else {
// Client error - only support POST
res.status(405).end();
}
})
Bear in mind that files saved to temp space occupy memory, so their sizes should be limited to a total of 10MB. For larger files, you should upload those to Cloud Storage and process them with a storage trigger.
Also bear in mind that the default selection of middleware added by Cloud Functions is not currently added to the local emulator via firebase serve. So this sample will not work (rawBody won't be available) in that case.
The team is working on updating the documentation to be more clear about what all happens during HTTPS requests that's different than a standard Express app.
Thanks to the answers above I've built a npm module for this (github)
It works with google cloud functions, just install it (npm install --save express-multipart-file-parser) and use it like this:
const fileMiddleware = require('express-multipart-file-parser')
...
app.use(fileMiddleware)
...
app.post('/file', (req, res) => {
const {
fieldname,
filename,
encoding,
mimetype,
buffer,
} = req.files[0]
...
})
I was able to combine both Brian's and Doug's response. Here's my middleware that end's up mimicking the req.files in multer so no breaking changes to the rest of your code.
module.exports = (path, app) => {
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: true }))
app.use((req, res, next) => {
if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')){
getRawBody(req, {
length: req.headers['content-length'],
limit: '10mb',
encoding: contentType.parse(req).parameters.charset
}, function(err, string){
if (err) return next(err)
req.rawBody = string
next()
})
} else {
next()
}
})
app.use((req, res, next) => {
if (req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')) {
const busboy = new Busboy({ headers: req.headers })
let fileBuffer = new Buffer('')
req.files = {
file: []
}
busboy.on('field', (fieldname, value) => {
req.body[fieldname] = value
})
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
file.on('data', (data) => {
fileBuffer = Buffer.concat([fileBuffer, data])
})
file.on('end', () => {
const file_object = {
fieldname,
'originalname': filename,
encoding,
mimetype,
buffer: fileBuffer
}
req.files.file.push(file_object)
})
})
busboy.on('finish', () => {
next()
})
busboy.end(req.rawBody)
req.pipe(busboy)
} else {
next()
}
})}
I have been suffering from the same problem for a few days, turns out that firebase team has put the raw body of multipart/form-data into req.body with their middleware. If you try console.log(req.body.toString()) BEFORE processing your request with multer, you will see your data. As multer creates a new req.body object which is overriding the resulting req, the data is gone and all we can get is an empty req.body. Hopefully the firebase team could correct this soon.
To add to the official Cloud Function team answer, you can emulate this behavior locally by doing the following (add this middleware higher than the busboy code they posted, obviously)
const getRawBody = require('raw-body');
const contentType = require('content-type');
app.use(function(req, res, next){
if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'] !== undefined && req.headers['content-type'].startsWith('multipart/form-data')){
getRawBody(req, {
length: req.headers['content-length'],
limit: '10mb',
encoding: contentType.parse(req).parameters.charset
}, function(err, string){
if (err) return next(err);
req.rawBody = string;
next();
});
}
else{
next();
}
});
Cloud functions pre-processes the request object before passing it on further. As such the original multer middleware doesn't work. Furthermore, using busboy is too low level and you need to take care of everything on your own which isn't ideal. Instead you can use a forked version of multer middleware for processing multipart/form-data on cloud functions.
Here's what you can do.
Install the fork
npm install --save emadalam/multer#master
Use startProcessing configuration for custom handling of req.rawBody added by cloud functions.
const express = require('express')
const multer = require('multer')
const SIZE_LIMIT = 10 * 1024 * 1024 // 10MB
const app = express()
const multipartFormDataParser = multer({
storage: multer.memoryStorage(),
// increase size limit if needed
limits: {fieldSize: SIZE_LIMIT},
// support firebase cloud functions
// the multipart form-data request object is pre-processed by the cloud functions
// currently the `multer` library doesn't natively support this behaviour
// as such, a custom fork is maintained to enable this by adding `startProcessing`
// https://github.com/emadalam/multer
startProcessing(req, busboy) {
req.rawBody ? busboy.end(req.rawBody) : req.pipe(busboy)
},
})
app.post('/some_route', multipartFormDataParser.any(), function (req, res, next) {
// req.files is array of uploaded files
// req.body will contain the text fields
})
I ran into this issue today, check here for more details on how to handle files on google cloud (basically you don't need multer).
Here is a middleware I use to extract files. This will keep all your files on request.files and other form fields on request.body for all POST with multipart/form-data content type. It will leave everything else the same for your other middlewares to handle.
// multiparts.js
const { createWriteStream } = require('fs')
const { tmpdir } = require('os')
const { join } = require('path')
const BusBoy = require('busboy')
exports.extractFiles = async(req, res, next) => {
const multipart = req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')
if (!multipart) return next()
//
const busboy = new BusBoy({ headers: req.headers })
const incomingFields = {}
const incomingFiles = {}
const writes = []
// Process fields
busboy.on('field', (name, value) => {
try {
// This will keep a field created like so form.append('product', JSON.stringify(product)) intact
incomingFields[name] = JSON.parse(value)
} catch (e) {
// Numbers will still be strings here (i.e 1 will be '1')
incomingFields[name] = value
}
})
// Process files
busboy.on('file', (field, file, filename, encoding, contentType) => {
// Doing this to not have to deal with duplicate file names
// (i.e. TIMESTAMP-originalName. Hmm what are the odds that I'll still have dups?)
const path = join(tmpdir(), `${(new Date()).toISOString()}-${filename}`)
// NOTE: Multiple files could have same fieldname (which is y I'm using arrays here)
incomingFiles[field] = incomingFiles[field] || []
incomingFiles[field].push({ path, encoding, contentType })
//
const writeStream = createWriteStream(path)
//
writes.push(new Promise((resolve, reject) => {
file.on('end', () => { writeStream.end() })
writeStream.on('finish', resolve)
writeStream.on('error', reject)
}))
//
file.pipe(writeStream)
})
//
busboy.on('finish', async () => {
await Promise.all(writes)
req.files = incomingFiles
req.body = incomingFields
next()
})
busboy.end(req.rawBody)
}
And now in your function, make sure that this is the first middleware you use.
// index.js
const { onRequest } = require('firebase-functions').https
const bodyParser = require('body-parser')
const express = require('express')
const cors = require('cors')
const app = express()
// First middleware I'm adding
const { extractFiles } = require('./multiparts')
app.use(extractFiles)
app.use(bodyParser.urlencoded({ extended: true }))
app.use(bodyParser.json())
app.use(cors({ origin: true }))
app.use((req) => console.log(req.originalUrl))
exports.MyFunction = onRequest(app);
I fixed some bugs G. Rodriguez's response. I add 'field' and 'finish' event for Busboy, and do next() in 'finish' event. This is work for me. As follow:
module.exports = (path, app) => {
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: true }))
app.use((req, res, next) => {
if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')){
getRawBody(req, {
length: req.headers['content-length'],
limit: '10mb',
encoding: contentType.parse(req).parameters.charset
}, function(err, string){
if (err) return next(err)
req.rawBody = string
next()
})
} else {
next()
}
})
app.use((req, res, next) => {
if (req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')) {
const busboy = new Busboy({ headers: req.headers })
let fileBuffer = new Buffer('')
req.files = {
file: []
}
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
file.on('data', (data) => {
fileBuffer = Buffer.concat([fileBuffer, data])
})
file.on('end', () => {
const file_object = {
fieldname,
'originalname': filename,
encoding,
mimetype,
buffer: fileBuffer
}
req.files.file.push(file_object)
})
})
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
});
busboy.on('finish', function() {
next()
});
busboy.end(req.rawBody)
req.pipe(busboy);
} else {
next()
}
})}
Thanks for everyone's help on this thread. I wasted a whole day trying every possible combination and all these different libraries... only to discover this after exhausting all other options.
Combined some of the above solutions to create a TypeScript and middleware capable script here:
https://gist.github.com/jasonbyrne/8dcd15701f686a4703a72f13e3f800c0
If you just want to get a single uploaded file from the request, use busboy to get the file as a readable stream:
const express = require('express')
const Busboy = require('busboy')
express().post('/', (req, res) => {
const busboy = new Busboy({ headers: req.headers })
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
// Do something with `file`, e.g. pipe it to an output stream.
// file.pipe(fs.createWriteStream('upload.pdf')
})
// The original input was moved to `req.rawBody`
busboy.write(req.rawBody)
})
Note that, on top of using Busboy on the server and parsing the rawReq, you may also need to add the following config to your Axios request:
{ headers: { 'content-type': `multipart/form-data; boundary=${formData._boundary}` }};
If you only specify the content-type and not the boundary you get a Boundary not found error on the server. If you remove the headers altogether, instead, Busboy won't parse the fields properly.
See: Firebase Cloud Functions and Busboy not parsing fields or files
I experience the same issue when i deployed my app using firebase function. I was using multer to upload image to amazon s3. I resolve this issue by using the above npm https://stackoverflow.com/a/48648805/5213790 created by Cristóvão.
const { mimetype, buffer, } = req.files[0]
let s3bucket = new aws.S3({
accessKeyId: functions.config().aws.access_key,
secretAccessKey: functions.config().aws.secret_key,
});
const config = {
Bucket: functions.config().aws.bucket_name,
ContentType: mimetype,
ACL: 'public-read',
Key: Date.now().toString(),
Body: buffer,
}
s3bucket.upload(config, (err, data) => {
if(err) console.log(err)
req.file = data;
next()
})
Note that this is for a single file image upload.
The next middleware will have the returned object from s3
{
ETag: '"cacd6d406f891e216f9946911a69aac5"',
Location:'https://react-significant.s3.us-west1.amazonaws.com/posts/1567282665593',
key: 'posts/1567282665593',
Key: 'posts/1567282665593',
Bucket: 'react-significant'
}
In this case, you might need the Location url before you save your data in the db.
I've tried Dougs answer, however the finish was never fired, so i tweaked the code a little bit and got this which works for me:
// It's very crucial that the file name matches the name attribute in your html
app.post('/', (req, res) => {
const busboy = new Busboy({ headers: req.headers })
// This object will accumulate all the uploaded files, keyed by their name
const uploads = {}
// This callback will be invoked for each file uploaded
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`)
// Note that os.tmpdir() is an in-memory file system, so should only
// be used for files small enough to fit in memory.
const filepath = path.join(os.tmpdir(), filename)
uploads[fieldname] = { file: filepath }
console.log(`Saving '${fieldname}' to ${filepath}`)
const stream = fs.createWriteStream(filepath)
stream.on('open', () => file.pipe(stream))
})
// This callback will be invoked after all uploaded files are saved.
busboy.on('finish', () => {
console.log('look im firing!')
// Do whatever you want here
res.end()
})
// The raw bytes of the upload will be in req.rawBody. Send it to busboy, and get
// a callback when it's finished.
busboy.end(req.rawBody)
})
I am trying to upload a file to Cloud Functions, using Express to handle requests there, but i am not succeeding. I created a version that works locally:
serverside js
const express = require('express');
const cors = require('cors');
const fileUpload = require('express-fileupload');
const app = express();
app.use(fileUpload());
app.use(cors());
app.post('/upload', (req, res) => {
res.send('files: ' + Object.keys(req.files).join(', '));
});
clientside js
const formData = new FormData();
Array.from(this.$refs.fileSelect.files).forEach((file, index) => {
formData.append('sample' + index, file, 'sample');
});
axios.post(
url,
formData,
{
headers: { 'Content-Type': 'multipart/form-data' },
}
);
This exact same code seems to break when deployed to Cloud Functions, where req.files is undefined. Does anyone have any idea what is happening here?
EDIT
I also had a go at using multer, which worked fine locally, but once uploaded to Cloud Functions, this got me an empty array (same clientside code):
const app = express();
const upload = multer();
app.use(cors());
app.post('/upload', upload.any(), (req, res) => {
res.send(JSON.stringify(req.files));
});
There was indeed a breaking change in the Cloud Functions setup that triggered this issue. It has to do with the way the middleware works that gets applied to all Express apps (including the default app) used to serve HTTPS functions. Basically, Cloud Functions will parse the body of the request and decide what to do with it, leaving the raw contents of the body in a Buffer in req.rawBody. You can use this to directly parse your multipart content, but you can't do it with middleware (like multer).
Instead, you can use a module called busboy to deal with the raw body content directly. It can accept the rawBody buffer and will call you back with the files it found. Here is some sample code that will iterate all the uploaded content, save them as files, then delete them. You'll obviously want to do something more useful.
const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');
exports.upload = functions.https.onRequest((req, res) => {
if (req.method === 'POST') {
const busboy = new Busboy({ headers: req.headers });
// This object will accumulate all the uploaded files, keyed by their name
const uploads = {}
// This callback will be invoked for each file uploaded
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`);
// Note that os.tmpdir() is an in-memory file system, so should only
// be used for files small enough to fit in memory.
const filepath = path.join(os.tmpdir(), fieldname);
uploads[fieldname] = { file: filepath }
console.log(`Saving '${fieldname}' to ${filepath}`);
file.pipe(fs.createWriteStream(filepath));
});
// This callback will be invoked after all uploaded files are saved.
busboy.on('finish', () => {
for (const name in uploads) {
const upload = uploads[name];
const file = upload.file;
res.write(`${file}\n`);
fs.unlinkSync(file);
}
res.end();
});
// The raw bytes of the upload will be in req.rawBody. Send it to busboy, and get
// a callback when it's finished.
busboy.end(req.rawBody);
} else {
// Client error - only support POST
res.status(405).end();
}
})
Bear in mind that files saved to temp space occupy memory, so their sizes should be limited to a total of 10MB. For larger files, you should upload those to Cloud Storage and process them with a storage trigger.
Also bear in mind that the default selection of middleware added by Cloud Functions is not currently added to the local emulator via firebase serve. So this sample will not work (rawBody won't be available) in that case.
The team is working on updating the documentation to be more clear about what all happens during HTTPS requests that's different than a standard Express app.
Thanks to the answers above I've built a npm module for this (github)
It works with google cloud functions, just install it (npm install --save express-multipart-file-parser) and use it like this:
const fileMiddleware = require('express-multipart-file-parser')
...
app.use(fileMiddleware)
...
app.post('/file', (req, res) => {
const {
fieldname,
filename,
encoding,
mimetype,
buffer,
} = req.files[0]
...
})
I was able to combine both Brian's and Doug's response. Here's my middleware that end's up mimicking the req.files in multer so no breaking changes to the rest of your code.
module.exports = (path, app) => {
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: true }))
app.use((req, res, next) => {
if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')){
getRawBody(req, {
length: req.headers['content-length'],
limit: '10mb',
encoding: contentType.parse(req).parameters.charset
}, function(err, string){
if (err) return next(err)
req.rawBody = string
next()
})
} else {
next()
}
})
app.use((req, res, next) => {
if (req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')) {
const busboy = new Busboy({ headers: req.headers })
let fileBuffer = new Buffer('')
req.files = {
file: []
}
busboy.on('field', (fieldname, value) => {
req.body[fieldname] = value
})
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
file.on('data', (data) => {
fileBuffer = Buffer.concat([fileBuffer, data])
})
file.on('end', () => {
const file_object = {
fieldname,
'originalname': filename,
encoding,
mimetype,
buffer: fileBuffer
}
req.files.file.push(file_object)
})
})
busboy.on('finish', () => {
next()
})
busboy.end(req.rawBody)
req.pipe(busboy)
} else {
next()
}
})}
I have been suffering from the same problem for a few days, turns out that firebase team has put the raw body of multipart/form-data into req.body with their middleware. If you try console.log(req.body.toString()) BEFORE processing your request with multer, you will see your data. As multer creates a new req.body object which is overriding the resulting req, the data is gone and all we can get is an empty req.body. Hopefully the firebase team could correct this soon.
To add to the official Cloud Function team answer, you can emulate this behavior locally by doing the following (add this middleware higher than the busboy code they posted, obviously)
const getRawBody = require('raw-body');
const contentType = require('content-type');
app.use(function(req, res, next){
if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'] !== undefined && req.headers['content-type'].startsWith('multipart/form-data')){
getRawBody(req, {
length: req.headers['content-length'],
limit: '10mb',
encoding: contentType.parse(req).parameters.charset
}, function(err, string){
if (err) return next(err);
req.rawBody = string;
next();
});
}
else{
next();
}
});
Cloud functions pre-processes the request object before passing it on further. As such the original multer middleware doesn't work. Furthermore, using busboy is too low level and you need to take care of everything on your own which isn't ideal. Instead you can use a forked version of multer middleware for processing multipart/form-data on cloud functions.
Here's what you can do.
Install the fork
npm install --save emadalam/multer#master
Use startProcessing configuration for custom handling of req.rawBody added by cloud functions.
const express = require('express')
const multer = require('multer')
const SIZE_LIMIT = 10 * 1024 * 1024 // 10MB
const app = express()
const multipartFormDataParser = multer({
storage: multer.memoryStorage(),
// increase size limit if needed
limits: {fieldSize: SIZE_LIMIT},
// support firebase cloud functions
// the multipart form-data request object is pre-processed by the cloud functions
// currently the `multer` library doesn't natively support this behaviour
// as such, a custom fork is maintained to enable this by adding `startProcessing`
// https://github.com/emadalam/multer
startProcessing(req, busboy) {
req.rawBody ? busboy.end(req.rawBody) : req.pipe(busboy)
},
})
app.post('/some_route', multipartFormDataParser.any(), function (req, res, next) {
// req.files is array of uploaded files
// req.body will contain the text fields
})
I ran into this issue today, check here for more details on how to handle files on google cloud (basically you don't need multer).
Here is a middleware I use to extract files. This will keep all your files on request.files and other form fields on request.body for all POST with multipart/form-data content type. It will leave everything else the same for your other middlewares to handle.
// multiparts.js
const { createWriteStream } = require('fs')
const { tmpdir } = require('os')
const { join } = require('path')
const BusBoy = require('busboy')
exports.extractFiles = async(req, res, next) => {
const multipart = req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')
if (!multipart) return next()
//
const busboy = new BusBoy({ headers: req.headers })
const incomingFields = {}
const incomingFiles = {}
const writes = []
// Process fields
busboy.on('field', (name, value) => {
try {
// This will keep a field created like so form.append('product', JSON.stringify(product)) intact
incomingFields[name] = JSON.parse(value)
} catch (e) {
// Numbers will still be strings here (i.e 1 will be '1')
incomingFields[name] = value
}
})
// Process files
busboy.on('file', (field, file, filename, encoding, contentType) => {
// Doing this to not have to deal with duplicate file names
// (i.e. TIMESTAMP-originalName. Hmm what are the odds that I'll still have dups?)
const path = join(tmpdir(), `${(new Date()).toISOString()}-${filename}`)
// NOTE: Multiple files could have same fieldname (which is y I'm using arrays here)
incomingFiles[field] = incomingFiles[field] || []
incomingFiles[field].push({ path, encoding, contentType })
//
const writeStream = createWriteStream(path)
//
writes.push(new Promise((resolve, reject) => {
file.on('end', () => { writeStream.end() })
writeStream.on('finish', resolve)
writeStream.on('error', reject)
}))
//
file.pipe(writeStream)
})
//
busboy.on('finish', async () => {
await Promise.all(writes)
req.files = incomingFiles
req.body = incomingFields
next()
})
busboy.end(req.rawBody)
}
And now in your function, make sure that this is the first middleware you use.
// index.js
const { onRequest } = require('firebase-functions').https
const bodyParser = require('body-parser')
const express = require('express')
const cors = require('cors')
const app = express()
// First middleware I'm adding
const { extractFiles } = require('./multiparts')
app.use(extractFiles)
app.use(bodyParser.urlencoded({ extended: true }))
app.use(bodyParser.json())
app.use(cors({ origin: true }))
app.use((req) => console.log(req.originalUrl))
exports.MyFunction = onRequest(app);
I fixed some bugs G. Rodriguez's response. I add 'field' and 'finish' event for Busboy, and do next() in 'finish' event. This is work for me. As follow:
module.exports = (path, app) => {
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: true }))
app.use((req, res, next) => {
if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')){
getRawBody(req, {
length: req.headers['content-length'],
limit: '10mb',
encoding: contentType.parse(req).parameters.charset
}, function(err, string){
if (err) return next(err)
req.rawBody = string
next()
})
} else {
next()
}
})
app.use((req, res, next) => {
if (req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')) {
const busboy = new Busboy({ headers: req.headers })
let fileBuffer = new Buffer('')
req.files = {
file: []
}
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
file.on('data', (data) => {
fileBuffer = Buffer.concat([fileBuffer, data])
})
file.on('end', () => {
const file_object = {
fieldname,
'originalname': filename,
encoding,
mimetype,
buffer: fileBuffer
}
req.files.file.push(file_object)
})
})
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
});
busboy.on('finish', function() {
next()
});
busboy.end(req.rawBody)
req.pipe(busboy);
} else {
next()
}
})}
Thanks for everyone's help on this thread. I wasted a whole day trying every possible combination and all these different libraries... only to discover this after exhausting all other options.
Combined some of the above solutions to create a TypeScript and middleware capable script here:
https://gist.github.com/jasonbyrne/8dcd15701f686a4703a72f13e3f800c0
If you just want to get a single uploaded file from the request, use busboy to get the file as a readable stream:
const express = require('express')
const Busboy = require('busboy')
express().post('/', (req, res) => {
const busboy = new Busboy({ headers: req.headers })
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
// Do something with `file`, e.g. pipe it to an output stream.
// file.pipe(fs.createWriteStream('upload.pdf')
})
// The original input was moved to `req.rawBody`
busboy.write(req.rawBody)
})
Note that, on top of using Busboy on the server and parsing the rawReq, you may also need to add the following config to your Axios request:
{ headers: { 'content-type': `multipart/form-data; boundary=${formData._boundary}` }};
If you only specify the content-type and not the boundary you get a Boundary not found error on the server. If you remove the headers altogether, instead, Busboy won't parse the fields properly.
See: Firebase Cloud Functions and Busboy not parsing fields or files
I experience the same issue when i deployed my app using firebase function. I was using multer to upload image to amazon s3. I resolve this issue by using the above npm https://stackoverflow.com/a/48648805/5213790 created by Cristóvão.
const { mimetype, buffer, } = req.files[0]
let s3bucket = new aws.S3({
accessKeyId: functions.config().aws.access_key,
secretAccessKey: functions.config().aws.secret_key,
});
const config = {
Bucket: functions.config().aws.bucket_name,
ContentType: mimetype,
ACL: 'public-read',
Key: Date.now().toString(),
Body: buffer,
}
s3bucket.upload(config, (err, data) => {
if(err) console.log(err)
req.file = data;
next()
})
Note that this is for a single file image upload.
The next middleware will have the returned object from s3
{
ETag: '"cacd6d406f891e216f9946911a69aac5"',
Location:'https://react-significant.s3.us-west1.amazonaws.com/posts/1567282665593',
key: 'posts/1567282665593',
Key: 'posts/1567282665593',
Bucket: 'react-significant'
}
In this case, you might need the Location url before you save your data in the db.
I've tried Dougs answer, however the finish was never fired, so i tweaked the code a little bit and got this which works for me:
// It's very crucial that the file name matches the name attribute in your html
app.post('/', (req, res) => {
const busboy = new Busboy({ headers: req.headers })
// This object will accumulate all the uploaded files, keyed by their name
const uploads = {}
// This callback will be invoked for each file uploaded
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`)
// Note that os.tmpdir() is an in-memory file system, so should only
// be used for files small enough to fit in memory.
const filepath = path.join(os.tmpdir(), filename)
uploads[fieldname] = { file: filepath }
console.log(`Saving '${fieldname}' to ${filepath}`)
const stream = fs.createWriteStream(filepath)
stream.on('open', () => file.pipe(stream))
})
// This callback will be invoked after all uploaded files are saved.
busboy.on('finish', () => {
console.log('look im firing!')
// Do whatever you want here
res.end()
})
// The raw bytes of the upload will be in req.rawBody. Send it to busboy, and get
// a callback when it's finished.
busboy.end(req.rawBody)
})
I have an issue to display a simple image on a web page. I have a node backend for the APIs and a node web server for the view pages.
The code was working very well but I need to add a sort of gateway in-between (that will handle authentication in the future) and it broke the code.
I am using express and gridfs to store and retrive from mongo the files.
Here is the code
HTML/Angular page
<img id="image" ng-src="http:localhost:3000/api/files/images/{{data.image}}" alt="" />
Gateway (node)
var request = require('request');
//settings - retrive the url of the api backend micro-service
var url = require('./configApiGw').url_service_api_contents;
//api to retrive a file stored in mongo using gridfs
app.get('/api/files/images/:id', function(req, res, next){
var uri = url+'/api/files/images/:'+req.params.id;
request({
uri: uri,
method: "GET",
timeout: 100000,
followRedirect: true,
maxRedirects: 10
}, function(error, response, body) {
res.send(response.body);
});
});
Backend API
app.get('/api/files/images/:id', function(req, res, next){
//call function to read the file using gridfs. call back function
readFile(req, res, function(file){
console.log('success');
});
});
function readFile(req,res,callback){
var fileId = req.params.id;
//delete the ':' that is added by the gateway
if(fileId.charAt(0) === ':'){
fileId = fileId.slice(1);
}
// streaming from gridfs
var rstream = gfs.createReadStream({filename: fileId});
var bufs = [];
rstream.on('data', function (chunk) {
bufs.push(chunk);
});
// done reading the file
rstream.on('end', function () {
var fbuf = Buffer.concat(bufs);
var file = (fbuf.toString('base64'));
callback(file);
});
//error handling, e.g. file does not exist
rstream.on('error', function (err) {
console.log('An error occurred!', err);
console.log(err);
res.send(500,err);
});
rstream.pipe(res);
}
The image does not display but I get a 200 OK response from both API backend and gateway.
When I look at the image details on the browser, I see the following data:
- Location: http://localhost:3000/api/files/images/file.jpeg
- Type: text/html
- Size: Unknown (not cached)
What am I doing wrong? thanks a lot.
Edit with Alexandr inputs
Gateway (Node) V2
var request = require('request');
//settings - retrive the url of the api backend micro-service
var url = require('./configApiGw').url_service_api_contents;
app.get('/api/files/images/:id', function(req, res, next){
var uri = url+'/api/files/images/:'+req.params.id;
request({
uri: uri,
method: "GET",
timeout: 100000,
followRedirect: true,
maxRedirects: 10
}, function(error, response, body) {
res.set('Content-Type', response.headers['content-type']);
res.send(response.body);
});
});
Backend API V2
//api to retrive a file stored in mongo using gridfs
app.get('/api/files/images/:id', function(req, res, next){
//call function to read the file using gridfs. call back function
db.readFile(req, res, function(file){
//res.send(file);
console.log("success");
});
});
readFile = function(req,res,callback){
var fileId = req.params.id;
//delete the ':' that is added by the gateway
if(fileId.charAt(0) === ':'){
fileId = fileId.slice(1);
}
//setHeaders content type for the file
setHeaders(fileId, function(contentType){
res.writeHead('200',{'Content-Type':contentType});
// streaming from gridfs
var rstream = gfs.createReadStream({filename: fileId});
var bufs = [];
rstream.on('data', function (chunk) {
bufs.push(chunk);
});
// done reading the file
rstream.on('end', function () {
var fbuf = Buffer.concat(bufs);
var file = (fbuf.toString('binary'));
callback(file);
});
//error handling, e.g. file does not exist
rstream.on('error', function (err) {
console.log('An error occurred!', err);
console.log(err);
res.send(500,err);
});
rstream.pipe(res);
});
};
function setHeaders(fileId, callback){
var ext = path.extname(fileId);
var contentType = 'text/html';
if (ext === '.gif') {
contentType = 'image/gif';
}
if (ext === '.jpeg') {
contentType = 'image/jepg';
}
if (ext === '.png') {
contentType = 'image/png';
}
if (ext === '.jpg') {
contentType = 'image/jpg';
}
callback(contentType);
}
The result is still not good: the image is not displayed. BUT now, the content-type is correctly set.
adding here the headers (postman):
Access-Control-Allow-Headers → Origin, X-Requested-With, Content-Type, Accept
Access-Control-Allow-Origin → *
Connection → keep-alive
Content-Length → 82360
Content-Type → image/jepg; charset=utf-8
Date → Fri, 20 Nov 2015 10:15:55 GMT
ETag → W/"141b8-Ox5qDdvc3kZTunf0uqMVQg"
X-Powered-By → Express
UPDATE
Try to set encoding property to null in the request object:
app.get('/api/files/images/:id', function(req, res, next){
var uri = url+'/api/files/images/:'+req.params.id;
request({
uri: uri,
method: "GET",
timeout: 100000,
followRedirect: true,
encoding: null,
maxRedirects: 10
}, function(error, response, body) {
res.set('Content-Type', response.headers['content-type']);
res.send(response.body);
});
});
Also, set image content-type headers to your responses:
app.get('/api/files/images/:id', function(req, res, next){
//call function to read the file using gridfs. call back function
readFile(req, res, function(file){
res.set('Content-Type', 'image/jpeg'); //it can different, depends on the image
res.send(file);
});
});
Gateway:
app.get('/api/files/images/:id', function(req, res, next){
var uri = url+'/api/files/images/:'+req.params.id;
request({
uri: uri,
method: "GET",
timeout: 100000,
followRedirect: true,
maxRedirects: 10
}, function(error, response, body) {
res.set('Content-Type', response.headers['content-type']);
res.send(response.body);
});
});
Yes, I am already using a controller for all the other components of the page but I call directly from the html page the API to retrive the image. You're right, I could change that.
The gateway is here also because I have multiple API components (micro-service architecture) and so the gateway is a great way to abstract all these different components for the web client.
I would prefer to keep the gateway to follow the microservices architecture pattern.
I am trying to setup a file API in my node.js application. My goal is to be able to write the file stream directly to gridfs, without needing to store the file to disk initially. It seems like my create code is working. I am able to save a file upload to gridfs. The problem is reading the file. When I try to download a saved file via a web browser window, I see that the file contents are wrapped with something like the following:
------WebKitFormBoundarye38W9pfG1wiA100l
Content-Disposition: form-data; name="file"; filename="myfile.txt"
Content-Type: text/javascript
***File contents here***
------WebKitFormBoundarye38W9pfG1wiA100l--
So my question is what do I need to do to strip the boundary information from the file stream before saving it to gridfs? Here's the code i'm working with:
'use strict';
var mongoose = require('mongoose');
var _ = require('lodash');
var Grid = require('gridfs-stream');
Grid.mongo = mongoose.mongo;
var gfs = new Grid(mongoose.connection.db);
// I think this works. I see the file record in fs.files
exports.create = function(req, res) {
var fileId = new mongoose.Types.ObjectId();
var writeStream = gfs.createWriteStream({
_id: fileId,
filename: req.query.name,
mode: 'w',
content_type: req.query.type,
metadata: {
uploadedBy: req.user._id,
}
});
writeStream.on('finish', function() {
return res.status(200).send({
message: fileId.toString()
});
});
req.pipe(writeStream);
};
// File data is returned, but it's wrapped with
// WebKitFormBoundary and has headers.
exports.read = function(req, res) {
gfs.findOne({ _id: req.params.id }, function (err, file) {
if (err) return res.status(400).send(err);
// With this commented out, my browser will prompt
// me to download the raw file where I can see the
// webkit boundary and request headers
//res.writeHead(200, { 'Content-Type': file.contentType });
var readstream = gfs.createReadStream({
_id: req.params.id
// I also tried this way:
//_id: file._id
});
readstream.pipe(res);
});
};
By the way, i'm not currently using any middleware for these routes, but am open to doing so. I just didn't want the file to hit the disk prior to being sent to gridfs.
Edit:
Per #fardjad, I added the node-multiparty module for multipart/form-data parsing and it kind of worked. But when I download an uploaded file and compare with an original (as text), there are lots of differences in the encoding, and the downloaded file won't open. Here's my latest attempt.
'use strict';
var mongoose = require('mongoose');
var _ = require('lodash');
var multiparty = require('multiparty');
var Grid = require('gridfs-stream');
Grid.mongo = mongoose.mongo;
var gfs = new Grid(mongoose.connection.db);
exports.create = function(req, res) {
var form = new multiparty.Form();
var fileId = new mongoose.Types.ObjectId();
form.on('error', function(err) {
console.log('Error parsing form: ' + err.stack);
});
form.on('part', function(part) {
if (part.filename) {
var writeStream = gfs.createWriteStream({
_id: fileId,
filename: part.filename,
mode: 'w',
content_type: part.headers['content-type'],
metadata: {
uploadedBy: req.user._id,
}
})
part.pipe(writeStream);
}
});
// Close emitted after form parsed
form.on('close', function() {
return res.status(200).send({
message: fileId.toString()
});
});
// Parse req
form.parse(req);
};
exports.read = function(req, res) {
gfs.findOne({ _id: req.params.id }, function (err, file) {
if (err) return res.status(400).send(err);
res.writeHead(200, { 'Content-Type': file.contentType });
var readstream = gfs.createReadStream({
_id: req.params.id
});
readstream.pipe(res);
});
};
Final Edit:
Here's a simple implementation that I copied from another developer and modified. This is working for me: (I'm still trying to figure out why it won't work in my original express app. Something seems to be interfering)
https://gist.github.com/pos1tron/094ac862c9d116096572
var Busboy = require('busboy'); // 0.2.9
var express = require('express'); // 4.12.3
var mongo = require('mongodb'); // 2.0.31
var Grid = require('gridfs-stream'); // 1.1.1"
var app = express();
var server = app.listen(9002);
var db = new mongo.Db('test', new mongo.Server('127.0.0.1', 27017));
var gfs;
db.open(function(err, db) {
if (err) throw err;
gfs = Grid(db, mongo);
});
app.post('/file', function(req, res) {
var busboy = new Busboy({ headers : req.headers });
var fileId = new mongo.ObjectId();
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('got file', filename, mimetype, encoding);
var writeStream = gfs.createWriteStream({
_id: fileId,
filename: filename,
mode: 'w',
content_type: mimetype,
});
file.pipe(writeStream);
}).on('finish', function() {
// show a link to the uploaded file
res.writeHead(200, {'content-type': 'text/html'});
res.end('download file');
});
req.pipe(busboy);
});
app.get('/', function(req, res) {
// show a file upload form
res.writeHead(200, {'content-type': 'text/html'});
res.end(
'<form action="/file" enctype="multipart/form-data" method="post">'+
'<input type="file" name="file"><br>'+
'<input type="submit" value="Upload">'+
'</form>'
);
});
app.get('/file/:id', function(req, res) {
gfs.findOne({ _id: req.params.id }, function (err, file) {
if (err) return res.status(400).send(err);
if (!file) return res.status(404).send('');
res.set('Content-Type', file.contentType);
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
var readstream = gfs.createReadStream({
_id: file._id
});
readstream.on("error", function(err) {
console.log("Got error while processing stream " + err.message);
res.end();
});
readstream.pipe(res);
});
});
See my comment on the issue you created on github. I had the same problem but I managed to debug the issue. I narrowed it down to where i was confident that the problem was a piece of express middleware modified the request. I disabled my middleware one by one until i found the unlikely culprit: connect-livereload
I commented out app.use(require('connect-livereload')()); and the problem went away.
I believe it was injecting the livereload script into the response (a binary image file).
Looks like the file has been uploaded through an HTML form, in that case you need to decode the multipart/form-data encoded data, re-assemble the parts if needed and save the file to GridFS. For parsing, you can use something like node-multiparty.
I am trying to download a File from google drive using Google SDK API using node.js.
But I am unable to write/save file at server side - node.js
Code:-
var GoogleTokenProvider = require("refresh-token").GoogleTokenProvider,
async = require('async'),
fs = require("fs"),
request = require('request'),
_accessToken;
var _XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var https = require('https');
const CLIENT_ID = "";
const CLIENT_SECRET = "";
const REFRESH_TOKEN = '';
const ENDPOINT_OF_GDRIVE = 'https://www.googleapis.com/drive/v2';
async.waterfall([
//-----------------------------
// Obtain a new access token
//-----------------------------
function(callback) {
var tokenProvider = new GoogleTokenProvider({
'refresh_token': REFRESH_TOKEN,
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET
});
tokenProvider.getToken(callback);
},
//--------------------------------------------
// Retrieve the children in a specified folder
//
// ref: https://developers.google.com/drive/v2/reference/files/children/list
//-------------------------------------------
function(accessToken, callback) {
_accessToken = accessToken;
request.get({
'url': ENDPOINT_OF_GDRIVE + '/files?' + "q='root' in parents and (mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')",
'qs': {
'access_token': accessToken
}
}, callback);
},
//----------------------------
// Parse the response
//----------------------------
function(response, body, callback) {
var list = JSON.parse(body);
if (list.error) {
return callback(list.error);
}
callback(null, list.items[0]);
},
//-------------------------------------------
// Get the file information of the children.
//
// ref: https://developers.google.com/drive/v2/reference/files/get
//-------------------------------------------
function(children, callback) {
var xhr = new _XMLHttpRequest();
xhr.open('GET', children.downloadUrl);
xhr.setRequestHeader('Authorization', 'Bearer ' + _accessToken);
xhr.onload = function() {
console.log("xhr.responseText", xhr.responseText)
fs.writeFile("download.docx", xhr.responseText)
callback(xhr.responseText);
};
xhr.onerror = function() {
callback(null);
};
xhr.send();
}
],
function(err, results) {
if (!err) {
console.log(results);
}
});
I am getting this in console:-
Content of xhr.responseText is something like that
��▬h��↕E6M��~��3�3∟�9�� � �►��/2�:���♂�4��]�♀I�R���►
$SB6Q���c↔��H�=;+
���►q�3Tdכ��#!T��hEl_�|�I�↨��h(�^:▬�[h̓D♠��f���♠*���ݾ��M→
�1⌂♦"N�↑�o�]�7U$��A6����♠�W��k`�f▬♫��K�Z�^‼�0{<Z�▼�]F�����
���J♥A♀��♣�a�}7�
"���H�w"�♥���☺w♫̤ھ�� �P�^����O֛���;�<♠�aYՠ؛`G�kxm��PY�[��g
Gΰino�/<���<�1��ⳆA$>"f3��\�ȾT��∟I S�������W♥����Y
Please help me to know what is the format of the data I am getting from Drive Api and write it in which format so that I got a complete .docx file
Edit
I am open to use any method other than xmlRequest if it helps me downloading the file(.docx).
node-XMLHttpRequest, it seems, does not support binary downloads - see this issue. What you are seeing is the file's binary contents converted into String which, in JavaScript, is an irreversible and destructive process for binary data (which means you cannot convert the string back to buffer and get the same data as the original contents).
Using request, you can download a binary file this way:
var request = require('request')
, fs = require('fs')
request.get(
{ url: 'your-file-url'
, encoding: null // Force Request to return the data as Buffer
, headers:
{ Authorization: 'Bearer ' + accessTokenHere
}
}
, function done (err, res) {
// If all is well, the file will be at res.body (buffer)
fs.writeFile('./myfile.docx', res.body, function (err) {
// Handle err somehow
// Do other work necessary to finish the request
})
}
)
Note: This will buffer the whole file into memory before it can be saved to disk. For small files, this is fine, but for larger files, you might want to look into implementing this as a streamed download. This SO question already answers that, I recommend you have a look.
More information about how to authorize your requests can be found on Google Developers docs.
Complete Working example: Downloading file from GoogleDrive - Node.js API
var GoogleTokenProvider = require("refresh-token").GoogleTokenProvider,
async = require('async'),
fs = require("fs"),
request = require('request'),
_accessToken;
const CLIENT_ID = "";
const CLIENT_SECRET = "";
const REFRESH_TOKEN = '';
const ENDPOINT_OF_GDRIVE = 'https://www.googleapis.com/drive/v2';
async.waterfall([
//-----------------------------
// Obtain a new access token
//-----------------------------
function(callback) {
var tokenProvider = new GoogleTokenProvider({
'refresh_token': REFRESH_TOKEN,
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET
});
tokenProvider.getToken(callback);
},
//--------------------------------------------
// Retrieve the children in a specified folder
//
// ref: https://developers.google.com/drive/v2/reference/files/children/list
//-------------------------------------------
function(accessToken, callback) {
_accessToken = accessToken;
request.get({
'url': ENDPOINT_OF_GDRIVE + '/files?' + "q='root' in parents and (mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')",
'qs': {
'access_token': accessToken
}
}, callback);
},
//----------------------------
// Parse the response
//----------------------------
function(response, body, callback) {
var list = JSON.parse(body);
if (list.error) {
return callback(list.error);
}
callback(null, list.items);
},
//-------------------------------------------
// Get the file information of the children.
//
// ref: https://developers.google.com/drive/v2/reference/files/get
//-------------------------------------------
function(children, callback) {
for(var i=0;i<children.length;i++) {
var file = fs.createWriteStream(children[i].title);
// Downnload and write file from google drive
(function(child) {
request.get(
{ url: child.downloadUrl
, encoding: null // Force Request to return the data as Buffer
, headers:
{ Authorization: 'Bearer ' + _accessToken
}
}
, function done (err, res) {
res.pipe(file)
// If all is well, the file will be at res.body (buffer)
fs.writeFile('./' + child.title, res.body, function (err) {
if(!err) {
console.log('done')
} else {
console.log(err)
}
// Handle err somehow
// Do other work necessary to finish the request
})
}
)
})(children[i])
}
}
],
function(err, results) {
if (!err) {
console.log(results);
}
});
I was just having issues with this, I've included an example of how I managed to get this working using the Google API Node.js library: https://gist.github.com/davestevens/6f376f220cc31b4a25cd