Develop Reference

How upload buffer files in array to express server?

I have array with buffer files. I have been trying upload buffers to express server using multer for whole day.
Problem is my files pass by req.body.image. But multer finds files from req.files. Multer can't found files and alerted error. How fix this problem?
Front End code:
let UserPhoto = ["data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAA"];
let formData = new FormData();
for (let i = 0; i < userPhoto.length; i++) {
formData.append("files", userPhoto[i]);
}
axios
.post(`v0/photo/${id}/User`, formData, {
headers: { "Content-Type": "multipart/form-data" },
})
.then((result) => {
console.log(result);
})
.catch((error) => {
console.log(error);
}
Back End code:
const fileFilter = (req, file, cb) => {
if (file.mimetype.substr(0, 6) === "image/") {
cb(null, true);
} else cb(new ErrorCatcher("Зөвхөн зураг upload хийнэ үү.", 400), false);
};
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, director);
},
filename: (req, file, cb) => {
cb(null, Date.now() + "." + file.mimetype.split("/")[1]);
},
});
const upload = multer({
storage: storage,
limits: { fileSize: process.env.IMAGE_MAX_SIZE },
fileFilter: fileFilter,
});
//authorize,
// permission("user", "operator", "manager", "admin"),
router
.route("/:objectId/:objectType")
.post(upload.array("files", 8), createImageConnect);
I was using formData. Result was same.

Multer is grabbing images but it is not saving my files in the correct directory

I have an express server. When the user goes to make a post request for the profiel. They submit a profile picture. I am trying to use multer to grab that photo/image that is being sent and store it in a folder on the backend directory.
I setup multer but for some reason it is not saving any of the photos that I want it to save locally so that I can retrieve it. How can I fix this issue. Does anyone have an idea on how to go about this.
controller:
const Profile = require("../../models/UserModels/Profiles.js")
const User = require('../../models/UserModels/Users.js')
const multer = require('multer')
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '../../client/images/profile/')
},
filename: function (req, file, cb) {
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9)
cb(null, file.fieldname + '-' + uniqueSuffix)
}
})
const upload = multer({ storage: storage })
...
post:(req, res) => {
console.log(req.body)
console.log(req.files)
upload.single(req.files.profile_pic.name) <----------------
let body = req.body
Profile.create({
profile_pic: req.files.profile_pic.name,
f_name: body.f_name,
l_name: body.l_name,
bio: body.bio,
location: body.location,
sm_facebook: body.sm_facebook,
sm_instagram: body.sm_instagram,
sm_twitter: body.sm_twitter,
sm_website: body.sm_website,
followers: body.followers,
following: body.following,
photos: body.photos,
downloads: body.downloads,
edits: body.edits,
userId: body.userId
})
.then((data) => {
res.send(data).status(200)
})
.catch((err) => {
console.error(err)
res.send(err).status(400)
})
},
Here is the direcory for this backend:
-----------update---------------
this is the react axios request from the frontend:
function createProfile(userId){
let data = new FormData()
data.append('f_name', firstName)
data.append('l_name', lastName)
data.append('bio', bio)
data.append('location', location)
data.append('sm_facebook', facebook)
data.append('sm_instagram', instagram)
data.append('sm_twitter', twitter)
data.append('sm_website', website)
data.append('followers', 0)
data.append('following', 0)
data.append('photos', 0)
data.append('edits', 0)
data.append('downloads', 0)
data.append('userId', userId)
data.append('profile_pic', profilePicture)
console.log(data)
axios({
method: "post",
url: "/api/profile/",
data: data,
headers: { "Content-Type": "multipart/form-data" },
})
.then((data) => {
return(<Navigate to='/' />)
})
.catch((err) => {
console.error(err)
})
}

You have to specify the storage path for multer as if you were located on the root folder of the project.
So instead of using '../../client/images/profile/' as storage path, you should use: './backend/client/images/profile'.
You'll code should end up as follows:
const multer = require('multer')
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, './backend/client/images/profile')
},
filename: function (req, file, cb) {
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9)
cb(null, file.fieldname + '-' + uniqueSuffix)
}
})

What is the correct way to make multer work with Node and Express here?

I am trying to create a route through which I can upload photos. However as I made so,e changes it stopped working and I am not sure how to make it work.
const multer = require('multer');
// MULTER STORAGE
const multerStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, '/upload');
},
filename: (req, file, cb) => {
const ext = file.mimetype.split('/')[1];
// Saving format: user-UserId-DateStamp.ext
//e.g user-608d55c7e512b74ee00791de-1621992912638.jpeg
cb(null, `user-${req.body.userId}-${Date.now()}.${ext}`);
},
});
//MULTER FILTER
const multerFilter = (req, file, cb) => {
//mimetype always starts with image/ then png or jpeg or..
if (file.mimetype.startsWith('image')) {
cb(null, true);
} else {
cb(new AppError('You are only allowed to upload image files.', 400), false);
}
};
const uploadDirectory = multer({
storage: multerStorage,
fileFilter: multerFilter,
});
//exports.uploadPhoto = uploadDirectory.single('photo');
//app.use(express.static('./uploads'));
// INCLUDE ERROR CLASS AND ERROR CONTROLLER
const AppError = require('../utils/appError.js');
const errorController = require('./errorController.js');
const { Mongoose } = require('mongoose');
The main problem Im guessing is in this block
//UPLOAD PHOTO
exports.uploadPhoto = uploadDirectory(async (req, res) => {
console.log(req.body);
console.log(req.file);
try {
const newPhoto = await photoModel.create(req.file);
newPhoto.save().then((result) => {
console.log('Saved');
res.status(201).json({
status: 'success',
// data: JSON.parse(JSON.stringify(newPhoto.file)),
});
});
} catch (err) {
console.log('Error in upload');
errorController.sendError(err, req, res);
}
}).single('photo');
Can anybody let me know how to correctly write the exports.uploadPhoto
Originally the last function looked like this
exports.uploadPhoto = async (req, res) => {
console.log(req.body);
console.log(req.file);
try {
const newPhoto = await photoModel.create(req.file);
newPhoto.save().then((result) => {
console.log('Saved');
res.status(201).json({
status: 'success',
// data: JSON.parse(JSON.stringify(newPhoto.file)),
});
});
} catch (err) {
console.log('Error in upload');
errorController.sendError(err, req, res);
}
};

The multer middleware function, in your case uploadDirectory, is usually used before other middleware functions/controllers where you have your business logic (e.g. uploadPhoto).
app.post('/upload', uploadDirectory.single('photo'), uploadPhoto);
Keep your original uploadPhoto function and with the above code you'll have access to the data and file through reg.body and req.file, respectively.
This Request Parsing in Node.js Guide (it's free) will help you with file uploads in Node.js.

How do I recover from failure when uploading multiple files via my server to S3?

New to NodeJS and S3, I wrote the following exploratory code to upload files to S3 via my NodeJS server without saving the file to disk or memory:
var express = require('express');
var Busboy = require('busboy');
var S3 = require('../utils/s3Util');
var router = express.Router(); // mounted at /uploads
router.post("/", function (req, res, next) {
let bb = new Busboy({ headers: req.headers });
const uploads = [];
bb.on('file', (fieldname, stream, filename, encoding, mimeType) => {
console.log(`Uploaded fieldname: ${fieldname}; filename: ${filename}, mimeType: ${mimeType}`);
uploads.push(S3.svc.upload({ Bucket: 'my-test-bucket', Key: filename, Body: stream }).promise());
});
bb.on('finish', () => {
console.log("# of promises:", uploads.length);
Promise.all(uploads).then(retVals => {
for (let i = 0; retVals && i < retVals.length; i++) {
console.log(`File ${i + 1}::`, retVals[i]);
}
res.end();
}).catch(err => {
console.log("Error::", err);
res.status(500).send(`${err.name}: ${err.message}`);
});
});
req.pipe(bb);
});
module.exports = router;
In the general failure case, how do I handle the scenario where the upload of 1 or more of x files being uploaded fails? Some uploads would have succeeded, some would have failed. However, in the catch clause I wouldn't know which ones have failed...
It would be good to be able to make this upload process somewhat transactional (i.e., either all uploads succeed, or none do). When errors happen, ideally I would be able to "rollback" the subset of successful uploads.

You could do it like this:
Push an object into uploads, with the data you need to retry, so:
uploads.push({
fieldname,
filename,
mimeType,
uploaded: S3.svc.upload({ Bucket: 'my-test-bucket', Key: filename, Body: stream })
.promise()
.then(() => true)
.catch(() => false)
});
...
const failed = await
(Promise.all(uploads.map(async upload => ({...upload, uploaded: await upload.uploaded})))).then(u => u.filter(upload => !upload.uploaded))
const failedFiles = failed.join(', ')
console.log(`The following files failed to upload: ${failedFiles}`);
You need to make your event handlers async to use await inside them, so, for example:
bb.on('file', async (fieldname, stream, filename, encoding, mimeType) => {

I finally went with the following code, which is an expansion of #JoshWulf's answer:
function handleUpload(req, res, bucket, key) {
let bb = new Busboy({ headers: req.headers });
const uploads = [];
bb.on('file', (fieldname, stream, filename, encoding, mimeType) => {
console.log(`Uploaded fieldname: ${fieldname}; filename: ${filename}, mimeType: ${mimeType}`);
const params = { Bucket: bucket, Key: key, Body: stream, ContentType: mimeType };
uploads.push({ filename, result: S3.svc.upload(params).promise().then(data => data).catch(err => err) });
});
bb.on('finish', async () => {
const results = await Promise.all(uploads.map(async (upload) => ({ ...upload, result: await upload.result })));
// handle success/failure with their respective objects
});
req.pipe(bb);
}
The difference here from #Josh Wulf's answer is that in my upload promise I am returning the returned data object (if successful) and the returned error object (in case of failure) as-is. This then enables me to later use them as I need.

Creating multiple buckets with GridFS

I am working with the GridFS library in express and node. I'm trying to create multiple buckets. For example I already have a bucket titled avatars, which stores images.
/* Start of mongo connection for uploading files */
const mongoURI = "mongodb://localhost:27017/PTAdata";
const conn = mongoose.createConnection(mongoURI);
let gfs;
conn.once('open', () => {
gfs = stream(conn.db, mongoose.mongo);
gfs.collection('avatars');
})
const storage = new GridFs({
url: "mongodb://localhost:27017/PTAdata",
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
file.user = req.body.username
const name = file.originalname
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: file.user,
bucketName: 'avatars'
};
resolve(fileInfo);
});
});
}
});
const upload = multer({ storage });
I now want to create another bucket called audio which will store mp3 files. I checked the documentation for GridFS at https://docs.mongodb.com/manual/core/gridfs/ and it states that "you can choose a different bucket name, as well as create multiple buckets in a single database." However it does not provide any insight or steps to how. Has anyone done any work with the GridFS library and know how to create multiple buckets?

You need to store another "new GridFS" object in a different variable, than pass it to multer as a different storage property. In your case, this should work:
const storage = new GridFs({
url: "mongodb://localhost:27017/PTAdata",
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
file.user = req.body.username
const name = file.originalname
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: file.user,
bucketName: 'avatars'
};
resolve(fileInfo);
});
});
}
});
const anotherStorage = new GridFs({
url: "mongodb://localhost:27017/PTAdata",
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
file.user = req.body.username
const name = file.originalname
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: file.user,
bucketName: 'mp3files'
};
resolve(fileInfo);
});
});
}
});
const upload = multer({ storage });
const uploadSongs = multer({ storage: anotherStorage });
Finally, you should choose between those buckets accordingly to your endpoints, for example:
app.post('/api/uploadAvatar', upload.any(), (req, res)=> {
... do stuff
}
app.post('/api/uploadMp3', uploadSongs.any(), (req, res)=> {
... do stuff
}
For me, it made sense to change gfs.collection() in each case (inside the file: (req, file) function), but it worked without changing as well. Be aware that any() is just an option, an it's not the safest one, but it's great for testing your code.