I'm attempting to handle file uploads using a Google Cloud Function. This function uses Busboy to parse the multipart form data and then upload to Google Cloud Storage.
I keep receiving a ERROR: { Error: ENOENT: no such file or directory, open '/tmp/xxx.png' error when triggering the function.
The error seems to occur within the finish callback function when storage.bucket.upload(file) attempts to open the file path /tmp/xxx.png.
Example code
const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');
const Storage = require('#google-cloud/storage');
const moment = require('moment');
const _ = require('lodash');
const projectId = 'xxx';
const bucketName = 'xxx';
const storage = new Storage({
projectId: projectId,
});
exports.uploadFile = (req, res) => {
if (req.method === 'POST') {
const busboy = new Busboy({
headers: req.headers
});
const uploads = []
const tmpdir = os.tmpdir();
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
const filepath = path.join(tmpdir, filename)
var obj = {
path: filepath,
name: filename
}
uploads.push(obj);
var writeStream = fs.createWriteStream(obj.path);
file.pipe(writeStream);
});
busboy.on('finish', () => {
_.forEach(uploads, function (file) {
storage
.bucket(bucketName)
.upload(file.path, {
name: moment().format('/YYYY/MM/DD/x') + '-' + file.name
})
.then(() => {
console.log(`${file.name} uploaded to ${bucketName}.`);
})
.catch(err => {
console.error('ERROR:', err);
});
fs.unlinkSync(file.path);
})
res.end()
});
busboy.end(req.rawBody);
} else {
res.status(405).end();
}
}
Solved this with a stream instead of a temporary file. Only handles a single file at the moment though.
https://gist.github.com/PatrickHeneise/8f2c72c16c4e68e829e58ade64aba553#file-gcp-function-storage-file-stream-js
function asyncBusboy(req, res) {
return new Promise((resolve, reject) => {
const storage = new Storage()
const bucket = storage.bucket(process.env.BUCKET)
const fields = []
const busboy = Busboy({
headers: req.headers,
limits: {
fileSize: 10 * 1024 * 1024
}
})
busboy.on('field', (key, value) => {
fields[key] = value
})
busboy.on('file', (name, file, fileInfo) => {
const { mimeType } = fileInfo
const destFile = bucket.file(fileName)
const writeStream = destFile.createWriteStream({
metadata: {
contentType: fileInfo.mimeType,
metadata: {
originalFileName: fileInfo.filename
}
}
})
file.pipe(writeStream)
})
busboy.on('close', function () {
return resolve({ fields })
})
if (req.rawBody) {
busboy.end(req.rawBody)
} else {
req.pipe(busboy)
}
})
}
Related
axios script.js file
const creatClient = async (client) => {
try {
const res = await axios({
method: 'POST',
withCredentials: true,
url: '/[url]',
data: client,
}).then(location.assign('/[newUrl]'));
} catch (error) {
console.log(error);
}
};
submitbtn.addEventListener('click', (e) => {
e.preventDefault;
const name = document.getElementById('name').value;
const phone = document.getElementById('phone').value;
const createdATT = new Date(document.getElementById('date').value);
const followUp = new Date(document.getElementById('date2').value);
const images = document.getElementById('img').value;
const insurance = document.getElementById('insurance').value;
const client = { name, phone, insurance, images, createdATT, followUp };
console.log(client);
client ? creatClient(...client) : console.log('no object created');
});
controller file
the console log for req.body [Object: null prototype] {*** the object ***}
const multer = require('multer');
const Client = require('../models/clientModel');
const multerStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, 'public/img');
},
filename: (req, file, cb) => {
const ext = file.mimetype.split('/')[1];
cb(null, `user-${Date.now()}.${ext}`);
},
});
const multerFilter = (req, file, cb) => {
if (file.mimetype.startsWith('image')) {
cb(null, true);
} else {
cd(console.log('select image'), false);
}
};
const upload = multer({
storage: multerStorage,
fileFilter: multerFilter,
});
exports.uploadImages = upload.single('images');
//
exports.createClients = async (req, res, next) => {
try {
if (req.file) req.body.images = req.file.filename;
const newClient = { ...req.body };
await Client.create(req.body).then(
res.status(200).json({
status: 'success',
newClient,
})
);
} catch (err) {
console.log(err);
}
};
also with postman sending request give success response with no errors
i've tried location.replace() but also it didn't work for me
and is there another trick from server to get to the desired location out from client side
then accepts a callback as a parameter.
then(() => location.assign('/[newUrl]'))
I have an app which calls a cloud function endpoint:
import './App.css';
import React from 'react';
import axios from 'axios';
function App() {
const [file, setFile] = React.useState(null);
function fileSelected(e)
{
setFile(()=> e.target.files[0]);
}
function uploadFile()
{
console.log(file)
const fd = new FormData();
fd.append('image', file, file.name);
console.log(file);
console.log(file.name);
axios.post('https://us-central1-athelasapp.cloudfunctions.net/uploadFile', fd)
.then(res => {
console.log(res)
});
}
return (
<div className="App">
<input type="file" onChange={fileSelected}/>
<input type="submit" onClick={uploadFile}/>
</div>
);
}
export default App;
and the endpoint tries to parse it with Busboy however, it throws a 500 error. I have Busboy imported but it throws:
xhr.js:220 POST https://us-central1-athelasapp.cloudfunctions.net/uploadFile 500`
const functions = require("firebase-functions");
const express = require("express");
const cors = require("cors");
const app = express();
const Busboy = require("busboy");
const os = require("os");
const path = require("path");
const fs = require("fs");
const gcconfig = {
projectId: "athelasapp",
keyFilename: "athelasapp-firebase-adminsdk-yojnp-1e9141a009.json",
};
const {Storage} = require("#google-cloud/storage");
const gcs = new Storage(gcconfig);
app.use(cors({origin: "http://localhost:3000"}));
// // Create and Deploy Your First Cloud Functions
// https://firebase.google.com/docs/functions/write-firebase-functions
exports.uploadFile = functions.https.onRequest(app);
app.post("/", (req, res) =>{
if (req.method !== "POST") {
return res.status(500).json({
message: "Method Does Not Work",
});
}
const busboy = new Busboy({headers: req.headers});
let uploadData = null;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const filepath = path.join(os.tmpdir(), filename);
uploadData = {file: filepath, type: mimetype};
file.pipe(fs.createWriteStream(filepath));
res.status(200).json({
imageDetails: uploadData,
});
});
busboy.on("finish", ()=>{
const bucket = gcs.bucket("athelasapp.appspot.com");
bucket.upload(uploadData.file, {
uploadType: "media",
metadata: {
metadata: {
contentType: uploadData.type,
},
},
});
}).then(() => {
res.status(200).json({
message: "Method Works!",
});
}).catch((err) =>{
res.status(500).json({
message: "Method Failed!",
});
});
busboy.end(req.rawBody);
res.status(200).json({
message: "Method Works",
});
});
I cant find any errors in my code or how it's implemented? Could it be I'm passing in the wrong things in the request? I think it might have to do with the nomenclature of Busboy
Here's a working gist streaming directly instead of creating the temporary file: https://gist.github.com/PatrickHeneise/8f2c72c16c4e68e829e58ade64aba553#file-gcp-function-storage-file-stream-js
function asyncBusboy(req, res) {
return new Promise((resolve, reject) => {
const storage = new Storage()
const bucket = storage.bucket(process.env.BUCKET)
const fields = []
const busboy = Busboy({
headers: req.headers,
limits: {
fileSize: 10 * 1024 * 1024
}
})
busboy.on('field', (key, value) => {
fields[key] = value
})
busboy.on('file', (name, file, fileInfo) => {
const { mimeType } = fileInfo
const destFile = bucket.file(fileName)
const writeStream = destFile.createWriteStream({
metadata: {
contentType: fileInfo.mimeType,
metadata: {
originalFileName: fileInfo.filename
}
}
})
file.pipe(writeStream)
})
busboy.on('close', function () {
return resolve({ fields })
})
if (req.rawBody) {
busboy.end(req.rawBody)
} else {
req.pipe(busboy)
}
})
}
frontend with React.js looks good i saw the other people who did same thing as i did but i don't know where is the problem ! anyone can help me ?
const [cours, setCours] = useState([]);
const [description, setDescription] = useState("")
const [title, setTitle] = useState("")
const coursHandle = (e) => { setCours([e.target.files]) }
const onsubmit = async (e) => {
e.preventDefault();
const formData = new FormData();
formData.append("description", description);
formData.append("title", title);
// cours.forEach((elem) => { formData.append("cours", elem) });
formData.append("cours", cours)
// for (let i = 0; i < cours.length; i++) {
// formData.append("cours", cours[i])
// }
await axios.post("http://localhost:5000/upload", formData)
.then((res) => console.log("successfully file post", res)).catch((err) =>
console.log("error with file post", err))
}
and backend with multer is here this code is in my app.js
app.use(express.static(path.join(__dirname, 'public')));
and the public folder is the same place as app.js
const multer = require("multer");
const path = require("path");
const MIME_TYPES = {
"file/pdf": "pdf",
"file/docx": "docx",
"file/txt": "txt",
"file/png": "png",
"file/jpeg": "jpg",
"file/jpg": "jpg",
}
const fileStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, "public");
},
filename: (req, file, cb) => {
const nameObject = path.parse(file.originalname);
// const nameObject = file.originalname.split(' ').join('_');
const extension = MIME_TYPES[file.mimetype];
cb(null, nameObject.name.split(" ").join("_") + Date.now() + "." + extension);
}
})
module.exports = { multerUpload: multer({ storage: fileStorage }).single("file") }
In last line
module.exports = { multerUpload: multer({ storage: fileStorage }).single("file")
the argument under single is file, but in your frontend form there is no input with name file maybe you have to change the argument of single function from file to cours
I found the answer for my question,
formData.append("cours", cours)
Here the key name as i try to send is "cours" and :
module.exports = { multerUpload: multer({ storage: fileStorage }).single("file")
Here the key name for single function is "file" !! that was a first problem and another one was this :
I've forgoten to add this line in my tag !
encType="multipart/form-data"
And finally my multer js is :
const fileStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, "public/files");
},
filename: (req, file, cb) => {
console.log("filename :", file);
const nameObject = path.parse(file.originalname);
cb(null, nameObject.name.split(" ").join("_") + Date.now() +
nameObject.ext);
}
})
module.exports = { multerUpload: multer({ storage: fileStorage }).array("cours") }
I want to pass functions with require. Here's the code:
multerModule.js
const path = require('path');
const multer = require('multer');
const crypto = require('crypto');
const GridFsStorage = require('multer-gridfs-storage');
// Create storage engine
const initStorage = (conn, bucketName) => new GridFsStorage({
db: conn,
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: bucketName
};
resolve(fileInfo);
});
});
}
});
const initUpload = (storage) => multer({
storage: storage,
fileFilter: function (req, file, callback) {
const ext = path.extname(file.originalname);
if (ext !== '.png' && ext !== '.jpg' && ext !== '.gif' && ext !== '.jpeg') {
return callback(new Error('Only images are allowed'))
}
callback(null, true)
}
});
module.exports = { initStorage, initUpload };
offers.js
const router = require('express').Router();
const auth = require('../../middleware/auth');
const mongoose = require('mongoose');
const Grid = require('gridfs-stream');
const collectionName = 'offers';
const bucketName = 'offers';
const { initStorage, initUpload } = require('../../modules/multerModule');
const conn = mongoose.connection;
Grid.mongo = mongoose.mongo;
// Init gfs
let gfs;
conn.once('open', () => {
// Init stream
gfs = Grid(conn.db);
gfs.collection(collectionName);
});
const storage = initStorage(conn, bucketName);
const upload = initUpload(storage);
And I get TypeError: storage is not a function
I think that I could to that by writing something like(correct me if I'm wrong):
module.exports = {
function1: function(params) {//do something},
function2: function(nextparams){//do something}
}
, but isn't a way to do that in this first way through these defined before module arrow functions ?
I am working with the GridFS library in express and node. I'm trying to create multiple buckets. For example I already have a bucket titled avatars, which stores images.
/* Start of mongo connection for uploading files */
const mongoURI = "mongodb://localhost:27017/PTAdata";
const conn = mongoose.createConnection(mongoURI);
let gfs;
conn.once('open', () => {
gfs = stream(conn.db, mongoose.mongo);
gfs.collection('avatars');
})
const storage = new GridFs({
url: "mongodb://localhost:27017/PTAdata",
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
file.user = req.body.username
const name = file.originalname
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: file.user,
bucketName: 'avatars'
};
resolve(fileInfo);
});
});
}
});
const upload = multer({ storage });
I now want to create another bucket called audio which will store mp3 files. I checked the documentation for GridFS at https://docs.mongodb.com/manual/core/gridfs/ and it states that "you can choose a different bucket name, as well as create multiple buckets in a single database." However it does not provide any insight or steps to how. Has anyone done any work with the GridFS library and know how to create multiple buckets?
You need to store another "new GridFS" object in a different variable, than pass it to multer as a different storage property. In your case, this should work:
const storage = new GridFs({
url: "mongodb://localhost:27017/PTAdata",
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
file.user = req.body.username
const name = file.originalname
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: file.user,
bucketName: 'avatars'
};
resolve(fileInfo);
});
});
}
});
const anotherStorage = new GridFs({
url: "mongodb://localhost:27017/PTAdata",
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
file.user = req.body.username
const name = file.originalname
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: file.user,
bucketName: 'mp3files'
};
resolve(fileInfo);
});
});
}
});
const upload = multer({ storage });
const uploadSongs = multer({ storage: anotherStorage });
Finally, you should choose between those buckets accordingly to your endpoints, for example:
app.post('/api/uploadAvatar', upload.any(), (req, res)=> {
... do stuff
}
app.post('/api/uploadMp3', uploadSongs.any(), (req, res)=> {
... do stuff
}
For me, it made sense to change gfs.collection() in each case (inside the file: (req, file) function), but it worked without changing as well. Be aware that any() is just an option, an it's not the safest one, but it's great for testing your code.