I'm trying to use S3, specifically multer-s3, for image upload for a traditional web app that currently has multer/file system file upload (GitHub repo with previous code before the failed S3 upload attempt can be found here). The app is deployed to Heroku, which has ephemeral file storage, so the old setup is a no-go.
I tried to use multer-s3 to do it based on this tutorial https://www.youtube.com/watch?v=ASuU4km3VHE&t=1364s, but got up to about 20 mins in, trying to send the POST request to the new image-upload route but am getting a 500 error, whereas in the tutorial an AWS image path is provided in the response.
Here's what I tried so far:
I created a bucket and get my access code and keys. In my S3 bucket settings, under Permissions -> Block public access, I set everything to off. I also added CORS config code as suggested by Heroku here (but I still get a 500 error without it).
In the util folder, I added a file named file-upload.js with this code (I'm using a Nodemon.json file for the config keys):
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const { uuid } = require('uuidv4');
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: 'us-west-2',
});
const s3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3,
bucket: 'nodejs-shop',
acl: 'public-read',
// Called when saving image to AWS
metadata(req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
// Called before saving image to AWS
key(req, file, cb) {
cb(null, uuid());
},
}),
});
module.exports = upload;
In the routes folder, I added a file named file-upload.js, with this code:
const express = require('express');
const router = express.Router();
const upload = require('../util/file-upload');
// Will send image under this key 'image' in request to server
const singleUpload = upload.single('image');
router.post('/image-upload', (req, res, next) => {
// Callback function called after image is uploaded or will get error from server
singleUpload(req, res, (err) => {
return res.json({ imageUrl: req.file.location });
});
});
module.exports = router;
In app.js, I imported the routes file const fileRoutes = require('./routes/file-upload'); and added the middleware after the authRoutes middleware app.use(fileRoutes);. I also commented out all the previously used multer code in app.js.
Current app.js code:
const path = require('path');
const fs = require('fs');
// const https = require('https');
const express = require('express');
const bodyParser = require('body-parser');
const mongoose = require('mongoose');
const session = require('express-session');
const MongoDBStore = require('connect-mongodb-session')(session);
const csrf = require('csurf');
const flash = require('connect-flash');
// const multer = require('multer');
// const { uuid } = require('uuidv4');
const helmet = require('helmet');
const compression = require('compression');
const morgan = require('morgan');
const errorController = require('./controllers/error');
const User = require('./models/user');
const MONGODB_URI =
// process object is globally available in Node app; part of Node core runtime. The env property contains all environment variables known by process object. Using nodemon.json to store environment variables, but could alternatively use dotenv package for this (see https://www.youtube.com/watch?v=17UVejOw3zA)
`mongodb+srv://${process.env.MONGO_USER}:${process.env.MONGO_PASSWORD}#cluster0-4yuid.mongodb.net/${process.env.MONGO_DEFAULT_DATABASE}`;
const app = express();
const store = new MongoDBStore({
uri: MONGODB_URI,
collection: 'sessions',
});
// Secret used for signing/hashing token is stored in session by default
const csrfProtection = csrf();
// Don't want to start server until file is read in, thus using synchronous version
// const privateKey = fs.readFileSync('server.key');
// const certificate = fs.readFileSync('server.cert');
// Commenting out original file upload method since changed to use AWS S3 for image upload/hosting
// const fileStorage = multer.diskStorage({
// destination: (req, file, cb) => {
// // First arg is for error message to throw to inform multer something is wrong with incoming file and it should not store it; with null, telling multer okay to store it
// cb(null, 'images');
// },
// filename: (req, file, cb) => {
// cb(null, uuid());
// },
// });
// const fileFilter = (req, file, cb) => {
// file.mimetype === 'image/png' ||
// file.mimetype === 'image/jpg' ||
// file.mimetype === 'image/jpeg'
// ? cb(null, true)
// : cb(null, false);
// };
app.set('view engine', 'ejs');
// Setting this explicity even though the views folder in main directory is where the view engine looks for views by default
app.set('views', 'views');
const adminRoutes = require('./routes/admin');
const shopRoutes = require('./routes/shop');
const authRoutes = require('./routes/auth');
const fileRoutes = require('./routes/file-upload');
// Create write stream (for passing to morgan, used to log request data), for logging request data in file instead of console
// flags: 'a': a is for append; new data will be appended to that file (additional log statements are added to end of existing file rather than overwriting it)
const accessLogStream = fs.createWriteStream(
path.join(__dirname, 'access.log'),
{ flags: 'a' }
);
// Set secure response header(s) with Helmet
// In my app, in developer tools (in the network tab) I can see it added one additional response header for localhost, Strict-Transport-Security. This HTTP header tells browsers to stick with HTTPS and never visit the insecure HTTP version. Once a browser sees this header, it will only visit the site over HTTPS for the next 60 days
app.use(helmet());
// Compress assets. Note: Compression is normally done by hosting providers, but deploying to Heroku which does offer it
app.use(compression());
// Log request data using writable file stream created above. Which data is logged and how to format it is passed into funtion
// Also normally handled by hosting providers
// app.use(morgan('combined', { stream: accessLogStream }));
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json());
// Commented out since changed to use AWS S3 for image upload/hosting
// app.use(multer({ storage: fileStorage, fileFilter }).single('image'));
app.use(express.static(path.join(__dirname, 'public')));
app.use('/images', express.static(path.join(__dirname, 'images')));
app.use(
session({
secret: 'my secret',
resave: false,
saveUninitialized: false,
store,
})
);
app.use(csrfProtection);
app.use(flash());
app.use((req, res, next) => {
// Locals field: Express feature for setting local variables that are passed into views. For every request that is executed, these fields are set for view that is rendered
res.locals.isAuthenticated = req.session.isLoggedIn;
res.locals.csrfToken = req.csrfToken();
next();
});
app.use((req, res, next) => {
// When you throw an error in synchronous places (outside of callbacks and promises), Express will detect this and execute next error handling middleware. But if error is thrown within async code (in then or catch block), Express error handling middleware won't be executed; app will simply crash; have to use next()
// throw new Error('sync dummy');
if (!req.session.user) {
return next();
}
User.findById(req.session.user._id)
.then((user) => {
if (!user) {
return next();
}
req.user = user;
next();
})
// catch block will be executed in the case of technical issue (e.g., database down, or insufficient permissions to execute findById())
.catch((err) => {
// Within async code snippets, need to use next wrapping error, outside you can throw error
next(new Error(err));
});
});
app.use('/admin', adminRoutes);
app.use(shopRoutes);
app.use(authRoutes);
app.use(fileRoutes);
app.get('/500', errorController.get500);
app.use(errorController.get404);
// Error-handling middleware. Express executes this middleware when you call next() with an error passed to it
app.use((error, req, res, next) => {
// res.status(error.httpStatusCode).render(...);
// res.redirect('/500');
res.status(500).render('500', {
pageTitle: 'Server Error',
path: '/500',
isAuthenticated: req.session.isLoggedIn,
});
});
mongoose
.connect(MONGODB_URI, { useUnifiedTopology: true, useNewUrlParser: true })
.then((result) => {
// First arg for createServer() configures server, second is request handler, in this case, Express application
// Commenting out because just as with request logging and asset compression, it's handled by hosting provider, and browsers don't accept custom/self-signed certificate; will be displayed as insecure with a message that connection is not private
// https
// .createServer({ key: privateKey, cert: certificate }, app)
// .listen(process.env.PORT || 3000);
app.listen(process.env.PORT || 3000);
})
.catch((err) => {
console.log(err);
});
This is my Postman request, similar to the one in the tutorial video, and as you can see I just get a 500 error.
Related
I am trying to deploy my React app onto Heroku however I'm running into some graphql errors. When I manually deploy the app, the server and client both start up successfully however upon performing any actions that require a database fetch, I get a failed to fetch error and a "Failed to load resource: net::ERR_CONNECTION_REFUSED" error. I think this is because the server is in the correct location (IP-wise) but this app was created to run locally so the client is expected to be on the localhost network, not the heroku url it has now. I have changed the ENV variables various different times to see what combination of addresses would work for the server and client however nothing changed. I will post my server side code and ENV variables below:
const { ApolloServer } = require('apollo-server-express');
const cors = require('cors');
const express = require('express');
const mongoose = require('mongoose');
const resolvers = require('./resolvers/root-resolver');
const { typeDefs } = require('./typedefs/root-def');
const serverOptions = require('./server-config');
require('dotenv').config();
const { MONGO_URI, BACKEND_PORT, CLIENT_LOCAL_ORIGIN, SERVER_LOCAL_DOMAIN } = process.env;
// create express server handling our middleware
const app = express();
// since we presume cors is enabled, this next step is not optional, so cors
// is enable here instead of in options
app.use(cors({ origin: CLIENT_LOCAL_ORIGIN, credentials: true }));
const corsPolicy = async(req, res, next) => {
/*
TODO for 316 students: res.set(), Access-Control-Allow-Origin and Access-Control-Allow-Credentials headers,
have them set these, inspect error messages, understand why they're needed
*/
res.set("Access-Control-Allow-Origin", req.headers.origin.toString());
res.set("Access-Control-Allow-Credentials", true);
next();
}
app.options('*', cors());
app.use(corsPolicy);
// middleware application is configured to happen in server-config.js
serverOptions(app);
const server = new ApolloServer({
typeDefs: typeDefs,
resolvers: resolvers,
context: ({req, res}) => ({ req, res })
});
// since the express server has cors configured, cors on the apollo server
// can be false; passing the same options as defined on the express instance
// works as well
server.applyMiddleware({ app , cors: false});
mongoose.connect(MONGO_URI, {useNewUrlParser: true , useUnifiedTopology: true})
.then(() => {
app.listen({ port: BACKEND_PORT }, CLIENT_LOCAL_ORIGIN, () => {
console.log(`Server ready at ${SERVER_LOCAL_DOMAIN}:${BACKEND_PORT}`);
})
})
.catch(error => {
console.log(error)
});
[ENV Variables][1]
[1]: https://i.stack.imgur.com/5nqmp.png
I'm using Vue with NodeJs, Vuetify and Express. I load user's file with the Vuetify's component:
<v-file-input
v-model="documentFile.value"
:error-messages="documentFile.errors"
accept="application/pdf"
/>
Then I want to pass the file (that is stored in this.documentFile.value) to my backend, so it will upload it to the Drive. I pass the data using Vue Recourse:
var params = {
"data": this.data.value
"score": this.score.value
//"document_file": this.documentFile.value,
"comments": this.comments.value
};
Vue.http.put('http://localhost:8081/api/new-document', {params: params}).then(
response => {
console.log("Sent data");
}, response => {
console.error(response);
}
);
In my NodeJS backend I have:
router.put('/new-document', function(request, response) {
console.log("New Document");
console.log(request.query);
// Upload file to drive
const oauth2Client = new google.auth.OAuth2(
CLIENT_ID,
CLIENT_SECRET,
REDIRECT_URI
);
response.status(200).send({});
});
How can I pass the file from the client to the backend?
EDIT: If I uncomment document_file, and try to print request.query, it prints:
{
data: { age: '27', name: 'robert' },
comments: 'comment',
"score": 89
}
For some reason, it ignores the document_file.
The code in my server.js:
const cors = require("cors");
const express = require("express");
const bodyParser = require("body-parser");
const routes = require('./routes');
const path = __dirname + '/../public/';
console.log("STARTED");
const app = express();
app.use(express.static(path));
var corsOptions = {
origin: "http://localhost:8080"
};
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
// map requests to routes
app.use('/api', routes);
// set port, listen for requests
const PORT = process.env.PORT || 8081;
app.listen(PORT, () => {
console.log(`Server is running on port ${PORT}.`);
});
To pass the file from the client to the backend, use the new FormData() object.
In your case you can do something like:
var params = new FormData();
params.append('data', this.data.value);
params.append('score', this.score.value);
params.append('document_file', this.documentFile.value)
params.append('comments', this.comments.value);
Then pass the object either using axios or Vue Recourse as you would like.
axios.put('url', params)
EDIT
You can use multer to upload your files either locally or to the cloud. In your case, you can upload to GoogleStorage
const multer = require('multer')
// You can start off by testing local uploads
const upload = multer({ dest: 'uploads/' })
// then use this to upload to Google Storage
const multerGoogleStorage = require('multer-google-storage')
const uploadHandler = multer({
storage: multerGoogleStorage.storageEngine({
autoRetry: true,
bucket: '<your_storage_bucket_name>',
projectId: '<your_project_ID>',
keyFilename: '<your_path_to_key_file>',
filename: (req, file, cb) => {
cb(null, `/<some_prefix_of_choice>/${Date.now()}_${file.originalname}`)
}
})
})
Upload the file either to local or Google Cloud
// local uploads (destination projectRoot/uploads)
router.put('/new-document', upload.single('document_file'), async (request, response) => {});
// or GoogleStorage
router.put('/new-document', uploadHandler.single('document_file'), async (request, response) => {});
Multiple files can also be uploaded
app.put('/new-document', upload.array('document_files', 12), function (req, res, next) {
// req.files is array of `photos` files
// req.body will contain the text fields, if there were any
})
The document file(s) can then be accessible on the Express server using
request.file
You can the upload the file. The other form objects can also be accessible through request.body e.g.
request.body.data
I am sending a simple get request (without any params) through chrome/postman When and after 2 minutes I got an error messsage.
I expected to get an answer "cannot get" on sending get request to localhost:3030
I expect to get the JSON file on my db when I send a get request to localhost:3030/api/task
In both cases I get the "loading /thinking" circle and nothing happens.
Error messsages (after 2 minutes of "loading"):
On chrome - After 2 minutes - I got " This site can't be reached" And On Postman "Could not get response
I wrote a console.log on the task controller file and it's even not reaching there.
I am attaching the server.js file. I suspect something is wrong there. I don't want to overload this question.
const cors = require('cors')
const path = require('path')
// const expressSession = require('express-session')
const app = express()
const http = require('http').createServer(app)
// const session = expressSession({
// secret: 'coding is amazing',
// resave: false,
// saveUninitialized: true,
// cookie: { secure: false }
// })
//Express App Config
app.use(express.json())
//app.use(session)
if (process.env.NODE_ENV === 'production') {
app.use(express.static(path.resolve(__dirname, 'public')))
} else {
const corsOptions = {
origin: ['http://127.0.0.1:8080', 'http://localhost:8080', 'http://127.0.0.1:3000', 'http://localhost:3000'],
credentials: true
}
app.use(cors(corsOptions))
}
const taskRoutes = require('./api/task/task.routes')
//const { connectSockets } = require('./services/socket.service')
// routes
const setupAsyncLocalStorage = require('./middlewares/setupAls.middleware')
app.all('*', setupAsyncLocalStorage)
// app.get('/api/setup-session', (req, res) => {
// req.session.connectedAt = Date.now()
// console.log('setup-session:', req.sessionID);
// res.end()
// })
app.use('/api/task', taskRoutes)
//connectSockets(http, session)
//connectSockets(http);
// Make every server-side-route to match the index.html
// so when requesting http://localhost:3030/index.html/car/123 it will still respond with
// // our SPA (single page app) (the index.html file) and allow vue/react-router to take it from there
// app.get('/**', (req, res) => {
// res.sendFile(path.join(__dirname, 'public', 'index.html'))
// })
const logger = require('./services/logger.service')
const port = process.env.PORT || 3030
http.listen(port, () => {
logger.info('Server is running on port: ' + port)
})
There are 2 possible reasons:
You aren't going to localhost:3030.
process.env.PORT is being set to something other than 3030.
Once you check these, it should go away.
Hi im triying to upload images to cloudinary using multer in my nodejs app it works perfectly on the localhost but when i upload it to zeit it doesn't work it says "ENOENT: no such file or directory, open (folder)"
I tried using #now/static to make the folder available but it doesn't work too. I will be happy if you can help me.
const express = require("express")
const multer = require("multer")
const cloudinary = require("cloudinary").v2
const cors = require("cors")
const config = require("../../config")
const response = require("../../network/response")
const Controller = require("./index")
const auth = require("./secure")
const router = express.Router()
cloudinary.config({
cloud_name: config.cloudinary.name,
api_key: config.cloudinary.api_key,
api_secret: config.cloudinary.api_secret,
})
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "./public/uploads")
},
filename: function (req, file, cb) {
cb(null, file.originalname)
},
})
const upload = multer({
storage: storage,
limits: { fileSize: 1024 * 1024 * 5 },
})
// Set routes
router.use(cors())
router.get("/", auth("list"), list)
router.post("/", auth("add"), upload.single("file"), upsert)
// Router Functions
function list(req, res, next) {
Controller.list()
.then((post) => {
response.success(req, res, post, 200)
})
.catch(next)
}
function upsert(req, res, next) {
const path = req.file.path
const uniqueFilename = new Date().toISOString()
cloudinary.uploader.upload(
path,
{ public_id: `public/${uniqueFilename}` }, // directory and tags are optional
function (err, image) {
if (err) return res.send(err)
console.log("file uploaded to Cloudinary")
// remove file from server
const fs = require("fs")
fs.unlinkSync(path)
}
)
Controller.upsert(req.body, uniqueFilename)
.then((post) => {
response.success(req, res, post, 201)
})
.catch(next)
}
module.exports = router
When working with Zeit/Vercel, your API code runs in a Lambda.
This lambda is meant to be disposable when it finishes executing,
therefore the filesystem that you use in your API is destroyed upon the end of your upload execution.
(it works locally because no one destroys your computer at the end :D).
To get this scenario working, you'll need to write the uploaded file somewhere persistent, such as AWS S3.
I dont know if im using i18n library wrong but I am pretty sure im doing it right.
I have been trying to change the locale. Here is my current code. As you can see I call setLocale on the glocal i81n variable but it still prints out en
I have tried to do res.setLocale(req.args.lang); in my controller BUT IT STILL PRINTS OUT the default language of en.
I honestly don't know if this library is just broken or I'm doing this completely wrong. Please help!
/**
* Main Express Application: set up express app
*/
'use strict';
// require third-party node modules
const express = require('express');
const sslRedirect = require('heroku-ssl-redirect');
const compression = require('compression');
const cookieParser = require('cookie-parser');
const bodyParser = require('body-parser');
const passport = require('passport');
const helmet = require('helmet');
const http = require('http');
const cors = require('cors'); // handle cors
const i18n = require('i18n'); // set up language
// env variables
const { NODE_ENV, COOKIE, REDIS_URL, REDIS_HOST, REDIS_PORT } = process.env;
// server
function server() {
// require custom
const models = require('./models'); // establish and grab db connection
const cfgPassport = require('./services/passport'); // configuration for passport
// require custom middleware
const args = require('./middleware/args');
const error = require('./middleware/error');
const exit = require('./middleware/exit');
const { attachJWTAuth } = require('./middleware/auth');
// set up express app
const app = express();
const newServer = http.createServer(app);
const io = require('socket.io')(newServer); // socket.io
// set up redis with socket
const redis = require('socket.io-redis');
const socket = require('./services/socket'); // require socket service to initiate socket.io
io.adapter(redis(REDIS_URL));
// enable ssl redirect in production
app.use(sslRedirect());
// only log requests in development
NODE_ENV === 'development' ? app.use(require('morgan')('dev')) : null;
// add middleware and they must be in order
app.use(compression()); // GZIP all assets
app.use(cors()); // handle cors
app.use(helmet()); // protect against vulnerabilities
// app.use(rawBody); // adds rawBody to req object
// set up language
i18n.configure({
locales: ['en', 'ko'], // set the languages here
defaultLocale: 'en',
queryParameter: 'lang', // query parameter to switch locale (ie. /home?lang=ch) - defaults to NULL
cookie: COOKIE,
directory: __dirname + '/locales'
});
// you will need to use cookieParser to expose cookies to req.cookies
app.use(cookieParser());
// i18n init parses req for language headers, cookies, etc.
// NOTE: If user is logged in, locale is set in verifyJWTAuth method
app.use(i18n.init);
i18n.setLocale('ko');
console.log(i18n.getLocale());
// save raw body
function rawBodySaver(req, res, buf, encoding) {
if (buf && buf.length) req.rawBody = buf.toString(encoding || 'utf8');
}
// body parser
app.use(bodyParser.json({ limit: '32mb', verify: rawBodySaver })); // raw application/json
app.use(bodyParser.urlencoded({ limit: '32mb', verify: rawBodySaver, extended: false })); // application/x-www-form-urlencoded
// NOTE: take this out because it interferes with multer
// app.use(bodyParser.raw({ limit: '32mb', verify: rawBodySaver, type: () => true }));
// only secure in production
if (NODE_ENV === 'production') app.set('trust proxy', 1); // get ip address using req.ip
// passport config, must be in this order!
app.use(passport.initialize());
cfgPassport(passport); // set up passport
// custom middleware
app.use(args.attach); // set req.args
app.use(attachJWTAuth(passport));
app.use(exit.middleware); // stops here if server is in the middle of shutting down
// host public files
// app.use(express.static(__dirname + '/public'));
// set up routes
const router = require('./routes')(passport); // grab routes
app.use('/', router); // place routes here
// error middleware MUST GO LAST
app.use(error);
// io connection, call socket.connect
io.on('connection', socket.connect);
return newServer; // return newServer
}
module.exports = server(); // return server app for testing
I figured it out by accident.
This really should be in the documentation but I will post it here so that other poor souls who ran into this issue can try this out.
The reason why nothing was working WAS BECAUSE my .json file had a trailing comma. Therefore it wasn't a valid .json file.
It took me two days to accidentally discover this. i18n SHOULD HAVE THROW AN ERROR in the console if the .json files are invalid!!!
Anyways, I removed the trailing comma and everything just worked.