Develop Reference

aws-sdk Signed URL Failing on Heroku during production but works in development

I have my code written below, and all of this generates my signed URL perfectly fine when on development and the files that I want to get and upload locally work.
const S3 = require("aws-sdk/clients/s3");
const fs = require("fs");
const s3 = new S3({
region: process.env.AWS_BUCKET_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECRET_KEY,
signatureVersion: "v2",
});
const uploadFile = (file, id, directory) => {
const fileStream = fs.createReadStream(file.path);
const uploadParams = {
Bucket: process.env.AWS_BUCKET_NAME,
Body: fileStream,
Key: directory + id,
MimeType: file.mimetype,
};
return s3.upload(uploadParams).promise();
};
exports.uploadFile = uploadFile;
const deleteFile = (id, directory) => {
const uploadParams = {
Bucket: process.env.AWS_BUCKET_NAME,
Key: directory + id,
};
return s3.deleteObject(uploadParams).promise();
};
exports.deleteFile = deleteFile;
const getFileStream = ({ key }) => {
if (key) {
const downloadParams = {
Key: key,
Bucket: process.env.AWS_BUCKET_NAME,
};
return s3.getObject(downloadParams).createReadStream();
}
};
exports.getFileStream = getFileStream;
function generatePreSignedPutUrl({ key, operation }) {
var params = { Bucket: process.env.AWS_BUCKET_NAME, Key: key, Expires: 60 };
let x = s3.getSignedUrl(operation, params);
return x;
}
exports.generatePreSignedPutUrl = generatePreSignedPutUrl;
These are the requests I make from the client
const getSignedURL = async ({ key, operation }) =>
client.post(`${endpoint}/get-signed-url`, { key, operation });
const result = await getSignedURL({
key: directory + "/" + key,
operation: "getObject",
});
let url = result.data.data.url;
console.log({ url });
and this is the route on my server.
router.post("/get-signed-url", requireKey, async (req, res) => {
const { key, operation } = req.body;
try {
console.log({ b: req.body });
let url = generatePreSignedPutUrl({ key, operation });
console.log({ url });
res.json({ success: true, data: { url } });
} catch (e) {
console.log({ e });
res.status(400).json({ error: "Internal Server Error" });
}
});
I followed all of S3's documentation to get this set up and I am trying to make sure all files can be downloaded and uploaded securely from my application. Does anyone know how I can get this to work in production when the server is hosted in Heroku and the client is a nextjs site?

Uploading a Blob held in State to AWS S3

I have implemented a mic-recorder-to-mp3 component in a ReactJS NextJS app which stores a voice-memo recorded in the browser by the user and saves the resulting blob to React state, as well as a resulting MP3 to state as well.
I am struggling to upload either the blob or the MP3 file to AWS S3 - the problem is evident in that I cannot parse the req.body string which is received by the API.
Here is some code! This is the function that stores the raw audio as state:
const [audioBlob, setAudioBlob] = useState(null)
const [blobURL, setBlobUrl] = useState(null)
const [audioFile, setAudioFile] = useState(null)
const stopRecording = () => {
recorder.current
.stop()
.getMp3()
.then(([buffer, blob]) => {
const file = new File(buffer, 'audio.mp3', {
type: blob.type,
lastModified: Date.now()
})
setAudioBlob(blob)
const newBlobUrl = URL.createObjectURL(blob)
setBlobUrl(newBlobUrl)
setIsRecording(false)
setAudioFile(file)
})
.catch(e => console.log(e))
}
And this is the function that sends the payload to the API:
const submitVoiceMemo = async () => {
const filename = encodeURIComponent(audioFile)
const res = await fetch(`/api/upload-url?file=${filename}`)
const { url, fields } = await res.json()
const formData = new FormData()
Object.entries({ ...fields, audioFile }).forEach(([key, value]) => {
formData.append(key, value)
})
const upload = await fetch(url, {
method: 'PUT',
body: formData
})
if (upload.ok) {
console.log('Uploaded successfully!')
} else {
console.error('Upload failed.')
console.error()
}
}
This is the upload-url API Route:
module.exports = async (req, res) => {
try {
aws.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_1,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY_ID,
region: 'us-east-2',
signatureVersion: 'v4'
})
const s3 = new aws.S3()
const post = await s3.createPresignedPost({
Bucket: 'waveforms',
Key: `voicememo/${req.query.file}`,
ContentType: 'audio/mpeg',
ACL: 'public-read',
Expires: 60
})
res.status(200).json(post
} catch (e) {
res.status(500).json({ error: e.message })
}
}
It currently returns a 400 Bad Request error.
This is an alternative solution, which does upload an MP3 successfully to S3, however the file is not playing when accessed via the S3 console, although it does have a filesize.
module.exports = requireAuth(async (req, res) => {
try {
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_1,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY_ID,
region: 'us-east-2',
signatureVersion: 'v4'
})
const s3 = new AWS.S3();
const uuid = randomUUID()
const s3Params = {
Bucket: 'waveforms',
Key: `voicememo/${uuid}.mp3`,
Body: req.body,
ContentType: 'audio/mp3',
ACL: 'public-read'
}
await s3.upload(s3Params).promise()
} catch (e) {
res.status(500).json({ error: e.message })
}
})
And this is the accompanying fetch request.
const submitVoiceMemo = async () => {
try {
await fetch('/api/createVoiceMemo', {
method: 'PUT',
body: audioBlob
})
} catch (error) {
console.error(error)
}
}

How to send file received through api to google drive nodejs

I am new to Nodejs and trying to send file received through multer to google drive. However I am setting
var media = {
mimeType: req.file.mimetype,
body:req.file
};
Which is giving me error as
const index = require('../index.js');
const { google } = require('googleapis');
let drive = google.drive('v3');
const receive = multer();
router.post("/insertNewProduct", auth, receive.single('productUrl'), async (req, res) => {
try {
const data = req.body;
var folderId = '1v3tdYH_GLtCOOxhQ8HXDdRAb-2Jh455_';
let fs = require('fs')
var fileMetadata = {
'name': new Date().toISOString()+req.file.filename,
parents: [folderId]
};
var media = {
mimeType: req.file.mimetype,
body:req.file
};
drive.files.create({
auth: index.jwtClient,
resource: fileMetadata,
publishAuto: true,
media: media,
fields: 'id'
},
async function (err, file) {
if (err) {
// Handle error
console.error(err);
}
else {
console.log('********File create success. File Id: ', file.data.id);
res.status(201).send({
message: "Record Create Successfully",
data: newRows
});
}
}
);
}
}
} catch (e) {
console.log("Inside catch: " + e);
res.status(404).send({
message: e,
data: {}
});
}
});
As I understand the error, how I set the body:req.file is wrong.
Here's how req.file look like.
How can I pass req.file to
var media = {
mimeType: req.file.mimetype,
body:req.file
};
I tried body:req.file.buffer but it didn't work.

Async Function working in Express but not NestJs

I initially created a little express server to run a report and file write function.
var ssrs = require('mssql-ssrs');
var fs = require('fs');
const express = require('express')
const app = express()
const port = 3001
app.get('/', (req, res) => {
reportCreation();
res.send('File Created');
})
app.get('/api', (req, res) => {
reportCreation();
res.json({'File Created': true});
})
app.listen(port, () => {
console.log(`Report Api listening at http://localhost:${port}`)
})
The function reportCreation() is an async function which gets a report from a SSRS. This works fine
async function reportCreation() {
var serverUrl = 'http://reportServerName/ReportServer/ReportExecution2005.asmx';
ssrs.setServerUrl(serverUrl);
var reportPath = '/ApplicationPortalReports/TestReportNew';
var fileType = 'word';
var parameters = { ApplicationId: 3, TrainingCardId: 267, PortalPersonId: 52 }
var auth = {
username: 'USERNAME',
password: 'PASSWORD',
domain: 'dmz'
};
try {
var report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
} catch (error) {
console.log(error);
}
console.log(report);
try {
fs.writeFile('ReportApiTest.doc', report, (err) => {
if (!err) console.log('Data written');
});
} catch (error) {
console.log(error);
}
I have been working a lot with NestJs recently and wanted to use the same function but within a NestJs service.
#Injectable()
export class AppService {
async getReport(): Promise<string> {
const serverUrl = 'http://reportServerName/ReportServer/ReportExecution2005.asmx';
ssrs.setServerUrl(serverUrl);
const reportPath = '/ApplicationPortalReports/TestReportNew';
const fileType = 'word';
// var parameters = {appId: 3, ReportInstanceId: 1 }
const parameters = {ApplicationId: 3, TrainingCardId: 267, PortalPersonId: 52 };
const auth = {
username: 'USERNAME',
password: 'PASSWORD',
domain: 'dmz'
};
try {
var report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
} catch (error) {
console.log(error);
}
console.log(report);
// excel = xlsx
// word = doc
// pdf = pdf
try {
fs.writeFile('ReportApiTest.doc', report, (err) => {
if (!err) { console.log('Data written');
return 'File Written Succesfully'}
});
} catch (error) {
console.log(error);
return 'File Write Error'
}
}
}
As you can see the files are almost identical, but when I run it through NestJs I get an error which looks like a problem with the line
var report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
not awaiting. Why does this work with Express and not NestJS? Below is the error from NestJs
buffer.js:219
throw new ERR_INVALID_ARG_TYPE(
^
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be one of type string, Buffer, ArrayBuffer,
Array, or Array-like Object. Received type undefined
at Function.from (buffer.js:219:9)
at new Buffer (buffer.js:179:17)
at Object.createType3Message (C:\Projects\SSRS-report-api\ssrs-report-api\node_modules\httpntlm\ntlm.js:172:19)
at sendType3Message (C:\Projects\SSRS-report-api\ssrs-report-api\node_modules\httpntlm\httpntlm.js:77:23)
at Immediate._onImmediate (C:\Projects\SSRS-report-api\ssrs-report-api\node_modules\httpntlm\httpntlm.js:101:4)
within the mssql-ssrs node package the getReportByURL looks like this
async function getReportByUrl(reportPath, fileType, params, auth) {
try {
var config = {
binary: true, // very important
username: auth.userName,
password: auth.password,
workstation: auth.workstation,
domain: auth.domain,
url: soap.getServerUrl()
+ "?" + (testReportPath(reportPath).replace(/\s/g, '+'))
+ "&rs:Command=Render&rs:Format=" + reportFormat(fileType)
+ formatParamsToUrl(params)
};
} catch (err) { report.errorHandler(err) }
return new Promise((resolve, reject) => {
config.url = encodeURI(config.url);
httpntlm.post(config, function (err, res) {
if (res.statusCode === 500) { reject(res) }
if (err || res.statusCode !== 200) { reject(err) }
else { resolve(res.body) }
})
})
}
Here is the app.controller.ts
#Controller()
export class AppController {
constructor(private readonly appService: AppService) {}
#Get()
getHello(): Promise<string> {
return this.appService.getReport();
}
}

This is not an answer for the question. But after I see your code, I can see an error you will face in future if await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth) failed. Actually you see above error because of this.
The way you used the try catch is really bad.
Here's the way I code it.
#Injectable()
export class AppService {
async getReport(): Promise<string> {
const serverUrl = 'http://reportServerName/ReportServer/ReportExecution2005.asmx';
ssrs.setServerUrl(serverUrl);
const reportPath = '/ApplicationPortalReports/TestReportNew';
const fileType = 'word';
// var parameters = {appId: 3, ReportInstanceId: 1 }
const parameters = {ApplicationId: 3, TrainingCardId: 267, PortalPersonId: 52 };
const auth = {
username: 'USERNAME',
password: 'PASSWORD',
domain: 'dmz'
};
const report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
return new Promise(function(resolve, reject) {
fs.writeFile('ReportApiTest.doc', report, , function(err) {
if (err) reject(err);
resolve("File Created");
});
});
}
And in my controller
#POST
async writeFile() {
try {
const res = await this.appService.getReport();
return res;
} catch(err) {
// handle your error
}
}

I had fudged the code in the node_module changing the userName variable to username and had not done the same in the NestJS version. I forgot I had done that so now it is working.

Amazon S3 Remote File Upload with Axios

I am trying to write a function that would:
Take a remote URL as a parameter,
Get the file using axios
Upload the stream to amazon s3
And finally, return the uploaded url
I found help here on stackoverflow. So far, I have this:
/*
* Method to pipe the stream
*/
const uploadFromStream = (file_name, content_type) => {
const pass = new stream.PassThrough();
const obj_key = generateObjKey(file_name);
const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
s3.upload(params, function(err, data) {
if(!err){
return data.Location;
} else {
console.log(err, data);
}
});
return pass;
}
/*
* Method to upload remote file to s3
*/
const uploadRemoteFileToS3 = async (remoteAddr) => {
axios({
method: 'get',
url: remoteAddr,
responseType: 'stream'
}).then( (response) => {
if(response.status===200){
const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
const content_type = response.headers['content-type'];
response.data.pipe(uploadFromStream(file_name, content_type));
}
});
}
But uploadRemoteFileToS3 does not return anything (because it's a asynchronous function). How can I get the uploaded url?
UPDATE
I have further improved upon the code and wrote a class. Here is what I have now:
const config = require('../config.json');
const stream = require('stream');
const axios = require('axios');
const AWS = require('aws-sdk');
class S3RemoteUploader {
constructor(remoteAddr){
this.remoteAddr = remoteAddr;
this.stream = stream;
this.axios = axios;
this.config = config;
this.AWS = AWS;
this.AWS.config.update({
accessKeyId: this.config.api_key,
secretAccessKey: this.config.api_secret
});
this.spacesEndpoint = new this.AWS.Endpoint(this.config.endpoint);
this.s3 = new this.AWS.S3({endpoint: this.spacesEndpoint});
this.file_name = this.remoteAddr.substring(this.remoteAddr.lastIndexOf('/')+1);
this.obj_key = this.config.subfolder+'/'+this.file_name;
this.content_type = 'application/octet-stream';
this.uploadStream();
}
uploadStream(){
const pass = new this.stream.PassThrough();
this.promise = this.s3.upload({
Bucket: this.config.bucket,
Key: this.obj_key,
ACL: this.config.acl,
Body: pass,
ContentType: this.content_type
}).promise();
return pass;
}
initiateAxiosCall() {
axios({
method: 'get',
url: this.remoteAddr,
responseType: 'stream'
}).then( (response) => {
if(response.status===200){
this.content_type = response.headers['content-type'];
response.data.pipe(this.uploadStream());
}
});
}
dispatch() {
this.initiateAxiosCall();
}
async finish(){
//console.log(this.promise); /* return Promise { Pending } */
return this.promise.then( (r) => {
console.log(r.Location);
return r.Location;
}).catch( (e)=>{
console.log(e);
});
}
run() {
this.dispatch();
this.finish();
}
}
But still have no clue how to catch the result when the promise is resolved. So far, I tried these:
testUpload = new S3RemoteUploader('https://avatars2.githubusercontent.com/u/41177');
testUpload.run();
//console.log(testUpload.promise); /* Returns Promise { Pending } */
testUpload.promise.then(r => console.log); // does nothing
But none of the above works. I have a feeling I am missing something very subtle. Any clue, anyone?

After an upload you can call the getsignedurl function in s3 sdk to get the url where you can also specify the expiry of the url as well. You need to pass the key for that function. Now travelling will update with example later.
To generate a simple pre-signed URL that allows any user to view the
contents of a private object in a bucket you own, you can use the
following call to getSignedUrl():
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myKey'};
s3.getSignedUrl('getObject', params, function (err, url) {
console.log("The URL is", url);
});
Official documentation link
http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/guide/node-examples.html
Code must be something like this
function uploadFileToS3AndGenerateUrl(cb) {
const pass = new stream.PassThrough();//I have generated streams from file. Using this since this is what you have used. Must be a valid one.
var params = {
Bucket: "your-bucket", // required
Key: key , // required
Body: pass,
ContentType: 'your content type',
};
s3.upload(params, function(s3Err, data) {
if (s3Err) {
cb(s3Err)
}
console.log(`File uploaded successfully at ${data.Location}`)
const params = {
Bucket: 'your-bucket',
Key: data.key,
Expires: 180
};
s3.getSignedUrl('getObject', params, (urlErr, urlData) => {
if (urlErr) {
console.log('There was an error getting your files: ' + urlErr);
cb(urlErr);
} else {
console.log(`url: ${urlData}`);
cb(null, urlData);
}
})
})
}

Please check i have update your code might its help you.
/*
* Method to upload remote file to s3
*/
const uploadRemoteFileToS3 = async (remoteAddr) => {
const response = await axios({
method: 'get',
url: remoteAddr,
responseType: 'stream'
})
if(response.status===200){
const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
const content_type = response.headers['content-type'];
response.data.pipe(uploadFromStream(file_name, content_type));
}
return new Promise((resolve, reject) => {
response.data.on('end', (response) => {
console.log(response)
resolve(response)
})
response.data.on('error', () => {
console.log(response);
reject(response)
})
})
};
*
* Method to pipe the stream
*/
const uploadFromStream = (file_name, content_type) => {
return new Promise((resolve, reject) => {
const pass = new stream.PassThrough();
const obj_key = generateObjKey(file_name);
const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
s3.upload(params, function(err, data) {
if(!err){
console.log(data)
return resolve(data.Location);
} else {
console.log(err)
return reject(err);
}
});
});
}
//call uploadRemoteFileToS3
uploadRemoteFileToS3(remoteAddr)
.then((finalResponse) => {
console.log(finalResponse)
})
.catch((err) => {
console.log(err);
});