I have an API that downloads multiple files from AWS S3, creates a zip which is saved to disk, and sends that zip back to the client. The API works, but I have no idea how to handle the response / download the zip to disk on the client side.
This is my API:
reports.get('/downloadMultipleReports/:fileKeys', async (req, res) => {
var s3 = new AWS.S3();
var archiver = require('archiver');
const { promisify } = require('util');
var str_array = req.params.fileKeys.split(',');
console.log('str_array: ',str_array);
for (var i = 0; i < str_array.length; i++) {
var filename = str_array[i].trim();
var filename = str_array[i];
var localFileName = './temp/' + filename.substring(filename.indexOf("/") + 1);
console.log('FILE KEY >>>>>> : ', filename);
const params = { Bucket: config.reportBucket, Key: filename };
const data = await (s3.getObject(params)).promise();
const writeFile = promisify(fs.writeFile);
await writeFile(localFileName, data.Body);
}
// create a file to stream archive data to.
var output = fs.createWriteStream('reportFiles.zip');
var archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
// listen for all archive data to be written
// 'close' event is fired only when a file descriptor is involved
output.on('close', function() {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
// This event is fired when the data source is drained no matter what was the data source.
// It is not part of this library but rather from the NodeJS Stream API.
// #see: https://nodejs.org/api/stream.html#stream_event_end
output.on('end', function() {
console.log('Data has been drained');
});
// good practice to catch warnings (ie stat failures and other non-blocking errors)
archive.on('warning', function(err) {
if (err.code === 'ENOENT') {
// log warning
} else {
// throw error
throw err;
}
});
// good practice to catch this error explicitly
archive.on('error', function(err) {
throw err;
});
// pipe archive data to the file
archive.pipe(output);
// append files from a sub-directory, putting its contents at the root of archive
archive.directory('./temp', false);
// finalize the archive (ie we are done appending files but streams have to finish yet)
// 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
archive.finalize();
output.on('finish', () => {
console.log('Ding! - Zip is done!');
const zipFilePath = "./reportFiles.zip" // or any file format
// res.setHeader('Content-Type', 'application/zip');
fs.exists(zipFilePath, function(exists){
if (exists) {
res.writeHead(200, {
"Content-Type": "application/octet-stream",
"Content-Disposition": "attachment; filename=" + "./reportFiles.zip"
});
fs.createReadStream(zipFilePath).pipe(res);
} else {
response.writeHead(400, {"Content-Type": "text/plain"});
response.end("ERROR File does not exist");
}
});
});
return;
});
And this is how I am calling the API / expecting to download the response:
downloadMultipleReports(){
var fileKeysString = this.state.awsFileKeys.toString();
var newFileKeys = fileKeysString.replace(/ /g, '%20').replace(/\//g, '%2F');
fetch(config.api.urlFor('downloadMultipleReports', { fileKeys: newFileKeys }))
.then((response) => response.body())
this.closeModal();
}
How can I handle the response / download the zip to disk?
This is what ended up working for me:
Server side:
const zipFilePath = "./reportFiles.zip";
fs.exists(zipFilePath, function(exists){
if (exists) {
res.writeHead(200, {
"Content-Type": "application/zip",
"Content-Disposition": "attachment; filename=" + "./reportFiles.zip"
});
fs.createReadStream(zipFilePath).pipe(res);
} else {
response.writeHead(400, {"Content-Type": "text/plain"});
response.end("ERROR File does not exist");
}
});
Client side:
downloadMultipleReports(){
var fileKeysString = this.state.awsFileKeys.toString();
var newFileKeys = fileKeysString.replace(/ /g, '%20').replace(/\//g, '%2F');
fetch(config.api.urlFor('downloadMultipleReports', { fileKeys: newFileKeys }))
.then((res) => {return res.blob()})
.then(blob =>{
download(blob, 'reportFiles.zip', 'application/zip');
this.setState({isOpen: false});
})
}
Related
I would download file on local the create a stream then send to an API.
In localhost files get created via blobClient.downloadToFile(defaultFile);
But When I deploy function it can not find file to stream, so I think that the download does not happen or in bad location.
I get this error
[Error: ENOENT: no such file or directory, open 'D:\home\site\wwwroot\importPbix\exampleName.pbix'
Here's my code
const blobServiceClient = BlobServiceClient.fromConnectionString(
process.env.CONNEXION_STRING
);
const containerClient = blobServiceClient.getContainerClient(
params.containerName
);
const blobClient = containerClient.getBlobClient(process.env.FILE_LOCATION); // get file from storage
let blobData;
var defaultFile = path.join(params.baseDir, `${params.reportName}.pbix`); // use path module
let stream;
try {
blobData = await blobClient.downloadToFile(defaultFile);
console.log(blobData);
stream = fs.createReadStream(defaultFile);
} catch (error) {
params.context.log(error);
console.log(error);
}
var options = {
method: "POST",
url: `https://api.powerbi.com/v1.0/myorg/groups/${params.groupId}/imports?datasetDisplayName=${params.reportName}`,
headers: {
"Content-Type": "multipart/form-data",
Authorization: `Bearer ${params.accessToken} `,
},
formData: {
"": {
value: stream,
options: {
filename: `${params.reportName}.pbix`,
contentType: null,
},
},
},
};
//check if file keep in mem
return new Promise(function (resolve, reject) {
request(options, function (error, response) {
if (error) {
params.context.log(error);
reject(error);
} else {
params.context.log(response);
resolve(response.body);
}
fs.unlinkSync(defaultFile);
});
});
I found this post having same issue , that's why I user path module and passed __dirname to function params.baseDir.
If you want to download a file from Azure blob and read it as a stream, just try the code below, in this demo, I try to download a .txt file to a temp folder(you should create it first on Azure function)and print its content from the stream for a quick test:
module.exports = async function (context, req) {
const { BlockBlobClient } = require("#azure/storage-blob")
const fs = require('fs')
const connStr = '<connection string>'
const container = 'files'
const blobName = 'test.txt'
const tempPath = 'd:/home/temp/'
const tempFilePath = tempPath + blobName
const blobClient = new BlockBlobClient(connStr,container,blobName);
await blobClient.downloadToFile(tempFilePath).then(async function(){
context.log("download successfully")
let stream = fs.createReadStream(tempFilePath)
//Print text content,just check if stream has been readed successfully
context.log("text file content:")
context.log(await streamToString(stream))
//You can call your API here...
})
function streamToString (stream) {
const chunks = [];
return new Promise((resolve, reject) => {
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('error', (err) => reject(err));
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
})
}
context.res = {
body: 'done'
}
}
Result
File has been downloaded:
read as stream successfully:
I am trying to make a api for parsing a simple pdf in firebase. I am able to upload a resume using my api under any given user (using Bearer token), but after uploading a pdf, I made a pdf url. Then using that pdf url, I was trying to using "resume-parser" library for parsing the pdf, but it seems like not responding when I check in postman.
Here is the code for making api:
const firebase = require("firebase");
firebase.initializeApp(config);
const dataBase = require("../models");
const axios = require("axios");
const moment = require("moment");
const User = dataBase.User;
//.... I have login and authentication code here and which are working fine.
// Then I want to use this pdf parsing code
exports.pdfparse2 = (req, res) => {
const BusBoy = require("busboy");
const path = require("path");
const os = require("os");
const fs = require("fs");
var busboy = new BusBoy({ headers: req.headers });
const bucket = admin.storage().bucket("mybucketname.appspot.com");
let mimtype;
var saveTo;
let pdfFileName;
busboy.on("file", function(name, file, filename, encoding, mimetype) {
console.log(
"File [" +
name +
"]: filename: " +
filename +
", encoding: " +
encoding +
", mimetype: " +
mimetype
);
const imageExtension = filename.split(".")[filename.split(".").length - 1];
var fname = filename + "." + imageExtension;
pdfFileName = filename;
saveTo = path.join(os.tmpdir(), filename);
file.pipe(fs.createWriteStream(saveTo));
mimtype = mimetype;
});
busboy.on("finish", async function() {
await bucket
.upload(saveTo, {
resumable: false,
gzip: true,
metadata: {
metadata: {
contentType: mimtype
}
}
})
.then(() => {
const pdfUrl = `https://storage.googleapis.com/mybucketname.appspot.com/${pdfFileName}`;
return db.doc(`/users/${req.user.userId}`).update({ pdfUrl });
ResumeParser.parseResumeUrl(pdfUrl) // url
.then(data => {
resumeData = {
link: pdfUrl
};
db.doc(`/users/${req.user.userId}`).set(
{
resumeList: admin.firestore.FieldValue.arrayUnion(resumeData)
},
{ merge: true }
);
//console.log('Yay! ', data);
return res.status(200).json({
resume_data: data,
resume_link: pdfUrl
});
})
.catch(error => {
console.error(error);
});
})
.then(() => {
return res.json({ message: "Image Uploaded Successfully" });
})
.catch(err => {
console.error(err);
return res
.status(400)
.send(JSON.stringify(err, ["message", "arguments", "type", "name"]));
});
res.end();
});
req.pipe(busboy);
};
Then I checked in postman, it is giving me only json output {"message": "Image Uploaded Successfully"}, but pdf parsing is now working.
postman.png
Could anybody help me with that?
The issue:
I need to download a PDF file from my server but getting either "No file" or empty file
Details:
Here is my server-side code:
let fileBuffered = '';
// authentication for downloading a file from Dropbox API to my server
const dropbox = dropboxV2Api.authenticate({
token: process.env.DEV_DROPBOX_SECRET_KEY
});
// configuring parameters
const params = Object.freeze({
resource: "files/download",
parameters: {
path: `/${customerFileFolder}/${fileName}`
}
});
let dropboxPromise = new Promise(function (resolve, reject) {
dropbox(params, function (err, result) {
if (err) {
reject(err);
} else {
resolve(result);
}
}).on('data',function(data) {
fileBuffered += data;
})
const file = fileBuffered;
res.setHeader("Content-Type", "application/octet-stream");
res.send(file);
The PDF file's that I'm trying to download size is 139,694 bytes. The length of the fileBuffered variable is 132,597. Here is the content of the variable as it is shown in the debugger:
Seems like a legit PDF file
Here is the client-side
function documentFileDownload(fileName) {
const ip = location.host;
let request = $.ajax({
type: "POST",
url: `${http() + ip}/documentFileDownload`,
headers: {
"Accept": "application/octet-stream"
},
data: {
fileName: fileName
},
error: function (err) {
console.log("ERROR: " + err);
}
});
console.log(request);
return request;
}
Problem:
Then I get the response on a client-side it looks like this:
Note the size of the responseText: 254Kb.
What I actually get in the browser is a "Failed - No file" message
What else I tried:
I tried to play with different Content-Types (application/pdf, text/pdf) on a server-side and tried to convert the variable to base64 buffer
const file = `data:application/pdf;base64, ${Buffer.from(fileBuffered).toString("base64")}`;
and added res.setHeader("Accept-Encoding", "base64");
but still getting the same result.
Any ideas?
I found a solution. I missed a .on("end", ) event while reading data from Dropbox stream. Here is a working solution:
Here is the server-side:
let chunk = [];
let fileBuffered = '';
// authentication for downloading a file from Dropbox API to my server
const dropbox = dropboxV2Api.authenticate({
token: process.env.DEV_DROPBOX_SECRET_KEY
});
// configuring parameters
const params = Object.freeze({
resource: "files/download",
parameters: {
path: `/${customerFileFolder}/${fileName}`
}
});
let dropboxPromise = new Promise(function (resolve, reject) {
dropbox(params, function (err, result) {
if (err) {
reject(err);
} else {
resolve(result);
}
}).on('data',function(data) {
fileBuffered += data;
}).on('end', () => {
// console.log("finish");\
// generate buffer
fileBuffered = Buffer.concat(chunk);
});
const file = `data:application/pdf;base64, ${Buffer.from(fileBuffered).toString("base64")}`;
res.setHeader("Content-Type", "application/pdf");
res.send(file);
Client-side:
function documentFileDownload(fileName) {
const ip = location.host;
let request = $.ajax({
type: "POST",
url: `${http() + ip}/documentFileDownload`,
responseType: "arraybuffer",
headers: {
"Accept": "application/pdf"
},
data: {
fileName: fileName
},
error: function (err) {
console.log("ERROR: " + err);
}
});
// console.log(request);
return request;
}
Try adding dataType: "blob" in your $.ajax method
and within the headers object add this 'Content-Type', 'application/json'.
According to official document, createReadStream can accept a buffer type as path argument.
But many Q&A only offer solutions of how to send argument by string, not buffer.
How do I set a properly buffer argument to meet path of createReadStream?
This is my code:
fs.access(filePath, (err: NodeJS.ErrnoException) => {
// Response with 404
if (Boolean(err)) { res.writeHead(404); res.end('Page not Found!'); return; }
// Create read stream accord to cache or path
let hadCached = Boolean(cache[filePath]);
if (hadCached) console.log(cache[filePath].content)
let readStream = hadCached
? fs.createReadStream(cache[filePath].content, { encoding: 'utf8' })
: fs.createReadStream(filePath);
readStream.once('open', () => {
let headers = { 'Content-type': mimeTypes[path.extname(lookup)] };
res.writeHead(200, headers);
readStream.pipe(res);
}).once('error', (err) => {
console.log(err);
res.writeHead(500);
res.end('Server Error!');
});
// Suppose it hadn't cache, there is a `data` listener to store the buffer in cache
if (!hadCached) {
fs.stat(filePath, (err, stats) => {
let bufferOffset = 0;
cache[filePath] = { content: Buffer.alloc(stats.size, undefined, 'utf8') }; // Deprecated: new Buffer
readStream.on('data', function(chunk: Buffer) {
chunk.copy(cache[filePath].content, bufferOffset);
bufferOffset += chunk.length;
//console.log(cache[filePath].content)
});
});
}
});
```
Use the PassThrough method from the inbuild stream library:
const stream = require("stream");
let readStream = new stream.PassThrough();
readStream.end(new Buffer('Test data.'));
// You now have the stream in readStream
readStream.once("open", () => {
// etc
});
ORIGINAL
I'm having problems to upload a file (image) to Dropbox from Node.js using the official dropbox.js.
I want to upload a picture that I have in another server. For example with the dropbpox icon (www.dropbox.com/static/images/new_logo.png).
client.writeFile(file, 'www.dropbox.com/static/images/new_logo.png', function(error, stat) {
if (error) {
return es.send(error.status); // Something went wrong.
}
res.send("File saved as revision " + stat.revisionTag);
});
I know that this only creates a text file with the url, but how I can upload the picture to Dropbox?
I also try to download the file using http.get and then upload this to dropbox but it doesn't work.
Thanks.
UPDATE WITH MORE INFO
First I download the image from a remote url with this code:
var request = http.get(options, function(res){
var imagedata = ''
res.setEncoding('binary')
res.on('data', function(chunk){
imagedata += chunk
})
res.on('end', function(){
console.log("Image downloaded!");
fs.writeFile(local, imagedata, 'binary', function(err){
if (err) throw err
console.log('File saved.')
})
})
})
The file is saved correctly.
Then I trie to things:
Sending the 'imagedata' to Dropbox:
console.log("Image downloaded!");
client.writeFile(file, imagedata, function(error, stat) {
if (error) {
return response.send(error.status); // Something went wrong.
}
response.send("File saved as revision " + stat.revisionTag);
});
And something is uploaded to Dropbox but it's nothing useful.
Then I also tried to read the file from disc and then send it to Dropbox but it doesn't work neither:
fs.readFile(file, function(err, data) {
Use dropbox-js 0.9.1-beta1 or above to upload binary files from node.js. You need to pass it Buffer or ArrayBuffer instances. Try this code:
var req = http.get(options, function(res) {
var chunks = [];
res.on('data', function(chunk) {
chunks.push(chunk);
});
res.on('end', function() {
console.log("Image downloaded!");
var imageData = Buffer.concat(chunks);
client.writeFile(file, imageData, function(error, stat) {
if (error) {
return response.send(error.status);
}
response.send("File saved as revision " + stat.revisionTag);
});
});
});
```
Original answer: the dropbox-js README mentions that binary files don't work in node.js just yet.
I had issue as well, I just copied and modified a bit on the old dropbox-node npm(which is now deprecated), but I added following function on dropbox.js.
Client.prototype.writeFileNodejs = function(path, data, callback) {
var self = this;
fs.readFile(data.path, function(err, data) {
if (err) return callback(err);
var uri = "" + self.urls.putFile + "/" + (self.urlEncodePath(path));
if (typeof data === 'function') callback = data, data = undefined;
var oauth = {
consumer_key: self.oauth.key
, consumer_secret: self.oauth.secret
, token: self.oauth.token
, token_secret: self.oauth.tokenSecret
};
var requestOptions = { uri: uri, oauth: oauth };
requestOptions.body = data;
return request['put'](requestOptions, callback ?
function(err, res, body) {
if (err) return callback(err);
var contentType = res.headers['content-type'];
// check if the response body is in JSON format
if (contentType === 'application/json' ||
contentType === 'text/javascript') {
body = JSON.parse(body);
if (body.error) {
var err = new Error(body.error);
err.statusCode = res.statusCode;
return callback(err);
}
} else if (errors[res.statusCode]) {
var err = new Error(errors[res.statusCode]);
err.statusCode = res.statusCode;
return callback(err);
}
// check for metadata in headers
if (res.headers['x-dropbox-metadata']) {
var metadata = JSON.parse(res.headers['x-dropbox-metadata']);
}
callback(null, body, metadata);
} : undefined);
});
};
As well you would like to require request and fs to do this.
var request = require('request'),
fs = require('fs');