I am trying to parse Excel file data in server using hapi.js. I am getting file but data is in binary stream.
This is the code I am using and it need to parse file data to JSON from binary stream. So, XLSX file is coming to server side API as a binary stream. Now I need to read that binary stream and create file back and save locally at server API
server.route({
method: 'POST',
path: `${path..'/xlsx/')}get`,
config: {
handler: function (request, reply) {
const payload = request.payload
console.log(payload);
reply ('done');
},
payload: {
output: 'stream',
parse: true,
allow: 'multipart/form-data'
},
if (data.file) {
var name = data.file.filename;
var path = dirname + "/uploads/" + name;
var file = fs.createWriteStream(path);
console.log(path);
file.on('error', function (err) {
console.error(err)
});
data.file.pipe(file);
data.file.on('end', function (err) {
var ret = {
filename: data.file.filename,
headers: data.file.headers
}
reply(JSON.stringify(ret));
});
next();
Related
I am trying to parse excel file data in the server and it is working fine but the problem is that it is saving file which we upload in the local disk in upload folder and then it is reading data.Here is the code which is working fine but when uploading large xlsx file is the issue because it will require extra memory and create same as another file in local disk.
server.route({
method: 'post',
path: `${(options.apiBase || '/xlsx/')}get`,
config: {
payload: {
output: 'stream',
parse: true,
allow: 'multipart/form-data'
}
},
handler: function (req, reply) {
try {
const data = req.payload;
if (data.file) {
let name = data.file.hapi.filename;
console.log("FIlename: " + name);
let path = __dirname + "/uploads/" + name;
let file = fs.createWriteStream(path);
data.file.pipe(file);
data.file.on('end', function (err) {
if (typeof require !== 'undefined')
XLSX = require('xlsx');
const workbook = XLSX.readFile(path);
console.log("row======>>>>");
const sheet_name_list = workbook.SheetNames;
const content =
XLSX.utils.sheet_to_json(workbook.Sheets[sheet_name_list[0]]);
console.log(content);
var ret = {
filename: data.file.hapi.filename,
headers: data.file.hapi.headers
}
reply(JSON.stringify(content));
})
} catch (err) {
console.log('Err----------------------' + err);
// error handling
reply(Boom.badRequest(err.message, err));
}
}
});
next();
You need to set config.payload.parse to false. This will return the raw stream. But realize that if you're uploading multipart/form-data, you will need to parse it yourself.
config: {
payload: {
output: 'stream',
parse: false,
allow: 'multipart/form-data'
}
}
This is handled by the subtext module. See https://github.com/hapijs/subtext/blob/0bf83af78e364518577913db1bbc9c27bc739d7a/lib/index.js#L67
I need to receive a file from the user in a React UI (using axios), send this file to a NodeJS method via POST using Express and then, I need to send this same file via POST directly to another API (.NET WebAPI) without saving on disk. I'm trying to pass the data file directly (req.files.file.data), but the .NET API doesn't recognise the file in the request.
Is there any possibility to create a temporary file within a "var" and send it as I do when I read a file from disk?
This example works fine with local files:
var formData = {
form: "{ abn: '666666666' }",
files: JSON.stringify([
{
name: "file1",
type: "1",
number: "6666"
},
{
name: "file2",
type: "3",
number: "3333"
},
]),
// Pass multiple values /w an Array
attachments: [
fs.createReadStream(__dirname + '/../file1.txt'),
fs.createReadStream(__dirname + '/../file2.txt')
],
};
var reqs = request.post({url:'https://tms-dev-api.micway.com.au/api/company/file', formData: formData, json: true}, function optionalCallback(err, response, body) {
if (err) {
return res.send(err);
}
return res.send({
status: response.statusCode,
body: body
});
});
I am creating a local file and then sending it through; however, I don't believe that it is the best way to do it. Like This:
router.post('/file', function(req, res) {
var fstream;
req.pipe(req.busboy);
req.busboy.on('file', function (fieldname, file, filename) {
fstream = fs.createWriteStream(__dirname + '/' + filename);
file.pipe(fstream);
var formData = {
attachments: [
fs.createReadStream(__dirname + '/' + filename)
],
};
var reqs = request.post({url:'https://tms-dev-api.micway.com.au/api/company/file', formData: formData, json: true}, function optionalCallback(err, response, body) {
if (err) {
return res.send(err);
}
console.log({
status: response.statusCode,
body: body
})
return res.send({
status: response.statusCode,
body: body
});
});
});
});
Thanks!
The solution that I find effective, is by saving it during execution, reading and manipulating the data in other to send it to the other API, and immediately delete it from Disk as soon as I get the response.
Create a PassThrough() stream instead of writing the file to the disk:
const { PassThrough } = require('stream')
...
fstream = new PassThrough()
file.pipe(fstream);
I created a node.js server that uses busboy to take requests, and pipe the files to Imgur for upload. However, I keep getting an "Uploading file too fast!" response from Imgur, and I'm not sure exactly what the problem is. Here is the code snippet involving busboy:
var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();
router.post('/u', function(req, res, next) {
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if(fieldname == 'image') {
var options = {
uri: 'https://api.imgur.com/3/image',
method: 'POST',
headers: {
'Authorization': 'Client-ID ' + clientID // put client id here
},
form: {
image: file,
type: 'file'
}
};
request(options)
.then(function(parsedBody) {
console.log(parsedBody);
})
.catch(function(err) {
console.log(err);
});
}
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('field');
});
busboy.on('finish', function() {
res.status(200).end();
});
req.pipe(busboy);
});
As you can see I'm piping the request file directly into my request for imgur. Providing a ReadStream by simply saving the file to disc and then using fs.createReadStream() works perfectly, so I'm not really sure why trying to pipe directly from request to request gives me the error. The exact response I'm getting from Imgur is:
StatusCodeError: 400 - {"data":{"error":"Uploading file too fast!","request":"\/3\/image","method":"POST"},"success":false,"status":400}
If anyone has encountered this before, it would be helpful...
The first issue is that you should be using formData instead of form for file uploads. Otherwise, the request library won't send the correct HTTP request.
The second issue is that the stream object won't have the correct content length until it's fully processed. We can buffer the data ourselves and pass it after the initial file stream from busboy has processed.*
This gives us something that looks like
var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();
router.post('/u', function(req, res, next) {
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if(fieldname == 'image') {
// the buffer
file.fileRead = [];
file.on('data', function(data) {
// add to the buffer as data comes in
this.fileRead.push(data);
});
file.on('end', function() {
// create a new stream with our buffered data
var finalBuffer = Buffer.concat(this.fileRead);
var options = {
uri: 'https://api.imgur.com/3/image',
method: 'POST',
headers: {
'Authorization': 'Client-ID ' + clientID // put client id here
},
formData: {
image: finalBuffer,
type: 'file'
}
};
request(options)
.then(function(parsedBody) {
console.log(parsedBody);
})
.catch(function(err) {
console.log(err);
});
});
}
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('field');
});
busboy.on('finish', function() {
res.status(200).end();
});
req.pipe(busboy);
});
Code for the buffering is from http://thau.me/2014/02/nodejs-streaming-files-to-amazons3/
Lastly, you may want to consider using the request library, as the request-promise library discourages the use of streams. See the github repo for more details: https://github.com/request/request-promise
save image in a folder using nodeJS and angularJS
Hi, i want save images in a folder, I have my backEnd in NodeJS and my frondEnd in AngularJS using ng-flow; my angular service code is the following
addImages : function (file) {
var uploadUrl = localPath + "addImages";
var fd = new FormData();
fd.append('file', file);
return $http.post(uploadUrl, fd, {
transformRequest: angular.identity,
headers: { 'Content-Type': undefined,
'file-name' : undefined,
'file-location' : undefined,
'user-id' : undefined
}
});
}
and i have the following POST service in Node
router.post('/addImages', function (req, res) {
console.log(req);
var tempPath = req.files.file.path;
console.log(tempPath);
var targetPath = path.resolve('/../static/image/image.png');
console.log(targetPath);
console.log(req.files.file.name);
if (path.extname(req.files.file.name).toLowerCase() === '.png') {
fs.rename(tempPath, targetPath, function(err) {
if (err) throw err;
console.log("Upload completed!");
});
} else {
fs.unlink(tempPath, function () {
if (err) throw err;
console.error("Only .png files are allowed!");
});
}
// ...
});
the error that i get is POST /assetImage/addImages 500
someone know a more easy form to get a image in angular and save in a folder using node?
Edit
when I print file using console.log i get o file object like File { name: "howls-moving-castle-40994-1920x1200…", lastModified: 1431122408000, lastModifiedDate: Date 2015-05-08T22:00:08.000Z, size: 910465, type: "image/jpeg" } but when print the fd get a empty object FormData { } I intentend pass file whitout using formData but in node get a Empty object to
If you are using Express.js, body-parser does not handle multipart bodies, due to their complex and typically large nature. For multipart bodies, you may be interested in the following modules:
busboy
multiparty
formidable
multer
See the suggested modules here for parsing multipart bodies.
I am trying to download a File from google drive using Google SDK API using node.js.
But I am unable to write/save file at server side - node.js
Code:-
var GoogleTokenProvider = require("refresh-token").GoogleTokenProvider,
async = require('async'),
fs = require("fs"),
request = require('request'),
_accessToken;
var _XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var https = require('https');
const CLIENT_ID = "";
const CLIENT_SECRET = "";
const REFRESH_TOKEN = '';
const ENDPOINT_OF_GDRIVE = 'https://www.googleapis.com/drive/v2';
async.waterfall([
//-----------------------------
// Obtain a new access token
//-----------------------------
function(callback) {
var tokenProvider = new GoogleTokenProvider({
'refresh_token': REFRESH_TOKEN,
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET
});
tokenProvider.getToken(callback);
},
//--------------------------------------------
// Retrieve the children in a specified folder
//
// ref: https://developers.google.com/drive/v2/reference/files/children/list
//-------------------------------------------
function(accessToken, callback) {
_accessToken = accessToken;
request.get({
'url': ENDPOINT_OF_GDRIVE + '/files?' + "q='root' in parents and (mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')",
'qs': {
'access_token': accessToken
}
}, callback);
},
//----------------------------
// Parse the response
//----------------------------
function(response, body, callback) {
var list = JSON.parse(body);
if (list.error) {
return callback(list.error);
}
callback(null, list.items[0]);
},
//-------------------------------------------
// Get the file information of the children.
//
// ref: https://developers.google.com/drive/v2/reference/files/get
//-------------------------------------------
function(children, callback) {
var xhr = new _XMLHttpRequest();
xhr.open('GET', children.downloadUrl);
xhr.setRequestHeader('Authorization', 'Bearer ' + _accessToken);
xhr.onload = function() {
console.log("xhr.responseText", xhr.responseText)
fs.writeFile("download.docx", xhr.responseText)
callback(xhr.responseText);
};
xhr.onerror = function() {
callback(null);
};
xhr.send();
}
],
function(err, results) {
if (!err) {
console.log(results);
}
});
I am getting this in console:-
Content of xhr.responseText is something like that
��▬h��↕E6M��~��3�3∟�9�� � �►��/2�:���♂�4��]�♀I�R���►
$SB6Q���c↔��H�=;+
���►q�3Tdכ��#!T��hEl_�|�I�↨��h(�^:▬�[h̓D♠��f���♠*���ݾ��M→
�1⌂♦"N�↑�o�]�7U$��A6����♠�W��k`�f▬♫��K�Z�^‼�0{<Z�▼�]F�����
���J♥A♀��♣�a�}7�
"���H�w"�♥���☺w♫̤ھ�� �P�^����O֛���;�<♠�aYՠ؛`G�kxm��PY�[��g
Gΰino�/<���<�1��ⳆA$>"f3��\�ȾT��∟I S�������W♥����Y
Please help me to know what is the format of the data I am getting from Drive Api and write it in which format so that I got a complete .docx file
Edit
I am open to use any method other than xmlRequest if it helps me downloading the file(.docx).
node-XMLHttpRequest, it seems, does not support binary downloads - see this issue. What you are seeing is the file's binary contents converted into String which, in JavaScript, is an irreversible and destructive process for binary data (which means you cannot convert the string back to buffer and get the same data as the original contents).
Using request, you can download a binary file this way:
var request = require('request')
, fs = require('fs')
request.get(
{ url: 'your-file-url'
, encoding: null // Force Request to return the data as Buffer
, headers:
{ Authorization: 'Bearer ' + accessTokenHere
}
}
, function done (err, res) {
// If all is well, the file will be at res.body (buffer)
fs.writeFile('./myfile.docx', res.body, function (err) {
// Handle err somehow
// Do other work necessary to finish the request
})
}
)
Note: This will buffer the whole file into memory before it can be saved to disk. For small files, this is fine, but for larger files, you might want to look into implementing this as a streamed download. This SO question already answers that, I recommend you have a look.
More information about how to authorize your requests can be found on Google Developers docs.
Complete Working example: Downloading file from GoogleDrive - Node.js API
var GoogleTokenProvider = require("refresh-token").GoogleTokenProvider,
async = require('async'),
fs = require("fs"),
request = require('request'),
_accessToken;
const CLIENT_ID = "";
const CLIENT_SECRET = "";
const REFRESH_TOKEN = '';
const ENDPOINT_OF_GDRIVE = 'https://www.googleapis.com/drive/v2';
async.waterfall([
//-----------------------------
// Obtain a new access token
//-----------------------------
function(callback) {
var tokenProvider = new GoogleTokenProvider({
'refresh_token': REFRESH_TOKEN,
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET
});
tokenProvider.getToken(callback);
},
//--------------------------------------------
// Retrieve the children in a specified folder
//
// ref: https://developers.google.com/drive/v2/reference/files/children/list
//-------------------------------------------
function(accessToken, callback) {
_accessToken = accessToken;
request.get({
'url': ENDPOINT_OF_GDRIVE + '/files?' + "q='root' in parents and (mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')",
'qs': {
'access_token': accessToken
}
}, callback);
},
//----------------------------
// Parse the response
//----------------------------
function(response, body, callback) {
var list = JSON.parse(body);
if (list.error) {
return callback(list.error);
}
callback(null, list.items);
},
//-------------------------------------------
// Get the file information of the children.
//
// ref: https://developers.google.com/drive/v2/reference/files/get
//-------------------------------------------
function(children, callback) {
for(var i=0;i<children.length;i++) {
var file = fs.createWriteStream(children[i].title);
// Downnload and write file from google drive
(function(child) {
request.get(
{ url: child.downloadUrl
, encoding: null // Force Request to return the data as Buffer
, headers:
{ Authorization: 'Bearer ' + _accessToken
}
}
, function done (err, res) {
res.pipe(file)
// If all is well, the file will be at res.body (buffer)
fs.writeFile('./' + child.title, res.body, function (err) {
if(!err) {
console.log('done')
} else {
console.log(err)
}
// Handle err somehow
// Do other work necessary to finish the request
})
}
)
})(children[i])
}
}
],
function(err, results) {
if (!err) {
console.log(results);
}
});
I was just having issues with this, I've included an example of how I managed to get this working using the Google API Node.js library: https://gist.github.com/davestevens/6f376f220cc31b4a25cd