Develop Reference

How to read sheet data of uploaded file with using sheetJS and node?

So the file uploaded is an excel file that sheetJS needs to read, otherwise it will show as {}.
app.post('/sendExcel', function(req, res) {
let data = req.body;
var workbook = sheetJS.read(data, {type: 'buffer'});
console.log(workbook.Sheets['Sheet1); //prints... "{ A1: { t: 's', v: '[object Object]' }, '!ref': 'A1' }"
let excel = workbook.Sheets['Sheet1']['A1']['v'][0]; //prints... "["
So I've tried various things including changing the type client side as I had problems with it being of type buffer. So now it works partially, but I still can't access the data in the sheet.
As an example, I used the file path instead here, and it's shown to work as normal.
app.get('/excel', function(err, res, data) {
var wb = sheetJS.readFile("data.xlsx");
let excel = wb.Sheets['Sheet1']['A1']['v'];
console.log(excel); //this prints "vehicle", which is what is supposed to happen, not "[".
res.send(excel)
});
I am supposed to get the excel data from the form upload. That's the issue. It is is now successful when sending to the db, but will not access the whole data. I believe I need to change it back to an array.
You can use:
var fileReader = new FileReader();
fileReader.readAsArrayBuffer(workbook);
But this will not run in app.js
Here is my other answer with client-side and server-side. It might be helpful to others.
Javascript Read Excel file on server with SheetJS

Don't use the file reader. Append the excel sheet to the form in the body normally.
Client side:
let excelInput = document.getElementById("fileToUpload");
//excelInput: this html element allows you to upload the excel sheet to it
let excelFile = excelInput.files[0];
let form = new FormData();
form.append("excel", excelFile);
fetch('/sendExcel', {method: "POST", body: form})
.then((data) => {
console.log('Success:', data);
})
.catch((error) => {
console.error('Error:', error);
});
Then use formidable server side.
Server side:
const sheetJS = require('xlsx');
const formidable = require('formidable');
app.post('/excel', function(req, res) {
let data = req.body;
const form = formidable({ multiples: true });
form.parse(req, (err, fields, files, next) => {
if (err) {
next(err);
return;
}
var f = files[Object.keys(files)[0]];
var workbook = sheetJS.readFile(f.path);
res.send(workbook);
});
});
So formidable has to be used otherwise it won't work. Then you can use sheetJS.readFile instead of sheetJS.read.

s3 isn't uploading file and getting error of SignatureDoesNotMatch

I'm trying to add images to my s3 bucket in aws, but it doesn't seem to work. I get the error of SignatureDoesNotMatch
Here's how I'm uploading the file/image:
FrontEnd
const file = e.target.files[0];
const fileParts = file.name.split('.');
const fileName = fileParts[0];
const fileType = fileParts[1];
const response = axios.post('api/aws/sign_s3', { fileName, fileType );
Backend
router.post('/sign_s3', async (req, res) => {
aws.config.update({
accessKeyId: config.aws.accessKey,
secretAccessKey: config.aws.secretKey,
region: 'us-west-1'
});
const s3 = new aws.S3(); // Create a new instance of S3
const fileName = req.body.fileName;
const fileType = req.body.fileType;
const s3Params = {
Bucket: config.aws.bucketName,
Key: fileName,
Expires: 500,
ContentType: fileType,
ACL: 'public-read'
};
s3.getSignedUrl('putObject', s3Params, (err, data) => {
if (err) return res.send(err);
const returnData = {
signedRequest: data,
url: `https://sim-to-do.s3.amazonaws.com/${fileName}`
};
res.json({ success: true, responseData: returnData });
});
});
I get two urls. When I go to the first one, I get the following error code:
SignatureDoesNotMatch
Error Message
The request signature we calculated does not match the signature you provided. Check your key and signing method.
What am I doing wrong? What's the correct way of uploading a file to aws s3?

I was able to fix this issue after removing the Content-Type from the headers.

If you get "Signature does not match", it's highly likely you used a wrong secret access key. Can you double-check access key and secret access key to make sure they're correct?
from awendt answer

Bot Framework : Error while creating a csv file in Node Js

I'm developing a chatbot using Microsoft Bot Framework with Node Js.
My purpose is to send the user a csv file while he asks me something.
I've implemented the following function but while I download the file 'prova.csv' the format is not recognized.
Inspecting well the format in output is of the type: "data"
Anyone can help me what's wrong? Thanks
function (session, results, next) {
if (results.response.entity === 'Si') {
const contentType = 'text/csv';
const response = session.dialogData.far_ric_formato.response.rows;
const csv = response.map(ric => `${ric.num};${ric.pin}`).join('\n');
session.send({
text: 'Ecco il CSV pronto per il download (MOCK)',
attachments: [
{
contentType: contentType,
contentUrl: `data:${contentType};base64,${Buffer.from(csv).toString('base64')}`,
name: 'prova.csv'
}
]
});

Base64 string of a .csv file can't be directly rendered on client side, as a workaround you can for example code like this:
var restify = require('restify');
var fs = require('fs');
bot.dialog('download', (session, result)=>{
fs.readFile('./files/test.csv', function(err, data){
var contentType = 'text/csv';
var base64 = Buffer.from(data).toString('base64');
var msg = new builder.Message(session)
.addAttachment({
contentUrl: 'http://localhost:3978/csv/'+base64, //replace with your server url + base64 string.
contentType: contentType,
name: 'MyTest.csv',
});
session.send(msg);
});
}).triggerAction({matches:/^download/i});
server.get('/csv/:base64code', (req, res, next)=>{
let base64code = req.params.base64code;
res.header('Content-disposition', 'inline; filename=test.csv');
res.header('Content-type', 'application/csv');
res.send(Buffer.from(base64code, 'base64'));
});
When user trigger the download dialog, it will send this file as attachment and when user click on this file, this .csv file will be downloaded in user's client side.

Create and Send Zip file -NODE JS

I'm trying to create and then send zip file to client. I know how to create it but I've got a problem with send it to client. I tried many ways.
I'm sending POST request from Client and as response I want to send a file.
This is my server-site example code
var Zip = require('node-zip');
router.post('/generator', function(req, res, next) {
var zip = new Zip;
zip.file('hello.txt', 'Hello, World!');
var options = {base64: false, compression:'DEFLATE'};
fs.writeFile('test1.zip', zip.generate(options), 'binary', function (error) {
console.log('wrote test1.zip', error);
});
res.setHeader('Content-disposition', 'attachment; filename=test1.zip');
res.download('test1.zip');
}
});
I also tried something like this:
res.setHeader('Content-disposition', 'attachment; filename=' + filename);
res.setHeader('Content-type', mimetype);
var filestream = fs.createReadStream(file);
filestream.pipe(res);
I tried to use such libraries as:
node-zip
archiver
Can anyone explain me how to do that ?

This module works fine too: https://www.npmjs.com/package/adm-zip
Example without creating temporary zip file in server:
var AdmZip = require('adm-zip');
router.get('/zipFilesAndSend', function(req, res) {
var zip = new AdmZip();
// add local file
zip.addLocalFile("./uploads/29/0046.xml");
// get everything as a buffer
var zipFileContents = zip.toBuffer();
const fileName = 'uploads.zip';
const fileType = 'application/zip';
res.writeHead(200, {
'Content-Disposition': `attachment; filename="${fileName}"`,
'Content-Type': fileType,
})
return res.end(zipFileContents);
});

Try this express-easy-zip npm package to generate a zip file from a local folder path and send it as a download to the client.
var zip = require('express-easy-zip');
var app = require('express')();
app.use(zip());
app.get('my-route/zip', async function(req, res) {
var dirPath = __dirname + "/uploads";
await res.zip({
files: [{
path: dirPath,
name: 'Package'
}],
filename: 'Package.zip'
});
});

I haven't worked with node-zip or archiver before (I usually just use the built-in zlib module), but one thing I noticed right away is that you should place res.download inside the callback of writeFile. That way it will only send the file once it has been fully written to disk.
fs.writeFile('test1.zip', zip.generate(options), 'binary', function (error) {
res.download('test1.zip');
});
I hope this solution works for you, if it doesn't feel free to comment.
Also, I think res.download sets the Content-disposition header for you, you don't need to set it manually. Not 100% sure on that one though.

Above solutions work.(above solutions generate zip and send it to frontend as data in response. In order to make it as downloadable following code will work) I was using express-zip. It is compressing files and sending data to frontend from backend(node). But in frontend I was getting only data in response. In my case I want user can be able to download the zip which sent by server. To solve this I followed following approach. For generating download window in browser i used downloadjs (we can follow another approach but i find this easy)
Front-End
const download = require('downloadjs')
return axios({
url:process.env.API_HOST+'/getuploadedfiles',
method:'get',
headers:{
'Content-Type': 'multipart/form-data',
withCredentials:true,
},
responseType:'arraybuffer' // If we don't mention we can't get data in desired format
})
.then(async response => {
console.log("got al files in api ");
let blob = await new Blob([response.data], { type: 'application/zip' }) //It is optional
download(response.data,"attachement.zip","application/zip") //this is third party it will prompt download window in browser.
return response.data;
})
Backe-End
const zip = require('express-zip');
app.use('/getuploadedfiles',function(req,res){
res.zip([
{path:'/path/to/file/file2.PNG',name:'bond.png'},
{path:'/path/to/file/file1.PNG',name:'james.png'}
])

nodejs web scraper for password protected website

I am trying to scrape a website using nodejs and it works perfectly on sites that do not require any authentication. But whenever I try to scrape a site with a form that requires username and password I only get the HTML from the authentication page (that is, if you would click 'view page source' on the authentication page it self, that is the HTML I get). I am able to get the desired HTML using curl
curl -d "username=myuser&password=mypw&submit=Login" URL
Here is my code...
var express = require('express');
var fs = require('fs'); //access to file system
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function(req, res){
url = 'myURL'
request(url, function(error, response, html){
// check errors
if(!error){
// Next, we'll utilize the cheerio library on the returned html which will essentially give us jQuery functionality
var $ = cheerio.load(html);
var title, release, rating;
var json = { title : "", release : "", rating : ""};
$('.span8 b').filter(function(){
// Let's store the data we filter into a variable so we can easily see what's going on.
var data = $(this);
title = data.first().text();
release = data.text();
json.title = title;
json.release = release;
})
}
else{
console.log("Error occurred: " + error);
}
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err){
console.log('File successfully written! - Check your project directory for the output.json file');
})
res.send('Check your console!')
})
})
app.listen('8081')
console.log('Magic happens on port 8081');
exports = module.exports = app;
I have tried the following...
var request = require('request',
username:'myuser',
password:'mypw');
This just returns the authentication page's HTML
request({form: {username:myuser, password:mypw, submit:Login}, url: myURL}, function(error, response, html){
...
...
...
}
This also just returns the authentication page's HTML
So my question is how do I achieve this using nodejs?

you shouldn't use .get but .post and put the post param (username and password) in your call
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: url,
body: "username=myuser&password=mypw&submit=Login"
}, function(error, response, html){
//do your parsing...
var $ = cheerio.load(html)
});