i’ve node.js app that I Need to zip all the current folder with command from
and get the zip on the root
For that I want to use the archiver npm package but I don’t understand the following:
where I put the current folder (since I want to zip all the application )
where should I put the name of the zip (the zip that should be created when execute the command)
My app have the following structure
MyApp
Node_modules
server.js
app.js
package.json
arc.js
In the arc.js I’ve put all the zip logic so I guess I need to provide zipPath (which in my case is ‘./‘)
and zip name like myZip…
I tried with the following without success, any idea ?
var fs = require('fs');
var archiver = require('archiver');
// create a file to stream archive data to.
var output = fs.createWriteStream(__dirname + '/.');
var archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
// listen for all archive data to be written
output.on('close', function() {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
archive.on('warning', function(err) {
if (err.code === 'ENOENT') {
// log warning
} else {
// throw error
throw err;
}
});
// good practice to catch this error explicitly
archive.on('error', function(err) {
throw err;
});
// pipe archive data to the file
archive.pipe(output);
I need that when I open the command line like
folder->myApp-> run zip arc and will create zipped file under the current path (which is the root in this case....)
You can use the glob method, but make sure to exclude *.zip files. Otherwise the zip file itself will be part of the archive.
Here is an example:
// require modules
var fs = require('fs');
var archiver = require('archiver');
// create a file to stream archive data to.
var output = fs.createWriteStream(__dirname + '/example.zip');
var archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
// listen for all archive data to be written
output.on('close', function () {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
// good practice to catch warnings (ie stat failures and other non-blocking errors)
archive.on('warning', function (err) {
if (err.code === 'ENOENT') {
// log warning
} else {
// throw error
throw err;
}
});
// good practice to catch this error explicitly
archive.on('error', function (err) {
throw err;
});
// pipe archive data to the file
archive.pipe(output);
archive.glob('**/*', { ignore: ['*.zip'] });
archive.finalize();
In the github of node-archiver there is an example folder
where I put the current folder (since I want to zip all the
application )
Example :
var file1 = __dirname + '/fixtures/file1.txt';
var file2 = __dirname + '/fixtures/file2.txt';
archive
.append(fs.createReadStream(file1), { name: 'file1.txt' })
.append(fs.createReadStream(file2), { name: 'file2.txt' })
.finalize();
Specifically about your case and directory you can use the .directory() method of archiver
where should I put the name of the zip (the zip that should be created
when execute the command)
Example :
var output = fs.createWriteStream(__dirname + '/example-output.zip');
Related
This is my code
var fs = require('fs'),
path = require('path');
cheerio = require('cheerio'),
newman = require('newman'),
os = require("os");
const directoryPath = path.join(__dirname, './payload');
fs.readdir(directoryPath, function(err, files) {
if (err) {
return console.log('Unable to scan directory: ' + err);
}
files.forEach(function(file) {
console.log("Files Read")
runNewman(file);
});
});
function runNewman(data) {
let logs=[];
var csvFile = "payload/" + data;
//var logFile = "payload/" + data + ".txt";
newman.run({
collection: require('./kmap_testing.postman_collection.json'),
environment: require('./kmap.postman_environment.json'),
globals: require('./My Workspace.postman_globals.json'),
reporters: 'cli',
iterationData:csvFile
}).on('start',function(err,args){
console.log('start');
}).on('console',function(err, args){
if(err){return;}
logs.push(args.messages);
}).on('done',function(err, summary){
if(err || summary.error){
console.error('collection run encounter an error');
}
else{
console.log('collection run completed');
}
fs.appendFileSync("results.csv",logs.join("\r\n"));
})
}
I have huge csv file with almost 100K+ data, I have split the data into 5K per csv file and saved them under Payload folder. However, newman.run - takes the files randomly or parallely and runs. The results.csv file ends up running two times and has more than 200K + results. Someone, please help me with this? I am a beginner with newman library.
I'm trying to add a local file to the zip so when the user downloads and unzips, he'll get a folder with a .dll and a config.json file:
var zip = new JSZip();
options.forEach(option => {
zip.folder("REST." + option + ".Connector")
.file("config.json", "//config for " + option)
// I want this file to be from a local directory within my project
// eg. {dir}\custom_rest_connector_repository\src\dlls\Connectors.RestConnector.dll
.file('../dlls/Connectors.RestConnector.dll', null);
});
zip.generateAsync({type:"blob"}).then(function (blob) {
FileSaver.saveAs(blob, "REST_Connectors_"
+ dateStr
+ ".zip");
});
I read through the JSZip documentation but couldn't find an example or any information whether this can actually be done.
If it can't, is there any other more robust library that does support this operation?
Found the answer to my own question using the jszip-utils
JSZipUtils.getBinaryContent("../dlls/Connectors.RestConnector.dll", function (err, data) {
if(err) {
throw err; // or handle the error
}
zip.file("../dlls/Connectors.RestConnector.dll", data, {binary:true});
});
I know this question has been asked but my mind has been blown by my inability to get this working. I am trying to upload multiple images to my server with the following code:
var formidable = require('formidable');
var fs = require('fs');
...
router.post('/add_images/:showcase_id', function(req, res){
if(!admin(req, res)) return;
var form = new formidable.IncomingForm(),
files = [];
form.uploadDir = global.__project_dirname+"/tmp";
form.on('file', function(field, file) {
console.log(file);
file.image_id = global.s4()+global.s4();
file.endPath = "/img/"+file.image_id+"."+file.type.replace("image/","");
files.push({field:field, file:file});
});
form.on('end', function() {
console.log('done');
console.log(files);
db.get("SOME SQL", function(err, image_number){
if(err){
console.log(err);
}
var db_index = 0;
if(image_number) db_index = image_number.image_order;
files.forEach(function(file, index){
try{
//this line opens the image in my computer (testing)
require("sys").exec("display " + file.file.path);
console.log(file.file.path);
fs.renameSync(file.file.path, file.file.endPath);
}catch (e){
console.log(e);
}
db.run( "SOME MORE SQL"')", function(err){
if(index == files.length)
res.redirect("/admin/gallery"+req.params.showcase_id);
});
});
});
});
form.parse(req);
});
The line that opens the image via system calls works just fine, however I continue to get:
Error: ENOENT, no such file or directory '/home/[username]/[project name]/tmp/285ef5276581cb3b8ea950a043c6ed51'
by the rename statement.
the value of file.file.path is:
/home/[username]/[project name]/tmp/285ef5276581cb3b8ea950a043c6ed51
I am so confused and have tried everything. What am I doing wrong?
Probably you get this error because the target path does not exist or you don't have write permissions.
The error you get is misleading due to a bug in nodejs, see:
https://github.com/joyent/node/issues/5287
https://github.com/joyent/node/issues/685
Consider adding:
console.log(file.file.endPath);
before the fs.renameSync call and check if the target path exist and is writable by your application
You stated form. Therefore note that Formidable doesn't work out of the box with just NodeJS. Unless you were to use something like the prompt module for input. If you are using HTML, you'll need something like Angular, React or Browserify to be able to give it access to your interface.
How would I go about creating a one time download link in nodeJS or Express?
I'm trying to find the simplest way to accomplish this. My ideas so far are:
Use fs stream to read and then delete the file
or
Somehow generate a link/route that gets removed once the download button is clicked
Are any of these implementations possible?
Is there a simpler way?
Any help or example code would be greatly appreciated!
-Thanks
Check this simple implementation:
You store the information of the download in a file. The filename is the download session id. The file content is the real path of the file to be downloaded.
Use these three functions to manage the lifecycle of the download sessions:
var fs = require('fs');
var crypto = require('crypto');
var path = require('path');
// Path where we store the download sessions
const DL_SESSION_FOLDER = '/var/download_sessions';
/* Creates a download session */
function createDownload(filePath, callback) {
// Check the existence of DL_SESSION_FOLDER
if (!fs.existsSync(DL_SESSION_FOLDER)) return callback(new Error('Session directory does not exist'));
// Check the existence of the file
if (!fs.existsSync(filePath)) return callback(new Error('File doest not exist'));
// Generate the download sid (session id)
var downloadSid = crypto.createHash('md5').update(Math.random().toString()).digest('hex');
// Generate the download session filename
var dlSessionFileName = path.join(DL_SESSION_FOLDER, downloadSid + '.download');
// Write the link of the file to the download session file
fs.writeFile(dlSessionFileName, filePath, function(err) {
if (err) return callback(err);
// If succeeded, return the new download sid
callback(null, downloadSid);
});
}
/* Gets the download file path related to a download sid */
function getDownloadFilePath(downloadSid, callback) {
// Get the download session file name
var dlSessionFileName = path.join(DL_SESSION_FOLDER, downloadSid + '.download');
// Check if the download session exists
if (!fs.existsSync(dlSessionFileName)) return callback(new Error('Download does not exist'));
// Get the file path
fs.readFile(dlSessionFileName, function(err, data) {
if (err) return callback(err);
// Return the file path
callback(null, data);
});
}
/* Deletes a download session */
function deleteDownload(downloadSid, callback) {
// Get the download session file name
var dlSessionFileName = path.join(DL_SESSION_FOLDER, downloadSid + '.download');
// Check if the download session exists
if (!fs.existsSync(dlSessionFileName)) return callback(new Error('Download does not exist'));
// Delete the download session
fs.unlink(dlSessionFileName, function(err) {
if (err) return callback(err);
// Return success (no error)
callback();
});
}
Use createDownload() to create download sessions wherever you need to. It returns the download sid, then you can use it to build your download URL like: http://your.server.com/download?sid=<RETURNED SID>.
Finally you can add a simple handler to your /download route:
app.get('/download', function(req, res, next) {
// Get the download sid
var downloadSid = req.query.sid;
// Get the download file path
getDownloadFilePath(downloadSid, function(err, path) {
if (err) return res.end('Error');
// Read and send the file here...
// Finally, delete the download session to invalidate the link
deleteDownload(downloadSid, function(err) {
// ...
});
});
});
With this method, you don't have to create/move/delete big download files, which could cause slow responses and unnecessary resource consumption.
You can delete routes from the app.routes object. See Remove route mappings in NodeJS Express for more info.
Here is my quick and not very well tested way of doing what you ask:
var express = require('express');
var app = express();
app.get('/download', function(req,res,next){
res.download('./path/to/your.file');
//find this route and delete it.
for(i = 0; i < app.routes.get.length; i++){
if(app.routes.get[i].path === '/download'){
app.routes.get.splice(i,1);
}
}
});
app.listen(80);
I'd probably map a single route to manage downloads, and then upon downloading the file, move or delete it. That way I can prevent a lot of cashing of routes, or a lot of small temp files from the other two answers, but YMMV. Something like this:
// say your downloads are in /downloads
app.get('/dl/:filename', function(req, res) {
var fileStream = fs.createReadStream('/downloads' + req.params.filename);
// error handler, ie. file not there...
fileStream.on('error', function(err) {
if(err) {
res.status(404); // or something
return res.end();
}
});
// here you ow pipe that stream to the response,
fileStream.on('data', downloadHandler);
// and here delete the file or move it to other folder or whatever, do cleanup
fileStream.on('end', deleteFileHandler);
}
Note: This is a possible security vulnerability, it could let the adversary download files outside your downloads location. That filename param is passed directly to fs.
I want to download a zip file from the internet and unzip it in memory without saving to a temporary file. How can I do this?
Here is what I tried:
var url = 'http://bdn-ak.bloomberg.com/precanned/Comdty_Calendar_Spread_Option_20120428.txt.zip';
var request = require('request'), fs = require('fs'), zlib = require('zlib');
request.get(url, function(err, res, file) {
if(err) throw err;
zlib.unzip(file, function(err, txt) {
if(err) throw err;
console.log(txt.toString()); //outputs nothing
});
});
[EDIT]
As, suggested, I tried using the adm-zip library and I still cannot make this work:
var ZipEntry = require('adm-zip/zipEntry');
request.get(url, function(err, res, zipFile) {
if(err) throw err;
var zip = new ZipEntry();
zip.setCompressedData(new Buffer(zipFile.toString('utf-8')));
var text = zip.getData();
console.log(text.toString()); // fails
});
You need a library that can handle buffers. The latest version of adm-zip will do:
npm install adm-zip
My solution uses the http.get method, since it returns Buffer chunks.
Code:
var file_url = 'http://notepad-plus-plus.org/repository/7.x/7.6/npp.7.6.bin.x64.zip';
var AdmZip = require('adm-zip');
var http = require('http');
http.get(file_url, function(res) {
var data = [], dataLen = 0;
res.on('data', function(chunk) {
data.push(chunk);
dataLen += chunk.length;
}).on('end', function() {
var buf = Buffer.alloc(dataLen);
for (var i = 0, len = data.length, pos = 0; i < len; i++) {
data[i].copy(buf, pos);
pos += data[i].length;
}
var zip = new AdmZip(buf);
var zipEntries = zip.getEntries();
console.log(zipEntries.length)
for (var i = 0; i < zipEntries.length; i++) {
if (zipEntries[i].entryName.match(/readme/))
console.log(zip.readAsText(zipEntries[i]));
}
});
});
The idea is to create an array of buffers and concatenate them into a new one at the end. This is due to the fact that buffers cannot be resized.
Update
This is a simpler solution that uses the request module to obtain the response in a buffer, by setting encoding: null in the options. It also follows redirects and resolves http/https automatically.
var file_url = 'https://github.com/mihaifm/linq/releases/download/3.1.1/linq.js-3.1.1.zip';
var AdmZip = require('adm-zip');
var request = require('request');
request.get({url: file_url, encoding: null}, (err, res, body) => {
var zip = new AdmZip(body);
var zipEntries = zip.getEntries();
console.log(zipEntries.length);
zipEntries.forEach((entry) => {
if (entry.entryName.match(/readme/i))
console.log(zip.readAsText(entry));
});
});
The body of the response is a buffer that can be passed directly to AdmZip, simplifying the whole process.
Sadly you can't pipe the response stream into the unzip job as node zlib lib allows you to do, you have to cache and wait the end of the response. I suggest you to pipe the response to a fs stream in case of big files, otherwise you will full fill your memory in a blink!
I don't completely understand what you are trying to do, but imho this is the best approach. You should keep your data in memory only the time you really need it, and then stream to the csv parser.
If you want to keep all your data in memory you can replace the csv parser method fromPath with from that takes a buffer instead and in getData return directly unzipped
You can use the AMDZip (as #mihai said) instead of node-zip, just pay attention because AMDZip is not yet published in npm so you need:
$ npm install git://github.com/cthackers/adm-zip.git
N.B. Assumption: the zip file contains only one file
var request = require('request'),
fs = require('fs'),
csv = require('csv')
NodeZip = require('node-zip')
function getData(tmpFolder, url, callback) {
var tempZipFilePath = tmpFolder + new Date().getTime() + Math.random()
var tempZipFileStream = fs.createWriteStream(tempZipFilePath)
request.get({
url: url,
encoding: null
}).on('end', function() {
fs.readFile(tempZipFilePath, 'base64', function (err, zipContent) {
var zip = new NodeZip(zipContent, { base64: true })
Object.keys(zip.files).forEach(function (filename) {
var tempFilePath = tmpFolder + new Date().getTime() + Math.random()
var unzipped = zip.files[filename].data
fs.writeFile(tempFilePath, unzipped, function (err) {
callback(err, tempFilePath)
})
})
})
}).pipe(tempZipFileStream)
}
getData('/tmp/', 'http://bdn-ak.bloomberg.com/precanned/Comdty_Calendar_Spread_Option_20120428.txt.zip', function (err, path) {
if (err) {
return console.error('error: %s' + err.message)
}
var metadata = []
csv().fromPath(path, {
delimiter: '|',
columns: true
}).transform(function (data){
// do things with your data
if (data.NAME[0] === '#') {
metadata.push(data.NAME)
} else {
return data
}
}).on('data', function (data, index) {
console.log('#%d %s', index, JSON.stringify(data, null, ' '))
}).on('end',function (count) {
console.log('Metadata: %s', JSON.stringify(metadata, null, ' '))
console.log('Number of lines: %d', count)
}).on('error', function (error) {
console.error('csv parsing error: %s', error.message)
})
})
If you're under MacOS or Linux, you can use the unzip command to unzip from stdin.
In this example I'm reading the zip file from the filesystem into a Buffer object but it works
with a downloaded file as well:
// Get a Buffer with the zip content
var fs = require("fs")
, zip = fs.readFileSync(__dirname + "/test.zip");
// Now the actual unzipping:
var spawn = require('child_process').spawn
, fileToExtract = "test.js"
// -p tells unzip to extract to stdout
, unzip = spawn("unzip", ["-p", "/dev/stdin", fileToExtract ])
;
// Write the Buffer to stdin
unzip.stdin.write(zip);
// Handle errors
unzip.stderr.on('data', function (data) {
console.log("There has been an error: ", data.toString("utf-8"));
});
// Handle the unzipped stdout
unzip.stdout.on('data', function (data) {
console.log("Unzipped file: ", data.toString("utf-8"));
});
unzip.stdin.end();
Which is actually just the node version of:
cat test.zip | unzip -p /dev/stdin test.js
EDIT: It's worth noting that this will not work if the input zip is too big to be read in one chunk from stdin. If you need to read bigger files, and your zip file contains only one file, you can use funzip instead of unzip:
var unzip = spawn("funzip");
If your zip file contains multiple files (and the file you want isn't the first one) I'm afraid to say you're out of luck. Unzip needs to seek in the .zip file since zip files are just a container, and unzip may just unzip the last file in it. In that case you have to save the file temporarily (node-temp comes in handy).
Two days ago the module node-zip has been released, which is a wrapper for the JavaScript only version of Zip: JSZip.
var NodeZip = require('node-zip')
, zip = new NodeZip(zipBuffer.toString("base64"), { base64: true })
, unzipped = zip.files["your-text-file.txt"].data;