Actually I am trying to do zip conversion and need to save zip at particular folder as zip_folder created with my project folder. This is happening when I call some api. I can't able to do but if I use __dirname its working properly. Can anyone help me to comeout from this by giving your solutions. Thank you.
const fs = require('fs');
const archiver = require('archiver');
var file1 = '../zip_folder/scorm.zip';
var onlyPath = require('path').dirname('C:\Users\is9115\Desktop\node_moodle');
const mysql = require('../shared/connection');
// create a file to stream archive data to.
const archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
async function createzip()
{
const output = fs.createWriteStream(file1); // this is not working at file location
const output = fs.createWriteStream(__dirname+'/scorm.zip');//working but creating at root folder itself
fs.readFile('imsmanifest.xml', 'utf-8',async function(err, data) {
if (err) throw err;
var newValue = data.replace(/Var_Title/g, 'Golf');
fs.writeFile('imsmanifest1.xml', newValue, 'utf-8', function(err, data) {
if (err) throw err;
console.log('Done!');
})
})
archive.pipe(output);
const file2 = __dirname + '/imsmanifest1.xml';
archive.append(fs.createReadStream(file2), { name: 'imsmanifest.xml' });
archive.append('string cheese!', { name: 'file2.txt' });
archive.directory('scorm12schemadefinition/', false);
archive.file('imsmainfest1.xml', { name: 'imsmanifest.xml' });
archive.finalize();
}
Related
I'm trying to start a script that itself creates a model file in json using fs.writeFile. The problem is when I run the script using node file.js. It is supposed to create a new file face-expression-model.json in directory /models but it doesn't create anything and doesn't show any errors.
I tried to use another library fs-extra not working as well, tried to make the script to create model directory fs.WriteDir not working eitheritried to add process.cwd() to bypass any authorisation when creating the file but didn't work. I also tried to add try/catch block to catch all errors but it doesn't show any errors and it appears that the file was created for the first while but NOPE, unfortunately.
Here is the code I'm using.
const axios = require("axios");
const faceapi = require("face-api.js");
const { FaceExpressions } = faceapi.nets;
const fs = require("fs");
async function trainModel(imageUrls) {
try {
await FaceExpressions.loadFromUri(process.cwd() + "/models");
const imageTensors = [];
for (let i = 0; i < imageUrls.length; i++) {
const response = await axios.get(imageUrls[i], {
responseType: "arraybuffer"
});
const image = new faceapi.Image();
image.constructor.loadFromBytes(new Uint8Array(response.data));
const imageTensor = faceapi.resizeImageToBase64Tensor(image);
imageTensors.push(imageTensor);
}
const model = await faceapi.trainFaceExpressions(imageTensors);
fs.writeFileSync("./models/face-expression-model.json", JSON.stringify(model), (err) => {
if (err) throw err;
console.log("The file has been saved!");
});
} catch (error) {
console.error(error);
}
}
const imageUrls = [
array of images urls here
];
trainModel(imageUrls);
I don't know exactly why but I had the same problem a while ago. Try using the "fs.writeFile" method. It worked for me.
fs.writeFile("models/face-expression-model.json", JSON.stringify(model), {}, (err) => {
if (err) throw err;
console.log("The file has been saved!");
});
Good luck with that!
I'm using the code from this question to archive files using node-archiver and transfer them to S3. My specific task requires me to download a large number of files from URLs, zip them to one archive, and transfer them to S3.
I'm using the "got" Javascript library to do this.
for (const file of files) {
const { resultFileName, fileUrl} = getFileNameAndUrl(file);
if (!fileUrl)
continue;
const downloadStream = got.stream(fileUrl, {
retry: {
limit: 5
}
});
archive.append(downloadStream, { name: resultFileName });
}
The rest of the code is pretty much the same as in the original question. The issue is that script doesn't work well with a huge amount of files (it just finishes execution at some point).
In the perfect world - I want this script to download files, append them to archive and transfer them to S3 using pipes. And the best is to download them in batches (something like Promise.map with concurrency in bluebird). I just don't get how to do it with Streams, as I do have not much experience with them.
archiver package processes one file at a time, so there is no point in downloading several in parallel with got. Follow the example by that link you provided and it should work.
Also, do not open a lot of streams to all files should be zipped. Do that one by one, since streams and archived package have timeouts on opened streams.
I hope this helps.
NOTE: I could not test this because I don't have access to aws s3.
This snippet should download webpages and saves it in zip file, which should contain fs.html & index.html file.
// file:main.mjs
import got from 'got'
import archiver from 'archiver'
import S3 from 'aws-sdk/clients/s3'
import { basename } from 'path'
try {
const urls = ['https://nodejs.org/api/fs.html', 'https://nodejs.org/api/index.html']
const gotconfig = {}
const archive = archiver('zip', {
zlib: { level: 9 },
})
archive.on('warning', function (err) {
if (err.code === 'ENOENT') {
} else {
throw err
}
})
archive.on('error', function (err) {
throw err
})
for (const url of urls) {
// const _url = new URL(url)
archive.append(got.stream(url, gotconfig), { name: basename(url) })
}
const s3 = new S3()
await s3.upload({ Bucket: 'buck', Key: 'key', Body: archive }).promise()
await archive.finalize()
} catch (error) {
console.error(error)
}
this one I have tested & it works. Similar to above but saves zip file in /tmp/test1.zip.
// file: local.mjs
import got from 'got'
import { createWriteStream } from 'fs'
import archiver from 'archiver'
import { basename } from 'path'
try {
const urls = ['https://nodejs.org/api/fs.html', 'https://nodejs.org/api/index.html']
const gotconfig = { }
const output = createWriteStream('/tmp/test1.zip')
const archive = archiver('zip', {
zlib: { level: 9 },
})
output.on('close', function () {
console.log(archive.pointer() + ' total bytes')
console.log('archiver has been finalized and the output file descriptor has closed.')
})
output.on('end', function () {
console.log('Data has been drained')
})
archive.on('warning', function (err) {
if (err.code === 'ENOENT') {
} else {
throw err
}
})
archive.on('error', function (err) {
throw err
})
archive.pipe(output)
for (const url of urls) {
archive.append(got.stream(url, gotconfig), { name: basename(url) })
}
await archive.finalize()
} catch (error) {
console.error(error)
}
I am working on post method in the server side to retrieve all files inside the requested directory (not recursive), and below is my code.
I am having difficulty sending the response back (res.json(pathContent);) with the updated pathContent without using the setTimeout.
I understand that this is due to the asynchronous behavior of the file system methods used (readdir and stat) and need to use some sort of callback, async, or promise technique.
I tried to use the async.waterfall with the entire body of readdir as one function and the res.json(pathContent) as the other, but it didn't send the updated array to the client side.
I know that there have been thousands of questions regarding this asynchronous operation but could not figure out how to solve my case after reading number of posts.
Any comments would be appreciated. Thanks.
const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');
var pathName = '';
const pathContent = [];
app.post('/api/files', (req, res) => {
const newPath = req.body.path;
fs.readdir(newPath, (err, files) => {
if (err) {
res.status(422).json({ message: `${err}` });
return;
}
// set the pathName and empty pathContent
pathName = newPath;
pathContent.length = 0;
// iterate each file
const absPath = path.resolve(pathName);
files.forEach(file => {
// get file info and store in pathContent
fs.stat(absPath + '/' + file, (err, stats) => {
if (err) {
console.log(`${err}`);
return;
}
if (stats.isFile()) {
pathContent.push({
path: pathName,
name: file.substring(0, file.lastIndexOf('.')),
type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
})
} else if (stats.isDirectory()) {
pathContent.push({
path: pathName,
name: file,
type: 'Directory',
});
}
});
});
});
setTimeout(() => { res.json(pathContent); }, 100);
});
The easiest and cleanest way would be use await/async, that way you can make use of promises and the code will almost look like synchronous code.
You therefor need a promisified version of readdir and stat that can be create by the promisify of the utils core lib.
const { promisify } = require('util')
const readdir = promisify(require('fs').readdir)
const stat = promisify(require('fs').stat)
async function getPathContent(newPath) {
// move pathContent otherwise can have conflicts with concurrent requests
const pathContent = [];
let files = await readdir(newPath)
let pathName = newPath;
// pathContent.length = 0; // not needed anymore because pathContent is new for each request
const absPath = path.resolve(pathName);
// iterate each file
// replace forEach with (for ... of) because this makes it easier
// to work with "async"
// otherwise you would need to use files.map and Promise.all
for (let file of files) {
// get file info and store in pathContent
try {
let stats = await stat(absPath + '/' + file)
if (stats.isFile()) {
pathContent.push({
path: pathName,
name: file.substring(0, file.lastIndexOf('.')),
type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
})
} else if (stats.isDirectory()) {
pathContent.push({
path: pathName,
name: file,
type: 'Directory',
});
}
} catch (err) {
console.log(`${err}`);
}
}
return pathContent;
}
app.post('/api/files', (req, res, next) => {
const newPath = req.body.path;
getPathContent(newPath).then((pathContent) => {
res.json(pathContent);
}, (err) => {
res.status(422).json({
message: `${err}`
});
})
})
And you should not concatenated paths using + (absPath + '/' + file), use path.join(absPath, file) or path.resolve(absPath, file) instead.
And you never should write your code in a way that the code executed for the request, relays on global variables like var pathName = ''; and const pathContent = [];. This might work in your testing environment, but will for sure lead to problems in production. Where two request work on the variable at the "same time"
Based on the initial comment I received and the reference, I used readdirSync and statSync instead and was able to make it work. I will review other answers as well and learn about other ways to implement this.
Thank you all for your kind inputs.
Here is my solution.
const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');
var pathName = '';
const pathContent = [];
app.post('/api/files', (req, res) => {
const newPath = req.body.path;
// validate path
let files;
try {
files = fs.readdirSync(newPath);
} catch (err) {
res.status(422).json({ message: `${err}` });
return;
}
// set the pathName and empty pathContent
pathName = newPath;
pathContent.length = 0;
// iterate each file
let absPath = path.resolve(pathName);
files.forEach(file => {
// get file info and store in pathContent
let fileStat = fs.statSync(absPath + '/' + file);
if (fileStat.isFile()) {
pathContent.push({
path: pathName,
name: file.substring(0, file.lastIndexOf('.')),
type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
})
} else if (fileStat.isDirectory()) {
pathContent.push({
path: pathName,
name: file,
type: 'Directory',
});
}
});
res.json(pathContent);
});
There is different way to do it :
You can first promisify the function with using new Promise() then second, use async/await or .then()
You can use the function ProsifyAll() of the Bluebird package (https://www.npmjs.com/package/bluebird)
You can use the synchrone version of the fs functions
https://nodejs.org/api/fs.html#fs_fs_readdirsync_path_options
https://nodejs.org/api/fs.html#fs_fs_statsync_path_options
Here's some options:
Use the synchronous file methods (check the docs, but they usually end with Sync). Slower, but a fairly simple code change, and very easy to understand.
Use promises (or util.promisify) to create a promise for each stat, and Promise.all to wait for all the stats to complete. After that, you can use async functions and await as well for easier to read code and simpler error handling. (Probably the largest code change, but it will make the async code easier to follow)
Keep a counter of the number of stats you have done, and if that number is the size you expect, then call res.json form inside the stat callback (smallest code change, but very error prone)
Readstreams for firebase storage:
I have files in my Google firebase storage for which I want to create a read stream (using javascript/node js). (I then intend to pipe this read stream to some middleware and then a write stream, but this is unimportant for the question.) The code snippet shows what I'm doing, but when I print the readStream to console I get a DestroyableTransform object instead of a ReadableStream. I feel like my code is very similar to the documentation. Does anyone know what might be wrong?
const filePath = 'image.png';
const getReadStream = (filePath) => {
let file;
try {
file = admin
.storage()
.bucket()
.file(filePath);
} catch (err) {
console.log(err);
}
const readStream = file.createReadStream()
.on('error', (err) => {
throw err;
});
console.log(readStream);
return readStream;
};
This is a possible answer.
const filePath = 'image.png';
const getReadStream = (filePath) => {
let file;
try {
file = admin
.storage()
.bucket(filePath);
} catch (err) {
console.log(err);
}
const readStream = file.createReadStream()
.on('error', (err) => {
throw err;
});
console.log(readStream);
return readStream;
};
You should exclude the inner "file" sentence.
I have a zip file (actually it's an epub file) I need to loop through the files in it and read them without unzipping them to the disk.
I tried to use a Node.js library called JSZip but the content of each file is stored in memory in Buffer and whenever I try to decode the buffer content to string the content returned is unreadable
Here's the code I tried:
const zip = new JSZip();
// read a zip file
fs.readFile(epubFile, function (err, data) {
if (err) throw err;
zip.loadAsync(data).then(function (zip) {
async.eachOf(zip.files, function (content, fileName, callback) {
if (fileName.match(/json/)) {
var buf = content._data.compressedContent;
console.log(fileName);
console.log((new Buffer(buf)).toString('utf-8'));
}
callback();
}, function (err) {
if (err) {
console.log(err);
}
});
});
});
Since unzip seems to be abandoned, I used node-stream-zip with pretty good success.
npm install node-stream-zip
Reading files be all like:
const StreamZip = require('node-stream-zip');
const zip = new StreamZip({
file: 'archive.zip',
storeEntries: true
});
zip.on('ready', () => {
// Take a look at the files
console.log('Entries read: ' + zip.entriesCount);
for (const entry of Object.values(zip.entries())) {
const desc = entry.isDirectory ? 'directory' : `${entry.size} bytes`;
console.log(`Entry ${entry.name}: ${desc}`);
}
// Read a file in memory
let zipDotTxtContents = zip.entryDataSync('path/inside/zip.txt').toString('utf8');
console.log("The content of path/inside/zip.txt is: " + zipDotTxtContents);
// Do not forget to close the file once you're done
zip.close()
});
npm install unzip
https://www.npmjs.com/package/unzip
fs.createReadStream('path/to/archive.zip')
.pipe(unzip.Parse())
.on('entry', function (entry) {
var fileName = entry.path;
var type = entry.type; // 'Directory' or 'File'
var size = entry.size;
if (fileName === "this IS the file I'm looking for") {
entry.pipe(fs.createWriteStream('output/path'));
} else {
entry.autodrain();
}
});