Download files, archive and transfer to S3 using streams - javascript

I'm using the code from this question to archive files using node-archiver and transfer them to S3. My specific task requires me to download a large number of files from URLs, zip them to one archive, and transfer them to S3.
I'm using the "got" Javascript library to do this.
for (const file of files) {
const { resultFileName, fileUrl} = getFileNameAndUrl(file);
if (!fileUrl)
continue;
const downloadStream = got.stream(fileUrl, {
retry: {
limit: 5
}
});
archive.append(downloadStream, { name: resultFileName });
}
The rest of the code is pretty much the same as in the original question. The issue is that script doesn't work well with a huge amount of files (it just finishes execution at some point).
In the perfect world - I want this script to download files, append them to archive and transfer them to S3 using pipes. And the best is to download them in batches (something like Promise.map with concurrency in bluebird). I just don't get how to do it with Streams, as I do have not much experience with them.


archiver package processes one file at a time, so there is no point in downloading several in parallel with got. Follow the example by that link you provided and it should work.
Also, do not open a lot of streams to all files should be zipped. Do that one by one, since streams and archived package have timeouts on opened streams.


I hope this helps.
NOTE: I could not test this because I don't have access to aws s3.
This snippet should download webpages and saves it in zip file, which should contain fs.html & index.html file.
// file:main.mjs
import got from 'got'
import archiver from 'archiver'
import S3 from 'aws-sdk/clients/s3'
import { basename } from 'path'
try {
const urls = ['https://nodejs.org/api/fs.html', 'https://nodejs.org/api/index.html']
const gotconfig = {}
const archive = archiver('zip', {
zlib: { level: 9 },
})
archive.on('warning', function (err) {
if (err.code === 'ENOENT') {
} else {
throw err
}
})
archive.on('error', function (err) {
throw err
})
for (const url of urls) {
// const _url = new URL(url)
archive.append(got.stream(url, gotconfig), { name: basename(url) })
}
const s3 = new S3()
await s3.upload({ Bucket: 'buck', Key: 'key', Body: archive }).promise()
await archive.finalize()
} catch (error) {
console.error(error)
}
this one I have tested & it works. Similar to above but saves zip file in /tmp/test1.zip.
// file: local.mjs
import got from 'got'
import { createWriteStream } from 'fs'
import archiver from 'archiver'
import { basename } from 'path'
try {
const urls = ['https://nodejs.org/api/fs.html', 'https://nodejs.org/api/index.html']
const gotconfig = { }
const output = createWriteStream('/tmp/test1.zip')
const archive = archiver('zip', {
zlib: { level: 9 },
})
output.on('close', function () {
console.log(archive.pointer() + ' total bytes')
console.log('archiver has been finalized and the output file descriptor has closed.')
})
output.on('end', function () {
console.log('Data has been drained')
})
archive.on('warning', function (err) {
if (err.code === 'ENOENT') {
} else {
throw err
}
})
archive.on('error', function (err) {
throw err
})
archive.pipe(output)
for (const url of urls) {
archive.append(got.stream(url, gotconfig), { name: basename(url) })
}
await archive.finalize()
} catch (error) {
console.error(error)
}

Related

having problems with `fs.writeFile` it doesn't create files

I'm trying to start a script that itself creates a model file in json using fs.writeFile. The problem is when I run the script using node file.js. It is supposed to create a new file face-expression-model.json in directory /models but it doesn't create anything and doesn't show any errors.
I tried to use another library fs-extra not working as well, tried to make the script to create model directory fs.WriteDir not working eitheritried to add process.cwd() to bypass any authorisation when creating the file but didn't work. I also tried to add try/catch block to catch all errors but it doesn't show any errors and it appears that the file was created for the first while but NOPE, unfortunately.
Here is the code I'm using.
const axios = require("axios");
const faceapi = require("face-api.js");
const { FaceExpressions } = faceapi.nets;
const fs = require("fs");
async function trainModel(imageUrls) {
try {
await FaceExpressions.loadFromUri(process.cwd() + "/models");
const imageTensors = [];
for (let i = 0; i < imageUrls.length; i++) {
const response = await axios.get(imageUrls[i], {
responseType: "arraybuffer"
});
const image = new faceapi.Image();
image.constructor.loadFromBytes(new Uint8Array(response.data));
const imageTensor = faceapi.resizeImageToBase64Tensor(image);
imageTensors.push(imageTensor);
}
const model = await faceapi.trainFaceExpressions(imageTensors);
fs.writeFileSync("./models/face-expression-model.json", JSON.stringify(model), (err) => {
if (err) throw err;
console.log("The file has been saved!");
});
} catch (error) {
console.error(error);
}
}
const imageUrls = [
array of images urls here
];
trainModel(imageUrls);

I don't know exactly why but I had the same problem a while ago. Try using the "fs.writeFile" method. It worked for me.
fs.writeFile("models/face-expression-model.json", JSON.stringify(model), {}, (err) => {
if (err) throw err;
console.log("The file has been saved!");
});
Good luck with that!

need to do a file system at particular file location in nodejs

Actually I am trying to do zip conversion and need to save zip at particular folder as zip_folder created with my project folder. This is happening when I call some api. I can't able to do but if I use __dirname its working properly. Can anyone help me to comeout from this by giving your solutions. Thank you.
const fs = require('fs');
const archiver = require('archiver');
var file1 = '../zip_folder/scorm.zip';
var onlyPath = require('path').dirname('C:\Users\is9115\Desktop\node_moodle');
const mysql = require('../shared/connection');
// create a file to stream archive data to.
const archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
async function createzip()
{
const output = fs.createWriteStream(file1); // this is not working at file location
const output = fs.createWriteStream(__dirname+'/scorm.zip');//working but creating at root folder itself
fs.readFile('imsmanifest.xml', 'utf-8',async function(err, data) {
if (err) throw err;
var newValue = data.replace(/Var_Title/g, 'Golf');
fs.writeFile('imsmanifest1.xml', newValue, 'utf-8', function(err, data) {
if (err) throw err;
console.log('Done!');
})
})
archive.pipe(output);
const file2 = __dirname + '/imsmanifest1.xml';
archive.append(fs.createReadStream(file2), { name: 'imsmanifest.xml' });
archive.append('string cheese!', { name: 'file2.txt' });
archive.directory('scorm12schemadefinition/', false);
archive.file('imsmainfest1.xml', { name: 'imsmanifest.xml' });
archive.finalize();
}

How could I check If a zip file is corrupted in NodeJS?

I would check if a ZIP file is corrupted using NodeJS using less CPU and memory as possible.
How to corrupt a ZIP file:
Download a ZIP file
Open the ZIP file using a text editor optimized like Notepad++
Rewrite the header. Only put random characters.
I am trying to reach this goal using the NPM library "node-stream-zip"
private async assertZipFileIntegrity(path: string) {
try {
const zip = new StreamZip.async({ file: path });
const stm = await zip.stream(path);
stm.pipe(process.stdout);
stm.on('end', () => zip.close());
} catch (error) {
throw new Error();
}
}
However, when I run the unit tests I receive an error inside an array:
Rejected to value: [Error]

import zip from 'yauzl';
import path from 'path';
const invalidZipPath = path.resolve('invalid.zip');
const validZipPath = path.resolve('valid.zip');
const isValidZipFile = (filePath) => {
return zip.open(filePath, { lazyEntries: true }, (err, stream ) => {
if (err) {
console.log('fail to read ', filePath);
return false;
}
console.log('success read ', filePath);
return true;
});
}
isValidZipFile(validZipPath);
isValidZipFile(invalidZipPath);

How to download pdf file from local app storage to phone storage in react native using expo

I am looking for a way to download binary pdf file, which I get through api. I have this file in local app file system, but I would like to download it on my phone.
const pdf = Buffer.from(res.data, "binary").toString("base64");
const fileUri =
FileSystem.documentDirectory + `${encodeURI("generated")}.pdf`;
FileSystem.writeAsStringAsync(fileUri, pdf, {
encoding: FileSystem.EncodingType.Base64
}).then((respond) => {
downloadPdf(fileUri);
//Sharing.shareAsync(fileUri); // Here I can share file by other apps
});
const downloadPdf = async (uri) => {
//Linking.openURL(uri) // #approach 1
// MediaLibrary.saveToLibraryAsync(uri).then(res => { // #approach 2
// console.log(res)
// }).catch(err => {
// console.log(err)
// })
const permissions = await MediaLibrary.getPermissionsAsync();
try {
const asset = await MediaLibrary.createAssetAsync(uri); // #approach 3
const album = await MediaLibrary.getAlbumAsync("Downloads");
if (album == null) {
await MediaLibrary.createAlbumAsync("Downloads", asset, false);
} else {
await MediaLibrary.addAssetsToAlbumAsync([asset], album, false);
}
} catch (e) {
console.log(e)
}
};
I have tried different ways to do it, but I've stopped here with expo-media-library, which gives me:
"Unable to copy file into external storage" error.
Is it a good direction to use it ? Maybe you have some better solutions?

Saving JSON in Electron

I am building an app using Electron. In this app, I am building a data structure using JSON. My data structure looks like this:
{
items: [
{ id:1, name:'football' },
{ id:2, name:'soccer ball' },
{ id:3, name:'basketball' }
]
}
I want to save this JSON to a file called "data.json". I want to save it to a file because I want to load the next time the application starts. My challenge is, I do not know how to save the data. In fact, I'm not sure where I should even save the file. Do I save it in the same directory as the app? Or is there some cross-platform approach I should use?
Currently, I have the following:
saveClick: function() {
var json = JSON.stringify(this.data);
// assume json matches the JSON provided above.
// Now, I'm not sure how to actually save the file.
}
So, how / where do I save JSON to the local file system for use at a later time?

Electron lacks an easy way to persist and read user settings for your application. electron-json-storage implements an API somehow similar to localStorage to write and read JSON objects to/from the operating system application data directory, as defined by app.getPath('userData').

Electron uses node.js as its core. You can use the following:
var fs = require("fs");
read_file = function(path){
return fs.readFileSync(path, 'utf8');
}
write_file = function(path, output){
fs.writeFileSync(path, output);
}
For write_file(), you can either pass "document.txt" as the path and it will write it to the same directory the html file it was run from. You can also put in a full path like "C:/Users/usern/document.txt" and it will write to the specific location you want.
Also, you can choose any file extention you want, (ie. ".txt", ".js", ".json", etc.). You can even make up your own!

I wrote a simple library that you can use, with a simple interface, it also creates subdirectories and works with promises/callbacks.
it will save the data into app.getPath("appData") as the root folder.
https://github.com/ran-y/electron-storage
Installation
$ npm install --save electron-storage
usage
const storage = require('electron-storage');
API
storage.get(filePath, (err, data) => {
if (err) {
console.error(err)
} else {
console.log(data);
}
});
storage.get(filePath)
.then(data => {
console.log(data);
})
.catch(err => {
console.error(err);
});
storage.set(filePath, data, (err) => {
if (err) {
console.error(err)
}
});
storage.set(filePath, data)
.then(data => {
console.log(data);
})
.catch(err => {
console.error(err);
});

`const fs = require('fs');
let student = {
name: 'Mike',
age: 23,
gender: 'Male',
department: 'English',
car: 'Honda'
};
let data = JSON.stringify(student, null, 2);
fs.writeFile('student-3.json', data, (err) => {
if (err) throw err;
console.log('Data written to file');
});
console.log('This is after the write call');`

There are multiple steps:
Step 1: As of version 5, the default for nodeIntegration changed from true to false. You can enable it when creating the Browser Window:
const createWindow = () => {
const win = new BrowserWindow({
width: 1000,
height: 800,
webPreferences: {
nodeIntegration: true,
contextIsolation: false,
}
})
}
Step 2:
function writetofile() {
let configsettings = {
break: output.innerHTML,
alwaysonoff: toggleoutput.innerHTML,
};
let settings_data = JSON.stringify(configsettings, null, 2);
const fs = require("fs");
fs.writeFileSync("assets/configs/settings.json", settings_data);
}

Categories

Resources