I am trying to download an .zip file from discord and extracting using decompress package, but it not return any errors and don't extract the package. (The file is saving and downloading correctly)
const decompress = require('decompress');
client.on("message", (msg) => {
if (msg.author.id === process.env.REIS_ID) {
if (msg.channel.id === config.channel_commit) {
if (config.file_status === 0) {
if (msg.attachments.first()) {
download_files(msg.attachments.first().url).then(res => {
console.log(res);
});
}
} else {
msg.reply("Upload ainda em processo.");
}
}
}
});
const download_files = async (url) => {
const stream = got.stream(url);
const file_ext = FileType.fromStream(stream).then(async (res) => {
got
.stream(url)
.pipe(
fs.createWriteStream("./testes/a/new." + res.ext, { overwrite: true})
);
console.log(res.ext);
if (res.ext === "zip") {
console.log("zip file");
const files = await decompress("./testes/a/new.zip", "./testes/a/new/");
}
});
};
Solution:
I download the unzip module from npm and make that the zip file is not saved to the folder, the request saves directly the extracted files.
Code:
client.on("message", (msg) => {
if (msg.author.id === process.env.REIS_ID) {
if (msg.channel.id === config.channel_commit) {
if (msg.attachments.first()) {
let res = download_files(msg.attachments.first().url);
}
}
}
});
const download_files = (url) => {
const stream = got.stream(url);
const file_ext = FileType.fromStream(stream).then(async (res) => {
got
.stream(url)
console.log(res.ext);
if (res.ext === "zip") {
got.stream(url).pipe(unzip.Extract({ path: "./testes/a/new/" }));
}
});
};
Related
am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};
I'm using react dropzone package in my next JS app. and i wanted to implement image compression automatically.
first after i got the images i converted the blob/ preview into a file reader. then i used compressorjs package. but after i did that the image is not getting compressed?
how can i compress an image/images in react when the image is passed from dropzone?
const [files, setFiles] = useState([]);
onDrop: (acceptedFiles) => {
const newFiles = acceptedFiles.map((file, index) => {
return Object.assign(file, {
preview: URL.createObjectURL(file),
});
});
if (files.length < 9) {
setFiles((prev) => [...prev, ...newFiles]);
files.map((file) => {
newFiles.forEach((newFile) => {
if (newFile.name == file.name) {
alert(newFile.name + " is a duplicate file");
setFiles(
files,
files.filter((val) => val !== newFile)
);
}
});
});
} else if (acceptedFiles.length >= 9) {
alert("select maximum of 9 images");
} else {
alert("maximum images to be selected is 9");
}
},
});
const removeFile = (file) => () => {
const newFiles = [...files];
newFiles.splice(newFiles.indexOf(file), 1);
setFiles(newFiles);
if (files.length == 1) {
setText("Drag and Drop or click here to upload Images");
}
};
useEffect(() => {
// Make sure to revoke the data uris to avoid memory leaks, will run on unmount
return () => files.forEach((file) => URL.revokeObjectURL(file.preview));
}, []);
the compressorjs code i used is
acceptedFiles.map((file, index) => {
const reader = new FileReader();
reader.onload = function (e) {
new Compressor(e.target.result, {
quality: 0.8,
success: (compressedFile) => {
setFiles((prev) => [
...prev,
compressedFile
]);
},
});
};
reader.readAsDataURL(file);
return file;
});
please any contribution on this.
I want to download an image from a URL.
const downloadImage = async () => {
const permissions = await StorageAccessFramework.requestDirectoryPermissionsAsync();
if (permissions.granted) {
const uri = permissions.directoryUri;
const files = await StorageAccessFramework.readDirectoryAsync(uri);
try {
await StorageAccessFramework.createFileAsync(permissions.directoryUri,
"myImage", "image/png")
.then((r) => {
console.log(r);
FileSystem.downloadAsync(
"my/image/url",
FileSystem.documentDirectory + "myImage.png"
)
.then(({ uri }) => {
console.log("Finished downloading to ", uri);
})
.catch((error) => {
console.error(error);
});
})
.catch((e) => {
console.log(e);
});
} catch {
console.log(e);
}
alert(`Files inside ${uri}:\n\n${JSON.stringify(files)}`);
}
};
Once the function runs everything goes well. A file with the name myImage.png is created. However, it's size is 0 Bytes and there is no image.
What am I missing?
Any help would be appreciated Thanks!
We can use MediaLibrary and FileSystem for this.
Check the below example,
import * as FileSystem from 'expo-file-system';
import * as Permissions from 'expo-permissions';
import * as MediaLibrary from 'expo-media-library';
import moment from 'moment';
import { Button, View } from 'react-native';
export default function DownloadImage() {
const imageUrl = { uri: "https://media.voguebusiness.com/photos/5ef6493adf1073db3375835d/master/pass/kanye-west-gap-news-voguebus-mert-alas-and-marcus-piggott-june-20-story.jpg" }
const handleDownload = async () => {
let date = moment().format('YYYYMMDDhhmmss')
let fileUri = FileSystem.documentDirectory + `${date}.jpg`;
try {
const res = await FileSystem.downloadAsync(imageUrl.uri, fileUri)
saveFile(res.uri)
} catch (err) {
console.log("FS Err: ", err)
}
}
const saveFile = async (fileUri) => {
const { status } = await Permissions.askAsync(Permissions.MEDIA_LIBRARY);
if (status === "granted") {
try {
const asset = await MediaLibrary.createAssetAsync(fileUri);
const album = await MediaLibrary.getAlbumAsync('Download');
if (album == null) {
await MediaLibrary.createAlbumAsync('Download', asset, false);
} else {
await MediaLibrary.addAssetsToAlbumAsync([asset], album, false);
}
} catch (err) {
console.log("Save err: ", err)
}
} else if (status === "denied") {
alert("please allow permissions to download")
}
}
return (
<View>
<Button
title="Download"
onPress={handleDownload}
/>
</View>
)
}
I download data from API in chunks decrypt it and than pass to ReadableStream.
But after last chunk, the file is not downloaded.
I work with axios and StreamSaver.js
Code:
Above in the code I declare:
this.filestream = streamSaver.createWriteStream('sample.jpg');
this.writer = await this.filestream.getWriter();
let readableStream;
readableStream = new ReadableStream({
start(ctrl) {
const nextChunk = async () => {
let fileDataResponse = await that.$api.post(
'endpoint', {
file_id: UUID,
chunk_index: index
}, {
headers: {
...
}
}
);
done =
fileDataResponse.data.length <=
fileDataResponse.data.current_index;
if (fileDataResponse.data.data) {
let data = await that.decryptData(fileDataResponse.data.data);
ctrl.enqueue(data);
}
if (!done) {
index += 1;
nextChunk();
} else {
ctrl.close();
}
};
nextChunk();
}
});
const reader = readableStream.getReader();
const close = () => {
that.writer.close();
};
const pump = () =>
reader.read().then((res) => {
if (!res.done) {
that.writer.write(res.value).then(pump);
} else {
close();
}
});
pump();
Where could be my error here?
Thank you a lot!
Issue was the res.value is not an Int8Array
Im fairly new in the programming world. I'm making an app where it should be possible to choose a directory, where to save some generated files.
I'm working with the ipc, and it seems like some of the code works, but it looks like i can't get the mainIpc to send the path back to the renderer.
I hope the hive can help, thanks in advance!
Renderer:
const electron = require("electron");
const ipc = require("electron").ipcRenderer;
createBtn.addEventListener("click", (event) => {
ipc.send("path:get");
});
ipc.on("path:selected", function (path) {
console.log("Full path: ", path);
});
Main
const ipc = require("electron").ipcMain;
const os = require("os");
const { dialog } = require("electron");
ipc.on("path:get", function (event) {
if (os.platform() === "linux" || os.platform() === "win32") {
dialog.showOpenDialog(
{
properties: ["openFile"],
},
function (files) {
if (files) win.webContents.send("path:selected", files[0]);
console.log("SENT");
}
);
} else {
dialog.showOpenDialog(
{
properties: ["openFile", "openDirectory"],
},
function (files) {
if (files) win.webContents.send("path:selected", files[0]);
console.log("SENT");
}
);
}
});
Edit: Adding the setup
Setup
const { app, BrowserWindow } = require("electron");
const ipc = require("electron").ipcMain;
const os = require("os");
const { dialog } = require("electron");
try {
require("electron-reloader")(module);
} catch (_) {}
let win;
function createWindow() {
win = new BrowserWindow({
width: 800,
height: 600,
webPreferences: {
nodeIntegration: true,
},
});
win.loadFile("./src/index.html");
}
app.whenReady().then(createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
app.quit();
}
});
app.on("activate", () => {
if (BrowserWindow.getAllWindows().length === 0) {
createWindow();
}
});
I figured it out with some kind help.
So if anyone needs the same procedure i'll try to explain what i got to.
So, in the main, i had to add a then, because the showDialog returns a promise
if (os.platform() === "linux" || os.platform() === "win32") {
dialog
.showOpenDialog({
properties: ["openFile", "openDirectory"],
})
.then((result) => {
if (result) win.webContents.send("path:selected", result.filePaths);
})
.catch((err) => {
console.log(err);
});
} else {
dialog
.showOpenDialog({
properties: ["openFile", "openDirectory"],
})
.then((result) => {
console.log(result.filePaths);
if (result) win.webContents.send("path:selected", result.filePaths);
})
.catch((err) => {
console.log(err);
});
}
});
This sends back an array with the path at [0]
in the renderer i forgot to add the event as an parameter.
ipc.on("path:selected", (event, path) => {
chosenPath = path;
console.log("Full path: ", chosenPath[0]);
});