I am trying to capture the download progress of a Fetch request and use that to change the width of a progress bar. I looked at ProgressEvent.lengthComputable as a potential solution but unsure if this can be used with the Fetch API.
without checking for errors (as in try/catch etc...)
const elStatus = document.getElementById('status');
function status(text) {
elStatus.innerHTML = text;
}
const elProgress = document.getElementById('progress');
function progress({loaded, total}) {
elProgress.innerHTML = Math.round(loaded/total*100)+'%';
}
async function main() {
status('downloading with fetch()...');
const response = await fetch('https://fetch-progress.anthum.com/30kbps/images/sunrise-baseline.jpg');
const contentLength = response.headers.get('content-length');
const total = parseInt(contentLength, 10);
let loaded = 0;
const res = new Response(new ReadableStream({
async start(controller) {
const reader = response.body.getReader();
for (;;) {
const {done, value} = await reader.read();
if (done) break;
loaded += value.byteLength;
progress({loaded, total})
controller.enqueue(value);
}
controller.close();
},
}));
const blob = await res.blob();
status('download completed')
document.getElementById('img').src = URL.createObjectURL(blob);
}
main();
<div id="status"> </div>
<h1 id="progress"> </h1>
<img id="img" />
adapted from here
Using this utility:
async function* streamAsyncIterable(stream) {
const reader = stream.getReader()
try {
while (true) {
const { done, value } = await reader.read()
if (done) return
yield value
}
} finally {
reader.releaseLock()
}
}
See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators
Then you can use for await...of loop:
const response = await fetch(url)
let responseSize = 0
for await (const chunk of streamAsyncIterable(response.body)) {
responseSize += chunk.length
}
But be aware that responseSize is response-size! Not necessarily download-size! What is the difference? There is no difference if there is no content-encoding (gzip, br, ...). But if a comperession was applied, final download-size will be the size of compressed data (the same content-length), and final response-size will be the size of uncompressed data.
See #ecthiender comment and this thread.
you can use axios instead
import axios from 'axios'
export async function uploadFile(file, cb) {
const url = `//127.0.0.1:4000/profile`
try {
let formData = new FormData()
formData.append("avatar", file)
const data = await axios.post(url, formData, {
onUploadProgress: (progressEvent) => {
console.log(progressEvent)
if (progressEvent.lengthComputable) {
let percentComplete = progressEvent.loaded / progressEvent.total;
if (cb) {
cb(percentComplete)
}
}
}
})
return data
} catch (error) {
console.error(error)
}
}
Related
am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};
I'm creating a web scraper using node, cheerio and calling the website using axios(async/await). I want the function to run every 30 seconds. I tried using setTimeout and setInterval but did not get the expected result. Instead got heap out of memory error. I want to run the mvcAppointmentSearch function in the while loop for every 30 seconds. Following is the code. also attaching the codepen link for better readability.
Code pen link
const express = require('express');
const request = require('request-promise');
const cheerio = require('cheerio');
const axios = require('axios');
const cssSelect = require('css-select');
const open = require('open');
// const mvcUrl = 'https://telegov.njportal.com/njmvc/AppointmentWizard/17/';
const mvcUrl = 'https://telegov.njportal.com/njmvc/AppointmentWizard/14/';
const mvcLocation = ['Edison', 'Rahway', 'SouthPlainfield'];
// const mvcLocationNumber = ['240', '252', '239'];
const mvcLocationNumber = ['163'];
const requiredMonths = ['September', 'October'];
const callUrl = async (url, locationNumberIndex) => {
try {
const response = await axios.get(url);
//console.log('call url', response.data);
getData(response.data, locationNumberIndex);
} catch (err) {
console.log(err);
}
};
const mvcAppointmentSearch = () => {
for (let i = 0; i < mvcLocationNumber.length; i++) {
const currentUrl = mvcUrl + mvcLocationNumber[i];
console.log(mvcLocationNumber[i]);
callUrl(currentUrl, i);
}
};
const getData = (html, locationNumberIndex) => {
let data = [];
let $ = cheerio.load(html);
console.log('datais ', $);
$.prototype.exists = function (selector) {
return this.find(selector).length > 0;
};
const checkerLength = $('div').exists('.alert-danger');
console.log(checkerLength);
if (checkerLength) {
console.log(
`No appointment available in ${mvcLocation[locationNumberIndex]}`
);
} else {
const dateString = $('.control-label').text();
const availableMonth = dateString.trim().split(' ')[7];
const exactDateAvailability = dateString.slice(24, -1);
console.log(availableMonth);
if (requiredMonths.includes(availableMonth)) {
console.log('Hurray there is an appointment available');
const message = `Appointment available for the location ${mvcLocation[locationNumberIndex]} on ${exactDateAvailability}`;
open(`${mvcUrl + mvcLocationNumber[locationNumberIndex]}`);
console.log(message);
} else {
console.log('required Month is not available still searching');
}
}
};
while (true) {
try {
// mvcAppointmentSearch();
// want to run the following function for every 30 seconds.
mvcAppointmentSearch();
} catch (err) {
console.log(`Error has Occured ${err}`);
}
}
I download data from API in chunks decrypt it and than pass to ReadableStream.
But after last chunk, the file is not downloaded.
I work with axios and StreamSaver.js
Code:
Above in the code I declare:
this.filestream = streamSaver.createWriteStream('sample.jpg');
this.writer = await this.filestream.getWriter();
let readableStream;
readableStream = new ReadableStream({
start(ctrl) {
const nextChunk = async () => {
let fileDataResponse = await that.$api.post(
'endpoint', {
file_id: UUID,
chunk_index: index
}, {
headers: {
...
}
}
);
done =
fileDataResponse.data.length <=
fileDataResponse.data.current_index;
if (fileDataResponse.data.data) {
let data = await that.decryptData(fileDataResponse.data.data);
ctrl.enqueue(data);
}
if (!done) {
index += 1;
nextChunk();
} else {
ctrl.close();
}
};
nextChunk();
}
});
const reader = readableStream.getReader();
const close = () => {
that.writer.close();
};
const pump = () =>
reader.read().then((res) => {
if (!res.done) {
that.writer.write(res.value).then(pump);
} else {
close();
}
});
pump();
Where could be my error here?
Thank you a lot!
Issue was the res.value is not an Int8Array
I want to use this package: sharp in beforeSaveFile trigger but it doesn't work. beforeSaveFile doesn't change anything.
My codes:
Parse.Cloud.define('test', async (req) => {
try {
const Resim = Parse.Object.extend('Resim')
const obj = new Resim()
const { photo } = req.params
let uploadedFile = await new Parse.File(
'galleryFile',
{ base64: photo },
'image/png'
)
obj.set('photo', uploadedFile)
const data = await obj.save()
return data
} catch (error) {
throw error
}
})
Parse.Cloud.beforeSaveFile(async (req) => {
const image = await sharp(req.file).resize(256)
return image
})
Thanks for help.
I figured out. This is the solution:
Parse.Cloud.beforeSaveFile(async (req) => {
const file = req.file
const fileData = await file.getData()
const str = fileData.toString('base64')
const imageBuffer = Buffer.from(str, 'base64')
const newImageBuffer = await sharp(imageBuffer)
.resize(800, 800)
.webp({ quality: 70, lossless: true })
.toBuffer()
return new Parse.File(
'image',
{ base64: newImageBuffer.toString('base64') },
'image/webp'
)
})
I have a requirement where the user wants to upload an image from a source URL let's say "https://homepages.cae.wisc.edu/~ece533/images/airplane.png" using Azure functions. Right now what I am trying to do is I am calling fetch method and pass the image URL and converting that into the blob but somehow that doesn't seem to be working. Below is the code. Is there any better way to achieve this
const { BlobServiceClient } = require("#azure/storage-blob");
const fetch = require("node-fetch");
const multipart = require("parse-multipart");
const AZURE_STORAGE_CONNECTION_STRING = process.env["AZURE_STORAGE_CONNECTION_STRING"];
module.exports = async function (context, req) {
context.log("JavaScript HTTP trigger function processed a request.");
const name =
req.query.name ||
(req.body &&
req.body.secure_url);
const responseMessage = name
? "Hello, " + name + ". This HTTP triggered function executed successfully."
: "This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.";
// context.log("requested body: ", req.body);
var images = "\""+ req.body.secure_url.toString() +"\"";
context.log("Image URL : ", images);
var bodyBuffer = Buffer.from(JSON.stringify(req.body));
let header = req.headers["content-type"]
let boundary = header.split(" ")[1]
boundary = header.split("=")[1]
// var boundary = multipart.getBoundary(req.headers['content-type']);
var parts = multipart.Parse(req.body, header);
var requestOptions = {
method: 'GET'
};
fetch(images, requestOptions)
.then((response) => {
context.log("Response Blob : ",response.blob())
response.blob()
}) // Gets the response and returns it as a blob
.then((blob) => { main(blob)
}).catch(error => console.log('error', error));
async function main(blob) {
const blobServiceClient = await BlobServiceClient.fromConnectionString(AZURE_STORAGE_CONNECTION_STRING);
const container = "sepik01-rdp-media-assets-migration";
const containerClient = await blobServiceClient.getContainerClient(container);
const blobName = images.toString().replace(/^.*[\\\/]/, "");
context.log("File Name: ", blobName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
// const uploadBlobResponse = await blockBlobClient.upload(parts[0].secure_url, parts[0].length);
const uploadBlobResponse = await blockBlobClient.upload(blob, blob.length);
context.res = { body : responseMessage };
context.done();
}
};
when we use package node-fetch to send HTTP request, the response.body will return as Readable stream. Then we can use the stream to upload data to azure blob.
For example
const fetch = require("node-fetch");
const { BlobServiceClient } = require("#azure/storage-blob");
const mime = require("mime");
const AZURE_STORAGE_CONNECTION_STRING =""
module.exports = async function (context, req) {
const images = "https://homepages.cae.wisc.edu/~ece533/images/airplane.png";
const requestOptions = {
method: "GET",
};
const response = await fetch(images, requestOptions);
if (!response.ok)
throw new Error(`unexpected response ${response.statusText}`);
const blobName = images.toString().replace(/^.*[\\\/]/, "");
const blobServiceClient = await BlobServiceClient.fromConnectionString(
AZURE_STORAGE_CONNECTION_STRING
);
const containerClient = await blobServiceClient.getContainerClient("image");
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadBlobResponse = await blockBlobClient.uploadStream(
response.body,
4 * 1024 * 1024,
20,
{
blobHTTPHeaders: {
blobContentType: mime.getType(blobName),
},
}
);
context.res = { body: uploadBlobResponse._response.status };
};