I download data from API in chunks decrypt it and than pass to ReadableStream.
But after last chunk, the file is not downloaded.
I work with axios and StreamSaver.js
Code:
Above in the code I declare:
this.filestream = streamSaver.createWriteStream('sample.jpg');
this.writer = await this.filestream.getWriter();
let readableStream;
readableStream = new ReadableStream({
start(ctrl) {
const nextChunk = async () => {
let fileDataResponse = await that.$api.post(
'endpoint', {
file_id: UUID,
chunk_index: index
}, {
headers: {
...
}
}
);
done =
fileDataResponse.data.length <=
fileDataResponse.data.current_index;
if (fileDataResponse.data.data) {
let data = await that.decryptData(fileDataResponse.data.data);
ctrl.enqueue(data);
}
if (!done) {
index += 1;
nextChunk();
} else {
ctrl.close();
}
};
nextChunk();
}
});
const reader = readableStream.getReader();
const close = () => {
that.writer.close();
};
const pump = () =>
reader.read().then((res) => {
if (!res.done) {
that.writer.write(res.value).then(pump);
} else {
close();
}
});
pump();
Where could be my error here?
Thank you a lot!
Issue was the res.value is not an Int8Array
Related
am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};
I have some code that when you console.log it, it looks like the image below:
The code I am running is as follows:
onClick={() => {
const stream = fetch(
'https://lichess.org/api/games/user/neio',
{ headers: { Accept: 'application/x-ndjson' } }
);
const onMessage = obj => {
console.log('test', obj);
};
const onComplete = () =>
console.log('The stream has completed');
stream.then(readStream(onMessage)).then(onComplete);
}}
export const readStream = processLine => response => {
const stream = response.body.getReader();
const matcher = /\r?\n/;
const decoder = new TextDecoder();
let buf = '';
const loop = () =>
stream.read().then(({ done, value }) => {
if (done) {
if (buf.length > 0) processLine(JSON.parse(buf));
} else {
const chunk = decoder.decode(value, {
stream: true,
});
buf += chunk;
const parts = buf.split(matcher);
buf = parts.pop();
for (const i of parts) processLine(JSON.parse(i));
return loop();
}
});
return loop();
};
export default readStream;
What I am trying to do is build a parent object that contains all these individual rows of data.
I'm new at promises and fetch etc. So currently, I have no idea on how to build this parent object that contains each individual row.
Any suggestions?
Can't you have a global array and add items to it like:
var arrCollection = [];
...
const onMessage = obj => {
arrCollection.push(obj);
};
You can have an object with those items doing like:
var objCollection = { items: arrCollection };
So I'm trying to use MP4Box.js. On its readme page it states: it has a demo: "A player that performs on-the-fly fragmentation".
However the onSegment callbacks which should feed MSE are not called at all:
const getVideo = async () => {
const data = await fetch(url, { headers: { range: 'bytes=0-567139' } });
let buff = await data.arrayBuffer();
console.log(buff)
mp4box = MP4Box.createFile();
mp4box.onError = function(e) {
console.log("mp4box failed to parse data.");
};
mp4box.onMoovStart = function () {
console.log("Starting to receive File Information");
};
mp4box.onReady = function(info) {
console.log(info.mime);
mp4box.onSegment = (id, user, buffer ) => {
setBuffer(buffer)
}
mp4box.setSegmentOptions(info.tracks[0].id, null, { nbSamples: 1000 });
var initSegs = mp4box.initializeSegmentation();
console.log(initSegs)
mp4box.start();
};
var nextBufferStart = 0;
buff.fileStart = nextBufferStart;
nextBufferStart = mp4box.appendBuffer(buff);
mp4box.flush();
const mediaSource = new MediaSource();
const mimeCodec = 'video/mp4; codecs="avc1.640029,mp4a.40.2"';
video.src = URL.createObjectURL(mediaSource);
function sourceopen() {
console.log('open')
const source = mediaSource.addSourceBuffer(mimeCodec);
const setBuffer = (buff) => {
source.appendBuffer(buff)
};
}
mediaSource.addEventListener('sourceopen', sourceopen, { once: true });
}
Now putting this into an HTML file would result this in the JavaScript console:
Starting to receive File Information
video/mp4; codecs="avc1.4d4028"; profiles="isom,iso2,avc1,iso6,mp41"
But nothing happens afterwards. Why is the onSegment not called here?
I have written a tiny indexedDB library but dosen't seem to work as expected. So my question is how do i fix or properly reconstruct the code bellow to work as expected with a promise. I'm out of idea.
Although the add and delete method works in terms of successfully adding and deleting item from the database but the delete method won't signal it .then function
class IDB {
constructor(name, version) {
this.name = name;
this.version = version;
this.db = null;
}
open(callback) {
const req = window.indexedDB.open(this.name, this.version);
req.onupgradeneeded = (e) => {
let db = e.target.result;
callback(db);
}
req.onsuccess = (e) => {
this.db = e.target.result;
this.db.onversionchange = () => {
this.db.close();
console.log('Tell user to reload the page');
}
}
req.onerror = (e) => {
let db = e.target.errorCode;
return;
}
}
result(req) {
return new Promise((resolve, reject) => {
if (req) {
req.onsuccess = (e) => {
let res = e.target.result;
if (res) {
resolve(res);
}
};
req.onerror = (e) => {
let err = e.target.errorCode;
if (err) {
reject(err);
}
};
}
})
}
add(store, Items) {
let req;
let tx = this.db.transaction([store], 'readwrite').objectStore(store);
Items.forEach(item => {
req = tx.add(item);
});
return this.result(req);
}
get(store, key) {
let req = this.db.transaction([store], 'readonly')
.objectStore(store).get(key);
return this.result(req);
}
cursor(store) {
let req = this.db.transaction(store).objectStore(store)
.openCursor();
return this.result(req);
}
delete(store, key) {
let req = this.db.transaction([store], 'readwrite')
.objectStore(store).delete(key);
return this.result(req);
}
}
Usage
const employeeData = [
{ id: "00-01", name: "gopal", age: 35, email: "gopal#tutorialspoint.com" },
{ id: "00-02", name: "prasad", age: 32, email: "prasad#tutorialspoint.com" }
];
const idb = new IDB('mydb', 1);
idb.open((db) => {
if (!db.objectStoreNames.contains('user')) {
db.createObjectStore('user', {keyPath: 'id'});
}
});
idb.add('user', employeeData).then(() => alert('Items added')).catch(() => alert('failed'));
But I would prefer the usage with the async/await code
await idb.add('..', ..) // blah blah
I am trying to capture the download progress of a Fetch request and use that to change the width of a progress bar. I looked at ProgressEvent.lengthComputable as a potential solution but unsure if this can be used with the Fetch API.
without checking for errors (as in try/catch etc...)
const elStatus = document.getElementById('status');
function status(text) {
elStatus.innerHTML = text;
}
const elProgress = document.getElementById('progress');
function progress({loaded, total}) {
elProgress.innerHTML = Math.round(loaded/total*100)+'%';
}
async function main() {
status('downloading with fetch()...');
const response = await fetch('https://fetch-progress.anthum.com/30kbps/images/sunrise-baseline.jpg');
const contentLength = response.headers.get('content-length');
const total = parseInt(contentLength, 10);
let loaded = 0;
const res = new Response(new ReadableStream({
async start(controller) {
const reader = response.body.getReader();
for (;;) {
const {done, value} = await reader.read();
if (done) break;
loaded += value.byteLength;
progress({loaded, total})
controller.enqueue(value);
}
controller.close();
},
}));
const blob = await res.blob();
status('download completed')
document.getElementById('img').src = URL.createObjectURL(blob);
}
main();
<div id="status"> </div>
<h1 id="progress"> </h1>
<img id="img" />
adapted from here
Using this utility:
async function* streamAsyncIterable(stream) {
const reader = stream.getReader()
try {
while (true) {
const { done, value } = await reader.read()
if (done) return
yield value
}
} finally {
reader.releaseLock()
}
}
See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators
Then you can use for await...of loop:
const response = await fetch(url)
let responseSize = 0
for await (const chunk of streamAsyncIterable(response.body)) {
responseSize += chunk.length
}
But be aware that responseSize is response-size! Not necessarily download-size! What is the difference? There is no difference if there is no content-encoding (gzip, br, ...). But if a comperession was applied, final download-size will be the size of compressed data (the same content-length), and final response-size will be the size of uncompressed data.
See #ecthiender comment and this thread.
you can use axios instead
import axios from 'axios'
export async function uploadFile(file, cb) {
const url = `//127.0.0.1:4000/profile`
try {
let formData = new FormData()
formData.append("avatar", file)
const data = await axios.post(url, formData, {
onUploadProgress: (progressEvent) => {
console.log(progressEvent)
if (progressEvent.lengthComputable) {
let percentComplete = progressEvent.loaded / progressEvent.total;
if (cb) {
cb(percentComplete)
}
}
}
})
return data
} catch (error) {
console.error(error)
}
}