Develop Reference

Upload Progress with Fetch in Vanilla JS [duplicate]

I'm struggling to find documentation or examples of implementing an upload progress indicator using fetch.
This is the only reference I've found so far, which states:
Progress events are a high level feature that won't arrive in fetch for now. You can create your own by looking at the Content-Length header and using a pass-through stream to monitor the bytes received.
This means you can explicitly handle responses without a Content-Length differently. And of course, even if Content-Length is there it can be a lie. With streams you can handle these lies however you want.
How would I write "a pass-through stream to monitor the bytes" sent? If it makes any sort of difference, I'm trying to do this to power image uploads from the browser to Cloudinary.
NOTE: I am not interested in the Cloudinary JS library, as it depends on jQuery and my app does not. I'm only interested in the stream processing necessary to do this with native javascript and Github's fetch polyfill.
https://fetch.spec.whatwg.org/#fetch-api

Streams are starting to land in the web platform (https://jakearchibald.com/2016/streams-ftw/) but it's still early days.
Soon you'll be able to provide a stream as the body of a request, but the open question is whether the consumption of that stream relates to bytes uploaded.
Particular redirects can result in data being retransmitted to the new location, but streams cannot "restart". We can fix this by turning the body into a callback which can be called multiple times, but we need to be sure that exposing the number of redirects isn't a security leak, since it'd be the first time on the platform JS could detect that.
Some are questioning whether it even makes sense to link stream consumption to bytes uploaded.
Long story short: this isn't possible yet, but in future this will be handled either by streams, or some kind of higher-level callback passed into fetch().

My solution is to use axios, which supports this pretty well:
axios.request({
method: "post",
url: "/aaa",
data: myData,
onUploadProgress: (p) => {
console.log(p);
//this.setState({
//fileprogress: p.loaded / p.total
//})
}
}).then (data => {
//this.setState({
//fileprogress: 1.0,
//})
})
I have example for using this in react on github.

fetch: not possible yet
It sounds like upload progress will eventually be possible with fetch once it supports a ReadableStream as the body. This is currently not implemented, but it's in progress. I think the code will look something like this:
warning: this code does not work yet, still waiting on browsers to support it
async function main() {
const blob = new Blob([new Uint8Array(10 * 1024 * 1024)]); // any Blob, including a File
const progressBar = document.getElementById("progress");
const totalBytes = blob.size;
let bytesUploaded = 0;
const blobReader = blob.stream().getReader();
const progressTrackingStream = new ReadableStream({
async pull(controller) {
const result = await blobReader.read();
if (result.done) {
console.log("completed stream");
controller.close();
return;
}
controller.enqueue(result.value);
bytesUploaded += result.value.byteLength;
console.log("upload progress:", bytesUploaded / totalBytes);
progressBar.value = bytesUploaded / totalBytes;
},
});
const response = await fetch("https://httpbin.org/put", {
method: "PUT",
headers: {
"Content-Type": "application/octet-stream"
},
body: progressTrackingStream,
});
console.log("success:", response.ok);
}
main().catch(console.error);
upload: <progress id="progress" />
workaround: good ol' XMLHttpRequest
Instead of fetch(), it's possible to use XMLHttpRequest to track upload progress — the xhr.upload object emits a progress event.
async function main() {
const blob = new Blob([new Uint8Array(10 * 1024 * 1024)]); // any Blob, including a File
const uploadProgress = document.getElementById("upload-progress");
const downloadProgress = document.getElementById("download-progress");
const xhr = new XMLHttpRequest();
const success = await new Promise((resolve) => {
xhr.upload.addEventListener("progress", (event) => {
if (event.lengthComputable) {
console.log("upload progress:", event.loaded / event.total);
uploadProgress.value = event.loaded / event.total;
}
});
xhr.addEventListener("progress", (event) => {
if (event.lengthComputable) {
console.log("download progress:", event.loaded / event.total);
downloadProgress.value = event.loaded / event.total;
}
});
xhr.addEventListener("loadend", () => {
resolve(xhr.readyState === 4 && xhr.status === 200);
});
xhr.open("PUT", "https://httpbin.org/put", true);
xhr.setRequestHeader("Content-Type", "application/octet-stream");
xhr.send(blob);
});
console.log("success:", success);
}
main().catch(console.error);
upload: <progress id="upload-progress"></progress><br/>
download: <progress id="download-progress"></progress>

Update: as the accepted answer says it's impossible now. but the below code handled our problem for sometime. I should add that at least we had to switch to using a library that is based on XMLHttpRequest.
const response = await fetch(url);
const total = Number(response.headers.get('content-length'));
const reader = response.body.getReader();
let bytesReceived = 0;
while (true) {
const result = await reader.read();
if (result.done) {
console.log('Fetch complete');
break;
}
bytesReceived += result.value.length;
console.log('Received', bytesReceived, 'bytes of data so far');
}
thanks to this link: https://jakearchibald.com/2016/streams-ftw/

As already explained in the other answers, it is not possible with fetch, but with XHR. Here is my a-little-more-compact XHR solution:
const uploadFiles = (url, files, onProgress) =>
new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', e => onProgress(e.loaded / e.total));
xhr.addEventListener('load', () => resolve({ status: xhr.status, body: xhr.responseText }));
xhr.addEventListener('error', () => reject(new Error('File upload failed')));
xhr.addEventListener('abort', () => reject(new Error('File upload aborted')));
xhr.open('POST', url, true);
const formData = new FormData();
Array.from(files).forEach((file, index) => formData.append(index.toString(), file));
xhr.send(formData);
});
Works with one or multiple files.
If you have a file input element like this:
<input type="file" multiple id="fileUpload" />
Call the function like this:
document.getElementById('fileUpload').addEventListener('change', async e => {
const onProgress = progress => console.log('Progress:', `${Math.round(progress * 100)}%`);
const response = await uploadFiles('/api/upload', e.currentTarget.files, onProgress);
if (response.status >= 400) {
throw new Error(`File upload failed - Status code: ${response.status}`);
}
console.log('Response:', response.body);
}
Also works with the e.dataTransfer.files you get from a drop event when building a file drop zone.

I don't think it's possible. The draft states:
it is currently lacking [in comparison to XHR] when it comes to request progression
(old answer):
The first example in the Fetch API chapter gives some insight on how to :
If you want to receive the body data progressively:
function consume(reader) {
var total = 0
return new Promise((resolve, reject) => {
function pump() {
reader.read().then(({done, value}) => {
if (done) {
resolve()
return
}
total += value.byteLength
log(`received ${value.byteLength} bytes (${total} bytes in total)`)
pump()
}).catch(reject)
}
pump()
})
}
fetch("/music/pk/altes-kamuffel.flac")
.then(res => consume(res.body.getReader()))
.then(() => log("consumed the entire body without keeping the whole thing in memory!"))
.catch(e => log("something went wrong: " + e))
Apart from their use of the Promise constructor antipattern, you can see that response.body is a Stream from which you can read byte by byte using a Reader, and you can fire an event or do whatever you like (e.g. log the progress) for every of them.
However, the Streams spec doesn't appear to be quite finished, and I have no idea whether this already works in any fetch implementation.

with fetch: now possible with Chrome >= 105 🎉
How to:
https://developer.chrome.com/articles/fetch-streaming-requests/
Currently not supported by other browsers (maybe that will be the case when you read this, please edit my answer accordingly)
Feature detection (source)
const supportsRequestStreams = (() => {
let duplexAccessed = false;
const hasContentType = new Request('', {
body: new ReadableStream(),
method: 'POST',
get duplex() {
duplexAccessed = true;
return 'half';
},
}).headers.has('Content-Type');
return duplexAccessed && !hasContentType;
})();
HTTP >= 2 required
The fetch will be rejected if the connection is HTTP/1.x.

Since none of the answers solve the problem.
Just for implementation sake, you can detect the upload speed with some small initial chunk of known size and the upload time can be calculated with content-length/upload-speed. You can use this time as estimation.

A possible workaround would be to utilize new Request() constructor then check Request.bodyUsed Boolean attribute
The bodyUsed attribute’s getter must return true if disturbed, and
false otherwise.
to determine if stream is distributed
An object implementing the Body mixin is said to be disturbed if
body is non-null and its stream is disturbed.
Return the fetch() Promise from within .then() chained to recursive .read() call of a ReadableStream when Request.bodyUsed is equal to true.
Note, the approach does not read the bytes of the Request.body as the bytes are streamed to the endpoint. Also, the upload could complete well before any response is returned in full to the browser.
const [input, progress, label] = [
document.querySelector("input")
, document.querySelector("progress")
, document.querySelector("label")
];
const url = "/path/to/server/";
input.onmousedown = () => {
label.innerHTML = "";
progress.value = "0"
};
input.onchange = (event) => {
const file = event.target.files[0];
const filename = file.name;
progress.max = file.size;
const request = new Request(url, {
method: "POST",
body: file,
cache: "no-store"
});
const upload = settings => fetch(settings);
const uploadProgress = new ReadableStream({
start(controller) {
console.log("starting upload, request.bodyUsed:", request.bodyUsed);
controller.enqueue(request.bodyUsed);
},
pull(controller) {
if (request.bodyUsed) {
controller.close();
}
controller.enqueue(request.bodyUsed);
console.log("pull, request.bodyUsed:", request.bodyUsed);
},
cancel(reason) {
console.log(reason);
}
});
const [fileUpload, reader] = [
upload(request)
.catch(e => {
reader.cancel();
throw e
})
, uploadProgress.getReader()
];
const processUploadRequest = ({value, done}) => {
if (value || done) {
console.log("upload complete, request.bodyUsed:", request.bodyUsed);
// set `progress.value` to `progress.max` here
// if not awaiting server response
// progress.value = progress.max;
return reader.closed.then(() => fileUpload);
}
console.log("upload progress:", value);
progress.value = +progress.value + 1;
return reader.read().then(result => processUploadRequest(result));
};
reader.read().then(({value, done}) => processUploadRequest({value,done}))
.then(response => response.text())
.then(text => {
console.log("response:", text);
progress.value = progress.max;
input.value = "";
})
.catch(err => console.log("upload error:", err));
}

I fished around for some time about this and just for everyone who may come across this issue too here is my solution:
const form = document.querySelector('form');
const status = document.querySelector('#status');
// When form get's submitted.
form.addEventListener('submit', async function (event) {
// cancel default behavior (form submit)
event.preventDefault();
// Inform user that the upload has began
status.innerText = 'Uploading..';
// Create FormData from form
const formData = new FormData(form);
// Open request to origin
const request = await fetch('https://httpbin.org/post', { method: 'POST', body: formData });
// Get amount of bytes we're about to transmit
const bytesToUpload = request.headers.get('content-length');
// Create a reader from the request body
const reader = request.body.getReader();
// Cache how much data we already send
let bytesUploaded = 0;
// Get first chunk of the request reader
let chunk = await reader.read();
// While we have more chunks to go
while (!chunk.done) {
// Increase amount of bytes transmitted.
bytesUploaded += chunk.value.length;
// Inform user how far we are
status.innerText = 'Uploading (' + (bytesUploaded / bytesToUpload * 100).toFixed(2) + ')...';
// Read next chunk
chunk = await reader.read();
}
});

const req = await fetch('./foo.json');
const total = Number(req.headers.get('content-length'));
let loaded = 0;
for await(const {length} of req.body.getReader()) {
loaded = += length;
const progress = ((loaded / total) * 100).toFixed(2); // toFixed(2) means two digits after floating point
console.log(`${progress}%`); // or yourDiv.textContent = `${progress}%`;
}

Key part is ReadableStream ≪obj_response.body≫.
Sample:
let parse=_/*result*/=>{
console.log(_)
//...
return /*cont?*/_.value?true:false
}
fetch('').
then(_=>( a/*!*/=_.body.getReader(), b/*!*/=z=>a.read().then(parse).then(_=>(_?b:z=>z)()), b() ))
You can test running it on a huge page eg https://html.spec.whatwg.org/ and https://html.spec.whatwg.org/print.pdf . CtrlShiftJ and load the code in.
(Tested on Chrome.)

Uploading Files: FileAPI vs fetch, equivalent of a progress hook in fetch? [duplicate]

I'm struggling to find documentation or examples of implementing an upload progress indicator using fetch.
This is the only reference I've found so far, which states:
Progress events are a high level feature that won't arrive in fetch for now. You can create your own by looking at the Content-Length header and using a pass-through stream to monitor the bytes received.
This means you can explicitly handle responses without a Content-Length differently. And of course, even if Content-Length is there it can be a lie. With streams you can handle these lies however you want.
How would I write "a pass-through stream to monitor the bytes" sent? If it makes any sort of difference, I'm trying to do this to power image uploads from the browser to Cloudinary.
NOTE: I am not interested in the Cloudinary JS library, as it depends on jQuery and my app does not. I'm only interested in the stream processing necessary to do this with native javascript and Github's fetch polyfill.
https://fetch.spec.whatwg.org/#fetch-api

Streams are starting to land in the web platform (https://jakearchibald.com/2016/streams-ftw/) but it's still early days.
Soon you'll be able to provide a stream as the body of a request, but the open question is whether the consumption of that stream relates to bytes uploaded.
Particular redirects can result in data being retransmitted to the new location, but streams cannot "restart". We can fix this by turning the body into a callback which can be called multiple times, but we need to be sure that exposing the number of redirects isn't a security leak, since it'd be the first time on the platform JS could detect that.
Some are questioning whether it even makes sense to link stream consumption to bytes uploaded.
Long story short: this isn't possible yet, but in future this will be handled either by streams, or some kind of higher-level callback passed into fetch().

My solution is to use axios, which supports this pretty well:
axios.request({
method: "post",
url: "/aaa",
data: myData,
onUploadProgress: (p) => {
console.log(p);
//this.setState({
//fileprogress: p.loaded / p.total
//})
}
}).then (data => {
//this.setState({
//fileprogress: 1.0,
//})
})
I have example for using this in react on github.

fetch: not possible yet
It sounds like upload progress will eventually be possible with fetch once it supports a ReadableStream as the body. This is currently not implemented, but it's in progress. I think the code will look something like this:
warning: this code does not work yet, still waiting on browsers to support it
async function main() {
const blob = new Blob([new Uint8Array(10 * 1024 * 1024)]); // any Blob, including a File
const progressBar = document.getElementById("progress");
const totalBytes = blob.size;
let bytesUploaded = 0;
const blobReader = blob.stream().getReader();
const progressTrackingStream = new ReadableStream({
async pull(controller) {
const result = await blobReader.read();
if (result.done) {
console.log("completed stream");
controller.close();
return;
}
controller.enqueue(result.value);
bytesUploaded += result.value.byteLength;
console.log("upload progress:", bytesUploaded / totalBytes);
progressBar.value = bytesUploaded / totalBytes;
},
});
const response = await fetch("https://httpbin.org/put", {
method: "PUT",
headers: {
"Content-Type": "application/octet-stream"
},
body: progressTrackingStream,
});
console.log("success:", response.ok);
}
main().catch(console.error);
upload: <progress id="progress" />
workaround: good ol' XMLHttpRequest
Instead of fetch(), it's possible to use XMLHttpRequest to track upload progress — the xhr.upload object emits a progress event.
async function main() {
const blob = new Blob([new Uint8Array(10 * 1024 * 1024)]); // any Blob, including a File
const uploadProgress = document.getElementById("upload-progress");
const downloadProgress = document.getElementById("download-progress");
const xhr = new XMLHttpRequest();
const success = await new Promise((resolve) => {
xhr.upload.addEventListener("progress", (event) => {
if (event.lengthComputable) {
console.log("upload progress:", event.loaded / event.total);
uploadProgress.value = event.loaded / event.total;
}
});
xhr.addEventListener("progress", (event) => {
if (event.lengthComputable) {
console.log("download progress:", event.loaded / event.total);
downloadProgress.value = event.loaded / event.total;
}
});
xhr.addEventListener("loadend", () => {
resolve(xhr.readyState === 4 && xhr.status === 200);
});
xhr.open("PUT", "https://httpbin.org/put", true);
xhr.setRequestHeader("Content-Type", "application/octet-stream");
xhr.send(blob);
});
console.log("success:", success);
}
main().catch(console.error);
upload: <progress id="upload-progress"></progress><br/>
download: <progress id="download-progress"></progress>

Update: as the accepted answer says it's impossible now. but the below code handled our problem for sometime. I should add that at least we had to switch to using a library that is based on XMLHttpRequest.
const response = await fetch(url);
const total = Number(response.headers.get('content-length'));
const reader = response.body.getReader();
let bytesReceived = 0;
while (true) {
const result = await reader.read();
if (result.done) {
console.log('Fetch complete');
break;
}
bytesReceived += result.value.length;
console.log('Received', bytesReceived, 'bytes of data so far');
}
thanks to this link: https://jakearchibald.com/2016/streams-ftw/

As already explained in the other answers, it is not possible with fetch, but with XHR. Here is my a-little-more-compact XHR solution:
const uploadFiles = (url, files, onProgress) =>
new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', e => onProgress(e.loaded / e.total));
xhr.addEventListener('load', () => resolve({ status: xhr.status, body: xhr.responseText }));
xhr.addEventListener('error', () => reject(new Error('File upload failed')));
xhr.addEventListener('abort', () => reject(new Error('File upload aborted')));
xhr.open('POST', url, true);
const formData = new FormData();
Array.from(files).forEach((file, index) => formData.append(index.toString(), file));
xhr.send(formData);
});
Works with one or multiple files.
If you have a file input element like this:
<input type="file" multiple id="fileUpload" />
Call the function like this:
document.getElementById('fileUpload').addEventListener('change', async e => {
const onProgress = progress => console.log('Progress:', `${Math.round(progress * 100)}%`);
const response = await uploadFiles('/api/upload', e.currentTarget.files, onProgress);
if (response.status >= 400) {
throw new Error(`File upload failed - Status code: ${response.status}`);
}
console.log('Response:', response.body);
}
Also works with the e.dataTransfer.files you get from a drop event when building a file drop zone.

I don't think it's possible. The draft states:
it is currently lacking [in comparison to XHR] when it comes to request progression
(old answer):
The first example in the Fetch API chapter gives some insight on how to :
If you want to receive the body data progressively:
function consume(reader) {
var total = 0
return new Promise((resolve, reject) => {
function pump() {
reader.read().then(({done, value}) => {
if (done) {
resolve()
return
}
total += value.byteLength
log(`received ${value.byteLength} bytes (${total} bytes in total)`)
pump()
}).catch(reject)
}
pump()
})
}
fetch("/music/pk/altes-kamuffel.flac")
.then(res => consume(res.body.getReader()))
.then(() => log("consumed the entire body without keeping the whole thing in memory!"))
.catch(e => log("something went wrong: " + e))
Apart from their use of the Promise constructor antipattern, you can see that response.body is a Stream from which you can read byte by byte using a Reader, and you can fire an event or do whatever you like (e.g. log the progress) for every of them.
However, the Streams spec doesn't appear to be quite finished, and I have no idea whether this already works in any fetch implementation.

with fetch: now possible with Chrome >= 105 🎉
How to:
https://developer.chrome.com/articles/fetch-streaming-requests/
Currently not supported by other browsers (maybe that will be the case when you read this, please edit my answer accordingly)
Feature detection (source)
const supportsRequestStreams = (() => {
let duplexAccessed = false;
const hasContentType = new Request('', {
body: new ReadableStream(),
method: 'POST',
get duplex() {
duplexAccessed = true;
return 'half';
},
}).headers.has('Content-Type');
return duplexAccessed && !hasContentType;
})();
HTTP >= 2 required
The fetch will be rejected if the connection is HTTP/1.x.

Since none of the answers solve the problem.
Just for implementation sake, you can detect the upload speed with some small initial chunk of known size and the upload time can be calculated with content-length/upload-speed. You can use this time as estimation.

A possible workaround would be to utilize new Request() constructor then check Request.bodyUsed Boolean attribute
The bodyUsed attribute’s getter must return true if disturbed, and
false otherwise.
to determine if stream is distributed
An object implementing the Body mixin is said to be disturbed if
body is non-null and its stream is disturbed.
Return the fetch() Promise from within .then() chained to recursive .read() call of a ReadableStream when Request.bodyUsed is equal to true.
Note, the approach does not read the bytes of the Request.body as the bytes are streamed to the endpoint. Also, the upload could complete well before any response is returned in full to the browser.
const [input, progress, label] = [
document.querySelector("input")
, document.querySelector("progress")
, document.querySelector("label")
];
const url = "/path/to/server/";
input.onmousedown = () => {
label.innerHTML = "";
progress.value = "0"
};
input.onchange = (event) => {
const file = event.target.files[0];
const filename = file.name;
progress.max = file.size;
const request = new Request(url, {
method: "POST",
body: file,
cache: "no-store"
});
const upload = settings => fetch(settings);
const uploadProgress = new ReadableStream({
start(controller) {
console.log("starting upload, request.bodyUsed:", request.bodyUsed);
controller.enqueue(request.bodyUsed);
},
pull(controller) {
if (request.bodyUsed) {
controller.close();
}
controller.enqueue(request.bodyUsed);
console.log("pull, request.bodyUsed:", request.bodyUsed);
},
cancel(reason) {
console.log(reason);
}
});
const [fileUpload, reader] = [
upload(request)
.catch(e => {
reader.cancel();
throw e
})
, uploadProgress.getReader()
];
const processUploadRequest = ({value, done}) => {
if (value || done) {
console.log("upload complete, request.bodyUsed:", request.bodyUsed);
// set `progress.value` to `progress.max` here
// if not awaiting server response
// progress.value = progress.max;
return reader.closed.then(() => fileUpload);
}
console.log("upload progress:", value);
progress.value = +progress.value + 1;
return reader.read().then(result => processUploadRequest(result));
};
reader.read().then(({value, done}) => processUploadRequest({value,done}))
.then(response => response.text())
.then(text => {
console.log("response:", text);
progress.value = progress.max;
input.value = "";
})
.catch(err => console.log("upload error:", err));
}

I fished around for some time about this and just for everyone who may come across this issue too here is my solution:
const form = document.querySelector('form');
const status = document.querySelector('#status');
// When form get's submitted.
form.addEventListener('submit', async function (event) {
// cancel default behavior (form submit)
event.preventDefault();
// Inform user that the upload has began
status.innerText = 'Uploading..';
// Create FormData from form
const formData = new FormData(form);
// Open request to origin
const request = await fetch('https://httpbin.org/post', { method: 'POST', body: formData });
// Get amount of bytes we're about to transmit
const bytesToUpload = request.headers.get('content-length');
// Create a reader from the request body
const reader = request.body.getReader();
// Cache how much data we already send
let bytesUploaded = 0;
// Get first chunk of the request reader
let chunk = await reader.read();
// While we have more chunks to go
while (!chunk.done) {
// Increase amount of bytes transmitted.
bytesUploaded += chunk.value.length;
// Inform user how far we are
status.innerText = 'Uploading (' + (bytesUploaded / bytesToUpload * 100).toFixed(2) + ')...';
// Read next chunk
chunk = await reader.read();
}
});

const req = await fetch('./foo.json');
const total = Number(req.headers.get('content-length'));
let loaded = 0;
for await(const {length} of req.body.getReader()) {
loaded = += length;
const progress = ((loaded / total) * 100).toFixed(2); // toFixed(2) means two digits after floating point
console.log(`${progress}%`); // or yourDiv.textContent = `${progress}%`;
}

Key part is ReadableStream ≪obj_response.body≫.
Sample:
let parse=_/*result*/=>{
console.log(_)
//...
return /*cont?*/_.value?true:false
}
fetch('').
then(_=>( a/*!*/=_.body.getReader(), b/*!*/=z=>a.read().then(parse).then(_=>(_?b:z=>z)()), b() ))
You can test running it on a huge page eg https://html.spec.whatwg.org/ and https://html.spec.whatwg.org/print.pdf . CtrlShiftJ and load the code in.
(Tested on Chrome.)

How to use promises with IndexedDB without transactions auto-committing?

Is there any way to use IndexedDB with promises and async/await without the transactions auto-committing? I understand that you can't do stuff like fetch network data in the middle of a transaction, but everything I was able to find online on the subject indicates that IndexedDB should still work if you simply wrap it in promises.
However, in my testing (Firefox 73), I found that simply wrapping the request's onsuccess method in a Promise is enough to cause the transaction to auto-commit before the promise executes, while the same code works when using the raw IndexedDB API. What can I do?
Here is a simplified minimal example of my code.
const {log, error, trace, assert} = console;
const VERSION = 1;
const OBJ_STORE_NAME = 'test';
const DATA_KEY = 'data';
const META_KEY = 'last-updated';
function open_db(name, version) {
return new Promise((resolve, reject) => {
const req = indexedDB.open(name, version);
req.onerror = reject;
req.onupgradeneeded = e => {
const db = e.target.result;
for (const name of db.objectStoreNames) {db.deleteObjectStore(name);}
db.createObjectStore(OBJ_STORE_NAME);
};
req.onsuccess = e => resolve(e.target.result);
});
}
function idbreq(objs, method, ...args) {
return new Promise((resolve, reject) => {
const req = objs[method](...args);
req.onsuccess = e => resolve(req.result);
req.onerror = e => reject(req.error);
});
}
async function update_db(db) {
const new_time = (new Date).toISOString();
const new_data = 42; // simplified for sake of example
const [old_data, last_time] = await (() => {
const t = db.transaction([OBJ_STORE_NAME], 'readonly');
t.onabort = e => error('trans1 abort', e);
t.onerror = e => error('trans1 error', e);
t.oncomplete = e => log('trans1 complete', e);
const obj_store = t.objectStore(OBJ_STORE_NAME);
return Promise.all([
idbreq(obj_store, 'get', DATA_KEY),
idbreq(obj_store, 'get', META_KEY),
]);
})();
log('fetched data from db');
// do stuff with data before writing it back
(async () => {
log('beginning write callback');
const t = db.transaction([OBJ_STORE_NAME], 'readwrite');
t.onabort = e => error('trans2 abort', e);
t.onerror = e => error('trans2 error', e);
t.oncomplete = e => log('trans2 complete', e);
const obj_store = t.objectStore(OBJ_STORE_NAME);
// This line works when using onsuccess directly, but simply wrapping it in a Promise causes the
// transaction to autocommit before the rest of the code executes, resulting in an error.
obj_store.get(META_KEY).onsuccess = ({result: last_time2}) => {
log('last_time', last_time, 'last_time2', last_time2, 'new_time', new_time);
// Check if some other transaction updated db in the mean time so we don't overwrite newer data
if (!last_time2 || last_time2 < new_time) {
obj_store.put(new_time, META_KEY);
obj_store.put(new_data, DATA_KEY);
}
log('finished write callback');
};
// This version of the above code using a Promise wrapper results in an error
// idbreq(obj_store, 'get', META_KEY).then(last_time2 => {
// log('last_time', last_time, 'last_time2', last_time2, 'new_time', new_time);
// if (!last_time2 || last_time2 < new_time) {
// obj_store.put(new_time, META_KEY);
// obj_store.put(new_data, DATA_KEY);
// }
// log('finished write callback');
// });
// Ideally, I'd be able to use await like a civilized person, but the above example
// shows that IndexedDB breaks when simply using promises, even without await.
// const last_time2 = await idbreq(obj_store, 'get', META_KEY);
// log('last_time', last_time, 'last_time2', last_time2, 'new_time', new_time);
// if (!last_time2 || last_time2 < new_time) {
// obj_store.put(new_time, META_KEY);
// obj_store.put(new_data, DATA_KEY);
// }
// log('finished write callback');
})();
return [last_time, new_time];
}
open_db('test').then(update_db).then(([prev, new_]) => log(`updated db timestamp from ${prev} to ${new_}`));

Orchestrate promises around transactions, not individual requests.
If that causes problems with your design, and you still want to use indexedDB, then design around it. Reevaluate whether you need transactional safety or whether you need to actually reuse a transaction for several requests instead of creating several transactions with only a couple requests per transaction.
There is little to no overhead in spawning a large number of transactions with a small number of requests per transaction in comparison to spawning a small number of transactions with a large number of requests. The only real concern is consistency.
Any await is a yield in disguise. indexedDB transactions timeout when no requests are pending. A yield causes a gap in time so the transactions will timeout.

It turns out that the problem was in a completely different part of my code.
At the end of my top level code, I had
.catch(e => {
error('caught error', e);
alert(e);
});
I'm not sure about the details, but showing an alert appears to cause all the transactions to autocommit, while the promises are still pending, leading the errors I saw once the user clicks "ok" on the alert popup and the pending promises continue. Removing the alert call from my global error handler fixed the issue.

Node Stream - Output multiple Transform streams to single PassThrough stream

I periodically have to download/parse a bunch of Json data, about 1000~1.000.000 lines.
Each request has a chunk limit of 5000. So I would like to fire of a bunch of request at the time, stream each output through its own Transfomer for filtering out the key/value's and then write to a combined stream that writes its output to the database.
But with every attempt it doesn't work, or it gives errors because to many event listeners are set. What seems correct if I understand the the 'last pipe' is always the reference next in the chain.
Here is some code (changed it lot of times so could make little sense).
The question is: Is it bad practice to join multiple streams to one? Google also doesn't show a whole lot about it.
Thanks!
brokerApi/getCandles.js
// The 'combined output' stream
let passStream = new Stream.PassThrough();
countChunks.forEach(chunk => {
let arr = [];
let leftOver = '';
let startFound = false;
let lastPiece = false;
let firstByte = false;
let now = Date.now();
let transformStream = this._client
// Returns PassThrough stream
.getCandles(instrument, chunk.from, chunk.until, timeFrame, chunk.count)
.on('error', err => console.error(err) || passStream.emit('error', err))
.on('end', () => {
if (++finished === countChunks.length)
passStream.end();
})
.pipe(passStream);
transformStream._transform = function(data, type, done) {
/** Treansform to typedArray **/
this.push(/** Taansformed value **/)
}
});
Extra - Other file that 'consumes' the stream (writes to DB)
DataLayer.js
brokerApi.getCandles(instrument, timeFrame, from, until, count)
.on('data', async (buf: NodeBuffer) => {
this._dataLayer.write(instrument, timeFrame, buf);
if (from && until) {
await this._mapper.update(instrument, timeFrame, from, until, buf.length / (10 * Float64Array.BYTES_PER_ELEMENT));
} else {
if (buf.length) {
if (!from)
from = buf.readDoubleLE(0);
if (!until) {
until = buf.readDoubleLE(buf.length - (10 * Float64Array.BYTES_PER_ELEMENT));
console.log('UNTIL TUNIL', until);
}
if (from && until)
await this._mapper.update(instrument, timeFrame, from, until, buf.length / (10 * Float64Array.BYTES_PER_ELEMENT));
}
}
})
.on('end', () => {
winston.info(`Cache: Fetching ${instrument} took ${Date.now() - now} ms`);
resolve()
})
.on('error', reject)

Check out the stream helpers from highlandjs, e.g. (untested, pseudo code):
function getCandle(candle) {...}
_(chunks).map(getCandle).parallel(5000).pipe(...)

How to upload multiple files to Firebase?

Is there a way to upload multiple files to Firebase storage. It can upload single file within single attempt as follows.
fileButton.addEventListener('change', function(e){
//Get file
var file = e.target.files[0];
//Create storage reference
var storageRef = firebase.storage().ref(DirectryPath+"/"+file.name);
//Upload file
var task = storageRef.put(file);
//Update progress bar
task.on('state_changed',
function progress(snapshot){
var percentage = snapshot.bytesTransferred / snapshot.totalBytes * 100;
uploader.value = percentage;
},
function error(err){
},
function complete(){
var downloadURL = task.snapshot.downloadURL;
}
);
});
How to upload multiple files to the Firebase storage.

I found the solution for my above question and I like to put it here because it can be useful for anyone.
//Listen for file selection
fileButton.addEventListener('change', function(e){
//Get files
for (var i = 0; i < e.target.files.length; i++) {
var imageFile = e.target.files[i];
uploadImageAsPromise(imageFile);
}
});
//Handle waiting to upload each file using promise
function uploadImageAsPromise (imageFile) {
return new Promise(function (resolve, reject) {
var storageRef = firebase.storage().ref(fullDirectory+"/"+imageFile.name);
//Upload file
var task = storageRef.put(imageFile);
//Update progress bar
task.on('state_changed',
function progress(snapshot){
var percentage = snapshot.bytesTransferred / snapshot.totalBytes * 100;
uploader.value = percentage;
},
function error(err){
},
function complete(){
var downloadURL = task.snapshot.downloadURL;
}
);
});
}

Firebase Storage uses Promise, so you can use Promises to achieve it.
Here's the firebase blog article that covers this subject:
Keeping our Promises (and Callbacks)
Give Promise.all() an "Array of Promises"
Promise.all(
// Array of "Promises"
myItems.map(item => putStorageItem(item))
)
.then((url) => {
console.log(`All success`)
})
.catch((error) => {
console.log(`Some failed: `, error.message)
});
Upload each file and return a Promise
putStorageItem(item) {
// the return value will be a Promise
return firebase.storage().ref("YourPath").put("YourFile")
.then((snapshot) => {
console.log('One success:', item)
}).catch((error) => {
console.log('One failed:', item, error.message)
});
}
YourPath and YourFile can be carried with myItems array (thus the item object).
I omitted them here just for readability, but you get the concept.

I believe there's a simpler solution:
// set it up
firebase.storage().ref().constructor.prototype.putFiles = function(files) {
var ref = this;
return Promise.all(files.map(function(file) {
return ref.child(file.name).put(file);
}));
}
// use it!
firebase.storage().ref().putFiles(files).then(function(metadatas) {
// Get an array of file metadata
}).catch(function(error) {
// If any task fails, handle this
});

let ad_images=["file:///data/user/0/..../IMG-20181216-WA00001.jpg",
"file:///data/user/0/..../IMG-20181216-WA00002.jpg",
"file:///data/user/0/..../IMG-20181216-WA00003.jpg"];
let firebase_images=[];
const ref = firebase.firestore().collection('ads').doc(newRecord.id);
putStorageItem = (url,index,ext) => {
return firebase.storage().ref('YOURFOLDER/'+ index +'.'+ext ).putFile(url)
.then((snapshot) => {
console.log(snapshot)
firebase_images[index] = snapshot.downloadURL;
//OR
//firebase_images.push(snapshot.downloadURL);
}).catch((error) => {
console.log('One failed:', error.message)
});
}
Promise.all(
ad_images.map( async (item,index) => {
let ext = item.split('/').pop().split(".").pop();
console.log(newRecord.id, item, index, ext);
await putStorageItem(newRecord.id, item, index, ext);
})
)
.then((url) => {
console.log(`All success`);
console.log(firebase_images);
})
.catch((error) => {
console.log(`Some failed: `, error.message)
});

This is a modification of the marked answer for those looking to wait for each upload to complete before the other starts.
As the marked answer stands, the promise is not resolved or rejected so when the upload begins from the loop everything just starts, the 1st file, 2nd.....
Think of 3 uploads each 20mb. The loop will call the upload function almost at the same time, making them run almost concurrently.
This answer solves this using async/await to handle the promises
fileButton.addEventListener('change', async function(e){
//Get files
for (var i = 0; i < e.target.files.length; i++) {
var imageFile = e.target.files[i];
await uploadImageAsPromise(imageFile).then((res)=>{
console.log(res);
});
}
});
//Handle waiting to upload each file using promise
async function uploadImageAsPromise (imageFile) {
return new Promise(function (resolve, reject) {
var storageRef = firebase.storage().ref(fullDirectory+"/"+imageFile.name);
var task = storageRef.put(imageFile);
//Update progress bar
task.on('state_changed',
function progress(snapshot){
var percentage = snapshot.bytesTransferred / snapshot.totalBytes *
100;
},
function error(err){
console.log(err);
reject(err);
},
function complete(){
var downloadURL = task.snapshot.downloadURL;
resolve(downloadURL);
}
);
});
}

#isuru, the guy who uploaded the question has a great solution provided below. But, some of the firebase functions have been updated. So, I have just updated the solution with the new updates in the Firebase.
//Firebase Storage Reference
const storageRef = firebase.storage().ref();
//Upload Image Function returns a promise
async function uploadImageAsPromise(imageFile) {
return new Promise(function (resolve, reject) {
const task = storageRef.child(imageFile.name).put(imageFile);
task.on(
"state_changed",
function progress(snapshot) {
const percentage = (snapshot.bytesTransferred / snapshot.totalBytes) * 100;
},
function error(err) {
reject(err);
},
async function complete() {
//The getDownloadURL returns a promise and it is resolved to get the image url.
const imageURL = await task.snapshot.ref.getDownloadURL();
resolve(imageURL);
}
);
});
}
//Handling the files
fileButton.addEventListener('change', function(e){
const promises = [];
for(const file of e.target.files){//Instead of e.target.files, you could also have your files variable
promises.push(uploadImageAsPromise(file))
}
//The Promise.all() will stop the execution, until all of the promises are resolved.
Promise.all(promises).then((fileURLS)=>{
//Once all the promises are resolved, you will get the urls in a array.
console.log(fileURLS)
})
});

Upload a file & get download URL
export const handleFileUploadOnFirebaseStorage = async (bucketName, file) => {
// 1. If no file, return
if (file === "") return "";
// 2. Put the file into bucketName
const uploadTask = await storage.ref(`/${bucketName}/${file.name}`).put(file);
// 3. Get download URL and return it as
return uploadTask.ref.getDownloadURL().then((fileURL) => fileURL);
};
Upload multiple files & get download URL
export const handleFilesUploadOnFirebaseStorage = async (bucketName, files) => {
// 1. If no file, return
if (files.length === 0) return [];
// 2. Create an array to store all download URLs
let fileUrls = [];
// 3. Loop over all the files
for (var i = 0; i < files.length; i++) {
// 3A. Get a file to upload
const file = files[i];
// 3B. handleFileUploadOnFirebaseStorage function is in above section
const downloadFileResponse = await handleFileUploadOnFirebaseStorage(bucketName, file);
// 3C. Push the download url to URLs array
fileUrls.push(downloadFileResponse);
}
return fileUrls;
};

all the promises get messy pretty quickly, why not use async and await instead?
Here, I have a function that keep tracks of all the images selected from the input/file control to be uploaded:
let images =[];
let imagePaths=[];
const trackFiles =(e)=>{
images =[];
imagePaths =[];
for (var i = 0; i < e.target.files.length; i++) {
images.push(e.target.files[i]);
}
}
And I have another function that will be triggered by a button that the user will click on when ready to do the actual upload:
const uploadFiles =()=>{
const storageRef = storage.ref();
images.map(async img =>{
let fileRef = storageRef.child(img.name);
await fileRef.put(img);
const singleImgPath = await fileRef.getDownloadURL();
imagePaths.push(singleImgPath);
if(imagePaths.length == images.length){
console.log("got all paths here now: ", imagePaths);
}
})
}
We basically loop through each image and perform the upload, and push the image paths into a separate imagePaths array one by one as each of them gets finished at its own pace, I then grab all the paths once we know they are all done by comparing the length of the images and their final paths.

We can Combine multiple Promises like this
Promise.all([promise1, promise2, promise3]).then(function(values) {
console.log(values);
});
And we can Chain Promise like this
return myFirstPromise.then( (returnFromFirst) => {
//Do something
return secondPromise();
}).then( (returnFromSecond) => {
//Do something
return thirdPromise();
}).then( (returnFromThird) => {
//All Done
}).catch( (e) =>{}
console.error("SOMETHING WENT WRONG!!!");
);
Idea is to combine upload file promises with Promise.all & chain them together to get download URLS after each upload
Promise.all(
//Array.map creates a new array with the results
// of calling a function for every array element.
//In this case Array of "Promises"
this.state.filesToUpload.map(item =>
this.uploadFileAsPromise(item))
)
.then(url => {
console.log(`All success`);
//Handle Success all image upload
})
.catch(error => {
console.log(`Some failed: `, error.message);
//Handle Failure some/all image upload failed
});
//return a promise which upload file & get download URL
uploadFileAsPromise(imageFile) {
// the return value will be a Promise
return storageRef
.child("images/users/" + imageFile.name)
.put(imageFile.file)
.then(snapshot => {
console.log("Uploaded File:", imageFile.name);
return snapshot.ref.getDownloadURL().then(downloadURL => {
//promise inside promise to get donloadable URL
console.log("File available at", downloadURL);
);
});
})
.catch(error => {
console.log("Upload failed:", imageFile.name, error.message);
});
}

This was a breeze implementing with rxjs's switchMap and combineLatest for the Angular fire