Develop Reference

Get image with fetch api and save through fs module

I can download any files that contains text using this function, but I also want to download images like .png and .jpg.
How do I do that using fetch?
async function fetchGithubFileContent(url) {
let res = await fetch(url).then((res) => res.text())
window.electronAPI.writeFile({ content: res, fileName: "res.js" })
}
async function writeFile({ content, fileName }) {
fs.writeFile(path.join(__dirname, fileName), content, () => console.log(`successfully saved ${fileName}`))
}
fetchGithubFileContent("https://raw.githubusercontent.com/Tetrax-10/Nord-Spotify/master/src/nord.js")
If it can't be with fetch, is it possible with axios?

Am answering my own ques as i solved it after a hour.
import { Buffer } from "buffer"
async function writeFile({ content, fileName }) {
fs.writeFile(path.join(__dirname, fileName), content, () => console.log(`successfully saved ${fileName}`))
}
async function writeImage({ binaryData, fileName }) {
let formatedBinaryData = binaryData.replace(/^data:image\/\w+;base64,/, "")
let buffer = Buffer.from(formatedBinaryData, "base64")
await writeFile({ content: buffer, fileName: fileName })
}
async function imageUrlToBase64(url) {
let response = await fetch(url)
let blob = await response.blob()
return new Promise((onSuccess, onError) => {
try {
const reader = new FileReader()
reader.onload = function () {
onSuccess(this.result)
}
reader.readAsDataURL(blob)
} catch (error) {
onError(error)
}
})
}
async function downloadGithubImage(url) {
let base64 = await imageUrlToBase64(url)
let fileName = url.split("/").pop()
window.electronAPI.writeImage({ binaryData: base64, fileName: fileName })
}
downloadGithubImage("https://raw.githubusercontent.com/Tetrax-10/Spicetify-Themes/master/assets/home.png")

Another method is by using node streams (better method)
import https from "https"
const { pipeline } = require("stream/promises")
import path from "path"
import fs from "fs"
async function download(url) {
return new Promise(async (onSuccess) => {
https.get(url, async (res) => {
let fileName = url.split("/").pop()
const fileWriteStream = fs.createWriteStream(path.join(__dirname, fileName), {
autoClose: true,
flags: "w",
})
await pipeline(res, fileWriteStream)
onSuccess("success")
})
})
}
await download("https://raw.githubusercontent.com/Tetrax-10/Spicetify-Themes/master/assets/artist-2.png")

ReactJS upload multiple image in base64 into an array

I want to create an image uplaoder based on base64 and I want to get results as an array but I got empty! array, I know maybe it's a asynchronous issue, but I don't know how to use async, await in map any idea?
const [array, setArray] = useState([]);
const fileBase64 = (img) => {
let result = [...img];
setUrlImage(img);
result && result.map(function (img){
let fileReader = new FileReader();
fileReader.readAsDataURL(img);
fileReader.onloadend = async () => {
let res = await fileReader.result;
setArray([...array, res])
};
})
console.log(array)
}
const handleImage = (e) => {
let image = [...e.target.files];
fileBase64(image);
}
<input type="file" multiple={true} onChange={handleImage}/>

Due to this asynchronous nature, state is being set i.e. push before data urls are set in array.
And that's the reason your your array return empty.
To fix it, you can use create Promise which gets resolved after load event of each file. And use Promise.all which would be resolved after each Promise has resolved and then use setArray:
fileBase64 = (img) => {
return new Promise((resolve, reject) => {
let fileReader = new FileReader();
fileReader.onerror = reject
fileReader.onload = function () {
resolve(fileReader.result)
}
fileReader.readAsDataURL(img)
})
}
handleImage = (e) => {
let image = e.target.files;
Promise.all(Array.from(image).map(this.readAsDataURL))
.then((urls) => {
setArray(urls)
})
.catch((error) => {
console.error(error)
})
}

Sharp Image Metadata Extraction Error - Input file contains unsupported image format

I am seeing the following error when trying to extract an image's metadata information with the Sharp module: "Input file contains unsupported image format".
This is only happening for certain signed image urls, particularly ones that contain xmp information in the metadata.
I am hoping someone can help me spot where the issue might be occurring in this code snippet.
Here is the exact code snippet I am using (insert the signed image URL where specified in the doStuff function to test):
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
This code snippet above fails with the "Input file contains unsupported image format" on the line that extracts metadata (imageMetadata = await manipulator.metadata();)
So the strange thing is, I am able to properly extract the metadata (with no errors) with this same code if I add a short Sleep after this line: const fileStream = fs.createWriteStream(localPath);
So this code snippet (all I'm doing here is adding a short sleep after fs.createWriteSteam) allows the image metadata to be extracted without issue:
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
await sleep(1000);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
Why would this Sleep resolve my issues? I don't see any asynchronous calls being run that I would need to be waiting for to complete. Perhaps fs.createWriteStream didn't have enough time to complete its operation? But I do not have the option to await the call to fs.createWriteStream, as it is not async.

web worker - data undefined when passed back to main thread

I'm trying to use a web worker to get some data and then pass them back to the main thread. I have tried with this code but it will not work as expected
onmessage = (e) => {
console.log(e);
if( e.data[0] === 'fetchData' ){
fetch('https://example.com/platform/api/v1/endpoint')
.then( (res) => res.json() )
.then( async (data) => {
const imagesData = await Promise.all(
data.map( async (item) => {
let res = await fetch(item.src);
let img = await res.blob();
let reader = new FileReader();
reader.readAsDataURL(img);
reader.onloadend = () => {
return {
title: item.title,
link: item.link,
src: reader.result
}
}
})
)
postMessage(imagesData);
})
}
}
The imagesData after a console.log will contain an array of seven undefined elements, how I can fix this?
UPDATE
I've changed the code in vue front-end and into the worker and now I'm able to get the data but sometimes the worker will not work and I will not able to get the data or I get just two entries but the expected number is seven items for the front-end. Here is how I've modified the code, maybe I need to terminate the work before use another one?
NB: I'm creating a tab override chrome extension
vue front-end code
<script>
const worker = new Worker('services.js');
export default {
name: 'App',
beforeCreate() {
worker.postMessage(['fetchData']);
},
created() {
this.init();
this.clock();
},
data() {
return {
mostVisited: [],
imagesData: [],
isLoading: true
}
},
methods: {
init() {
worker.onmessage = (e) => {
console.log(e);
this.imagesData = e.data;
this.isLoading = false;
}
browser.topSites.get().then( (sites) => this.mostVisited = sites );
} //end init
}
</script>
web worker code
onmessage = (e) => {
console.log(e);
if( e.data[0] === 'fetchData' ){
fetch('https://example.com/platform/api/v1/endpoint')
.then( (res) => res.json() )
.then( async (data) => {
let imagesData = [];
await Promise.all(
data.map( async (item) => {
let res = await fetch(item.src);
let img = await res.blob();
let reader = new FileReader();
reader.readAsDataURL(img);
reader.onloadend = () => {
imagesData.push({ title: item.title, link: item.link, src: reader.result });
}
})
)
postMessage(imagesData);
}); // end then(data)
}
}

You are not waiting for the asynchronous FileReader to have read your files before resolving the outer Promise, so the Promise.all Promise resolves after let img = await res.blob(); but before onloadend does.
Since you are in a Worker context, you can use the synchronous FileReaderSync API, which would give something like
.then( async (data) => {
let imagesData = [];
await Promise.all(
data.map( async (item) => {
let res = await fetch(item.src);
let img = await res.blob();
let reader = new FileReaderSync();
const result = reader.readAsDataURL(img);
imagesData.push({ title: item.title, link: item.link, src: result });
})
)
postMessage(imagesData);
});
But I'm 99% confident that you don't even need that data: URL, and that it will do more harm than anything.
Remember that the data: URL from a FileReader always encode to base64, which will produce a DOMString containing 134% of the original data, that you multiply per 2 since DOMStrings are stored in UTF-16.
Then, to pass that data to the main context, the browser will have to serialize the data using the structured clone algorithm, and for DOMStrings that means a simple copy in memory. Each image now takes in memory about 5 times its real size, and it's even before the main context starts parsing it again to binary data so it can build the pixels data...
Instead, simply pass the Blob you get as img.
Blobs' inner data is passed by reference by the structured clone algorithm, so all you copy is the small js wrapper around.
Then when you need to display these images on the main context, use URL.createObejctURL(img) which will return a blob: URL directly pointing to that same Blob's data, which is still stored only once in memory.
.then( async (data) => {
let imagesData = [];
await Promise.all(
data.map( async (item) => {
let res = await fetch(item.src);
let img = await res.blob();
const url = URL.createObjectURL(img);
imagesData.push({ title: item.title, link: item.link, src: url, file: img });
})
)
postMessage(imagesData);
});

await not waiting for async function to return value

I'm using react with firebase where I need to upload some pictures to firebase storage and then save the download url that gets returned from the upload function to store that value on firestore.
This is my image uploading function
const imageUpload = async (image) => {
const uploadTask = storage.ref(`images/${image.name}`).put(image);
uploadTask.on(
'state_changed',
(snapshot) => {},
(error) => {
console.log(error);
},
() => {
storage
.ref('images')
.child(image.name)
.getDownloadURL()
.then((url) => {
setImageUrl(url);
console.log(url);
return url;
});
}
);
};
And this is my on submit handler
const handleSubmit = async (e) => {
e.preventDefault();
let entry = {
author: currentUser.email,
body,
};
if (image) {
await imageUpload(image).then(async (url) => {
console.log(url);
entry = {
author: currentUser.email,
body,
imageUrl,
};
try {
await createEntry(entry).then(() => {
setBody('');
setShowSnackbar(true);
});
} catch (error) {
console.log(error);
}
});
}
try {
await createEntry(entry).then(() => {
setBody('');
setShowSnackbar(true);
});
} catch (error) {
console.log(error);
}
};
However this doesn't work because the console shows undefined first and then the url which means that the await is not waiting for the url to be returned. How do i resolve this?

I think you are mixing things.
You don't need to use then on your promises if you are using async / await
Using the async/await idiom your code should look more like
async function handleSubmit(e) {
e.preventDefault();
let entry = {
author: currentUser.email,
body,
};
if (image) {
const url = await imageUpload(image);
entry = {
author: currentUser.email,
body,
imageUrl,
};
try {
await createEntry(entry);
setBody("");
setShowSnackbar(true);
} catch (error) {
console.log(error);
}
}
try {
await createEntry(entry);
setBody("");
setShowSnackbar(true);
} catch (error) {
console.log(error);
}
}
async function imageUpload(image) {
const uploadTask = storage.ref(`images/${image.name}`).put(image);
return new Promise((resolve, reject) => {
uploadTask.on(
"state_changed",
(snapshot) => {},
(error) => {
reject(error);
},
() => {
storage
.ref("images")
.child(image.name)
.getDownloadURL()
.then((url) => {
setImageUrl(url);
resolve(url);
});
}
);
});
}
async/await is actually intended to make programming with promises somehow 'feel' synchronous. Using then and callbacks, beside the fact that the code would not work, makes it taking no benefit from the syntax.
see https://developer.mozilla.org/fr/docs/Learn/JavaScript/Asynchronous/Concepts

The issue is primarily because of await and then are used together.
By converting your code to use await only can help.
Imagine a scenario where a function c is called when function a call to an asynchronous function b resolves:
const a = () => {
b().then(() => c());
};
Here’s the same program, written using async/await instead of promise:
const a = async () => {
await b();
c();
};
So your logic of image upload can look like code below and you can convert rest of the code:
const url = await imageUpload(image)
console.log(url);
entry = {
author: currentUser.email,
body,
imageUrl,
};
and the imageUpload function can look like,
async function imageUpload(image) {
try {
const storageRef = firebase.storage().ref();
// Create the file metadata
const metadata = { contentType: "image/jpeg" };
const fileRef = storageRef.child(`${this.name}/` + image.name);
const uploadTaskSnapshot = await fileRef.put(file, metadata);
const downloadURL = await uploadTaskSnapshot.ref.getDownloadURL();
setImageUrl(url);
console.log(url);
return downloadURL;
} catch (error) {
console.log("ERR ===", error);
}
}