Develop Reference

How to save data fetched from API with JavaScript

I want to save fetched data to a file, but when program is finished the file contains only two curly brackets "{}". No errors, successfully compiled.
When I try to save data to a variable, the variable contains Promise { <pending> }.
But if we run getData().then(data => console.log(data.foods[0].foodNutrients));, everything works fine and we see the data in console.
How to save the data?
const fetch = require("node-fetch");
const fs = require('fs');
const params = {
api_key: 'i8BGQ3wZNm7Urp1Vb5ly2mfVPcHprcweMGPasvXD',
query: 'carrot, raw',
dataType: ['Survey (FNDDS)'],
pagesize: 1,
}
const api_url =
`https://api.nal.usda.gov/fdc/v1/foods/search?api_key=${encodeURIComponent(params.api_key)}&query=${encodeURIComponent(params.query)}&dataType=${encodeURIComponent(params.dataType)}&pageSize=${encodeURIComponent(params.pagesize)}`
function getData() {
return fetch(api_url).then(response => response.json())
};
const nutrients = getData().then(data => data.foods[0].foodNutrients);
fs.writeFile('/Users/Mark/Desktop/scrap/output.txt', JSON.stringify(nutrients), function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});

Fetch is async so data is received inside the then() block, you can write your file in there:
const fetch = require("node-fetch");
const fs = require('fs');
const params = {
api_key: 'i8BGQ3wZNm7Urp1Vb5ly2mfVPcHprcweMGPasvXD',
query: 'carrot, raw',
dataType: ['Survey (FNDDS)'],
pagesize: 1,
}
const api_url =
`https://api.nal.usda.gov/fdc/v1/foods/search?api_key=${encodeURIComponent(params.api_key)}&query=${encodeURIComponent(params.query)}&dataType=${encodeURIComponent(params.dataType)}&pageSize=${encodeURIComponent(params.pagesize)}`
function getData() {
return fetch(api_url).then(response => response.json())
};
const nutrients = getData().then(data => {
let myData = data.foods[0].foodNutrients;
fs.writeFile('/Users/Mark/Desktop/scrap/output.txt', JSON.stringify(myData), function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
});

Sharp Image Metadata Extraction Error - Input file contains unsupported image format

I am seeing the following error when trying to extract an image's metadata information with the Sharp module: "Input file contains unsupported image format".
This is only happening for certain signed image urls, particularly ones that contain xmp information in the metadata.
I am hoping someone can help me spot where the issue might be occurring in this code snippet.
Here is the exact code snippet I am using (insert the signed image URL where specified in the doStuff function to test):
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
This code snippet above fails with the "Input file contains unsupported image format" on the line that extracts metadata (imageMetadata = await manipulator.metadata();)
So the strange thing is, I am able to properly extract the metadata (with no errors) with this same code if I add a short Sleep after this line: const fileStream = fs.createWriteStream(localPath);
So this code snippet (all I'm doing here is adding a short sleep after fs.createWriteSteam) allows the image metadata to be extracted without issue:
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
await sleep(1000);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
Why would this Sleep resolve my issues? I don't see any asynchronous calls being run that I would need to be waiting for to complete. Perhaps fs.createWriteStream didn't have enough time to complete its operation? But I do not have the option to await the call to fs.createWriteStream, as it is not async.

nodeJS cannot see local path when using API endpoint

I have the code below that runs fine when run in a standalone file. But when I try to use it in my API endpoint and send a request with the postman, it can't seem to work.
I can't understand why. I also have the same issue when trying to write an excel file with the same API endpoint. If I specify a path, it won't find it. If I just use the filename, it will write fine in the current directory.
What am I missing? the code below is from when I use it in a standalone file. If I use it inside this route below, it won't work.
exports.download_excel = async (req, res) => {....
full code below
// modules import
const path = require('path');
const fs = require('fs');
const util = require('util');
const csv = require('#fast-csv/parse');
const { forEach } = require('p-iteration');
// const path = '../csv/';
const casesBO = [];
const readCSVFiles = async function() {
try {
const allLocalFiles = path.join('csv/');
const readDir = util.promisify(fs.readdir);
await readDir(allLocalFiles, async function(err, file) {
forEach(file, async item => {
fs.createReadStream('csv/' + item)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', async row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', async rowCount => {});
});
});
} catch (error) {
console.log(error);
}
};

You are awaiting readDir, but you are also giving it a callback function. You can't both await a Promise and also give it a callback. For that matter, Promises don't take a callback as argument at all.
Also you are writing async everywhere, this is useless for functions that don't await anything inside.
const readCSVFiles = async function () {
try {
const allLocalFiles = path.join('csv/');
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream('csv/' + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => { });
}
} catch (error) {
console.log(error);
}
};

I need help in using double Promises in Javascript

Here is the code that I tried.
// To get base64 code of file
const toBase64 = file => new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onloaded = () => resolve(reader.result.replace(/^data:.+;base64,/, ''));
reader.onerror = error => reject(error);
})
// To make an array of files
const getAttachments = async files => {
let documents;
try {
documents = files.map(async file => {
let base64 = await toBase64(file)
return {
doc: base64,
documentName: file.name,
documentType: file.type
}
})
} catch {
console.error('Failed to get files as base64')
}
return Promise.resolve(documents)
}
And I just tried to get an object array as a result by using the above 2 functions.
Like the following;
getAttachments(Array.from(event.target.files)).then(documents => {
console.info(documents)
}
But the result is
Logged out result in Console
I'd love to know how I can get what I want.
Thanks.

Instead of returning an array of promises try returning an array of resolved promise using the await keyword.
try this
const getAttachments = async files => {
let documents;
try {
documents = files.map(async file => {
let base64 = await toBase64(file)
return {
doc: base64,
documentName: file.name,
documentType: file.type
}
})
return await Promise.all(documents);
} catch {
console.error('Failed to get files as base64')
}
}

await not waiting for async function to return value

I'm using react with firebase where I need to upload some pictures to firebase storage and then save the download url that gets returned from the upload function to store that value on firestore.
This is my image uploading function
const imageUpload = async (image) => {
const uploadTask = storage.ref(`images/${image.name}`).put(image);
uploadTask.on(
'state_changed',
(snapshot) => {},
(error) => {
console.log(error);
},
() => {
storage
.ref('images')
.child(image.name)
.getDownloadURL()
.then((url) => {
setImageUrl(url);
console.log(url);
return url;
});
}
);
};
And this is my on submit handler
const handleSubmit = async (e) => {
e.preventDefault();
let entry = {
author: currentUser.email,
body,
};
if (image) {
await imageUpload(image).then(async (url) => {
console.log(url);
entry = {
author: currentUser.email,
body,
imageUrl,
};
try {
await createEntry(entry).then(() => {
setBody('');
setShowSnackbar(true);
});
} catch (error) {
console.log(error);
}
});
}
try {
await createEntry(entry).then(() => {
setBody('');
setShowSnackbar(true);
});
} catch (error) {
console.log(error);
}
};
However this doesn't work because the console shows undefined first and then the url which means that the await is not waiting for the url to be returned. How do i resolve this?

I think you are mixing things.
You don't need to use then on your promises if you are using async / await
Using the async/await idiom your code should look more like
async function handleSubmit(e) {
e.preventDefault();
let entry = {
author: currentUser.email,
body,
};
if (image) {
const url = await imageUpload(image);
entry = {
author: currentUser.email,
body,
imageUrl,
};
try {
await createEntry(entry);
setBody("");
setShowSnackbar(true);
} catch (error) {
console.log(error);
}
}
try {
await createEntry(entry);
setBody("");
setShowSnackbar(true);
} catch (error) {
console.log(error);
}
}
async function imageUpload(image) {
const uploadTask = storage.ref(`images/${image.name}`).put(image);
return new Promise((resolve, reject) => {
uploadTask.on(
"state_changed",
(snapshot) => {},
(error) => {
reject(error);
},
() => {
storage
.ref("images")
.child(image.name)
.getDownloadURL()
.then((url) => {
setImageUrl(url);
resolve(url);
});
}
);
});
}
async/await is actually intended to make programming with promises somehow 'feel' synchronous. Using then and callbacks, beside the fact that the code would not work, makes it taking no benefit from the syntax.
see https://developer.mozilla.org/fr/docs/Learn/JavaScript/Asynchronous/Concepts

The issue is primarily because of await and then are used together.
By converting your code to use await only can help.
Imagine a scenario where a function c is called when function a call to an asynchronous function b resolves:
const a = () => {
b().then(() => c());
};
Here’s the same program, written using async/await instead of promise:
const a = async () => {
await b();
c();
};
So your logic of image upload can look like code below and you can convert rest of the code:
const url = await imageUpload(image)
console.log(url);
entry = {
author: currentUser.email,
body,
imageUrl,
};
and the imageUpload function can look like,
async function imageUpload(image) {
try {
const storageRef = firebase.storage().ref();
// Create the file metadata
const metadata = { contentType: "image/jpeg" };
const fileRef = storageRef.child(`${this.name}/` + image.name);
const uploadTaskSnapshot = await fileRef.put(file, metadata);
const downloadURL = await uploadTaskSnapshot.ref.getDownloadURL();
setImageUrl(url);
console.log(url);
return downloadURL;
} catch (error) {
console.log("ERR ===", error);
}
}