Develop Reference

Wait for a Javascript function to finish executing and fetch the response from it

I've a array of images and I am uploading these images to firebase storage.
data = {
...data,
downloadedUrl: [],
};
if (data?.image?.length) {
for (const image of data?.image) {
await uploadFile(image, data);
}
}
uploadFile handles the logic for uploading the image to firebase.
const uploadFile = useCallback((file, data) => {
if (!file) return;
const storageRef = ref(storage, `/images/${file.name}`);
const uploadTask = uploadBytesResumable(storageRef, file);
uploadTask.on(
"state_changed",
(snap_shot) => {},
(err) => console.log(err),
async () => {
await getDownloadURL(uploadTask.snapshot.ref).then((url) => {
data.downloadedUrl.push(url);
});
}
);
}, []);
It takes few seconds to get the downloadedUrl from uploadTask and I want to store this downloadedUrl in firebase firestore when I get all the urls.
Issue with the current approach is that before getting the urls, the other function start executing and I am not able to upload this data on firestore with the downloadedUrl
Here's the full function when someone clicks on form submit
const handleFormSubmit = useCallback(
async (data) => {
setLoading(true);
data = {
...data,
downloadedUrl: [],
};
if (data?.image?.length) {
for (const image of data?.image) {
await uploadFile(image, data);
}
}
if (data.downloadedUrl.length) {
uploadDataToFirestore(data);
}
if (!data.downloadedUrl?.length) {
dispatch(handleAlertState({ message: "Error Occured!!!" }));
router.push("/services");
return;
}
setLoading(false);
router.push("/");
},
[dispatch, router, uploadDataToFirestore, uploadFile]
);
const uploadDataToFirestore = useCallback(
async (data) => {
await setDoc(doc(db, "form-responses"), data)
.then((response) => {
console.log("response", response);
dispatch(
handleAlertState({
message: "Success. Your request has been sent. Thank You.",
})
);
})
.catch((error) => {
console.log("error", error);
});
},
[dispatch]
);
This bellow block of code executes the code while images are being uploaded to the cloud storage.
I want to wait for the downloadedUrl and then upload the urls to firebase firestore.
if (!data.downloadedUrl?.length) {
dispatch(handleAlertState({ message: "Error Occured!!!" }));
router.push("/services");
return;
}

Create array of promises
Use Promise.all to watch for every promise
const uploadFile = useCallback((file, data) => {
return new Promise((resolve, reject) => {
if (!file) reject();
const storageRef = ref(storage, `/images/${file.name}`);
const uploadTask = uploadBytesResumable(storageRef, file);
uploadTask.on(
'state_changed',
snap_shot => {},
err => reject(err),
() => resolve(getDownloadURL(uploadTask.snapshot.ref)),
);
});
}, []);
let allPromises = [];
if (data?.image?.length) {
for (const image of data?.image) {
allPromises.push(uploadFile(image, data));
}
}
let uploadedUrls = await Promise.all(allPromises);
console.log(uploadedUrls);
await Promise.all take an array of promises, we created a new array which holds the promise returned from uploadFile function. when all promises resolved then promise.all resolved as well and return array of urls. we await for Promise.all so it will not execute next line until resolved

Sharp Image Metadata Extraction Error - Input file contains unsupported image format

I am seeing the following error when trying to extract an image's metadata information with the Sharp module: "Input file contains unsupported image format".
This is only happening for certain signed image urls, particularly ones that contain xmp information in the metadata.
I am hoping someone can help me spot where the issue might be occurring in this code snippet.
Here is the exact code snippet I am using (insert the signed image URL where specified in the doStuff function to test):
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
This code snippet above fails with the "Input file contains unsupported image format" on the line that extracts metadata (imageMetadata = await manipulator.metadata();)
So the strange thing is, I am able to properly extract the metadata (with no errors) with this same code if I add a short Sleep after this line: const fileStream = fs.createWriteStream(localPath);
So this code snippet (all I'm doing here is adding a short sleep after fs.createWriteSteam) allows the image metadata to be extracted without issue:
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
await sleep(1000);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
Why would this Sleep resolve my issues? I don't see any asynchronous calls being run that I would need to be waiting for to complete. Perhaps fs.createWriteStream didn't have enough time to complete its operation? But I do not have the option to await the call to fs.createWriteStream, as it is not async.

nodeJS cannot see local path when using API endpoint

I have the code below that runs fine when run in a standalone file. But when I try to use it in my API endpoint and send a request with the postman, it can't seem to work.
I can't understand why. I also have the same issue when trying to write an excel file with the same API endpoint. If I specify a path, it won't find it. If I just use the filename, it will write fine in the current directory.
What am I missing? the code below is from when I use it in a standalone file. If I use it inside this route below, it won't work.
exports.download_excel = async (req, res) => {....
full code below
// modules import
const path = require('path');
const fs = require('fs');
const util = require('util');
const csv = require('#fast-csv/parse');
const { forEach } = require('p-iteration');
// const path = '../csv/';
const casesBO = [];
const readCSVFiles = async function() {
try {
const allLocalFiles = path.join('csv/');
const readDir = util.promisify(fs.readdir);
await readDir(allLocalFiles, async function(err, file) {
forEach(file, async item => {
fs.createReadStream('csv/' + item)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', async row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', async rowCount => {});
});
});
} catch (error) {
console.log(error);
}
};

You are awaiting readDir, but you are also giving it a callback function. You can't both await a Promise and also give it a callback. For that matter, Promises don't take a callback as argument at all.
Also you are writing async everywhere, this is useless for functions that don't await anything inside.
const readCSVFiles = async function () {
try {
const allLocalFiles = path.join('csv/');
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream('csv/' + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => { });
}
} catch (error) {
console.log(error);
}
};

Write Blocking Js code to write = csv file (using csv-parser) after scraping data from web

In writeFile fn the records returned by makeCsv function are empty (used await still :/ empty) how to make all the code inside makeCsv blocking such that it i get all entries in records array when i call fn(makeCsv) Expected Code Flow
makeCsv -> reads a csv from local storage and calls function getPubKey
getPubKey-> fetches a key against each call for the account name passed by makeCsv by making a request to url and returns that key
makeCsv-> appends key property to object pushes it to results array and returns it
writeFile -> calls makeCsv and takes array to write new Csv with keys included
Issue : As soon as call to getPubkey function is made end event is triggred resolving promise with empty array and thus writeFile receives empty array because it runs before makeCsv has ended creating all requests and then adding keys to data objects
Code
const csv = require('csv-parser')
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
const fetch = require('node-fetch');
const fs = require('fs');
let nkas = []
async function makeCsv(results) {
const readable = fs.createReadStream('/home/vector/Desktop/test.csv')
return new Promise((resolve, reject) => {
readable.pipe(csv(['acc', 'balance']))
.on('data', async (data) => {
data.key = await getPubkey(data.acc)
results.push(data)
})
.on('end', () => {
return resolve(results);
});
})
}
async function getPubkey(account) {
const body = { account_name: account }
let key;
await fetch('https://eos.greymass.com/v1/chain/get_account', {
method: 'post',
body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' },
})
.then(res => res.json())
.then(json => key = json.permissions[1].required_auth.keys[0].key)
.catch(err => console.log(err))
if (key && key.length > 0)
return key
else
nkas.push(account);
console.log(nkas);
}
async function writeFile() {
const csvWriter = createCsvWriter({
path: 'out.csv',
header: [
{ id: 'acc', title: 'Account' },
{ id: 'balance', title: 'Balance' },
{ id: 'key', title: 'PubKey' }
]
});
let records = []
records = await makeCsv(records)
console.log(records)
csvWriter.writeRecords(records) // returns a promise
.then(() => {
console.log('...Done');
});
}
writeFile();

You need to wait for the getPubKey() promises to resolve before resolving the makeCsv() promise:
async function makeCsv() {
const readable = fs.createReadStream('/home/vector/Desktop/test.csv')
const pending = [];
return new Promise((resolve, reject) => {
readable.pipe(csv(['acc', 'balance']))
.on('data', (data) => {
const getKey = async (data) => {
data.key = await getPubkey(data.acc)
return data
}
pending.push(getKey(data))
})
.on('end', () => {
Promise.all(pending).then(results => resolve(results));
});
})
}

Internal server error aws lambda function nodejs

I am trying out a demo with AWS lambda function using Axios and Cheerio, I am getting back response after calling the endpoint as {message: Internal Server Error}
exports.lambdaHandler = async (event, context) => {
try {
const axios = require('axios');
const cheerio = require('cheerio');
axios.get('https://www.kitco.com').then((response) => {
const html = response.data;
const $ = cheerio.load(html);
const ask = $('#AU-ask').text();
const bid = $('#AU-bid').text();
const resbid = bid.slice(0,7);
const resask = ask.slice(0,7);
const result = {
"ask": resask,
"bid": resbid
}
return result;
});
response = {
'statusCode': 200,
'body': result
}
} catch (err) {
console.log(err);
return err;
}
return response
};

result is clearly not in response scope, therefore this will result in a typical undefined error.
The solution would be to handle the logic inside axios.get callback, try this:
const axios = require('axios');
const cheerio = require('cheerio');
exports.lambdaHandler = (event, context) => {
axios.get('https://www.kitco.com')
.then((response) => {
const html = response.data;
const $ = cheerio.load(html);
const ask = $('#AU-ask').text();
const bid = $('#AU-bid').text();
const resbid = bid.slice(0, 7);
const resask = ask.slice(0, 7);
const result = {
statusCode: 200,
body: {
ask: resask,
bid: resbid
}
};
console.log(result);
})
.catch(err => {
console.log(err);
});
};

You can get error detail in monitor tab of Lambda console web. I guest you get back an error like response is undefined in return response line.
With your code, return response line will be execute immediately when you call the function, but response did not defined in the lambdaHandler scope.
I recommended that, don’t mix async/await syntax with Promise syntax (.then .catch), just use one of them, I suggest use async/await syntax.
The function will like:
exports.lambdaHandler = async (event, context) => {
try {
const axios = require('axios');
const cheerio = require('cheerio');
const response = await axios.get('https://www.kitco.com'); // wait until we get the response
const html = response.data;
const $ = cheerio.load(html);
const ask = $('#AU-ask').text();
const bid = $('#AU-bid').text();
const resbid = bid.slice(0, 7);
const resask = ask.slice(0, 7);
const result = {
"ask": resask,
"bid": resbid
}
return {
statusCode: 200,
body: JSON.stringify(result), // If you working with lambda-proxy-integrations, the `body` must be a string
}; // return to response the request
} catch (err) {
console.log(err);
return {
statusCode: 500, // Example, http status will be 500 when you got an exception
body: JSON.stringify({error: err}),
}
}
};