Develop Reference

How to use Async Await in Node.js

I am new to Scraping.
This is my PDF downloading code.
I want to use Async Await in this code.
I don't know where I have to use async await in my code.
function scrapPdf(config, search_url, message) {
console.log('PDF downloading');
got(search_url).then(response => {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}).catch(err => {
console.log(err);
});
}

Here's one possibility assuming you only want to use async/await on the Promise returned from the .then:
async function scrapPdf(config, search_url, message) {
let response = null
console.log('PDF downloading');
try {
response = await got(search_url);
} catch (err) {
console.log(err);
}
if (response) {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}
}

Calling a Promise Method from another Promise

Hope everyone of you are doing well. I am stuck in a bit of problem. Any help will be highly appreciated.
I am using ldapJS and I want to call another promise from searchEntry method of LDAPJS. When i write the code same as below. it throws an error
exports.getADUsers = async (reqst, resp, next) => {
let flag = false;
var finalList = [];
let adConnStatus = await ConnectAD()
.then().catch((errConnStatus) => {
console.log("Error in connection with Active directory " + errConnStatus);
});
if (adConnStatus.status == 0) {
let client = adConnStatus.client;
const opts = {
filter: '(ObjectClass=*)',
scope: 'sub',
attributes: ['cn', 'sid', "objectSid"]
};
client.search('dc=domain,dc=com', opts, (err, res) => {
if (err) {
console.log("Error: " + err);
} else {
res.on('searchEntry', (entry) => {
finalList = await searchEntry(entry);
});
res.on('searchReference', (referral) => {
console.log('referral: ' + referral.uris.join());
});
res.on('error', (err) => {
console.error('error: ' + err.message);
});
res.on('end', (result) => {
resp.send(finalList);
console.log(result);
});
}
});
}
}
Error:
finalList = await searchEntry(entry);
^^^^^
SyntaxError: await is only valid in async function
Please help! how to call another promise from one promise. Though the method is async why it shows this message? what am i doing wrong?
EDIT
After adding the keyword async as suggested by #Punth. Also modifying a bit of the code. my new code is as follows.
exports.getADUsers = async (reqst, resp, next) => {
let adConnStatus = await ConnectAD()
.then().catch((errConnStatus) => {
console.log("Error in connection with Active directory " + errConnStatus);
});
if (adConnStatus.status == 0) {
var adUsersList = [];
let client = adConnStatus.client;
const opts = {
filter: '(ObjectClass=*)',
scope: 'sub',
attributes: ['cn', 'sid', "objectSid"]
};
client.search('dc=domain,dc=com', opts, (err, res) => {
if (err) {
console.log("Error: " + err);
} else {
res.on('searchEntry', async (entry) => {
var raw = entry.raw;
if (raw.objectSid != "undefined" && raw.objectSid != null && entry.object.cn != null && entry.object.cn != "undefined") {
let userData = {
"Name": entry.object.cn,
"SSID": sidBufferToString(raw.objectSid)
}
var lmn = await ConnectAndGetUsersList(userData.SSID);
userData["XYZ"] = lmn.xyz;
userData["ABC"] = lmn.abc;
adUsersList.push(userData);
}
});
res.on('searchReference', (referral) => {
console.log('referral: ' + referral.uris.join());
});
res.on('error', (err) => {
console.error('error: ' + err.message);
});
res.on('end', (result) => {
console.log(result);
resp.send(adUsersList);
});
}
});
}
}
By running the above code it doesn't shows me anything. res.on('end' ....) is called prior to res.on("searchEntry"....) Therefore the array of adUsersList is null.
Now my question is how to resp.send the final arraylist???

Update this line as follows, to invoke promises using async/await syntax inside a callback you need to make it async
res.on('searchEntry', async (entry) => {
finalList = await searchEntry(entry);
});

How to add async/await to my functions in nodejs?

I tried to make the code asynchronous but I couldn't. What i need to do?
This is my functions:
1.
router.post('/urls', (req, response) => {
count = 2;
webUrl = req.body.url;
depth = req.body.depth;
letstart(webUrl, response);
});
function letstart(urlLink, response) {
request(urlLink, function (error, res, body) {
console.error('error:', error); // Print the error if one occurred
console.log('statusCode:', res && res.statusCode); // Print the response status code if a response was received
//console.log('body:', body); // Print the HTML for the Google homepage.
if (!error) {
getLinks(body);
if (!ifFinishAll) {
GetinsideLinks(linkslinst, response);
}
else {
console.log("Finish crawl");
}
}
else {
console.log("sorry");
return "sorry";
}
});
}
function GetinsideLinks(list, response) {
count++;
if (count <= depth) {
for (let i = 0; i < list.length; i++) {
const link = list[i].toString();
var includeUrl = link.includes(webUrl);
if (!includeUrl) {
request(link, function (error, res, body) {
console.error('error2:', error); // Print the error if one occurred
console.log('statusCode2:', res && res.statusCode); // Print the response status code if a response was received
if (!error) {
getLinks(body);
}
else {
console.log("sorry2");
}
});
}
}
ifFinishAll = true;
}
else {
console.log("finish");
ifFinishAll = true;
response.status(200).send(resArray);
};
return resArray;
}
function getLinks(body) {
const html = body;
const $ = cheerio.load(html);
const linkObjects = $('a');
const links = [];
linkObjects.each((index, element) => {
countLinks = linkObjects.length;
var strHref = $(element).attr('href');
var strText = $(element).text();
var existUrl = linkslinst.includes(strHref);
var existText = textslist.includes(strText);
if (strText !== '' && strText !== "" && strText !== null && strHref !== '' && strHref !== "" && strHref !== null && strHref !== undefined && !existUrl && !existText) {
var tel = strHref.startsWith("tel");
var mail = strHref.startsWith("mailto");
var linkInStart = isUrlValid(strHref);
if (!tel && !mail) {
if (linkInStart) {
links.push({
text: $(element).text(), // get the text
href: $(element).attr('href'), // get the href attribute
});
linkslinst.push($(element).attr('href'));
textslist.push($(element).text());
}
else {
links.push({
text: $(element).text(), // get the text
href: webUrl.toString() + $(element).attr('href'), // get the href attribute
});
linkslinst.push(webUrl.toString() + $(element).attr('href'))
textslist.push($(element).text());
}
}
}
});
const result = [];
const map = new Map();
for (const item of links) {
if (!map.has(item.text)) {
map.set(item.text, true); // set any value to Map
result.push({
text: item.text,
href: item.href
});
}
}
if (result.length > 0) {
resArray.push({ list: result, depth: count - 1 });
}
console.log('res', resArray);
return resArray;
}
I want to return/response finally to the "resArray". I tried to add async and await to function number 1 and number 2 but it didn't succeed. Maybe I need to add async/await to all functions? How can I fix that?

You can achieve your goal by using async-await.
An async function is a function declared with the async keyword, and the await keyword is permitted within them. The async and await keywords enable asynchronous, promise-based behavior to be written in a cleaner style, avoiding the need to explicitly configure promise chains.
Basic example:
function resolveImmediately() {
return new Promise(resolve => {
resolve(true);
});
}
function resolveAfter2Seconds() {
return new Promise(resolve => {
setTimeout(() => {
resolve('resolved');
}, 2000);
});
}
async function asyncCall() {
console.log('calling');
const result = await resolveImmediately();
console.log(result);
if(result) {
const anotherResult = await resolveAfter2Seconds();
console.log(anotherResult);
}
}
asyncCall();
Note: Your code is too long to debug. As a result, to make you understand about the approach (what & how to do), i have added a simple example into my answer.

Why is this asynchronous function being called twice

I am trying to create a user with email and password using firebase, but when I call the function that creates it, it is called twice and I get an error because I am trying to register the email that is already in use.
I noticed that the console.log('CALLED') is called once, I don't understand why RegisterWithEmail is called twice. My auth flow only creates the userDocument in the confirmation phase, for this reason userSnap.length equals zero in the second call and tries to create again.
How can I call this function once?
FILE: emailconfirm.page.tsx
registerEmail = async data => {
const { setRegStatus, createDoc } = this.props;
console.log('CALLED')
await RegisterWithEmail(data).then(res => {
console.log('Final response ', res)
if(res === 'EMAIL_VERIFIED') {
createDoc()
setRegStatus({ status: 'created', data: res })
}
else if(res === 'SOMETHING_WENT_WRONG'){
setRegStatus({ status: 'error', data: res })
}
}).catch(err => {
console.log('Error ', err)
setRegStatus({ status: 'error', data: err })
})
}
FILE: firebase.utils.tsx
export const RegisterWithEmail = async user => {
console.log("Called Once...");
if(!user) return 'SOMETHING_WENT_WRONG';
else {
const snap = await firestore.collection('users').where('email', '==', user.email).get();
const docs = snap.docs.map((doc) => doc.data());
if (docs.length !== 0) return 'EMAIL_HAS_ALREADY_BEEN_TAKEN';
try {
console.log("Trying to register email...");
return await auth.createUserWithEmailAndPassword(user.email, user.password).then(async usr => {
await usr.user.updateProfile({
displayName: user.name
}) // SETTING NAME
const sendVerifyEmail = usr.user.sendEmailVerification().then(() => setTimer(usr.user, 5))
return await sendVerifyEmail.then(msg => {
console.log('Finishing...', msg)
if(msg.txt !== 'waiting') {
if(msg.error) {
throw msg.txt
}
else return msg.txt
}
}).catch(() => {
throw 'EMAIL_NOT_SENT'
})
}).catch(() => {
throw 'USER_NOT_CREATED'
})
} catch (err) {
throw 'USER_ALREADY_REGISTERED'
}
}
}
Developer console:

You shouldn't be mixing and matching .then()s in async functions for your own sanity's sake.
Something like
export const RegisterWithEmail = async (user) => {
if (!user) return false;
const snap = await firestore.collection("users").where("email", "==", user.email).get();
const docs = snap.docs.map((doc) => doc.data());
if (docs.length !== 0) return false;
console.log("Trying to register email...");
try {
const resp = await auth.createUserWithEmailAndPassword(user.email, user.password);
// then ...
return true;
} catch (err) {
// catch ...
}
};
might work better for you.

I need more code to be sure, but I think you should add await
registerEmail = async data => {
console.log('CALLED')
await RegisterWithEmail(data)
}

How to properly format nested try/catch blocks inside a function that returns a server response

I have a function that is called that must return a response to a server. Inside this function are two await function calls that are nested. To track error handling, I added try/catch blocks. Is there a way to avoid having nested try catch blocks to track all cases where the function might fail so I can send back an error server response?
Here's my function, it queries for a user's unique device id's and sends a push notification to each one. If a token becomes invalid, then I delete it from my database:
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).pipe(
filter((userDevices) => userDevices && userDevices.length > 0),
take(1)
).subscribe( async (devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens !== undefined && userDeviceTokens.length != 0) {
try {
message['tokens'] = userDeviceTokens;
const pushResponse = await admin.messsaging().sendMulticast(message);
if (pushResponse.failureCount > 0) {
const failedTokens = [];
pushResponse.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(userDeviceTokens[idx]);
}
});
failedTokens.forEach( async (token) => {
var tokenInstanceID = token.split(':')[0];
try {
await deleteOldToken(tokenInstanceID);
console.log(`Token ${tokenInstanceID} deleted`)
} catch {
return res.status(500).send("err");
}
})
return res.status(200).send("ok");
} else {
return res.status(200).send("ok");
}
} catch {
return res.status(500).send("err");
}
} else {
return res.status(200).send("ok");
}
})
}
It just feels a bit excessive with all the returns I must have. Where can I improve?
EDIT, broke apart code into three blocks to prevent arrow coding
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).pipe(
filter((userDevices) => userDevices && userDevices.length > 0),
take(1)
).subscribe(async (devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens !== undefined && userDeviceTokens.length != 0) {
try {
message['tokens'] = userDeviceTokens;
const response = await admin.messaging().sendMulticast(message);
const oldTokensArray = checkOldTokens(response, userDeviceTokens);
if (oldTokensArray.length > 0) {
await deleteOldTokens(oldTokensArray);
return res.status(200).send("ok");
} else {
return res.status(200).send("ok");
}
} catch (err) {
return res.status(500).send(err);
}
} else {
return res.status(200).send("ok");
}
})
}
function checkOldTokens(response, userDeviceTokens) {
if (response.failureCount > 0) {
const failedTokens = [];
response.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(userDeviceTokens[idx]);
}
});
return failedTokens;
} else {
return [];
}
}
async function deleteOldTokens(tokenArray) {
for (const token of tokenArray) {
await fb.firestore().collection('devices').doc(token).delete();
}
}