Develop Reference

How to use Async Await in Node.js

I am new to Scraping.
This is my PDF downloading code.
I want to use Async Await in this code.
I don't know where I have to use async await in my code.
function scrapPdf(config, search_url, message) {
console.log('PDF downloading');
got(search_url).then(response => {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}).catch(err => {
console.log(err);
});
}

Here's one possibility assuming you only want to use async/await on the Promise returned from the .then:
async function scrapPdf(config, search_url, message) {
let response = null
console.log('PDF downloading');
try {
response = await got(search_url);
} catch (err) {
console.log(err);
}
if (response) {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}
}

Async/Await in for loop NodeJS Not blocking the loop execuation

I know that old school for loop works in the traditional way - that it waits for the await to finish getting results.
But in my use case, I need to read a file from local/s3 and process it line by line, and for each line I need to call an External API.
Generally I use await inside the loop because all are running inside a lambda and I don't want to use all memory for running it parallelly.
Here I am reading the file using a stream.on() method, and in order to use await inside that, I need to add async in read method, like so:
stream.on('data',async () =>{
while(data=stream.read()!==null){
console.log('line');
const requests = getRequests(); // sync code,no pblms
for(let i=0;i<requests.length;i++){
const result = await apiCall(request[i);
console.log('result from api')
const finalResult = await anotherapiCall(result.data);
}
}
});
This is working but order in which the lines are processed is not guaranteed. I need all in a sync manner. Any help?
Complete Code
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
console.log('Read line');
console.log(id);
try {
for (let i = 0;i < XMLRequests.length;i++) {
totalRequestsCount++;
console.log('Sending request');
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log("got response");
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
failedRequestsCount++;
} else {
successRequestsCount++;
console.log('Response from the Lost Server');
console.log(response[i].data);
}
}
} catch (err) {
console.log(err);
}
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
});
}
async function sendLoSTRequest (options, params) {
const { axios } = options;
const { url, data } = params;
if (url) {
return axios.post(url, data);
// eslint-disable-next-line no-else-return
} else {
console.log('URL is not found');
return null;
}
}
Code needs to flow like so:
read a line in a sync way
process the line and transform the line into an array of two members
for every member call API and do stuff
once line is complete, look for another line, all done in order
UPDATE: Got a workaround..but it fires stream.end() without waiting stream to finish read
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const { ERR_DIR, fs, xml2js, LOST_URL, byline, event } = options;
const { key } = params;
const responseObject = {};
let totalRecordsCount = 0;
let totalRequestsCount = 0;
let failedRequestsCount = 0;
let successRequestsCount = 0;
let dataObject = {};
const queue = (() => {
let q = Promise.resolve();
return fn => (q = q.then(fn));
})();
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
// eslint-disable-next-line no-loop-func
queue(async () => {
try {
for (let i = 0;i < XMLRequests.length;i++) {
console.log('Sending request');
console.log(id);
totalRequestsCount++;
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log('got response');
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
// console.log('Response have the error:');
// await handleError(options, { err: responseObj, id });
failedRequestsCount++;
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
} else {
console.log('Response from the Lost Server');
console.log(response[i].data);
successRequestsCount++;
}
}
} catch (err) {
console.log(err);
}
});
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
Object.assign(responseObject, {
failedRequestsCount,
successRequestsCount,
totalRecordsCount,
totalRequestsCount
});
});
}
Thank You

The sample code at the top of your question could be rewritten like
const queue = (() => {
let q = Promise.resolve();
return (fn) => (q = q.then(fn));
})();
stream.on('data', async() => {
while (data = stream.read() !== null) {
console.log('line');
const requests = getRequests(); // sync code,no pblms
queue(async () => {
for (let i = 0; i < requests.length; i++) {
const result = await apiCall(request[i]);
console.log('result from api');
const finalResult = await anotherapiCall(result.data);
}
});
}
});
Hopefully that will be useful for the complete code

If anyone want a solution for synchronisely process the file, ie, linebyline read and execute some Async call, it's recommended to use inbuilt stream transform. There we can create a transform function and return a callback when finishes.
That's will help of any one face this issues.
Through2 is a small npm library that also can be used for the same.

Promise should return data after end two for loop

which get data from API:
const countries = [
'Spain',
'England',
];
const getLeagueID = () => {
const newData = [];
return new Promise(resolve => {
for (let i = 0; i < countries.length; i++) {
getApi(`https://www.thesportsdb.com/api/v1/json/1/search_all_leagues.php?c=${countries[i]}`)
.then(({ countrys }) => {
countrys.forEach((league, index) => {
if (league.strSport === 'Soccer') {
const getData = {
strSport: league.strSport,
strLeague: league.strLeague,
};
newData.push(getData);
}
if (index === countrys.length - 1 && i === countries.length - 1) {
resolve(newData);
}
});
})
.catch(err => {
console.log(err);
});
}
});
};
In first for loop I doing increment by countries from list.
When Api return data I create second foreach method. Inside this method I get data and push it to arra newData. The problem is with resolve:
if (index === countrys.length - 1 && i === countries.length - 1) {
resolve(newData);
}
I don't know how to write instruction if, which wait for the end foreach and for loop. My instruction if is wrong because not return all data. First time return 3 records, next time 7 records.

It's work, but sure that it can be improved
const getLeagueID = () => {
return new Promise((resolve, reject) => {
const promises = [];
for (let i = 0; i < countries.length; i++) {
promises.push(
getApi(`https://www.thesportsdb.com/api/v1/json/1/search_all_leagues.php?c=${countries[i]}`)
);
}
Promise.all(promises)
.then(res => {
const newData = [];
res.map(row => {
const data = JSON.parse(row);
const {countrys} = data;
countrys.forEach((league, index) => {
if (league.strSport === 'Soccer') {
const getData = {
strSport: league.strSport,
strLeague: league.strLeague,
};
newData.push(getData);
}
});
});
resolve(newData);
})
.catch(err => {
console.log(err);
reject(err);
});
});
}

Function return null, await result not applied

I am using the below function within post method. The async-await is used but in transferAmount totalBalance is not updated when I call the function inside the post route. The return from the function is not proper. I need guidance so that it returns the object with updated values.
async function transferAmount(fromAccountId, toAccountId, amount) {
const session = await mongoose.startSession();
const options= {session, new:true}
let sourceAccount, destinationAccount;
const BASICSAVNGS_MAX_BALANCE = 1500;
const result = {
newSrcBalance: 0,
totalDestBalance:0,
transferedAt:moment.now()
}
try {
session.startTransaction();
const source= await Account.findByIdAndUpdate(
{_id:sourceAccount._id},
{$inc:{balance:-amount}},
options
);
if(source.balance <0) {
// Source account should have the required amount for the transaction to succeed
const errorMessage='Insufficient Balance with Sender:';
throw new ErrorHandler(404,errorMessage);
}
const destination = await Account.findByIdAndUpdate(
{_id:destinationAccount._id},
{$inc:{balance:amount}},
options
);
// The balance in ‘BasicSavings’ account type should never exceed Rs. 50,000
if((destination.accountType.name === 'BasicSavings') && (destination.balance > BASICSAVNGS_MAX_BALANCE)) {
const errorMessage=`Recepient's maximum account limit reached`;
throw new ErrorHandler(404,errorMessage);
}
await session.commitTransaction();
result.transferedAt= moment.now() //*UPDATE THE TRANSFER TIMESTAMP
result.newSrcBalance = source.balance; //*UPDATE THE SOURCE BALANCE
session.endSession();
// finding total balance in destination account
await User.findById(destination.user.id, async function(err,user) {
if(err) {
const errorMessage=`Recepient not found!`;
console.log(err);
throw new ErrorHandler(404,errorMessage);
} else {
if(user.accounts) {
await Account.find({
'_id' :{$in:user.accounts}
}, function(err,userAccounts) {
totalDestBalance = userAccounts.reduce( (accumulator,obj) => accumulator+obj.balance,0);
result.totalDestBalance = totalDestBalance; //*UPDATE THE TOTAL BALANCE
console.log(result);
return result;
});
}
}
});
}
catch (error) {
// Abort transaction and undo any changes
await session.abortTransaction();
session.endSession();
throw new ErrorHandler(404,error);
} finally {
if(session) {
session.endSession();
}
}
}
module.exports = transferAmount;
Result of above function is
{
newSrcBalance: 940,
totalDestBalance: 1060,
transferedAt: 1594982541900
}
But inside the post request below it is {}
const result = await transferAmount(fromAccountId, toAccountId, amount);

You are not returning something inside the function.
User.findById - this receives a callback for returning something.
You can convert it as async/await syntax or have to resolve the result with promise.
Like below:
try {
const user = await User.findById(destination.user.id);
if (user.accounts) {
const userAccounts = await Account.find({ _id: { $in: user.accounts } });
totalDestBalance = userAccounts.reduce((accumulator, obj) => accumulator + obj.balance, 0);
result.totalDestBalance = totalDestBalance; //*UPDATE THE TOTAL BALANCE
console.log(result);
return result;
}
} catch (err) {
const errorMessage = `Recepient not found!`;
console.log(err);
throw new ErrorHandler(404, errorMessage);
}
Or:
return new Promise((resolve, reject) => {
User.findById(destination.user.id, async function(err, user) {
if (err) {
const errorMessage = `Recepient not found!`;
console.log(err);
reject(err);
} else {
if (user.accounts) {
await Account.find(
{
_id: { $in: user.accounts },
},
function(err, userAccounts) {
totalDestBalance = userAccounts.reduce((accumulator, obj) => accumulator + obj.balance, 0);
result.totalDestBalance = totalDestBalance; //*UPDATE THE TOTAL BALANCE
console.log(result);
resolve(result);
}
);
}
}
});
});

I may be wrong but i cannot see a return statement in you transferAmount function.

MongoDB Find queries slow while updating/inserting schema

I'm doing a big loop once a day - which updating existing documents in the database (and also inserting new documents).
this loop get executed in a separate server ( prevents from the main server to be slow ), but the main problem is that all the find queries on the Data base (while the loop is executed) are very slow (the loop slows it down significantly).
This is a very big issue in my website ( this loop must be executed once a day ) and iv'e been trying to find a solution online - but i couldn't manage to find something.
Is there any way to prevent the find queries from being so slow while inserting/updating the database??
uploadProductsManually = async (name, products, map, valuesMap) => {
return new Promise(async function (resolve, reject) {
const company = await Company.findOne({ name }).exec();
if (!company) return reject(new errors.NotFound("Company not found"));
const rows = products;
const parsedRows = [];
const findCorrectKey = (key) => {
const correctKey = key.trim();
if (productFields[correctKey]) return productFields[correctKey];
const category = map.find((item) => {
return item.options.some((option) => {
return option.trim().toLowerCase() === correctKey.toLowerCase();
});
});
const categoryName = category && category.name;
return productFields[categoryName];
};
const hashProductValues = (product) => {
let valueToHash;
if (product.productId) {
valueToHash = product.productId;
} else if (product.certificateId) {
valueToHash = product.certificateId;
} else {
valueToHash = JSON.stringify(
product.size + product.color
);
}
return base64encode(valueToHash);
};
rows.forEach(function (row, i) {
var newProduct = {};
for (var key in row) {
var val = row[key];
if (val) {
let normalizedKey = findCorrectKey(key);
if (normalizedKey) {
newProduct[normalizedKey] = val;
}
let normalizedValue = normalizeValue(normalizedKey, val,valuesMap);
newProduct[normalizedKey] = normalizedValue;
}
}
newProduct.channels = [];
if (newProduct.productId) {
parsedRows.push(newProduct);
}
});
fetchProducts();
function fetchProducts() {
Product.find({ company: company._id }).exec(function (err, products) {
if (err) console.log(err);
var map = {};
if (products) {
products.forEach(function (product) {
const productIdentifier = hashProductValues(product);
map[productIdentifier] = product;
if (product.productStatus == "manual") {
// product.isAvailable = false;
// product.save();
} else {
product.status = "deleted";
product.save();
}
});
}
mergeData(map);
});
}
async function mergeData(map) {
let created = 0;
let updated = 0;
let manual = 0;
async.each(
parsedRows,
function (row, callback) {
const productIdentifier = hashProductValues(row);
let product = map[productIdentifier];
if (product) {
map[productIdentifier] = undefined;
Product.findByIdAndUpdate(id, { $set: updatedProduct }, function (
err,
updatedProd
) {
if (err) {
// errors.push(productIdentifier);
console.log("err is:", err);
}
updated++;
callback();
});
} else {
row = new Product(row);
row.save(function (err) {
if (err) {
// errors.push(productIdentifier);
console.log(err);
}
created++;
callback();
});
}
},
(err) => {
if (err) return reject(err);
Company.findByIdAndUpdate(
company._id,
{ lastUpdate: new Date() },
function (err, comp) {
if (err) console.log(err);
}
);
console.log(
`Created: ${created}\nUpdated: ${updated} \manual: ${manual}`
);
resolve({
created,
updated,
manual,
errors,
});
}
);
}
});
};