Develop Reference

Callback function inside for loop - Nodejs

Hope you're having a good day. I recently discovered that it is not as easy to handle callbacks inside a for loop. I have tried a few things but couldn't find a solution.
Here is the code:
var book = new Array;
var chapters = Object.keys(epub.spine.contents).length;
for (let i = 0; i < chapters; i++) {
let cacheArray = [];
epub.getChapter(epub.spine.contents[i].id, function (err, data) {
if (err) {
return console.log(err);
}
//remove html tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
})
}
console.log(book)//returns empty array ofc
After this code is executed, I need to loop over the array to search its contents. If that was not the case, my approach would be to just send it to a db.
My approach was the following:
var recursiveChapter = function (n) {
var book = new Array;
if (n < chapters) {
// chapter function
epub.getChapter(epub.spine.contents[n].id, function (err, data) {
if (err) {
throw err
}
//remove HTML tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
recursiveChapter(n + 1)
});
}
}
//start the recursive function
recursiveChapter(0)
console.log(book)//returns an empty array
I am stuck and can't think of a way of using this data without storing it in a db.
Any help would be appreciated .

There are a few ways you can tackle this. One way is to use the async library, this allows to to run async. calls in parallel, or in series.
For example, we can use async.mapSeries() to get the result of a series of asynchronous calls as an array.
You input your array of ids, then the callback will return an array of the data returned, for example, I've mocked out the getChapter() function to give you an idea of how it would work:
// Mock out epub object
const epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
console.log("Calling async.mapSeries for ids:", ids);
async.mapSeries(ids, epub.getChapter, (err, result) => {
if (err) {
console.error(err);
} else {
console.log("Result:", result)
}
})
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/3.2.0/async.min.js" integrity="sha512-6K6+H87tLdCWvY5ml9ZQXLRlPlDEt8uXmtELhuJRgFyEDv6JvndWHg3jadJuBVGPEhhA2AAt+ROMC2V7EvTIWw==" crossorigin="anonymous"></script>
You could also promisify the epub call and use Promise.all to get the result, like so:
epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
function getChapterPromisified(id) {
return new Promise((resolve, reject) => {
epub.getChapter(id, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
})
})
}
// Create a list of promises, one for each call
const promises = ids.map(id => getChapterPromisified(id))
Promise.all(promises)
.then(result => console.log("Result:", result))
.catch(error => console.error("An error occurred:", err));

Trouble with Async/Await with mySQL result in Node

I have the following code, which is an API call through Express. I need to return an array of quotes, and an array of the parts associated with those quotes. To do so, I'm using async/await, which is working properly in my quotes variable, but I'm getting an empty array in my parts variable.
I believe this has to do with the parts results being pending promises, because when I console log my results within the loop, I get the appropriate values.
I suspect I must be misunderstanding how either async/await works, or how promises/Promise.all works.
Any help understanding why I'm not getting fulfilled promise results is appreciated!
router.get('/:pagenum/:dealerno/:modelyear/', async (req, res) => {
let quotes = await getQuotes(
req.params.pagenum,
req.params.dealerno,
req.params.modelyear
);
console.log('MY QUOTES:');
console.log(quotes); //This returns an array of my quotes, as expected
let parts = await getQuoteParts(quotes);
console.log('MY PARTS:');
console.log(parts); //This returns an array of undefined values
});
function getQuotes(pagenum, dealerno, modelyear) {
// return new Promise(function (resolve, reject) {
var q_quote =
"SELECT quote.id, DATE_FORMAT(quote.timestamp, '%b %d, %Y') AS timestamp, quote.quotenum, quote.quotetype, quote.configtype, quote.sid, quote.tractortype, customer.cus_id, customer.customername, parts.partno, units.type AS fmtype, combos.type AS mmtype FROM GH_savedquoteinfo AS quote JOIN GH_internal_customers AS customer ON quote.cusid=customer.cus_id JOIN GH_savedquoteparts AS parts ON (quote.quotenum = parts.quotenum AND parts.type='unit') LEFT JOIN " +
modelyear +
"_fm_units AS units ON (parts.partno = units.model_no AND quote.tractortype='FM') LEFT JOIN " +
modelyear +
"_mm_combos AS combos ON (parts.partno = combos.model_no AND quote.tractortype='MM') WHERE quote.dealerno=? AND quote.modelyear=? AND quote.quotetype!='multi' ORDER BY quote.quotenum LIMIT 7 OFFSET 0;";
let resultHolder = [];
return new Promise((resolve, reject) => {
connection.query(q_quote, [dealerno, modelyear], function (
err_quote,
result_quote
) {
if (err_quote) {
console.log('error');
} else {
try {
resultHolder.push(result_quote);
} catch (e) {
console.log('error');
} finally {
resolve(resultHolder);
}
}
});
});
}
async function getQuoteParts(quoteResult) {
let quoteParts = [];
let quotePartsResult = await Promise.all(
quoteResult[0].map(async (quote, index) => {
var q_parts =
"SELECT * FROM GH_savedquoteparts WHERE quotenum=? AND (type='' OR type='pvcomplete') ORDER BY id";
connection.query(q_parts, [quote.quotenum], function (
err_parts,
result_parts
) {
if (err_parts) {
console.log('error in parts' + err.message);
} else {
console.log(result_parts); //This returns an array of my parts
quoteParts.push(result_parts);
}
});
})
);
return quoteParts;
}

The reason you're getting an array of undefined values is because you're returning quotePartsResult instead of quoteParts in which you're pushing values. Since you're running a .map on quoteResult and not returning anything inside the map function, it resolves to undefined.
I would like to refactor your code for a better understanding. You should have an array of promises in your Promise.all() call.
Something like:
const promises = qouteResult.map(async (quote) => {
// Your query calls. You can also promisify the callbacks in here using
util.promisify()
});
OR
const promises = qouteResult.map(qoute => {
return new Promise((resolve,reject) => {
// Your query calls.
if(success) {
resolve()
} else {
reject()
}
});
});
const result = await Promise.all(promises);
return result;

As someone in the comments mentioned, the main thing I was missing was a new Promise wrapper around my second connection. I also needed a Promise.all within that second function, getQuoteParts. Here's what the final code looked like:
router.get('/:pagenum/:dealerno/:modelyear/', async (req, res) => {
//QUOTES ARRAY
const quotes = await getQuotes(
req.params.pagenum,
req.params.dealerno,
req.params.modelyear
);
const parts = await getQuoteParts(quotes);
const details = await getPartDescrips(parts, quotes);
Promise.all([quotes, parts, details]).then((results) => {
res.send(results);
});
});
function getQuotes(pagenum, dealerno, modelyear) {
// return new Promise(function (resolve, reject) {
const qQuote =
"SELECT quote.id, DATE_FORMAT(quote.timestamp, '%b %d, %Y') AS timestamp, quote.quotenum, quote.quotetype, quote.configtype, quote.sid, quote.tractortype, customer.cus_id, customer.customername, parts.partno, units.type AS fmtype, combos.type AS mmtype FROM GH_savedquoteinfo AS quote JOIN GH_internal_customers AS customer ON quote.cusid=customer.cus_id JOIN GH_savedquoteparts AS parts ON (quote.quotenum = parts.quotenum AND parts.type='unit') LEFT JOIN " +
modelyear +
"_fm_units AS units ON (parts.partno = units.model_no AND quote.tractortype='FM') LEFT JOIN " +
modelyear +
"_mm_combos AS combos ON (parts.partno = combos.model_no AND quote.tractortype='MM') WHERE quote.dealerno=? AND quote.modelyear=? AND quote.quotetype!='multi' ORDER BY quote.quotenum DESC LIMIT 7 OFFSET 0;";
let resultHolder = [];
return new Promise((resolve, reject) => {
connection.query(qQuote, [dealerno, modelyear], function (
errQuote,
resultQuote
) {
if (errQuote) {
console.log(errQuote.message);
} else {
try {
resolve(resultQuote);
} catch (e) {
console.log(e.message);
}
}
});
});
}
function getQuoteParts(quoteResult) {
var partResults = quoteResult.map((quote, index) => {
return new Promise((resolve, reject) => {
const qParts =
"SELECT * FROM GH_savedquoteparts WHERE quotenum=? AND (type='' OR type='pvcomplete') ORDER BY type='pvcomplete', type='', id";
// new Promise((resolve, reject) => {
connection.query(qParts, [quote.quotenum], function (
errParts,
resultParts
) {
if (errParts) {
console.log('error in PARTS');
} else {
resolve(resultParts);
}
});
});
});
return Promise.all(partResults);
}
I wrote an article outlining how it works here:
https://medium.com/#amymurphy_40966/node-mysql-chained-promises-vs-async-await-9d0c8e8f5ee1

Cannot await for sqlite3.Database.get() function completion in Node.js

I'm struggling with some basic async/await problem in node.js using node-sqlite3.
My objective is to select some value from SQLite DB, check it for some condition and take some actions in case the condition is met. Here's the code:
const sqlite3 = require('sqlite3').verbose();
main();
async function main() {
let ordersDb = await createDbConnection('./ProcessedOrders.db');
var orderProcessed = await orderAlreadyProcessed(ordersDb, "555");
console.log("orderProcessed = " + orderProcessed);
if (!orderProcessed) {
console.log("So condition is met!");
}
}
async function orderAlreadyProcessed(ordersDb, orderNumberStr) {
console.log('starting orderAlreadyProcessed function'); //DEBUG
var result;
var query = 'select count(SoldOrderNumber) as "recsCount" from ProcessedSoldOrders where SoldOrderNumber = ?;';
await ordersDb.get(query
,[orderNumberStr]
,(err, row) => {
console.log('Row with count = ' + row); //DEBUG
console.log('row.recsCount = ' + row.recsCount); //DEBUG
result = typeof row !== 'undefined' && row.recsCount > 0;
});
console.log('Returning ' + result); //DEBUG
return result;
}
async function createDbConnection(dbFileName) {
let db = new sqlite3.Database(dbFileName, (err) => {
if (err) {
console.log(err.message);
}
});
return db;
}
But what I get is code executing further, not awaiting for Database.get() method at all! As a result, here's what I see printing in console:
starting orderAlreadyProcessed function
Returning undefined
orderProcessed = undefined
So IF condition met!
Row with count = [object Object]
row.recsCount = 1
As we can see, we return from orderAlreadyProcessed too early with return value = 'undefined'. So condition is met, actions taken, and only then Database.get() returns. But if it was properly awaited, condition would not be met.
How can I make it await for result value?

Since you want to use async/await, and the node-sqlite3 (sqlite3) library does not support the Promise API, you need to use the node-sqlite (sqlite) library, which is a wrapper over sqlite3 and adds support for the Promise API. Then, your code will look something like this:
const sqlite3 = require('sqlite3');
const { open } = require('sqlite');
async function main() {
try {
sqlite3.verbose();
const ordersDb = await createDbConnection('./ProcessedOrders.db');
const orderProcessed = await orderAlreadyProcessed(ordersDb, "555");
console.log("orderProcessed = " + orderProcessed);
if (!orderProcessed) {
console.log("So condition is met!");
}
} catch (error) {
console.error(error);
}
}
async function orderAlreadyProcessed(ordersDb, orderNumberStr) {
try {
console.log('Starting orderAlreadyProcessed function');
const query = 'SELECT COUNT(SoldOrderNumber) as `recsCount` from ProcessedSoldOrders where SoldOrderNumber = ?;'
const row = await ordersDb.get(query, [orderNumberStr]);
console.log('Row with count =', row);
console.log('row.recsCount =', row.recsCount);
const result = typeof row !== 'undefined' && row.recsCount > 0;
console.log('Returning ' + result);
return result;
} catch (error) {
console.error(error);
throw error;
}
}
function createDbConnection(filename) {
return open({
filename,
driver: sqlite3.Database
});
}
main();
I specifically did not remove your console.log and other parts of the code so as not to confuse the original logic of your program.

If we don't to use another library
then we can return a new Promise function & use await, as below:
Note: Below has example for INSERT/run, instead of SELECT/get, but promise/await works same
const sqlite3 = require("sqlite3").verbose();
let db;
db = new sqlite3.Database('./Chinook.db');
function insert() {
return new Promise((resolve, reject) => { // return new Promise here <---
const userId = uuid4();
let sql = `INSERT INTO Users(id) VALUES (?)`; // INSERT <----
let params = [userId];
return db.run(sql, params, function (err, res) { // .run <----
if (err) {
console.error("DB Error: Insert failed: ", err.message);
return reject(err.message);
}
return resolve("done");
});
});
}
let result = await insert(); // now await works fine <------
res.send({ result });

Async/Await with map or foreach

I'm trying to retrieve a bunch of product prices from my database and assumed I could just map, or foreach, through them and += the price onto a variable as below:
// Get Total
exports.getTotal = (req,res) => {
let productList = req.body;
let total = 0;
const results = productList.map(async (product) => {
Product.findById(product._id)
.select('price')
.exec((err, foundProduct) => {
if (err){
console.log(`Error: Product with id ${product._id} not found.`);
} else {
console.log('Product price. ', foundProduct.price);
total += foundProduct;
}
})
});
Promise.all(results).then(data => console.log('Total is', total));
};
However the console.log for the total always returns 0. I suspect it's an issue that the console.log is running before the map and database lookup promise has complete.
Any guidance appreciated.

You are using exec in a wrong way Exec returns you a promise. You can simplify this like
// Get Total
exports.getTotal = (req, res) => {
const productList = req.body;
let total = 0;
const results = productList.map(async product => {
const foundProduct = await Product.findById(product._id).exec();
total += foundProduct;
return foundProduct;
});
Promise.all(results).then(data => console.log("Total is", total));
};

you forgot to await an asynchronous function await Product.findById(product._id).
Also writing in a functional way removes clutter for this kind of task by using array reduce.
// Get Total
exports.getTotal = (req, res) => {
const results = req.body.reduce(async (total, product) => {
await Product.findById(product._id)
.select('price')
.exec((err, foundProduct) => {
if (err){
console.log(`Error: Product with id ${product._id} not found.`);
return total
} else {
console.log('Product price. ', foundProduct.price);
return total += foundProduct;
}
})
}, 0);
Promise.all(results).then(data => console.log('Total is', data));
};

Array's map(), foreach() and other iteration methods do not use promises. So making the callbacks async does not make the internal iteration wait before the doing the next iteration call to the callback. Also your map would have been filled with promises that never get resolved as you never return anything, it would be equivalent to having
Promises.all([new Promise(), new Promise(), new Promise()])
You can change your code to create a proper map of new promises and then calculate the total after everything has been fetched. Note since exec() itself returns a promise you just return that:
const results = productList.map(product=> {
return Product.findById(product._id).select('price').exec();
});
Promise.all(results).then(values => {
let total = values.reduce((a, b) => a + b);
console.log(`Total: ${total}`)
});
You could also just use a for loop in a standalone async function and use await
async function getTotal(){
let total = 0;
for(let product of productList){
let price = await Product.findById(product._id).select('price').exec();
total+=price;
}
return total;
}
getTotal().then(total=>console.log(`Total: ${total}`));

Terminate mysql connection after multiple queries have executed

I have some node.js code which fetches data from an API in a loop and runs mutliple mysql queries to update some rows.
The issue I have is that the script keeps running until I terminate the mysql connection with connection.end(). I am a newbie in asynchronous code. Where do I call the termination function so that it executes when all the queries have finished executing? What's the right design pattern for this? Would waterfall be any good?
This is a snippet from the code I have at the moment (error handling removed for simplicity):
var connection = mysql.createConnection({ host, user, etc... });
for (var i = 0; i < 10; i++) {
var url = "http://api.com?i="+i;
request(url, function(error, response, body) {
var data = JSON.parse(body);
for (el in data) {
connection.query(
"UPDATE table SET col = ? WHERE symbol = ?",
[
data[el].col,
el
]
);
}
});
}
// this will run before all queries have executed
// resulting in an error
connection.end();

So, the problem here is that you are cycling in a synchronized way through the data here:
var data = JSON.parse(body);
for (el in data) {
connection.query(
"UPDATE table SET col = ? WHERE symbol = ?",
[
data[el].col,
el
]
);
}
while the mysql module handles the query in a callback style:
connection.query(query, function(error, rows, fields) {
if (error) {
return callback(error);
} else {
return callback(null,rows);
}
});
where callback has the signature callback(error,rows), so that you can handle the results in this way supposed to have a reusable function:
var executeQuery = function(query,callback) {
var self=this;
this.connection.query(query, function(error, rows, fields) {
if (error) {
return callback(error);
} else {
return callback(null,rows);
}
});
}
and you can call in your code like
executeQuery(statement, function(error,rows) {
//...
})
That said, you must consider that you are doing multiple queries to your database and it is not recommended to do this in for loop cycle. You should consider to use a better solution that could be a waterfall as you say or a promise all using the Promise paradigma.
Suppose that to have this nice function:
var promiseAllP = function(items, block) {
var promises = [];
items.forEach(function(item,index) {
promises.push( function(item,i) {
return new Promise(function(resolve, reject) {
return block.apply(this,[item,index,resolve,reject]);
});
}(item,index))
});
return Promise.all(promises);
}
that takes as input an array of items and a execution function that is function(item,index,resolve,reject) that has a resolve and reject functions of a Promise, so let's turn your executeQuery function in a Promise as well:
var executeQueryP = function(query) {
var self=this;
return new Promise(function(resolve, reject) {
self.connection.query(query, function(error, rows, fields) {
if (error) {
return reject(error);
} else {
return resolve(null,rows);
}
});
}
Now you can process your data in a totally async way promisyfied:
promiseAllP(data,(item,index,resolve,reject) => {
var query= "UPDATE table SET col = %s WHERE symbol = %s";
// example: prepare the query from item in the data
query = replaceInString(query,item.col,item);
executeQueryP(query)
.then(result => resolve(result))
.catch(error => reject(error))
})
.then(results => { // all execution completed
console.log(results)
})
.catch(error => { // some error occurred while executing
console.error(error)
})
where the replaceInString will help you to prepare the statement
var replaceInString = function() {
var args = Array.prototype.slice.call(arguments);
var rep= args.slice(1, args.length);
var i=0;
var output = args[0].replace(/%s|%d|%f|%#/g, function(match,idx) {
var subst=rep.slice(i, ++i);
return( subst );
});
return(output);
},//replace,
This is what we have done here:
Used native Promise only
Turned your mysql query in a promise
Called the statements against your data in a completely asynchronous way
Used a Promise and Promise all paradigma, that let you collect the results of the Promise and return to the caller when all the functions are completed.
Catched errors in all the statements execution
Added a simply way to fulfill statements with parameters
Also notice the arrow function syntax (param1, param2) => that simplify the way to write a function, that can help a lot with the Promise paradigma.

For anyone interested, I ended up solving it by a mixture of promises and counting the queries, something along the lines of this (not sure if this code actually works but the idea is there):
function fetchFromAPI() {
return new Promise((resolve, reject)=>{
var urls = [];
for (var i = 0; i < 10; i++) {
urls.push("http://api.com?i="+i);
}
var data = [];
var requestedUrls=0;
urls.forEach(url=>{
request(url, (err, response, body) {
if(err) reject(err);
data.push(JSON.parse(body));
requestedUrls++;
if(requestedUrls==urls.length) resolve(data);
};
});
}
}
fetchFromAPI().then(data=>{
mysql.createConnection({ user, hostname, etc... });
var processedKeys=0;
data.forEach(el=> {
mysql.query("UPDATE table SET name = ? WHERE id = ?", [el.name, el.id], (err, rows, fields) => {
processedKeys++;
if(processedKeys==data.length) {
connection.end();
}
});
}
}).catch(err=>{
console.error(err);
});