Develop Reference

Callback function inside for loop - Nodejs

Hope you're having a good day. I recently discovered that it is not as easy to handle callbacks inside a for loop. I have tried a few things but couldn't find a solution.
Here is the code:
var book = new Array;
var chapters = Object.keys(epub.spine.contents).length;
for (let i = 0; i < chapters; i++) {
let cacheArray = [];
epub.getChapter(epub.spine.contents[i].id, function (err, data) {
if (err) {
return console.log(err);
}
//remove html tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
})
}
console.log(book)//returns empty array ofc
After this code is executed, I need to loop over the array to search its contents. If that was not the case, my approach would be to just send it to a db.
My approach was the following:
var recursiveChapter = function (n) {
var book = new Array;
if (n < chapters) {
// chapter function
epub.getChapter(epub.spine.contents[n].id, function (err, data) {
if (err) {
throw err
}
//remove HTML tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
recursiveChapter(n + 1)
});
}
}
//start the recursive function
recursiveChapter(0)
console.log(book)//returns an empty array
I am stuck and can't think of a way of using this data without storing it in a db.
Any help would be appreciated .

There are a few ways you can tackle this. One way is to use the async library, this allows to to run async. calls in parallel, or in series.
For example, we can use async.mapSeries() to get the result of a series of asynchronous calls as an array.
You input your array of ids, then the callback will return an array of the data returned, for example, I've mocked out the getChapter() function to give you an idea of how it would work:
// Mock out epub object
const epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
console.log("Calling async.mapSeries for ids:", ids);
async.mapSeries(ids, epub.getChapter, (err, result) => {
if (err) {
console.error(err);
} else {
console.log("Result:", result)
}
})
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/3.2.0/async.min.js" integrity="sha512-6K6+H87tLdCWvY5ml9ZQXLRlPlDEt8uXmtELhuJRgFyEDv6JvndWHg3jadJuBVGPEhhA2AAt+ROMC2V7EvTIWw==" crossorigin="anonymous"></script>
You could also promisify the epub call and use Promise.all to get the result, like so:
epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
function getChapterPromisified(id) {
return new Promise((resolve, reject) => {
epub.getChapter(id, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
})
})
}
// Create a list of promises, one for each call
const promises = ids.map(id => getChapterPromisified(id))
Promise.all(promises)
.then(result => console.log("Result:", result))
.catch(error => console.error("An error occurred:", err));

Solved Why do MySQL statements work normally in Node REPL when typed line by line, but not in a function? [duplicate]

In the code
var stuff_i_want = '';
stuff_i_want = get_info(parm);
And the function get_info:
get_info(data){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
console.log(stuff_i_want); // Yep. Value assigned..
}
in the larger scope
stuff_i_want = null
What am i missing regarding returning mysql data and assigning it to a variable?
============ New code per Alex suggestion
var parent_id = '';
get_info(data, cb){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
return cb(results[0].objid); // Scope is larger than function
}
==== New Code in Use
get_data(parent_recording, function(result){
parent_id = result;
console.log("Parent ID: " + parent_id); // Data is delivered
});
However
console.log("Parent ID: " + parent_id);
In the scope outside the function parent_id is null

You're going to need to get your head around asynchronous calls and callbacks with javascript, this isn't C#, PHP, etc...
Here's an example using your code:
function get_info(data, callback){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
return callback(results[0].objid);
})
}
//usage
var stuff_i_want = '';
get_info(parm, function(result){
stuff_i_want = result;
//rest of your code goes in here
});
When you call get_info this, in turn, calls connection.query, which takes a callback (that's what function(err, results) is
The scope is then passed to this callback, and so on.
Welcome to javascript callback hell...
It's easy when you get the hang of it, just takes a bit of getting used to, coming from something like C#

I guess what you really want to do here is returning a Promise object with the results. This way you can deal with the async operation of retrieving data from the DBMS: when you have the results, you make use of the Promise resolve function to somehow "return the value" / "resolve the promise".
Here's an example:
getEmployeeNames = function(){
return new Promise(function(resolve, reject){
connection.query(
"SELECT Name, Surname FROM Employee",
function(err, rows){
if(rows === undefined){
reject(new Error("Error rows is undefined"));
}else{
resolve(rows);
}
}
)}
)}
On the caller side, you use the then function to manage fulfillment, and the catch function to manage rejection.
Here's an example that makes use of the code above:
getEmployeeNames()
.then(function(results){
render(results)
})
.catch(function(err){
console.log("Promise rejection error: "+err);
})
At this point you can set up the view for your results (which are indeed returned as an array of objects):
render = function(results){ for (var i in results) console.log(results[i].Name) }
Edit
I'm adding a basic example on how to return HTML content with the results, which is a more typical scenario for Node. Just use the then function of the promise to set the HTTP response, and open your browser at http://localhost:3001
require('http').createServer( function(req, res){
if(req.method == 'GET'){
if(req.url == '/'){
res.setHeader('Content-type', 'text/html');
getEmployeeNames()
.then(function(results){
html = "<h2>"+results.length+" employees found</h2>"
html += "<ul>"
for (var i in results) html += "<li>" + results[i].Name + " " +results[i].Surname + "</li>";
html += "</ul>"
res.end(html);
})
.catch(function(err){
console.log("Promise rejection error: "+err);
res.end("<h1>ERROR</h1>")
})
}
}
}).listen(3001)

Five years later, I understand asynchronous operations much better.
Also with the new syntax of async/await in ES6 I refactored this particular piece of code:
const mysql = require('mysql2') // built-in promise functionality
const DB = process.env.DATABASE
const conn = mysql.createConnection(DB)
async function getInfo(data){
var sql = "SELECT a from b where info = data"
const results = await conn.promise().query(sql)
return results[0]
}
module.exports = {
getInfo
}
Then, where ever I need this data, I would wrap it in an async function, invoke getInfo(data) and use the results as needed.

This was a situation where I was inserting new records to a child table and needed the prent record key, based only on a name.
This was a good example of understanding the asynchronous nature of node.
I needed to wrap the all the code affecting the child records inside the call to find the parent record id.
I was approaching this from a sequential (PHP, JAVA) perspective, which was all wrong.

Easier if you send in a promise to be resolved
e.g
function get_info(data, promise){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
promise.resolve(results[0].objid);
}
}
This way Node.js will stay fast because it's busy doing other things while your promise is waiting to be resolved

I've been working on this goal since few weeks, without any result, and I finally found a way to assign in a variable the result of any mysql query using await/async and promises.
You don't need to understand promises in order to use it, eh, I don't know how to use promises neither anyway
I'm doing it using a Model class for my database like this :
class DB {
constructor(db) {
this.db = db;
}
async getUsers() {
let query = "SELECT * FROM asimov_users";
return this.doQuery(query)
}
async getUserById(array) {
let query = "SELECT * FROM asimov_users WHERE id = ?";
return this.doQueryParams(query, array);
}
// CORE FUNCTIONS DON'T TOUCH
async doQuery(queryToDo) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
async doQueryParams(queryToDo, array) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, array, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
}
Then, you need to instantiate your class by passing in parameter to constructor the connection variable given by mysql. After this, all you need to do is calling one of your class methods with an await before. With this, you can chain queries without worrying of scopes.
Example :
connection.connect(function(err) {
if (err) throw err;
let DBModel = new DB(connection);
(async function() {
let oneUser = await DBModel.getUserById([1]);
let allUsers = await DBModel.getUsers();
res.render("index.ejs", {oneUser : oneUser, allUsers : allUsers});
})();
});
Notes :
if you need to do another query, you just have to write a new method in your class and calling it in your code with an await inside an async function, just copy/paste a method and modify it
there are two "core functions" in the class, doQuery and doQueryParams, the first one only takes a string as a parameter which basically is your mysql query. The second one is used for parameters in your query, it takes an array of values.
it's relevant to notice that the return value of your methods will always be an array of objects, it means that you'll have to do var[0] if you do a query which returns only one row. In case of multiple rows, just loop on it.

How can I execute an mySQL query in another MySQL query with express.js

we have an issue when we want to execute a query that takes the output from another query.
connection.query(`SELECT * from Database.table1`, function (err, rows, fields) {
for (var item of rows) {
allcounts += item.count
number = item.number
console.log(allcounts)
console.log(number)
connection.query(`DELETE from Database.table2 where numbers = ${number}`, function (err, rows, fields) {
console.log("delete successfull")
})
console.log("Step One finished !")
}
})
The way it should work is, that rows in the first query is returning two JSON objects. Each object stands for a specific product with a count(count) and a product number(number). This products should be deleted in another table in the database.
You can see that we want to loop through the output(rows) of our first statement and execute another statement with this output(rows). This works but express is executing the code in the wrong direction...
Heres the output:
0.89
12345
Step One finished!
2.28
32598
Step One finished!
delete successfull
delete successfull
Its showing us the number and is adding the counts correctly!
The strange thing is that both queries are executed AFTER both counts are added and the number is shown...
We tried several solutions but no one was the right for us.
It would be great if anyone could help us soon :)

You can try with 2 solutions other than promises
1) using anonymous function which blocks the for loop for next iteration untill the query is executed
connection.query(`SELECT * from Database.table1`, function (err, rows, fields) {
for (var item of rows) {
(function (item){
allcounts += item.count
number = item.number
console.log(allcounts)
console.log(number)
connection.query(`DELETE from Database.table2 where numbers = ${number}`, function (err, rows, fields) {
console.log("delete successfull")
})
console.log("Step One finished !")
})(item);
}
})
2) using IN in where clause
at takes the output from another query.
connection.query(`SELECT * from Database.table1`, function (err, rows, fields) {
let numbers = [];
for (var item of rows) {
numbers.push(item.number);
}
connection.query(`DELETE from Database.table2 where numbers IN ("+numbers+")`, function (err, rows, fields) {
connection.release();
console.log("delete successfull")
})
console.log("Step One finished !")
})

The problem is that your for-loop doesn't wait for the asynchronous connection.query() to finish before continuing with the next iteration.
You can use promises to solve that issue.
I've included some commented sample code using promises below
(untested, so you may have to squash a bug) .
Note:
the second query in the loop doesn't make sense in my opinion.
It's better to do everything in a single query.
Search around and read up on using promises.
You'll be glad you did when you got the basics.
function getNumbers() {
return new Promise(function(resolve, reject) {
connection.query(`SELECT * from Database.table1`,
function(err, rows, fields) {
// if an error occured we let the caller know it didn't work out with reject()
// if everthing went well, we pass on the result data with resolve()
return (err) ? reject(err) : resolve(rows);
});
});
}
getNumbers()
// below code be executed AFTER the first query succesfully completed
.then(function(rows) {
// use Array.prototype.map() with Array.prototype.join()
// to create string of numbers for deletion query,
// while also calculating total
let totalCount = 0;
const numbersTotal = rows
.map(item => {
numbersTotal += item.number;
return item.number;
})
.join();
console.log("String of numbers: " + numbers);
console.log("numbersTotal: " + totalCount);
// now we can achieve the same result
// using only 2 queries instead of a loop
numbers.length > 0 && connection.query(
`DELETE from Database.table2 where numbers in ${numbers}`,
function(err, rows, fields) {
console.log("delete successfull");
}
);
})
.catch(err =>
// stop immediatly when a mysql error occurs
setImmediate(() => {
throw err;
})
);

How to write an arbitrarily long Promise chain

I receive an object bigListFromClient that includes an arbitrary number of objects each of which may have an arbitrary number of children. Every object needs to be entered into my database, but the DB needs to assign each of them a unique ID and child objects need to have the unique ID of their parents attached to them before they are sent off to the DB.
I want to create some sort of Promise or other calling structure that would call itself asynchronously until it reached the last object in bigListFromClient but I'm having trouble figuring out how to write it.
for(let i = 0; i < bigListFromClient.length; i++){
makeDbCallAsPromise(bigListFromClient[i].queryString, console.log); //I'm not just accepting anything from a user here, but how I get my queryString is kind of out of scope for this question
for(let j = 0; j < bigListFromClient[i].children.length; j++){
//the line below obviously doesn't work, I'm trying to figure out how to do this with something other than a for loop
makeDbCallAsPromise(bigListFromClient[i].children[j].queryString + [the uniqueID from the DB to insert this correctly as a child], console.log);
}
}
//this promise works great
makeDbCallAsPromise = function(queryString){
return new Promise((resolve, reject) => {
connection = mysql.createConnection(connectionCredentials);
connection.connect();
query = queryString;
connection.query(query, function (err, rows, fields) {
if (!err) {
resolve(rows);
} else {
console.log('Error while performing Query.');
console.log(err.code);
console.log(err.message);
reject(err);
}
});
connection.end();
})
};
My attempts at solving this on my own are so embarrassingly bad that even describing them to you would be awful.
While I could defer all the calls to creating children until the parents have been created in the DB, I wonder if the approach I've described is possible.

There are essentially two ways to do this. One is making the database calls sequential and the other one is making the calls parallel.
Javascript has a built-in function for parallel called Promise.all, you pass it an array of Promise instances and it returns a Promise instance containing the array.
In your case your code would look like this:
const result = Promise.all(
bigListFromClient.map(item =>
makeDbCallAsPromise(item.queryString).then(result =>
Promise.all(
item.children.map(item =>
makeDbCallAsPromise(item.queryString + [result.someId])
)
)
])
})
result will now contain a Promise that resolves to an array of arrays. These arrays contain the result of intserting children.
Using a more modern approach (with async await), sequential and with all results in a flat array:
const result = await bigListFromClient.reduce(
async (previous, item) => {
const previousResults = await previous
const result = await makeDbCallAsPromise(item.queryString)
const childResults = await item.children.reduce(
async (result, item) =>
[...(await result), await makeDbCallAsPromise(item.queryString + [result.someId])],
[]
)
return [...previousResults, result, ...childResults)
]),
[]
})
Depending on what you want to achieve and how you want to structure your code you can pick and choose from the different approaches.

For this sort of operation, try looking into bulk inserting. If you are intent on performing a single DB query/transaction per iteration, loop recursively over each parent and/or execute the same procedure for each child.
const dbCall = async (elm) => {
elm.id = Math.random().toString(36).substring(7)
if (elm.children) {
await Promise.all(elm.children.map(child => {
child.parentId = elm.id
return dbCall(child)
}))
}
return elm
}
const elms = [
{
queryString: '',
children: [
{
queryString: ''
}
]
}
]
Promise.all(elms.map(dbCall)).then(elm => /* ... */)

Setting variable via asynchronous function inside for loop

I've come across a situation where I have an asynchronous function inside of a for loop. I've done my prerequisite searching and now know that forEach() or map() might be the solution I need. But all of the examples I see just console.log() the result of each async function. How would it work to have to set the results of each async function into a variable, and then only return the variable?
Here's some slimmed down code of what I'm doing: (this is all in node, by the way)
var clients={
"102323":{stuff about this client},
"242341":{stuff about that client}
};
var messages={};
for (var id_client in clients) {
mysql.query("SELECT * FROM messages WHERE id_client='"+id_client+"' ORDER BY date", function(err, rows) {
if (typeof rows !== 'undefined') messages[id_client]=rows;
});
}
//do other stuff with messages variable
With this, messages predictably is null. Which I understand.
But even when I transition this to using map() instead of for(), like this...
var messages={};
Object.keys(clients).map(function(id_client){
mysql.query("SELECT * FROM messages WHERE id_client='"+id_client+"' ORDER BY date", function(err, rows) {
if (typeof rows !== 'undefined') messages[id_client]=rows;
});
});
...messages ends up being null.
Finally, just want to note that I do know how to wrap mysql.query() into another function with a callback and all that to get around the whole asynchronous thing. I just don't know how that all works if being iterated on inside of a loop.

You can use Promise.
var messages = {};
Promise.all(Object.keys(clients).map((id_client) => {
return new Promise((resolve, reject) => {
mysql.query("SELECT * FROM messages WHERE id_client='" + id_client + "' ORDER BY date", function(err, rows) {
if (typeof rows !== 'undefined') {
messages[id_client] = rows;
resolve(rows);
}
});
});
})).then(results => {
console.log(messages);
});

Here is a pure JavaScript version:
var clients={
"102323":{stuff about this client},
"242341":{stuff about that client}
};
var messages={};
var cnt = 2;
function done() {
console.log(messages);
}
mysql.query("SELECT * FROM messages WHERE id_client='"+id_client+"' ORDER BY date", function(err, rows) {
cnt--;
if (typeof rows !== 'undefined') messages[id_client]=rows;
if(!cnt) done();
});

I presume you are console.log()ing just after map. But map didn't finished yet, since it's callback is asynchonous.
You can use async to get a asynchnous version of map, which will work:
'use strict';
let async = require('async'),
messages = {};
async.map(Object.keys(clients), (id_client) => {
mysql.query("SELECT * FROM messages WHERE id_client='"+id_client+"' ORDER BY date", (error, rows) => {
if (typeof rows !== 'undefined') {
messages[id_client]=rows;
}
callback(error);
});
}, (error) => {
if(error) {
console.log(error);
}
console.log(messages);
});