I have some node.js code which fetches data from an API in a loop and runs mutliple mysql queries to update some rows.
The issue I have is that the script keeps running until I terminate the mysql connection with connection.end(). I am a newbie in asynchronous code. Where do I call the termination function so that it executes when all the queries have finished executing? What's the right design pattern for this? Would waterfall be any good?
This is a snippet from the code I have at the moment (error handling removed for simplicity):
var connection = mysql.createConnection({ host, user, etc... });
for (var i = 0; i < 10; i++) {
var url = "http://api.com?i="+i;
request(url, function(error, response, body) {
var data = JSON.parse(body);
for (el in data) {
connection.query(
"UPDATE table SET col = ? WHERE symbol = ?",
[
data[el].col,
el
]
);
}
});
}
// this will run before all queries have executed
// resulting in an error
connection.end();
So, the problem here is that you are cycling in a synchronized way through the data here:
var data = JSON.parse(body);
for (el in data) {
connection.query(
"UPDATE table SET col = ? WHERE symbol = ?",
[
data[el].col,
el
]
);
}
while the mysql module handles the query in a callback style:
connection.query(query, function(error, rows, fields) {
if (error) {
return callback(error);
} else {
return callback(null,rows);
}
});
where callback has the signature callback(error,rows), so that you can handle the results in this way supposed to have a reusable function:
var executeQuery = function(query,callback) {
var self=this;
this.connection.query(query, function(error, rows, fields) {
if (error) {
return callback(error);
} else {
return callback(null,rows);
}
});
}
and you can call in your code like
executeQuery(statement, function(error,rows) {
//...
})
That said, you must consider that you are doing multiple queries to your database and it is not recommended to do this in for loop cycle. You should consider to use a better solution that could be a waterfall as you say or a promise all using the Promise paradigma.
Suppose that to have this nice function:
var promiseAllP = function(items, block) {
var promises = [];
items.forEach(function(item,index) {
promises.push( function(item,i) {
return new Promise(function(resolve, reject) {
return block.apply(this,[item,index,resolve,reject]);
});
}(item,index))
});
return Promise.all(promises);
}
that takes as input an array of items and a execution function that is function(item,index,resolve,reject) that has a resolve and reject functions of a Promise, so let's turn your executeQuery function in a Promise as well:
var executeQueryP = function(query) {
var self=this;
return new Promise(function(resolve, reject) {
self.connection.query(query, function(error, rows, fields) {
if (error) {
return reject(error);
} else {
return resolve(null,rows);
}
});
}
Now you can process your data in a totally async way promisyfied:
promiseAllP(data,(item,index,resolve,reject) => {
var query= "UPDATE table SET col = %s WHERE symbol = %s";
// example: prepare the query from item in the data
query = replaceInString(query,item.col,item);
executeQueryP(query)
.then(result => resolve(result))
.catch(error => reject(error))
})
.then(results => { // all execution completed
console.log(results)
})
.catch(error => { // some error occurred while executing
console.error(error)
})
where the replaceInString will help you to prepare the statement
var replaceInString = function() {
var args = Array.prototype.slice.call(arguments);
var rep= args.slice(1, args.length);
var i=0;
var output = args[0].replace(/%s|%d|%f|%#/g, function(match,idx) {
var subst=rep.slice(i, ++i);
return( subst );
});
return(output);
},//replace,
This is what we have done here:
Used native Promise only
Turned your mysql query in a promise
Called the statements against your data in a completely asynchronous way
Used a Promise and Promise all paradigma, that let you collect the results of the Promise and return to the caller when all the functions are completed.
Catched errors in all the statements execution
Added a simply way to fulfill statements with parameters
Also notice the arrow function syntax (param1, param2) => that simplify the way to write a function, that can help a lot with the Promise paradigma.
For anyone interested, I ended up solving it by a mixture of promises and counting the queries, something along the lines of this (not sure if this code actually works but the idea is there):
function fetchFromAPI() {
return new Promise((resolve, reject)=>{
var urls = [];
for (var i = 0; i < 10; i++) {
urls.push("http://api.com?i="+i);
}
var data = [];
var requestedUrls=0;
urls.forEach(url=>{
request(url, (err, response, body) {
if(err) reject(err);
data.push(JSON.parse(body));
requestedUrls++;
if(requestedUrls==urls.length) resolve(data);
};
});
}
}
fetchFromAPI().then(data=>{
mysql.createConnection({ user, hostname, etc... });
var processedKeys=0;
data.forEach(el=> {
mysql.query("UPDATE table SET name = ? WHERE id = ?", [el.name, el.id], (err, rows, fields) => {
processedKeys++;
if(processedKeys==data.length) {
connection.end();
}
});
}
}).catch(err=>{
console.error(err);
});
Related
Hope you're having a good day. I recently discovered that it is not as easy to handle callbacks inside a for loop. I have tried a few things but couldn't find a solution.
Here is the code:
var book = new Array;
var chapters = Object.keys(epub.spine.contents).length;
for (let i = 0; i < chapters; i++) {
let cacheArray = [];
epub.getChapter(epub.spine.contents[i].id, function (err, data) {
if (err) {
return console.log(err);
}
//remove html tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
})
}
console.log(book)//returns empty array ofc
After this code is executed, I need to loop over the array to search its contents. If that was not the case, my approach would be to just send it to a db.
My approach was the following:
var recursiveChapter = function (n) {
var book = new Array;
if (n < chapters) {
// chapter function
epub.getChapter(epub.spine.contents[n].id, function (err, data) {
if (err) {
throw err
}
//remove HTML tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
recursiveChapter(n + 1)
});
}
}
//start the recursive function
recursiveChapter(0)
console.log(book)//returns an empty array
I am stuck and can't think of a way of using this data without storing it in a db.
Any help would be appreciated .
There are a few ways you can tackle this. One way is to use the async library, this allows to to run async. calls in parallel, or in series.
For example, we can use async.mapSeries() to get the result of a series of asynchronous calls as an array.
You input your array of ids, then the callback will return an array of the data returned, for example, I've mocked out the getChapter() function to give you an idea of how it would work:
// Mock out epub object
const epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
console.log("Calling async.mapSeries for ids:", ids);
async.mapSeries(ids, epub.getChapter, (err, result) => {
if (err) {
console.error(err);
} else {
console.log("Result:", result)
}
})
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/3.2.0/async.min.js" integrity="sha512-6K6+H87tLdCWvY5ml9ZQXLRlPlDEt8uXmtELhuJRgFyEDv6JvndWHg3jadJuBVGPEhhA2AAt+ROMC2V7EvTIWw==" crossorigin="anonymous"></script>
You could also promisify the epub call and use Promise.all to get the result, like so:
epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
function getChapterPromisified(id) {
return new Promise((resolve, reject) => {
epub.getChapter(id, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
})
})
}
// Create a list of promises, one for each call
const promises = ids.map(id => getChapterPromisified(id))
Promise.all(promises)
.then(result => console.log("Result:", result))
.catch(error => console.error("An error occurred:", err));
I am not experienced with async functions and I would like to perform a request in a for loop. Here's my code:
app.route('/friendlist').post((req, res) => {
var body = req.body;
var list = "";
con.query(`SELECT * FROM player_friends WHERE main_user_id = '${body.player_id}'`, (err, row, fields) => {
if (err) throw err;
async function queryOutUserData(data) {
var rows = await new Promise((resolve, reject) => {
con.query(`SELECT * FROM players WHERE player_id = '${data.player_id}'`, (error, player, field) => {
if (error) {
console.log(error);
return reject(error);
}
resolve(player);
});
});
rows.then(message => {
return message
});
}
for (var i = 0; i <= row.length; i++) {
console.log(row[i].main_user_id);
var result = await queryOutUserData(row[i]);
list = list + ";" + result[0].player_id + ":" + result[0].player_username;
}
console.log(list);
return list;
});
});
Actually here's the full problem: I did some debugging and apparently value i in for loop increases before the promise is resolved. Also as I mentioned I am not familiar with async functions, could you provide me a descriptive resource about how promises and async functions work?
Thanks
NOTE: For better indentation, here's the code: https://hastebin.com/fovayucodi.js
Instead of using async/await I suggest doing everything in one query using WHERE IN rather than one query per player. See if the following fits your needs:
app.route('/friendlist').post((req,res) => {
var body = req.body;
var list = "";
con.query(`SELECT * FROM player_friends WHERE main_user_id = '${body.player_id}'`, (err, row, fields) => {
if (err) throw err;
const playerIds = row.map(player => player.player_id);
con.query(`SELECT * FROM players WHERE player_id IN ${playerIds}`, (error, players, field) => {
for (let player of players) {
list += `;${player.player_id}:${player.player_username}`;
}
});
console.log(list);
return list;
});
});
If you await a promise, it evaluates to the result of that promise, so rows is not a promise, it's the result. So this:
rows.then(message => {return message});
Doesn't make much sense, just do:
return message;
Also, you have an await inside of a regular function, thats a syntax error.
Additionally return list; doesn't do much (if that is express), you might want to return res.json({ list });.
: I did some debugging and apparently value i in for loop increases before the promise is resolved.
I doubt that you can debug code if you can't actually run it because of the syntax errors.
try to use for-of instead just a for.
something like this:
Async Function:
async function test() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(true);
return
}, 1000)
})
}
Here another function using for and waiting for the finish of loop
async function myFunction() {
const data = [1,2,3,4,5,6,7,8]
for(let i of data) {
const value = await test();
console.log(value)
}
console.log("finish");
}
I have a problem.
I have to do two different SOAP calls to retrieve two list of vouchers and then use these lists to do a check on them and to do some job.
I put the two calls in different Promise functions because I want start the job on the lists after the call returned its result.
This is the first Promise call:
let vouchers = function(voucherTypeList){
return new Promise(function(resolve,reject){
const categoryId = "1000";
let args = {
"tns:CategoryId": categoryId
};
var header = {
"tns:Authenticate": {
"tns:UserName": soapVoucherWsdlUsername,
"tns:Password": soapVoucherWsdlPassword
}
};
// let voucherTypeList;
voucherClient.addSoapHeader(header);
voucherClient.GetVouchers(args, function(err, result) {
console.log("DENTRO GET VOUCHERS");
if (err) {
console.log(err);
writeResponse(res, '200', err);
} else {
//++++++++++++++++++++++
//voucherTypeList is what I want to return to the main function
voucherTypeList = mapGetVoucherTypeListResponse(result);
//++++++++++++++++++++++
}
resolve("done 1");
});
});
}
This is the second Promise call:
let issuedVouchers = function(accountId) {
return new Promise(function (resolve, reject) {
const categoryId = "1000";
let args = {
"tns:CategoryId": categoryId,
"tns:CheckRedeem": true,
"tns:IncludeRedeemed": false,
"tns:CardId": accountId
};
var header = {
"tns:Authenticate": {
"tns:UserName": soapVoucherWsdlUsername,
"tns:Password": soapVoucherWsdlPassword
}
};
let issuedVoucherList;
voucherClient.addSoapHeader(header);
voucherClient.GetVouchers(args, function (err, result) {
console.log("DENTRO GET ISSUED VOUCHERS");
if (err) {
console.log(err);
writeResponse(res, '200', err);
} else {
//++++++++++++++++++++++
//issuedTypeList is what I want to return to the main function
issuedTypeList = mapGetVoucherTypeListResponse(result);
//++++++++++++++++++++++
}
resolve("done 2");
});
});
}
And this is the main function, with the Promise flow:
function getAvailableVoucherTypes(req, res) {
var accountId = req.params.accountId;
vouchers(voucherTypeList).
then(issuedVouchers(accountId)).
then(function() {
//here I want to use voucherTypeList and issuedTypeList
//and do some jobs on them
console.log("OK");
});
}
How can I do this? I tried many solutions, but I'm not able to see voucherTypeList and issuedTypeList in the main function.
The then callbacks are getting the value of what you pass to the resolve function in your promises. You are currently passing arbitrary strings, which is useless... But for the demonstration, let's keep those and just log their values in your main script:
function getAvailableVoucherTypes(req, res) {
var accountId = req.params.accountId;
vouchers(voucherTypeList).
then(function(result){
console.log(result); //done 1
return issuedVouchers(accountId);
}).
then(function(result) {
console.log(result); //done 2
//here I want to use voucherTypeList and issuedTypeList
//and do some jobs on them
console.log("OK");
});
}
I'll let you play with your promises to pass the right variables...
Now, it seems that your 2 calls do not need to be sequential, so let's make them parallel, it's gonna be slightly easier for us too.
function getAvailableVoucherTypes(req, res) {
var accountId = req.params.accountId;
var promises = [vouchers(),issuedVouchers(accountId)]
Promise.all(promises).then(function(results){
//In Promise.all, the results of each promise are passed as array
//the order is the same as the order of the promises array.
var voucherTypeList = results[0];
var issuedTypeList = results[1];
});
}
BONUS: I do not want to complicate this task too much before you grasp it correctly. So I won't add more code. But note that you should use reject too, instead of handling your errors in every promise, you should reject them when things go wrong. Just reject(err) and add a second callback to your main script's then to handle any error that may happen. If you keep resolving your promises that did not work, you will not be passing the elements you are expecting and you'll need to add checks over every step.
Let's modify the GetVouchers callback to fit what I suggest.
voucherClient.GetVouchers(args, function (err, result) {
console.log("DENTRO GET ISSUED VOUCHERS");
if (err) {
reject(err);
} else {
resolve(mapGetVoucherTypeListResponse(result));
}
});
Once it is done on both your promises, we can change your main script to handle the error accordingly.
Promise.all(promises).then(function(results){
//Handle success like above.
},function(err){
//Handle error.
console.log(err.stack || err);
writeResponse(res, '200', err);
});
I am trying to perform sql queries based on the callback results in if conditions but i am unable to write the code .so please provide som information in code
app.get('/resell-property', function(req, res) {
var data = {}
data.unit_price_id = 1;
function callback(error, result) {
if (result.count == 0) {
return hp_property_sell_request.create(data)
}
else if (result.count > 0) {
return hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
}
}
hp_property_sell_request.findAndCountAll({
where: {
unit_price_id: data.unit_price_id
}
}).then(function (result) {
if (result) {
callback(null, result);
}
});
});
In this how can i write the callbacks for
hp_property_sell_request.create(data) ,hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
In that after returning result again i have to handle callbacks and perform this query
if(result.request_id){
return hp_unit_price.findAll({
where:{
unit_price_id:result.unit_price_id,
hp_property_id:result.property_id,
hp_unit_details_id:result.unit_details_id
}
}).then(function (result){
if(result.is_resale_unit==0 && result.sold_out==0){
return Sequelize.query('UPDATE hp_unit_price SET resale_unit_status=1 WHERE hp_unit_details_id='+result.unit_details_id+' and hp_property_id='+result.property_id)
}
})
}
The promise resolve function takes only one input argument, so if you need to pass in multiple stuff, you have to enclose them in a single object. Like, if you have to go with something like:
database.openCollection()
.then(function(collection){
var result = collection.query(something);
var resultObject = { result: result, collection: collection };
})
.then(function(resultObject){
doSomethingSyncronousWithResult(resultObject.result);
resultObject.collection.close();
});
You can't use Promise all if all of your stuff isn't a result of a promise resolve, you might need to go with something like this.
Disclaimer: The code example is a very poor one, but it explains the concept.
I would suggest you to learn about Promises, particularly Bluebird.
You can promisify traditional callback methods.
I would also create model level functions in different files. Here's an example.
parent.js
const db = require("./connections/database"); // connection to database
const getChildForParent = function (parentId, childId, callback) {
db.find({parent: parentId, child_id: childId}, "childrenTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
children.js
const db = require("./connections/database"); // connection to database
const getToysForChild = function (childId, callback) {
db.find({toy_belongs_to: parentId}, "toysTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
Then in controller you can do something like this:
const Bluebird = require("bluebird");
const Parent = require("./parent.js");
const Child = require("./child.js");
// Promisifying adds "Async" at the end of your methods' names (these are promisified)
Bluebird.promisifyAll(Parent);
Bluebird.promisifyAll(Child);
// Just an example.
app.get("/parent/:parentId/children/:childId", function(req, res) {
return Bluebird.try(function() {
return User.getChildForParentAsync(req.params.parentId, req.params.childId);
}).then(function(child) {
return Child.getToysForChildAsync(child.child_id);
}).then(function(toys) {
// Do something with toys.
});
});
Of course you can do much more with this and this is not the only way.
Also you can use Promise.all(). This method is useful for when you want to wait for more than one promise to complete.
Let's say you have a list of urls that you want to fetch and process the results after all the data has been fetched.
var urls = [url1, url2, url3, url4, url5 .......... ];
var Bluebird = require("bluebird");
var request = require("request"); // callback version library
Bluebird.promisifyAll(request);
// create a list which will keep all the promises
var promises = [];
urls.forEach(function(url) {
promises.push(request.getAsync(url1));
});
// promises array has all the promises
// Then define what you want to do on completion.
Bluebird.all(promises).then(function(results) {
// results is an array with result a url in an index
// process results.
});
I would recommend to use Promises to solve that. If you need all results of all Requests, when they are all done Promise.all() will do that for you. Your basic could look like that:
var req1 = new Promise(function(res, rej){
var req = new XMLHttpRequest()
…
req.addEventListener('load', function (e) {
res(e);
})
var req2 = //similar to the above
Promise.all([req1, req2, …]).then(function(values){
//all requests are done here and you can do your stuff
});
You can also use the new fetch api, which creates Promises like so:
var req1 = fetch(…);
var req2 = fetch(…);
Promise.all([req1, re2, …]).then(…);
this is a follow up question to Asynchron Errorhandling inside $.each. As mentioned in the comments there, i want to handle data after the last async job from a $.each loop.
So for instance:
var errors = 0;
var started = 0;
var successful = 0;
$.each(..., function(){
started++;
connection.query('INSERT INTO tableName SET ?', post, function(err, result)
{
if (err) {
if (err.code === 'ER_DUP_ENTRY')
{ errors++; }
else
{ throw err; }
} else { successful++;}
if (started == successful + errors) {
// all done
console.log(errors + " errors occurred");
}
});
});
In this case everything logs out properly when the // all done comment is reached. But what if i want to use this data later on instead of just logging it out.
Is there a way to wait for this data outside of the $.each scope? Or do i always have to handle everything in the nested function?
You can use promises instead
var promises = [];
$.each(..., function() {
var promise = new Promise(function(resolve, reject) {;
connection.query('INSERT INTO tableName SET ?', post, function(err, result) {
if (err) {
resolve(err.code);
} else {
resolve(result);
}
});
});
promises.push(promise);
});
var result = Promise.all(promises);
And then when you want to use the data, you do
result.then(function(data) {
// use data array looking like ["result data", "result data", "ER_DUP_ENTRY" .. etc]
})