I am currently building a Nodejs, Express, Sequelize (w. PostgreSQL) app, and have run into a few problems with using promises together with transactions and loops.
I am trying to figure out how to use a for loops in a transaction. I am trying to loop through a list of members and create a new user in the database for each of them.
I know the following code is wrong but it shows what I am trying to do.
Can anyone point me in the right direction?
var members = req.body.members;
models.sequelize.transaction(function (t) {
for (var i = 0; i < members.length; i++) {
return models.User.create({'firstname':members[i], 'email':members[i], 'pending':true}, {transaction: t}).then(function(user) {
return user.addInvitations([group], {transaction: t}).then(function(){}).catch(function(err){return next(err);});
})
};
}).then(function (result) {
console.log("YAY");
}).catch(function (err) {
console.log("NO!!!");
return next(err);
});
You should use a Promise.all
var members = req.body.members;
models.sequelize.transaction(function (t) {
var promises = []
for (var i = 0; i < members.length; i++) {
var newPromise = models.User.create({'firstname':members[i], 'email':members[i], 'pending':true}, {transaction: t});
promises.push(newPromise);
};
return Promise.all(promises).then(function(users) {
var userPromises = [];
for (var i = 0; i < users.length; i++) {
userPromises.push(users[i].addInvitations([group], {transaction: t});
}
return Promise.all(userPromises);
});
}).then(function (result) {
console.log("YAY");
}).catch(function (err) {
console.log("NO!!!");
return next(err);
});
I don't believe you need to catch within sequelize transactions as I think it jumps out to the catch on the transaction
Sorry for formatting. On mobile.
Promise.all will wait for all promises to return (or fail) before running the .then, and the .then callback will be all the promise data from each array
You'll need to use the built in looping constructs of bluebird which ships with sequelize:
var members = req.body.members;
models.sequelize.transaction(t =>
Promise.map(members, m => // create all users
models.User.create({firstname: m, email: m, 'pending':true}, {transaction: t})
).map(user => // then for each user add the invitation
user.addInvitations([group], {transaction: t}) // add invitations
)).nodeify(next); // convert to node err-back syntax for express
Depending on your implementation of Node.js this may help. I have the same setup using express, POSTGRES and sequelize.
Personally I'd prefer the async/await (ES6) implementation over then/catch as it is easier to read. Also creating a function that can be called externally improves re-usability.
async function createMemeber(req) {
let members = req.body.members;
for (var i = 0; i < members.length; i++) {
// Must be defined inside loop but outside the try to reset for each new member;
let transaction = models.sequelize.transaction();
try {
// Start transaction block.
let user = await models.User.create({'firstname':members[i], 'email':members[i], 'pending':true}, {transaction});
await user.addInvitations([group], {transaction}));
// if successful commit the record. Else in the catch block rollback the record.
transaction.commit();
// End transaction block.
return user;
} catch (error) {
console.log("An unexpected error occurred creating user record: ", error);
transaction.rollback();
// Throw the error back to the caller and handle it there. i.e. the called express route.
throw error;
}
}
}
if someone is looking for a solution with typescript v4.0.5 using async and await here is what worked out for me. Maybe you can use it on your javascript aplication too but it will depend on the version of it.
const array = ['one','two','three'];
const createdTransaction = sequelize.transaction();
const promises = array.map(async item => {
await model.create({
name: item,
},
{ transaction: createdTransaction },
);
});
Promise.all(promises).then(async values => {
await createdTransaction.commit();
});
First: https://caolan.github.io/async/docs.html
So, easily:
// requiring...
const async = require('async');
// exports...
createAllAsync: (array, transaction) => {
return new Promise((resolve, reject) => {
var results = [];
async.forEachOf(array, (elem, index, callback) => {
results.push(models.Model.create(elem, {transaction}));
callback();
}, err => {
if (err) {
reject(err);
}
else {
resolve(results);
}
});
});
}
Related
I have a javascript, mongoose function which should return an array. So basically what I have is a user schema, and a shopping list schema. User schema has a parameter which is authorizedLists, an array of shopping lists ids. What I want is, that by passing the user id as a parameter, I can show those lists the user is authorized in. This function right now returns an empty array, and what I expected was an array with the shopping list objects.
I know it is not working due to Javascript asynchronous way of reading code, but I´m really stucked here.
Also I´m relatively new to nodejs and mongoose so I may have made other mistakes but haven´t realized yet.
Code:
function getUserAuthLists(req,res){
const { userId } = req.params;
var list = [];
let i = 0;
User.findById( userId,async function(err, user){
if(err) return res.status(404).send(err);
console.log(user);
const array = user.authorizedLists;
console.log(array);
array.forEach( function(element){
console.log(element);
ShoppingList.findById(element , (error, result)=>{
if(error) {
console.log(error);
return res.status(404).send(error);
}
list.push(result);
console.log(list);
});
i++;
});
if(i === array.length) {
console.log(list);
return res.status(202).send(list);
}
});
}
Thanks in advance.
Refactor your function to use promises, collect all authorizedLists and pass it in one-go using the $in operator instead of repeatedly doing findById
async function getUserAuthLists(req, res){
try {
const { userId } = req.params;
let i = 0;
const user = await User.findById(userId).exec();
const list = await ShoppingList.find({
_id: {
$in: user.authorizedLists
}
}).exec();
return res.status(202).send(list);
} catch (err) {
return res.status(404).send(err);
}
}
Since ShoppingList.findById is an i/o call, control does not wait by default to finish this call and directly jump to list.push(result); with list empty values.
You need to use promise to make it wait until it gets results from i/o call and then you should send back response. That way list will be updated for each array item call.
Something like this-
async function getUserAuthLists(req, res) {
User.findById(userId, async function (err, user) {
if (err) return res.status(404).send(err);
console.log(user);
const array = user.authorizedLists;
console.log(array);
await ShoppingListFromDB(array)
.then((list) => {
console.log("list--", list);
return res.status(202).send(list);
})
.catch((error) => {
console.log(error);
return res.status(404).send(err);
})
});
}
function ShoppingListFromDB(array) {
return new Promise((resolve, reject) => {
let i = 0
var list = [];
for (const element of array) {
console.log("element ", element);
ShoppingList.findById(element, (error, result) => {
if (err) {
console.log(err)
return reject(error);
}
i++
list.push(result)
if (i == array.length) {
return resolve(list)
}
})
}
})
}
I have created a sample that simulated your requirement.
https://github.com/ajaysikdar/test_server
server
client
I know this is a common problem for people, and I've looked up some guides and looked at the docs, but I'm not understanding what is happening, and would love some help.
Here is the controller function I am working on:
exports.appDetail = function (req, res) {
appRepo.findWithId(req.params.id, (err, myApp) => {
if (err) {
console.log(err)
}
getDeviceData(myApp.devices, (err, myDeviceData) => {
if (err) console.log(err)
console.log(JSON.stringify(myDeviceData) + ' || myDeviceData')
// construct object to be returned
let appsDataObject = {
name: myApp.name,
user_id: myApp.user_id,
devices: myDeviceData,
permissions: myApp.permissions
}
return res.status(200).send(appsDataObject)
})
})
}
// write async function here
const getDeviceData = function (devices, callback) {
let devicesDataArray = []
async.each(devices, function (device, cb) {
deviceRepo.findById(new ObjectID(device), (err, myDevice) => {
if (err) {
cb(err)
}
// get device data, push to devices array
let deviceObj = {
name: myDevice.name,
version: myDevice.version
}
devicesDataArray.push(deviceObj)
console.log(JSON.stringify(devicesDataArray) + ' || devicesDataAray after obj push')
})
cb(null, devicesDataArray)
}, function (err) {
// if any of the file processing produced an error, err would equal that error
if (err) console.log(err)
})
callback(null, devicesDataArray)
}
I originally wrote this with a for loop and a callback, but I think it was impossible to do that way (I'm not sure about that though). If there is a better way to make an asynchronous loop, please let me know.
On ~line 8 there is a log statement myDeviceData. This should be returning the data I want through a callback, but this log statement always comes back empty. And since the other log statements show that the data is being formatted correctly, the problem must be with returning the data I need through the callback of getDeviceData(). Presumably, the callback(null, devicesDataArray) should do this.
I'm not understanding how these async functions are supposed to work, clearly. Can someone please help me understand how I should get values from these async.each functions? Thank you.
EDIT:
I refactored the code to try and make it clearer and approach the problem better, and I have pinpointed where the problem is. At the beginning the this function I define devicesDataArray as an empty array, and at the end I return the array. Everything inside happens as it should, but I don't know how to tell the return to wait until the array isn't empty, if that makes sense, Here is the new code:
let getData = async function (devices) {
let devicesDataArray = []
for (let i = 0; i < devices.length; i++) {
deviceRepo.findById(new ObjectID(devices[i]), async (err, myDevice) => {
if (err) {
console.log(err)
}
console.log(JSON.stringify(myDevice) + ' || myDevice')
let deviceObj = await {
name: myDevice.name,
version: myDevice.version
}
console.log(JSON.stringify(deviceObj) + ' || deviceObj')
await devicesDataArray.push(deviceObj)
console.log(JSON.stringify(devicesDataArray) + ' || devicesDataArray after push')
})
}
console.log(JSON.stringify(devicesDataArray) + ' || devicesDataArray before return')
return Promise.all(devicesDataArray) // problem is here.
}
Any help towards understanding this is appreciated.
Here we are using a Promise
The Promise object is returned immediately and we perform our async code inside there.
const someFuncThatTakesTime = () => {
// Here is our ASYNC
return new Promise((resolve, reject) => {
// In here we are synchronous again.
const myArray = [];
for(x = 0; x < 10; x++) {
myArray.push(x);
}
// Here is the callback
setTimeout(() => resolve(myArray), 3000);
});
};
someFuncThatTakesTime().then(array => console.log(array));
first, I would prefer to avoid using callback because it will make your code unreadable,
also you can handle async using javascript methods without any needs to use other modules.
the reason for your problem is your async.each function executed without waiting what it has inside
and you have here deviceRepo.findById which is async function and need to wait for it
I refactored your code using async await and I wish this will simplify the problem
exports.appDetail = function async(req, res) {
try {
const myApp = await appRepo.findWithId(req.params.id)
const myDeviceData = await getDeviceData(myApp.device)
console.log(JSON.stringify(myDeviceData) + ' || myDeviceData')
// construct object to be returned
let appsDataObject = {
name: myApp.name,
user_id: myApp.user_id,
devices: myDeviceData,
permissions: myApp.permissions
}
return res.status(200).send(appsDataObject)
} catch (err) {
console.log(err)
}
}
// write async function here
const getDeviceData = function async(devices) {
let devicesDataArrayPromises = devices.map(async(device)=>{
const myDevice= await deviceRepo.findById(new ObjectID(device))
let deviceObj = {
name: myDevice.name,
version: myDevice.version
}
return deviceObj
})
const devicesDataArray = await Promise.all(devicesDataArrayPromises)
console.log(JSON.stringify(devicesDataArray) + ' || devicesDataAray after obj push')
return devicesDataArray
}
I am working on AWS Lambda using nodejs environment. I have one API in which I am using recursive function for some functionality.
Actually most people say avoid recursive function but as per my functionality I need this. My functionality is as follows.
I have one table table1 in which there are two columns pid and cid which is defined as unique constraint. Which means combination of two columns should be unique.
So if I am inserting any combination in table1 and it that combination already exist then it gives me duplicate entry error which is correct as per my functionality.
So in order to handle this duplicate entry error I have used try..catch block. So in catch block I have checked if duplicate entry error occurred then I am calling one recursive function which try different combination until new entry is created in table1.
And my recursive function is promise based. but when new entry in created successfully then I am resolving the promise. But promise does not get returned from where I have called my recursive function for very first time. And because of that timeout occurs.
So please someone suggest me solution so that my promise got resolved and my functionality will continue from point where I have called my recursive function for very first time. So the timeout will not come.
I am providing my code for reference.
var mysql = require('mysql');
var con = mysql.createConnection({
"host": "somehost.com",
"user": "myusername",
"password": "mypassword",
"database": "mydatabase"
});
exports.handler = async (event, context) => {
try {
con.connect();
} catch (error) {
throw error;
return 0;
}
let tableId = '';
let count = '';
try {
var tempUsageData = {
user_id: userId,
code: code,
platform: source,
some_id: some_id,
count: count
};
dbColumns = 'user_id, code, platform, added_on, count, some_id';
let usageData = [
[userId, code, source, new Date(), count, some_id]
];
var tableInsert = await databaseInsert(con, constants.DB_CONSTANTS.DB_USAGE, dbColumns, usageData);
tableId = tableInsert.insertId;
} catch (error) {
console.log('## error insert table1 ##', error);
if (error.errno == 1062) {
try {
// calling recursive function here
let newTableData = await createTableEntry(con, tempUsageData);
tableId = newTableData.new_usage_id;
count = newTableData.new_count;
} catch (error) {
console.log('Error', error);
return 0;
}
} else {
return 0;
}
};
console.log('## EXECUTION DONE ##');
return 1;
}
var createTableEntry = (con, dataObject) => {
return new Promise(async function (resolve, reject) {
console.log('createTableEntry Called for count', dataObject.count);
try {
var newCounter = await getDataFromDatabase(con, dataObject.some_id);
dbColumns = 'user_id, code, platform, added_on, count, some_id';
let tableData = [
[userId, code, source, new Date(), Number(newCounter[0].counter + 1), some_id]
];
var tableInsert = await databaseInsert(con, 'table1', dbColumns, tableData);
let response = {
new_table_id: tableInsert.insertId,
new_count: Number(newCounter[0].counter + 1)
}
return resolve(response);
//function not returning from here once successful entry done and timeout occures
} catch (error) {
console.log('## ERROR ##', error);
if (error.errno == 1062) {
console.log('## CALL FUNCTION AGAIN ##');
dataObject.count = Number(newCounter[0].counter + 1);
await createTableEntry(con, dataObject);
} else {
return reject(error);
}
}
});
};
My final output should be message "EXECUTION DONE" should be displayed once execution done.
Please suggest me good solution for this. Thanks in advance.
Update your catch block
catch (error) {
console.log('## ERROR ##', error);
if (error.errno == 1062) {
console.log('## CALL FUNCTION AGAIN ##');
dataObject.count = Number(newCounter[0].counter + 1);
let result = await createTableEntry(con, dataObject);
return resolve(result);
} else {
return reject(error);
}
}
I'm trying to use apolloFetch inside a Promise.all in my Node.js microservice but keep getting an error that the query is empty. The reason for using apolloFetch is to call another micro service and pass it an array of queries. Can someone give me some direction? My code is as follows:
const uri = "dsc.xxx.yyyy.com/abc/def/graphql";
const apolloFetch = CreateApolloFetch({uri});
const QryAllBooks = {
type: new GraphQLList(BookType),
args: {},
resolve() {
return new Promise((resolve, reject) => {
let sql = singleLineString`
select distinct t.bookid,t.bookname,t.country
from books_tbl t
where t.ship_status = 'Not Shipped'
`;
pool.query(sql, (err, results) => {
if (err) {
reject(err);
}
resolve(results);
const str = JSON.stringify(results);
const json = JSON.parse(str);
const promises = [];
for (let p = 0; p < results.length; p++) {
const book_id = json[p].bookid;
const query = `mutation updateShipping
{updateShipping
(id: ${book_id}, input:{
status: "Shipped"
})
{ bookid
bookname }}`;
promises.push(query);
}
//Below is the Promise.all function with the
//apolloFetch that calls another graphql endpoint
//an array of queries
Promise.all(promises.map(p => apolloFetch({p}))).then((result) => {
//this is the problem code^^^^^^^^^^^^^^^^^^^^^
resolve();
console.log("success!");
}).catch((e) => {
FunctionLogError(29, "Error", e);
});
});
});
}
};
module.exports = {
QryAllBooks,
BookType
};
It looks like apolloFetch requires query - you are passing p
change
Promise.all( promises.map(p=>apolloFetch({p})) )
to
Promise.all( promises.map(query=>apolloFetch({query})) )
You also call resolve twice
To resolve all errors or success
const final_results = []
Promise.all(promises.map(query => apolloFetch({
query,
}))).then((result) => {
final_results.push(result)
}).catch((e) => {
final_results.push(e)
}).then(() => {
resolve(final_results)
});
You immediately resolve or rejects once the pool.query() callback starts:
if(err){ reject(err);}resolve(results);
So unless the query fails, you never resolve with the results from the apolloFetch calls, since the promise is already resolved with the pool.query() results. I guess you're missing an else block:
if( err ) {
reject();
}
else {
const promises = ...
}
PS: you can try using node.js' util.promisify() to turn pool.query() into a promise as well so you can just write something resembling: query(...).then(results=>results.map(apolloFetch) instead of ahving to mix callbacks and promises.
Here is a sample code which am using for selenium WebDriver JS axe integration to test my website for accessibility -
var AxeBuilder = require('axe-webdriverjs');
var WebDriver = require('selenium-webdriver');
var driver = new WebDriver.Builder()
.forBrowser('firefox')
.build();
driver
.get('https://dequeuniversity.com/demo/mars/')
.then(function() {
AxeBuilder(driver).analyze(function(err, results) {
if (err) {
// Handle error somehow
}
console.log(results);
});
});
There is one URL that is being parsed here. Can someone help me with how I can parse multiple urls? I would want the print the results of all the urls being given as input to driver.get(). Any help would be appreciated !
Reference - https://github.com/dequelabs/axe-webdriverjs
Use Promise.all and map a url array.
const urlArray = [url1,url2,url3,...];
const finalResult = await Promise.all(urlArray.map(async url=>{
return await driver.get(url);
}))
you will get all the result in the array finalResult.
So I'll format the #CertainPerformance's comment as an answer.
The simplest approach is using the modern async/await syntax:
for (const url of [url1, url2, url3]) {
await driver
.get(url)
.then(async function() {
await AxeBuilder(driver).analyze(function(err, results) {
if (err) {
// Handle error somehow
}
console.log(results);
});
});
}
Do not forget to replace url1, url2, url3 with your urls.
P.S. as #jfriend00 stated (in comments below), we don't know if AxeBuilder function is actually returns a promise or not. So the await before it (and async) may be unnecessary in the latter case.
the above solution will work, but it will be serialized i.e. you will get the result of the driver.get promise and then analyze the result of one url before moving to the next. maybe you could use promise.all to do it all in parallel. Something on the lines of
function executeGetPromises() {
var getPromises = [];
var drivers = [];
for (const url of [url1, url2, url3]) {
var driver = new WebDriver.Builder().forBrowser('firefox').build();
getPromises.push(driver.get(url));
drivers.push(driver);
}
var analysePromises = [];
int index = 0;
Promise.all(getPromises.map(p => p.catch(e => e)))
.then(results => {
for (int i=0; i< results.length; i++) {
var result = results[i];
if (!(result instanceof Error)) {
analysePromises.push(AxeBuilder(drivers[i]).analyze);
}
}
executeAnalysePromises(analysePromises);
});
}
function executeAnalysePromises (analysePromises) {
Promise.all(analysePromises.map(p => p.catch(e => e)))
.then(results => {
results.forEach(result => {
if (!(result instanceof Error)) {
console.log(result);
}
});
});
}
here we are keeping a track of all the drivers, and all the driver.get promises are run in parallel and once all the getPromises return (either resolved/rejected), the analysePromises get executed in parallel.
EDIT: A more simpler approach using async functions. The above is a bit complicated, you can achieve the same using async functions
async function executeTask (driver, url) {
try{
await driver.get(url);
let result = await AxeBuilder(driver).analyze();
return Promise.resolve(result);
}
catch(err) {
return Promise.reject(err);
}
}
function iterateThroughUrls(urls) {
urls.forEach(url => {
var driver = new WebDriver.Builder().forBrowser('firefox').build();
executeTask(driver, url).then(result => {
console.log(result);
}).catch(err => {
//handle errors
});
});
}