Develop Reference

Knex.js design pattern for using transactions [duplicate]

I'm trying to couple ES7's async/await with knex.js transactions.
Although I can easily play around with non-transactional code, I'm struggling to get transactions working properly using the aforementioned async/await structure.
I'm using this module to simulate async/await
Here's what I currently have:
Non-transactional version:
works fine but is not transactional
app.js
// assume `db` is a knex instance
app.post("/user", async((req, res) => {
const data = {
idUser: 1,
name: "FooBar"
}
try {
const result = await(user.insert(db, data));
res.json(result);
} catch (err) {
res.status(500).json(err);
}
}));
user.js
insert: async (function(db, data) {
// there's no need for this extra call but I'm including it
// to see example of deeper call stacks if this is answered
const idUser = await(this.insertData(db, data));
return {
idUser: idUser
}
}),
insertData: async(function(db, data) {
// if any of the following 2 fails I should be rolling back
const id = await(this.setId(db, idCustomer, data));
const idCustomer = await(this.setData(db, id, data));
return {
idCustomer: idCustomer
}
}),
// DB Functions (wrapped in Promises)
setId: function(db, data) {
return new Promise(function (resolve, reject) {
db.insert(data)
.into("ids")
.then((result) => resolve(result)
.catch((err) => reject(err));
});
},
setData: function(db, id, data) {
data.id = id;
return new Promise(function (resolve, reject) {
db.insert(data)
.into("customers")
.then((result) => resolve(result)
.catch((err) => reject(err));
});
}
Attempt to make it transactional
user.js
// Start transaction from this call
insert: async (function(db, data) {
const trx = await(knex.transaction());
const idCustomer = await(user.insertData(trx, data));
return {
idCustomer: idCustomer
}
}),
it seems that await(knex.transaction()) returns this error:
[TypeError: container is not a function]

I couldn't find a solid answer for this anywhere (with rollbacks and commits) so here's my solution.
First you need to "Promisify" the knex.transaction function. There are libraries for this, but for a quick example I did this:
const promisify = (fn) => new Promise((resolve, reject) => fn(resolve));
This example creates a blog post and a comment, and rolls back both if there's an error with either.
const trx = await promisify(db.transaction);
try {
const postId = await trx('blog_posts')
.insert({ title, body })
.returning('id'); // returns an array of ids
const commentId = await trx('comments')
.insert({ post_id: postId[0], message })
.returning('id');
await trx.commit();
} catch (e) {
await trx.rollback();
}

Here is a way to write transactions in async / await.
It is working fine for MySQL.
const trx = await db.transaction();
try {
const catIds = await trx('catalogues').insert({name: 'Old Books'});
const bookIds = await trx('books').insert({catId: catIds[0], title: 'Canterbury Tales' });
await trx.commit();
} catch (error) {
await trx.rollback(error);
}

Async/await is based around promises, so it looks like you'd just need to wrap all the knex methods to return "promise compatible" objects.
Here is a description on how you can convert arbitrary functions to work with promises, so they can work with async/await:
Trying to understand how promisification works with BlueBird
Essentially you want to do this:
var transaction = knex.transaction;
knex.transaction = function(callback){ return knex.transaction(callback); }
This is because "async/await requires the either a function with a single callback argument, or a promise", whereas knex.transaction looks like this:
function transaction(container, config) {
return client.transaction(container, config);
}
Alternatively, you can create a new async function and use it like this:
async function transaction() {
return new Promise(function(resolve, reject){
knex.transaction(function(error, result){
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
}
// Start transaction from this call
insert: async (function(db, data) {
const trx = await(transaction());
const idCustomer = await(person.insertData(trx, authUser, data));
return {
idCustomer: idCustomer
}
})
This may be useful too: Knex Transaction with Promises
(Also note, I'm not familiar with knex's API, so not sure what the params are passed to knex.transaction, the above ones are just for example).

For those who come in 2019.
After I updated Knex to version 0.16.5. sf77's answer doesn't work anymore due to the change in Knex's transaction function:
transaction(container, config) {
const trx = this.client.transaction(container, config);
trx.userParams = this.userParams;
return trx;
}
Solution
Keep sf77's promisify function:
const promisify = (fn) => new Promise((resolve, reject) => fn(resolve));
Update trx
from
const trx = await promisify(db.transaction);
to
const trx = await promisify(db.transaction.bind(db));

I think I have found a more elegant solution to the problem.
Borrowing from the knex Transaction docs, I will contrast their promise-style with the async/await-style that worked for me.
Promise Style
var Promise = require('bluebird');
// Using trx as a transaction object:
knex.transaction(function(trx) {
var books = [
{title: 'Canterbury Tales'},
{title: 'Moby Dick'},
{title: 'Hamlet'}
];
knex.insert({name: 'Old Books'}, 'id')
.into('catalogues')
.transacting(trx)
.then(function(ids) {
return Promise.map(books, function(book) {
book.catalogue_id = ids[0];
// Some validation could take place here.
return knex.insert(book).into('books').transacting(trx);
});
})
.then(trx.commit)
.catch(trx.rollback);
})
.then(function(inserts) {
console.log(inserts.length + ' new books saved.');
})
.catch(function(error) {
// If we get here, that means that neither the 'Old Books' catalogues insert,
// nor any of the books inserts will have taken place.
console.error(error);
});
async/await style
var Promise = require('bluebird'); // import Promise.map()
// assuming knex.transaction() is being called within an async function
const inserts = await knex.transaction(async function(trx) {
var books = [
{title: 'Canterbury Tales'},
{title: 'Moby Dick'},
{title: 'Hamlet'}
];
const ids = await knex.insert({name: 'Old Books'}, 'id')
.into('catalogues')
.transacting(trx);
const inserts = await Promise.map(books, function(book) {
book.catalogue_id = ids[0];
// Some validation could take place here.
return knex.insert(book).into('books').transacting(trx);
});
})
await trx.commit(inserts); // whatever gets passed to trx.commit() is what the knex.transaction() promise resolves to.
})
The docs state:
Throwing an error directly from the transaction handler function automatically rolls back the transaction, same as returning a rejected promise.
It seems that the transaction callback function is expected to return either nothing or a Promise. Declaring the callback as an async function means that it returns a Promise.
One advantage of this style is that you don't have to call the rollback manually. Returning a rejected Promise will trigger the rollback automatically.
Make sure to pass any results you want to use elsewhere to the final trx.commit() call.
I have tested this pattern in my own work and it works as expected.

Adding to sf77's excellent answer, I implemented this pattern in TypeScript for adding a new user where you need to do the following in 1 transaction:
creating a user record in the USER table
creating a login record in the LOGIN table
public async addUser(user: User, hash: string): Promise<User> {
//transform knex transaction such that can be used with async-await
const promisify = (fn: any) => new Promise((resolve, reject) => fn(resolve));
const trx: knex.Transaction = <knex.Transaction> await promisify(db.transaction);
try {
let users: User [] = await trx
.insert({
name: user.name,
email: user.email,
joined: new Date()})
.into(config.DB_TABLE_USER)
.returning("*")
await trx
.insert({
email: user.email,
hash
}).into(config.DB_TABLE_LOGIN)
.returning("email")
await trx.commit();
return Promise.resolve(users[0]);
}
catch(error) {
await trx.rollback;
return Promise.reject("Error adding user: " + error)
}
}

Querying multiple promises with a callback

In node.js i have a databaseMapper.js file, that uses the Ojai node MapR api. to extract data. So far i have it working with single documents, but since this is an async api, i have a bit of issues with querying multiple documents.
This is what i have so far:
function queryResultPromise(queryResult) {
//this should handle multiple promises
return new Promise((resolve, reject) => {
queryResult.on("data", resolve);
// ...presumably something here to hook an error event and call `reject`...
});
}
const getAllWithCondition = async (connectionString, tablename, condition) =>{
const connection = await ConnectionManager.getConnection(connectionString);
try {
const newStore = await connection.getStore(tablename);
const queryResult = await newStore.find(condition);
return await queryResultPromise(queryResult);
} finally {
connection.close();
}
}
here it will only return the first because queryResultPromise will resolve on the first document.. however the callback with "data" may occur multiple times, before the queryResult will end like this queryResult.on('end', () => connection.close())
i tried using something like Promise.all() to resolve all of them, but I'm not sure how i include the queryResult.on callback into this logic

This will work
const queryResultPromise = (queryResult) => {
return new Promise((resolve, reject) => {
let result = [];
queryResult.on('data', (data) => {
result.push(data)
});
queryResult.on('end', (data) => {
resolve(result);
});
queryResult.on('error', (err) => {
reject(err);
})
});
};

Javascript promise chain hell

I'm having a problem with promise chains that I don't know how to resolve. Summary of my code: I do a mongoose query for a specific user, fetch his CarIds and then query each car for his details and return these details via JSON response.
let carsDetails = [];
User.findById(userId)
.then(user => {
const carIds = user.carsDetails;
carIds.forEach((carId) => {
Car.findById(carId)
.then(car => {
console.log(car);
carsDetails.push(car);
})
.catch(err => { throw err; });
});
return res.status(200).json({ data: carsDetails });
})
.catch(error => {
throw error;
});
The problem is that no cars get actually pushed onto carsDetails array on the carsDetails.push(car); line, because it jumps to return statement before it manages to fill up the array. Is there a workaround that could do those queries and return a result in a form of an array, object...? I tried writing everything in async await form too with self-made async forEach statement, but it doesn't help me. Any suggestions? I already tried with Promise.all(), but didn't manage to fix the issue. Thanks!

You'll need to collect the promises of your Car.findById(carId) calls and use Promise.all() to wait for all of them before responding. You can use array.map() to map each ID to a promise from Car.findById().
User.findById(userId)
.then(user => {
const carIds = user.carsDetails;
const carPromises = carIds.map(carId => Car.findById(carId))
return Promise.all(carPromises)
})
.then(cars => {
res.status(200).json({ data: cars })
})
.catch(error => {
throw error
})

If you can have a lot of cars to find, you may want to do your query in a single request, no need to stack multiple promises :
User.findById(userId)
.then(user => {
const carIds = user.carsDetails;
// if carsDetails is not an array of objectIds do this instead :
// const carIds = user.carsDetails.map(id => mongoose.Types.ObjectId(id));
return Car.find({ _id: { $in: carIds });
})
.then(userCars => {
res.status(200).json({ data: userCars })
})

await/async is the way to go, with await/async you use regular for ... of loops instead of forEach.
async function getCarDetails() {
let carsDetails = [];
let user = await User.findById(userId);
const carIds = user.carsDetails;
for (let carID of carIds) {
let car = await Car.findById(carId)
console.log(car);
carsDetails.push(car);
}
return res.status(200).json({
data: carsDetails
});
}
Or you use Promise.all and map instead of for ... of
async function getCarDetails() {
let user = await User.findById(userId);
const carIds = user.carsDetails;
let carsDetails = await Promise.all(carIds.map(carID => Car.findById(carID)));
return res.status(200).json({
data: carsDetails
});
}
Those two solutions are slightly different. The second version with the map will send all requests to the DB at once, and then waits until they are all resolved. The first one will send the request one after another. Which one is better depends on the use-case, the second one could lead to request peeks, and might be easier be abused for DDoS.

Try to use async/await to solve this problem. It will be more readable and clean
async function getCarDetail() {
let carsDetails = []
try {
const user = await User.findById(userId)
const carIds = user.carsDetails
for (let i = 0; i < carIds.length; i++) {
const carId = carIds[i]
const car = await Car.findById(carId)
carsDetails.push(car)
}
return res.status(200).json({ data: carsDetails })
} catch(error) {
console.log(error)
}
}

Promise.all is returning undefined

When I asynchronously map an array, Promise.all was supposed to make the function wait until all the promises are resolved. However, Promise.all shows as undefined. Here is my code. Please can someone tell me what I am doing wrong? Thanks.
router.get("/vehicle_reports/interior-pictures", auth, async (req, res) => {
const fileKeysObj = await Report.findOne({ orderId: req.params.jobId }, {
"vehicleInterior": 1
})
const fileKeysArray = fileKeysObj.interior
console.log("fileKeysArray: ", fileKeysArray);
//Retrieve the files from S3
const files = fileKeysArray.map(async (item) => {
const params = {
Bucket: process.env.AWS_BUCKET_NAME,
Key: item.fileKey
}
await s3.getObject(params, async (err, data) => {
if (err) {
throw new Error()
}
else {
const base64Data = base64Arraybuffer.encode(data.Body)
const contentType = data.ContentType
const fileName = item.fileName
return { base64Data, contentType, fileName }
}
})
})
console.log( files) //Pending promise
await Promise.all(files)
console.log( files) //Undefined
res.send(files) //Sends empty array
})

I wish people would stop hyping async/await. The await keyword was designed to work with Promises. And not all asynchronous functions return promises. Lots of APIs (such as S3) use callbacks instead. Also, architectures where you can expect multiple/infinite data return such as servers listening to incoming connection or reading a stream are not good fit for Promises which are basically single-shot. For those EventEmitters are more appropriate.
The async keyword does not convert a function to a Promise. It does return a promise but does not have the ability to convert callback based functions to Promises that can be used with await. For that you need to use the original Promise constructor. Therefore, the correct way to get an array of promises is as follows:
const files = fileKeysArray.map((item) => { /* note: async keyword is useless here */
const params = {
Bucket: process.env.AWS_BUCKET_NAME,
Key: item.fileKey
}
// We are returning a Promise, so no need to force it to further
// return a promise by marking this function as "async" above:
return new Promise((perfectly_fine, oops_something_went_wrong) => {
s3.getObject(params, async (err, data) => {
if (err) {
// Normally people would name this function "reject"
// but I'm illustrating that the name is really up to you
// and is not part of the syntax:
oops_something_went_wrong(err)
}
else {
const base64Data = base64Arraybuffer.encode(data.Body)
const contentType = data.ContentType
const fileName = item.fileName
// This is how you "return" to a promise:
perfectly_fine({ base64Data, contentType, fileName })
}
})
})
})
Now you can await the result. But you are using await wrong. The correct way to await is as follows:
let resulting_files = await Promise.all(files);
console.log(resulting_files);
You can also choose to not use await. Instead you can use .then():
Promise.all(files).then(resulting_files => {
// use resulting_files here:
console.log(resulting_files);
});

Replace the inside of the map call to look like this.
const params = {
Bucket: process.env.AWS_BUCKET_NAME,
Key: item.fileKey
}
return new Promise((res, rej) => {
s3.getObject(params, async (err, data) => {
if (err) {
rej('FAILED');
} else {
const base64Data = base64Arraybuffer.encode(data.Body)
const contentType = data.ContentType
const fileName = item.fileName
res( { base64Data, contentType, fileName });
}
})
});

How to access the body of a request outside in an async function?

So I have a post route which is an async await function, and I have a reuest inside it which pulls some data from the api and I want to save the content of that body in a variable outside the request function
I have tried using promises but I am not really familiar with that.
//#route POST api/portfolio/stock
//#desc Create or update a user's stock portfolio
//#access private
router.post(
"/stock",
auth,
[
check("symbol", "symbol is require or incorrect.")
.not()
.isEmpty(),
check("qty", "Quantity of the stock purchased is required")
.not()
.isEmpty()
],
async (req, res) => {
const errors = validationResult(req);
if (!errors.isEmpty) {
return res.status(400).json({ errors: errors.array() });
}
const { stock, qty } = req.body;
const newPortfolio = {};
newPortfolio.user = req.user.id;
newPortfolio.stocks = [];
if(stock) newPortfolio.stocks.stock = stock;
if(qty) newPortfolio.stocks.qty = qty;
request(`https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=${stock}&apikey=${config.get(
"API_KEY")}`, (error, response, body) => {
if (error) console.error(error);
if (response.statusCode !== 200) {
res.status(404).json({msg: 'No stock found'});
}
let content = JSON.parse(body);
let quote = content['Global Quote'];
});
newPortfolio.stocks.stockInfo = quote;
try {
let portfolio = await Portfolio.findOne({ user: user.req.id });
//update if exists
if(portfolio) {
portfolio = await Portfolio.findOneAndUpdate(
{ user: user.req.id },
{ $push: { stocks: newPortfolio.stocks }}
);
return res.json(portfolio);
}
//create if not found
portfolio = new Portfolio(newPortfolio);
await portfolio.save();
res.json(portfolio);
} catch (err) {
console.error(err.message);
res.status(500).send("Server Error");
}
}
);
I want to save myPortfolio.stocks.stockInfo using body of that request.

How to access the body of a request outside in an async function?
You don't. Asynchronous results are only available in the asynchronous context. For an asynchronous function that returns the result via a callback, that means you consume/use the results INSIDE the callback. Any code that needs access to those results goes in the callback. In traditional async Javascript programming, you just continue the code of your function inside the callback.
Fortunately, promises and the invention of async and await can make the coding a bit simpler. For asynchronous functions that return a promise (instead of taking a callback), you can use await to get their result and you can write code that looks more like the sequential model, even though it's still asynchronous.
For example, this is what a rewrite of your function might look like where we switch the request-promise library (same as the request library, but returns a promise instead of uses a callback) and then we use await on the result:
const rp = require('request-promise');
router.post(
"/stock",
auth,
[
check("symbol", "symbol is require or incorrect.")
.not()
.isEmpty(),
check("qty", "Quantity of the stock purchased is required")
.not()
.isEmpty()
],
async (req, res) => {
const errors = validationResult(req);
if (!errors.isEmpty) {
return res.status(400).json({
errors: errors.array()
});
}
const {
stock,
qty
} = req.body;
const newPortfolio = {};
newPortfolio.user = req.user.id;
newPortfolio.stocks = [];
if (stock) newPortfolio.stocks.stock = stock;
if (qty) newPortfolio.stocks.qty = qty;
try {
let body = await rp(`https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=${stock}&apikey=${config.get("API_KEY")}`);
let content = JSON.parse(body);
newPortfolio.stocks.stockInfo = content['Global Quote'];
} catch (e) {
console.error(e);
res.status(404).json({
msg: 'No stock found'
});
return;
}
try {
let portfolio = await Portfolio.findOne({
user: user.req.id
});
//update if exists
if (portfolio) {
portfolio = await Portfolio.findOneAndUpdate({
user: user.req.id
}, {
$push: {
stocks: newPortfolio.stocks
}
});
return res.json(portfolio);
}
//create if not found
portfolio = new Portfolio(newPortfolio);
await portfolio.save();
res.json(portfolio);
} catch (err) {
console.error(err.message);
res.status(500).send("Server Error");
}
}
);
Note: your adding properties to newPortfolio.stocks which is an array is quite unusual. While it's not technically wrong, you would usually declare newPortfolio.stocks to be an object, not an array if you were just going to add properties to it and not use it like an actual array. It can often be confusing to people reading your code if you use the same variable as both an object and an array. Usually, you want a variable (or property) to behave as one or the other, not both.