I'm trying to reverse engineer a Node script from a library (research project) to do the following tasks sequentially:
1) Open and read a file (e.g., 'input.txt'). For simplicity, assume that the contents are properly formatted SQL queries)
2) Create a connection to a MySQL database
3) Execute the queries (constructed from (1) -- assume queries are properly defined in the file)
4) Terminate connection with the database
I want these tasks to be executed in order (i.e., 1--4). I don't have much experience in using Promises (Bluebird). Here is an excerpt of the code I have so far:
//Read the input file
function readFilePromise(){
return new Promise(function(resolve, reject){
var filePath = path.join(__dirname, filename);
//asynchronous read
fs.readFile(filePath, 'utf8', function (err, text){
if (err)
reject(err);
else
resolve(text.split('\n'));
});
})
}
//create connection
function createConnectionPromise(){
return new Promise(function (resolve, reject){
var connection = mysql.createConnection(connectionOptions);//global
connection.connect(function(err){
if(err){
console.log('Error connecting to Db');
reject(err);
}
else{
console.log('Connection established');
resolve(connection);
}
});
})
}
//do transactions
function doTransactionsPromise (data){
return new Promise(function (resolve, reject){
var connection = data[0];
var lines = data[1];
var topPromises = [];
lines.forEach(function(sSQL){
var p = new Promise(function(resolve,reject){
console.log('Add: ' + sSQL);
makeTransaction(connection, sSQL);
return connection;
});
topPromises.push(p);
});
resolve(topPromises);
});
}
//make transaction
function makeTransaction(connection, sSQL){
connection.beginTransaction(function(err){
function treatErro(err, connection) {
console.log('Failed to insert data in the database . Undoing!');
connection.rollback();
}
function final() {
connection.commit(function(err) {
if(err) {
treatErro(err, connection);
}
else {
console.log('Added: ' + sSQL);
return connection;
}
});
}
if(err) {
treatErro(err, connection);
}
else {
connection.query(sSQL, function (err, result) {
if(err) {
console.log(sSQL);
treatErro(err, connection);
}
else {
id = result.insertId;
}
});
final();
}
});
}
Promise.all([createConnectionPromise(), readFilePromise()])
.then(doTransactionsPromise)
.then(function(promises){
Promise.all(promises)
.then(function(data){
var connection = data[0];
connection.end();
});
})
.catch(function(error) {
console.log('Error occurred!', error);
});
The queries are executed fine but the connection to the DB does not terminate. Any help is appreciated.
PS: I'm sure the code can be improved massively.
The possible problem I see in your code is in function doTransaction
function doTransactionsPromise (data){
return new Promise(function (resolve, reject){
var connection = data[0];
var lines = data[1];
var topPromises = [];
lines.forEach(function(sSQL){
var p = new Promise(function(resolve,reject){
console.log('Add: ' + sSQL);
makeTransaction(connection, sSQL);
return connection;
});
// P is never fullfilled.
//Either transfer the responsibility to full-fill the promise to makeTransaction
// or makeTransaction function should return the promise which is full-filled by itself.
topPromises.push(p);
});
resolve(topPromises);
});
}
I have not tested the code, but following code should do
//Read the input file
function readFilePromise(){
return new Promise(function(resolve, reject){
var filePath = path.join(__dirname, filename);
//asynchronous read
fs.readFile(filePath, 'utf8', function (err, text){
if (err)
reject(err);
else
resolve(text.split('\n'));
});
})
}
//create connection
function createConnectionPromise(){
return new Promise(function (resolve, reject){
var connection = mysql.createConnection(connectionOptions);//global
connection.connect(function(err){
if(err){
console.log('Error connecting to Db');
reject(err);
}
else{
console.log('Connection established');
resolve(connection);
}
});
})
}
//do transactions
function doTransactionsPromise (data){
var connection = data[0];
var lines = data[1];
var topPromises = [];
topPromise = lines.map(function(sSQL){
return makeTransaction(connection, sSQL);
});
return Promise.all(topPromises).then( function(){
return connection;
},function(){
return connection;
});
}
//make transaction
function makeTransaction(connection, sSQL){
return new Promise(resolve, reject, function(){
connection.beginTransaction(function(err){
function treatErro(err, connection) {
console.log('Failed to insert data in the database . Undoing!');
connection.rollback();
reject(connection);
}
function final() {
connection.commit(function(err) {
if(err) {
treatErro(err, connection);
}
else {
console.log('Added: ' + sSQL);
resolve(connection);
return connection;
}
});
}
if(err) {
treatErro(err, connection);
}
else {
connection.query(sSQL, function (err, result) {
if(err) {
console.log(sSQL);
treatErro(err, connection);
}
else {
id = result.insertId;
}
});
final();
}
});
})
}
Promise.all([createConnectionPromise(), readFilePromise()])
.then(doTransactionsPromise)
.then(function(connection){
return connection.end();
})
.catch(function(error) {
console.log('Error occurred!', error);
});
Related
I'm trying to rewrite my code to incorporate promises. I know that mongo already incorporates promises but i'd like to understand promises a bit more first. I don't understand the error message because I have used await in the async function. I found this articles that seems to do it similarly, but I still wasn't able to get it working.
What am i doing incorrectly here?
error message
SyntaxError: await is only valid in async function
code
app.post('/search/word',urlencodedParser, async function(req, res){
try{
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
let dbo = db.db("words");
//Declare promise
let searchWord = function(){
return new Promise(function(resolve, reject){
dbo.collection("word").find({"$text": {"$search": req.body.word}})
.toArray(function(err, result) {
err ? reject(err) : resolve(result);
});
});
};
result = await searchWord();
db.close();
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(result));
});
} catch(e) {
console.log(e);
}
});
The callback functions needs to be async
app.post('/search/word',urlencodedParser, async function(req, res){
try{
MongoClient.connect(url, { useNewUrlParser: true }, async function(err, db) {
if (err) throw err;
let dbo = db.db("words");
//Declare promise
let searchWord = function(){
return new Promise(function(resolve, reject){
dbo.collection("word").find({"$text": {"$search": req.body.word}})
.toArray(function(err, result) {
err ? reject(err) : resolve(result);
});
});
};
result = await searchWord();
db.close();
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(result));
});
} catch(e) {
console.log(e);
}
});
A file contains json data with details of database.
For each database connection, a series of queries need to be executed.
Currently, the map function is waiting for the database connection.
Below is the start function
function start() {
console.log('function initiated');
try {
let jsonData = fs.readFileSync('../request.json');
let jsonString = JSON.parse(jsonData);
//jsonString['request'].forEach(async function(json) {
jsonString['request'].map(async json => {
dbdetails = json.dbdetails;
//dbdetails.forEach(async function(db){
await dbbdetails.map(async db => {
console.log('pdbdetails: ' + db);
connString = json.connString;
//makes the DB connection
await connectDB(db.userId, db.Password, connString)
.then(async conn => {
await execution(conn, pdbDetails, vmUser, vmPassword, ip);
})
.catch(err => {
console.log(err);
});
console.log('after each execution');
//}
});
});
} catch (err) {
console.log(err.message);
return;
}
}
Below function is to make a database connection and return the connection
function connectDB(oUser, oPassword, connString) {
console.log('inside connectDB');
return new Promise((resolve, reject) => {
oracledb.getConnection(
{
user: oUser,
password: oPassword,
connectString: connString
},
function(err, connection) {
if (err) {
console.error(err.message);
reject(err);
//throw err;
}
console.log('returning connection');
//console.log(connection);
resolve(connection);
//return connection;
}
);
});
}
below is the function which executes servies of queries on database
function execution() {
/// series of sql query execution
}
Not sure what you’re trying to do exactly, but sounds like the problem is that .map doesn’t wait for your async functions. If you have to do them one at a time, use a for loop:
for ( var item of array ) {
await item.something();
}
To do them all at once:
var results = await Promise.all( array.map( item => item.something() )
Below program is to read all the files under a folder and check if all the files contains email information.
The below test should be failed(i.e.., promise to be rejected and status should be failed) when the condition in the second "if" statement is true. Execution is getting stopped, but the test is getting passed instead of getting failed.
So, How do i make the below test fail.
Thanks in Advance.
var checkFileContent = function(directory) {
var results = [];
fs.readdir(directory, function(err, list) {
// if (err) return done(err);
console.log("The folder or list of file names : " + list);
var i = 0;
(function next()
{
var file = list[i++];
// if (!file) return done(null, results);
file = directory + '/' + file;
fs.stat(file, function(err, stat)
{
if (stat && stat.isDirectory())
{
checkFileContent(file, function(err, res)
{
results = results.concat(res);
next();
});
} else
{
fs.readFile(file, "utf8", function(err, data)
{
// if ( err )
// { throw err;}
console.log(file +" file content is");
console.log(data);
console.log( data.toLowerCase().indexOf('email'));
return new Promise((resolve, reject) => {
if(data.toLowerCase().indexOf('email') != -1)
{
return reject(file + 'contains email ');
}
else
{
return new Promise((resolve, reject) =>
{
fs.readFile(file, "utf8", function(err, data)
{
// if ( err )
// { throw err;}
console.log(file +" file content is");
console.log(data);
console.log( data.toLowerCase().indexOf('email'));
//return newfunc(file,data);
if(data.toLowerCase().indexOf('email') != -1)
{
reject(file + 'contains email ');
}
else
{
console.log(file + 'doesnt contain email');
resolve(true);
}
}).catch ((err) =>
{
//Execution is getting stopped here, but the test is getting passed instead of getting failed.
return reject(err);
});
});
results.push(file);
//console.log("List of files under the current Log folder are : " + results);
next();
} // else closure
}); // fs.stat() closure
})(); });
}
The above function is being called from another JS File using Mocha as shown below :
it('should read Log files', function () {
return ----
.then((abc) => {
------
}).then(()=>
{
return JSFileName.checkFileContent(directory);
}).catch((err) => {
return Promise.reject(err);
})
})
I'm trying to write a wrapper class called Mongo. When I call getCollection() inside insert(), but I'm getting 'TypeError: this.getCollection is not a function'.
const mongoClient = require('mongodb').MongoClient;
const connectionString = process.env.MONGODB_CONNECTION_STRING;
const mongoOptions = {
connectTimeoutMS: 500,
autoReconnect: true
};
function Mongo(dbName, collectionName) {
this.dbName = dbName;
this.collectionName = collectionName;
this.db = null;
this.collectionCache = {};
this.getDB = function () {
return new Promise(function (resolve, reject) {
if (this.db == null) {
mongoClient.connect(connectionString, mongoOptions, function (err, db) {
if (err) reject(err);
this.db = db.db(this.dbName);
resolve(this.db);
});
} else {
resolve(this.db);
}
});
};
this.getCollection = function () {
return new Promise(function (resolve, reject) {
if (this.collectionName in this.collectionCache) {
resolve(this.collectionCache[this.collectionName]);
} else {
getDB().then(function(db) {
db.collection(this.collectionName, function (err, collection) {
if (err) reject(err);
this.collectionCache[this.collectionName] = collection;
resolve(collection);
});
}, function (err) {
reject(err);
});
}
});
};
this.insert = function(docs) {
return new Promise(function (resolve, reject) {
this.getCollection().then(function(collection) {
collection.insert(docs, function(err, results) {
if (err) reject(err);
resolve(results);
});
});
}, function (err) {
reject(err);
});
}
}
module.exports = Mongo;
How this class is instantiated, and the insert method is called.
const assert = require('assert');
const Mongo = require('../data/mongo');
describe('MongoTest', function() {
it('TestInsert', function() {
var mongo = new Mongo('testdb', 'humans');
var testDoc = {
_id: 1100,
name: 'tommy',
tags: ['cool', 'interesting']
};
mongo.insert(testDoc).then(function(result){
assert.equal(result._id, 1100);
});
})
})
Where you're calling the getCollection function, this no longer refers to the Mongo object, it refers to the callback function you've passed as a parameter to the Promise.
There are many ways of fixing that. Here are a few:
Keeping a reference to the object:
this.insert = function(docs) {
var self = this;
return new Promise(function (resolve, reject) {
self.getCollection().then(function(collection) {
collection.insert(docs, function(err, results) {
if (err) reject(err);
resolve(results);
});
});
}, function (err) {
reject(err);
});
}
Binding the context:
this.insert = function(docs) {
var callbackFunction = function (resolve, reject) {
this.getCollection().then(function(collection) {
collection.insert(docs, function(err, results) {
if (err) reject(err);
resolve(results);
});
});
};
return new Promise(callbackFunction.bind(this), function (err) {
reject(err);
});
}
Update your insert function as follows -
this.insert = function(docs) {
var _self = this;
return new Promise(function (resolve, reject) {
_self.getCollection().then(function(collection) {
collection.insert(docs, function(err, results) {
if (err) reject(err);
resolve(results);
});
});
}, function (err) {
reject(err);
});
}
You are getting this error because you are calling getCollection inside callback function where this does not refer to Mongo class. So you need to store the Mongo class reference in a variable and then use it.
I'm using the mssql module to connect to a sql server database using node. Bluebird has a feature that's similar to resource management in c#. It has a 'using' method to avoid having to use try/catch/finall to dispose of the resources. They have examples for pg and mysql, but they don't have an example for mssql which doesn't create a connection the same way as pg and mysql. Here's an example of how to use it:
using(getConnection(),
fs.readFileAsync("file.sql", "utf8"), function(connection, fileContents) {
return connection.query(fileContents);
}).then(function() {
console.log("query successful and connection closed");
});
But to be able to use this method, you need to create a connection method which describes how to close the connection. Here's an example for pg:
function getSqlConnection(connectionString) {
var close;
return pg.connectAsync(connectionString).spread(function(client, done) {
close = done;
return client;
}).disposer(function(client) {
if (close) close(client);
});
}
The problem I'm having with mssql module is that the connect method doesn't return a connection object like pg or even the mysql module. Has anyone been able to do this with mssql?
Update 1:
Here's how I made the transaction disposer:
function getTransaction(connection) {
return new Promise(function(resolve, reject) {
var tx = sql.Transaction(connection);
tx.beginAsync().then(function(err) {
if(err) {
tx = null;
return reject(err);
}
return resolve(tx);
});
}).disposer(function(tx, promise) {
if(promise.isFulfilled()) {
return tx.commitAsync();
}
else {
return tx.rollbackAsync();
}
});
}
It seems to be working, but not sure if this is efficient. Now I need to figure out how to catch errors on a query.
This is how I'm doing a transaction:
using(getConnection(), function(connection) {
return using(getTransaction(connection), function(tx) {
return query(queryString, tx).then(function() {
console.log('first query in transaction completed.');
console.log('starting second query in transaction.');
return query(anotherQueryString, tx);
});
});
});
If I tag a single catch to the outer 'using', will that catch all errors from the whole transaction?
Good question, mssql has really tricky API (constructors taking callbacks!) so this is good addition to the documentation.
var Promise = require("bluebird");
var sql = Promise.promisifyAll(require("mssql"));
global.using = Promise.using;
function getConnection(config) {
var connection;
return new Promise(function(resolve, reject)
connection = new sql.Connection(config, function(err) {
if (err) {
connection = null;
return reject(err);
}
resolve(connection);
});
}).disposer(function() {
if (connection) connection.close();
});
}
var config = {
user: '...',
password: '...',
server: 'localhost',
database: '...',
};
using(getConnection(config), function(connection) {
var request = new sql.Request(connection);
return request.queryAsync("select 1 as number").then(function(recordSet) {
console.log("got record set", recordSet);
return request.queryAsync("select 10 as number");
});
}).then(function(recordSet) {
console.log("got record set", recordSet);
})
To use the transaction, try implementing getTransaction like:
function getTransaction(connection) {
var tx = new sql.Transaction(connection);
return tx.beginAsync().thenReturn(tx).disposer(function(tx, promise) {
return promise.isFulfilled() ? tx.commitAsync() : tx.rollbackAsync();
});
}
And using it like:
using(getConnection(), function(connection) {
return using(getTransaction(connection), function(tx) {
var request = new sql.Request(tx);
return request.queryAsync("INSERT 1...").then(function() {
return request.queryAsync("INSERT 2...");
}).then(function() {
return request.queryAsync("INSERT 3...");
});
});
});
Error handling:
using(getConnection(), function(connection) {
return using(getTransaction(connection), function(tx) {
var request = new sql.Request(tx);
return request.queryAsync("INSERT...");
});
}).catch(sql.TransactionError, function(e) {
console.log("transaction failed", e);
}).catch(sql.ConnectionError, function(e) {
console.log("connection failed", e);
}).catch(sql.RequestError, function(e) {
console.log("invalid query", e);
});