I need to query some data from Redshift in my aws lambda code and I do it several times with different parameters, pretty soon I get the error:
ActiveStatementsExceededException: Active statements exceeded the allowed quota (200).
How can I send thousands of queries to Redshift without hitting a limit?
Here is my JS code:
...
let results = await redshiftQuery(connectionInfo, "SELECT * from db.games where game_id=:game_id", {game_id: 12345})
...
async function redshiftQuery(conInfo, query, params = {}) {
let rsParams = []
for(let key in params) {
rsParams.push( { name: key, value: params[key].toString() })
}
const executeStatementInput = {
ClusterIdentifier: conInfo.clusterId,
Database: conInfo.database,
SecretArn: conInfo.secret,
WithEvent: false,
Sql: query
}
if(rsParams.length) {
executeStatementInput.Parameters = rsParams
}
let results = []
try {
let exResponse = await redshiftDataApiClient.executeStatement(executeStatementInput).promise()
if(!exResponse.Id) return results
let describeStatementInfo = null
while(true) {
let { Status: queryStatus, ...rest } = await redshiftDataApiClient.describeStatement({ Id: exResponse.Id }).promise()
describeStatementInfo = rest
if(["FAILED", "FINISHED"].includes(queryStatus)) {
break
}
await sleepMillis(200)
}
if (describeStatementInfo && describeStatementInfo.HasResultSet) {
let result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id }).promise()
convertResult(result, results)
while(result.NextToken) {
result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id, NextToken: result.NextToken }).promise()
convertResult(result, results)
}
}
} catch (e) {
console.error("Redshift Error", e)
}
return results
}
I am trying to use nodejs mssql package for bulk insert into existing table.It give error as Invalid Object Name 'my_test' even my table is already there and i have tried 'db.schema.tablename' or 'schema.tablename' both. Please help me on that and Thanks in advance. My code part is like below - :
async function getPool(name) {
if (!Object.prototype.hasOwnProperty.call(pools, name)) {
const pool = process.env.NODE_ENV.trim() === 'local' ? new sql.ConnectionPool(configLocal) : new sql.ConnectionPool(config);
const close = pool.close.bind(pool);
pool.close = (...args) => {
delete pools[name]
return close(...args)
}
await pool.connect();
pools[name] = pool;
}
return pools[name];
}
const pool = await getPool('default');
const table = new sql.Table('my_test'); // or temporary table, e.g. #temptable
table.create = false;
table.columns.add('id', sql.Int, { nullable: false,primary:true,identity:true});
table.columns.add('name', sql.VarChar(50), { nullable: false });
table.rows.add(10, 'test');
table.rows.add(11, 'test 2');
const request = new sql.Request(pool);
request.bulk(table, (err, result) => {
console.log("Result ", result, err);//return result;
}); ```
I cannot find clear information on how to manage SQL server database connections from an Azure function written in Javascript.
I am using a connection pool code -
const pool = new sql.ConnectionPool(config);
const poolConnect = pool.connect();
pool.on('error', err => {
// ... error handler
})
and I am using the poolConnect object from the function which is executing the query
export const selectQuery = function() {
const connectionPool = await mssqlDBPoolConnect;
const request = connectionPool.request();
await request.query('select query');
}
So how can I use the same connection pool across all azure functions.
Create two folder named config and toolkit under your root path. Put your db.js in config folder, and create a sql helper class to export a function named sqltools.js in toolkit folder.
So you could use the same connection pool by calling sqltools in your function's code. This step help you to reduce using the same code in every function.
Try use the db.js code below:
const sql = require('mssql')
const config = {
user: 'yourusername',
password: 'yourpassword',
server: 'yoursqlserver.database.windows.net', // You can use 'localhost\\instance' to connect to named instance. Do not use TCP.
database: 'yourdb',
"options": {
"encrypt": true,
"enableArithAbort": true
}
}
const poolPromise = new sql.ConnectionPool(config)
.connect()
.then(pool => {
console.log('Connected to MSSQL')
return pool
})
.catch(err => console.log('Database Connection Failed! Bad Config: ', err))
module.exports = {
sql, poolPromise
}
The sqltools.js class:
const { poolPromise } = require('../config/db')
module.exports.sqltools = {
ExecSqlQuery : async function(arg){
const pool = await poolPromise
//SELECT *FROM SYSOBJECTS WHERE xtype = \'U\'
var result=null;
try {
result = await pool.request()
.query(arg)
} catch (error) {
console.log(error.message);
}
return result;
},
ExecProce : function (arg2, arg3, arg4){
console.log(arg2,arg3,arg4);
}
}
Here is my HttpTrigger1 index.js code, call ExecSqlQuery to exec sqlstrings:
const { sqltools } = require('../toolkit/sqltools');
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
var result=null;
try {
// call ExecSqlQuery func
result = await sqltools.ExecSqlQuery('SELECT *FROM SYSOBJECTS WHERE xtype = \'U\'');
} catch (error) {
console.log(error.message);
}
const responseMessage ="Func 1 Result : TableName= " + result.recordset[0].name;
context.res = {
// status: 200, /* Defaults to 200 */
body: responseMessage
};
}
I've created an API which calls get cloudWatch AWS API and gives back datapoints that can be graphed on my app. I have separate routes for each package (as shown in the routing code below). This API uses REST MVC Method.
So a couple things I'm doing with my function.
Reading in EC2 Instance data from a SQLite3 database to grab
information about a running instance (IP, instance_id,
instance_launchtime) so that I can put it in the parameters required
for the getMetricStatistics API from the AWS SDK.
This data from step1 is then put into an array of parameters (3 that respond with 3 different metric datapoints). This loops through each parameter, inserting it into the getMetricStatistics API (ONE BY ONE SINCE getMetricStatistics doesn't accept multiple metrics at once) to grab data points for that instance and push them to an array.
For the database is async I believe, that is why I've attached a promise to it. When I load in the endpoint into my browser, it just keeps loading and won't show any data. When I do refresh the page, however, it shows all the results correctly...
This is my controller for the API:
// Return results sent from Cloud Watch API
const InsightModel = require('../models/insight.model.js');
const cloudWatch = InsightModel.cloudWatch;
const CWParams = InsightModel.CWParams;
const packageById = InsightModel.packageById;
let cpuUtilParam;
let cpuCBParam;
let cpuCUParam;
let insightParams = [];
let metricResults = [];
exports.getAnalytics = (req, res) => {
const currentDate = new Date().toISOString();
let promise1 = new Promise((resolve, reject) => {
packageById(req.params.packageKey, (err, data) => {
if (err) {
reject(
res.status(500).send({
message:
err.message ||
'Error while getting the insight configuration data.',
})
);
} else {
cpuUtilParam = new CWParams(
currentDate,
'CPUUtilization',
'AWS/EC2',
data[0].launch_time,
data[0].instance_id
);
cpuCBParam = new CWParams(
currentDate,
'CPUCreditBalance',
'AWS/EC2',
data[0].launch_time,
data[0].instance_id
);
cpuCUParam = new CWParams(
currentDate,
'CPUCreditUsage',
'AWS/EC2',
data[0].launch_time,
data[0].instance_id
);
insightParams = [cpuUtilParam, cpuCBParam, cpuCUParam];
resolve(insightParams);
}
});
})
let promise2 = new Promise((resolve, reject) => {
insightParams.forEach(metric => {
cloudWatch.getMetricStatistics(metric, function(err, data) {
if (err) {
reject(
res.status(500).send({
messaage:
err.message ||
'Error occured while running cloudWatch getMetricStatistcs API: ',
})
);
} else {
metricResults.push(data);
if (metricResults.length === insightParams.length)
resolve(metricResults);
}
});
});
});
Promise.all([promise1, promise2])
.then(metricResults => {
res.send(metricResults);
console.log('AWS CW API successful');
})
.catch(err => {
res.status(500).send({
messaage:
err.message ||
'Error occured while reading in a promise from cloudWatch getMetricStatistcs API: ',
})
});
metricResults = [];
};
The model for the API:
// Call AWS Cost Explorer API
const AWS = require('aws-sdk');
const config = require('./AWSconfig');
const database = require('./db');
const insightdb = database.insightdb;
AWS.config.update({
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region,
});
//Linking AWS CloudWatch Service
var cloudWatch = new AWS.CloudWatch();
const packageById = (packageId, callback) => {
insightdb.all(
'SELECT * FROM ec2Instance WHERE package_id == ?',
packageId,
(err, rows) => {
if (err) {
callback(err, null);
} else {
callback(null, rows);
}
}
);
};
// Parameter class to feed into the CloudWatch getMetricStatistics function
const CWParams = function(reqDate, metricName,service,launchTime,instanceId) {
(this.EndTime = reqDate) /* required */,
(this.MetricName = metricName) /* required */,
(this.Namespace = service) /* required */,
(this.Period = 3600) /* required */,
(this.StartTime = launchTime) /* ${createDate}`, required */,
(this.Dimensions = [
{
Name: 'InstanceId' /* required */,
Value: instanceId /* required */,
},
]),
(this.Statistics = ['Maximum']);
};
//Exports variables to the controller (so they can be re-used)
module.exports = { cloudWatch, CWParams, packageById };
The route for the API:
module.exports = app => {
const insight = require('../controllers/insight.controller.js');
app.get('/insights/aws/:packageKey', insight.getAnalytics);
};
As it stands, in the second Promise constructor, insightParams is guaranteed not to have been composed yet because insightParams = [.....] is in a callback that is called asynchronously. Therefore, the program flow needs to ensure all the "promise2" stuff happens only after "promise1" is fulfilled.
Things become a lot simpler in the higher level code if asynchronous functions are "promisified" at the lowest possible level. So do two things in the model:
Promisify cloudWatch.getMetricStatistics()
Write packageById() to return Promise rather than accepting a callback.
The model thus becomes:
const AWS = require('aws-sdk'); // no change
const config = require('./AWSconfig'); // no change
const database = require('./db'); // no change
const insightdb = database.insightdb; // no change
AWS.config.update({
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region
}); // no change
var cloudWatch = new AWS.CloudWatch(); // no change
// Promisify cloudWatch.getMetricStatistics() as cloudWatch.getMetricStatisticsAsync().
cloudWatch.getMetricStatisticsAsync = (metric) => {
return new Promise((resolve, reject) => {
cloudWatch.getMetricStatistics(metric, function(err, data) {
if (err) {
if(!err.message) { // Probably not necessary but here goes ...
err.message = 'Error occured while running cloudWatch getMetricStatistcs API: ';
}
reject(err); // (very necessary)
} else {
resolve(data);
}
});
});
};
// Ensure that packageById() returns Promise rather than accepting a callback.
const packageById = (packageId) => {
return new Promise((resolve, reject) => {
insightdb.all('SELECT * FROM ec2Instance WHERE package_id == ?', packageId, (err, rows) => {
if (err) {
reject(err);
} else {
resolve(rows);
}
});
});
};
Now getAnalytics() can be written like this:
exports.getAnalytics = (req, res) => {
packageById(req.params.packageKey)
.then(data => {
const currentDate = new Date().toISOString();
let insightParams = [
new CWParams(currentDate, 'CPUUtilization', 'AWS/EC2', data[0].launch_time, data[0].instance_id),
new CWParams(currentDate, 'CPUCreditBalance', 'AWS/EC2', data[0].launch_time, data[0].instance_id),
new CWParams(currentDate, 'CPUCreditUsage', 'AWS/EC2', data[0].launch_time, data[0].instance_id)
];
// Composition of `insightParams` is synchronous so you can continue
// with the `cloudWatch.getMetricStatisticsAsync()` stuff inside the same .then().
return Promise.all(insightParams.map(metric => cloudWatch.getMetricStatisticsAsync(metric))); // Simple because of the Promisification above.
}, err => {
// This callback handles error from packageById() only,
// and is probably unnecessary but here goes ...
if(!err.message) {
err.message = 'Error while getting the insight configuration data.';
}
throw err;
})
.then(metricResults => {
res.send(metricResults);
console.log('AWS CW API successful');
})
.catch(err => {
// Any async error arising above will drop through to here.
res.status(500).send({
'message': err.message
}));
});
};
Note that multiple catches each with res.status(500).send() are not necessary. Error propagation down the Promise chain allows a single, terminal .catch()
I'm trying to get some data from a pg database to my api endpoint , I can print the results to the console but I can't get them to display in the browser with res.send. I'm guessing the problem is with global and local scope however I've not been able to figure it out. I'm using ES6 but transpiling with babel. Here's a snippet.
app.get('/', (request, response) => {
const { Pool, Client } = require('pg');
const config = {
user: '',
host: '',
database: '',
password: '',
port: ,
}
const pool = new Pool(config);
const client = new Client(config);
let whole = [];
client.connect();
const text = "SELECT * FROM entries where id='1'";
client.query(text)
.then(res => {
console.log(res.rows[0]);
whole.push(res.rows[0]);
})
.catch(e => console.error(e.stack));
response.send(whole);
client.end;
});
This logs to the console
{ id: 1, title: 'First title', body: 'beautiful body' }
However the browser only displays []
This is what babel transpiles it to which is the script I run in node.
var whole = [];
client.connect();
var text = "SELECT * FROM entries where id='1'";
client.query(text).then(function (res) {
console.log(res.rows[0]);
whole.push(res.rows[0]);
}).catch(function (e) {
return console.error(e.stack);
});
response.send(whole);
client.end;
response.send is called outside of the async promise .then resolver, and is therefore executed before you push the row data into the array. Moving response.send into the promise resolver should fix it.
client.query(text).then(res => {
whole.push(res.rows[0]);
client.end();
response.send(whole);
}).catch((e) => {
console.error(e.stack);
});
Alternatively, you can use async/await depending on your babel version and presets/plugins.
const { Client } = require("pg");
const config = {...};
const queryText = "SELECT * FROM entries where id='1'";
app.get("/", async (request, response) => {
const client = new Client(config);
await client.connect();
try {
const queryResponse = await client.query(queryText);
// Send response without pushing to array
response.send(queryResponse.rows[0]);
client.end();
} catch (e) {
console.error(e.stack);
}
});