I'm using a snippet example for Amazon Athena just to test inserting some data. I can't tell why it isn't working and CloudWatch logs does not show any output when the statement execution is completed. Even when I change it to a simple select statement I can't see any output. I know the query, database and table is fine, because when I test it using the Athena query editor it executes without a problem.
module.exports.dlr = async event => {
let awsFileCreds = {
accessKeyId: "XXX",
secretAccessKey: "XXX"
};
let creds = new AWS.Credentials(awsFileCreds);
AWS.config.credentials = creds;
let client = new AWS.Athena({ region: "eu-west-1" });
let q = Queue((id, cb) => {
startPolling(id)
.then(data => {
return cb(null, data);
})
.catch(err => {
console.log("Failed to poll query: ", err);
return cb(err);
});
}, 5);
const sql = "INSERT INTO delivery_receipts (status, eventid, mcc, mnc, msgcount, msisdn, received, userreference) VALUES ('TestDLR', 345345, 4353, '5345435', 234, '345754', 234, '8833')"
makeQuery(sql)
.then(data => {
console.log("Row Count: ", data.length);
console.log("DATA: ", data);
})
.catch(e => {
console.log("ERROR: ", e);
});
function makeQuery(sql) {
return new Promise((resolve, reject) => {
let params = {
QueryString: sql,
ResultConfiguration: { OutputLocation: ATHENA_OUTPUT_LOCATION },
QueryExecutionContext: { Database: ATHENA_DB }
};
client.startQueryExecution(params, (err, results) => {
if (err) return reject(err);
q.push(results.QueryExecutionId, (err, qid) => {
if (err) return reject(err);
return buildResults(qid)
.then(data => {
return resolve(data);
})
.catch(err => {
return reject(err);
});
});
});
});
}
function buildResults(query_id, max, page) {
let max_num_results = max ? max : RESULT_SIZE;
let page_token = page ? page : undefined;
return new Promise((resolve, reject) => {
let params = {
QueryExecutionId: query_id,
MaxResults: max_num_results,
NextToken: page_token
};
let dataBlob = [];
go(params);
function go(param) {
getResults(param)
.then(res => {
dataBlob = _.concat(dataBlob, res.list);
if (res.next) {
param.NextToken = res.next;
return go(param);
} else return resolve(dataBlob);
})
.catch(err => {
return reject(err);
});
}
function getResults() {
return new Promise((resolve, reject) => {
client.getQueryResults(params, (err, data) => {
if (err) return reject(err);
var list = [];
let header = buildHeader(
data.ResultSet.ResultSetMetadata.ColumnInfo
);
let top_row = _.map(_.head(data.ResultSet.Rows).Data, n => {
return n.VarCharValue;
});
let resultSet =
_.difference(header, top_row).length > 0
? data.ResultSet.Rows
: _.drop(data.ResultSet.Rows);
resultSet.forEach(item => {
list.push(
_.zipObject(
header,
_.map(item.Data, n => {
return n.VarCharValue;
})
)
);
});
return resolve({
next: "NextToken" in data ? data.NextToken : undefined,
list: list
});
});
});
}
});
}
function startPolling(id) {
return new Promise((resolve, reject) => {
function poll(id) {
client.getQueryExecution({ QueryExecutionId: id }, (err, data) => {
if (err) return reject(err);
if (data.QueryExecution.Status.State === "SUCCEEDED")
return resolve(id);
else if (
["FAILED", "CANCELLED"].includes(data.QueryExecution.Status.State)
)
return reject(
new Error(`Query ${data.QueryExecution.Status.State}`)
);
else {
setTimeout(poll, POLL_INTERVAL, id);
}
});
}
poll(id);
});
}
function buildHeader(columns) {
return _.map(columns, i => {
return i.Name;
});
}
return { message: 'Go Serverless v1.0! Your function executed successfully!', event };
};
Figured it out. Using aws lambda events with athena is easy using the athena-express package. You can specify your configuration and query the athena database like you normally would with significantly less code than what's provided in the amazon athena nodejs example.
This is the code I used to achieve a result:
"use strict";
const AthenaExpress = require("athena-express"),
aws = require("aws-sdk");
const athenaExpressConfig = {
aws,
db: "messaging",
getStats: true
};
const athenaExpress = new AthenaExpress(athenaExpressConfig);
exports.handler = async event => {
const sqlQuery = "SELECT * FROM delivery_receipts LIMIT 3";
try {
let results = await athenaExpress.query(sqlQuery);
return results;
} catch (error) {
return error;
}
};
Related
I'm making a request to my database, I set the functions as asynchronous and to wait, but it still returns me undefined or Promise { pending }
how do I just return it when I have the result?
export const getGerente = async (req, res) => {
var query = "SELECT * FROM inventory;"
const r = await select(query)
console.log(r)
return res.json({message:"teste"})
}
export async function select(query) {
var teste = await client.connect(() =>{
client
.query(query)
.then((resultado) => {
console.log('sucess!!');
return resultado.rows
/*
const rows=resultado.rows
rows.map(x =>{
console.log(x.name)
})*/
})
.catch((erro) => {
console.log("erro: " + erro.message);
})
.then((teste) => {
console.log('Finished execution, exiting now');
process.exit();
});
})
}
result: Promise { pending }
I'm calling her for a request
Your select function is not awaiting the client.connect properly.
Try this for select function -
export async function select(query) {
const promisifiedRows = new Promise((resolve, reject) => {
client.connect((err) => {
if (err) {
reject(err); // err in connecting
} else {
console.log('Connected!');
client.query(query, (err, rows) => {
if (err) {
reject(err); // err while exceuting the query
} else {
resolve(rows);
}
});
}
});
});
const rows = await promisifiedRows();
return rows;
}
I am using googleapis to create a google sheet. My code
const { google } = require("googleapis");
const keys = require("./credentials.json");
var spreadsheetId = null;
var gsapi = null;
const title = "SheetExample1!";
createConnection = () => {
return new Promise((resolve, reject) => {
const client = new google.auth.JWT(
keys.client_email,
null,
keys.private_key,
["https://www.googleapis.com/auth/spreadsheets"]
);
client.authorize((err, res) => {
if (err) {
reject(err);
} else {
gsapi = google.sheets({
version: "v4",
auth: client,
});
console.log("Connection Created Successfully");
resolve(res);
}
});
});
};
createSheet = () => {
return new Promise((resolve, reject) => {
const resource = {
properties: {
title,
},
};
gsapi.spreadsheets.create(
{
resource,
fields: "spreadsheetId",
},
(err, response) => {
if (err) {
reject(err);
} else {
console.log(
"Sheet Created with spreadsheetId============",
response.data.spreadsheetId
);
resolve(response.data);
}
}
);
});
};
functionHelper = async () => {
await createConnection();
await createSheet();
};
functionHelper();
In response, I am getting spreadsheetId. But I have no idea where all these sheets are stored and how I can open a spreadsheet using the generated spreadsheet Id.
I checked I can't see these files inside https://docs.google.com/spreadsheets/u/0/
Any suggestion?
I'm trying to retrieve ALL the data on all the Fleets in our GameLift, to be displayed on a React-powered CMS site.
Currently our page retrieves the Fleets (via listFleets), the Fleet Attributes (via describeFleetAttributes), and Game Sessions (via describeGameSessions).
Here's the code for all of that:
requestGameLiftData = async () => {
const gamelift = new AWS.GameLift();
try {
const { FleetIds } = await new Promise((resolve, reject) => { // Get Fleet IDs
gamelift.listFleets({}, function(err, data) {
if (err) { reject("Fleet id error"); }
else { resolve(data); }
});
});
const { FleetAttributes } = await new Promise((resolve, reject) => { // Get Fleet Attributes by IDs
gamelift.describeFleetAttributes(
{ FleetIds: FleetIds },
function(err, data) {
if (err) { reject("Fleet attributes error"); }
else { resolve(data); }
}
);
});
await new Promise((resolve, reject) => { // Save Fleet Attributes to state
this.setState(
{ fleetList: [...FleetAttributes] },
() => { resolve(); }
);
});
const instancePromiseArr = [];
const gameSessionPromiseArr = [];
const playerSessionPromiseArr = [];
const { fleetList } = this.state;
for (let fleet of fleetList) {
instancePromiseArr.push(this.getFleetInstances(fleet, gamelift));
gameSessionPromiseArr.push(this.getFleetGameSessions(fleet, gamelift));
}
let instanceData; // Get all Instances of every Fleet
try { instanceData = await Promise.all(instancePromiseArr); }
catch (err) { throw new Error("Fleet instances error"); }
let gameSessionData; // Get all Game Sessions of every Fleet
try { gameSessionData = await Promise.all(gameSessionPromiseArr); }
catch (err) { throw new Error("Fleet game session error"); }
fleetList.forEach((fleet, index) => { // Nesting game sessions and instances inside their respective fleets
fleet["Instances"] = instanceData[index].Instances;
fleet["GameSessions"] = gameSessionData[index].GameSessions;
});
await new Promise((resolve, reject) => {
this.setState(
{ fleetList: [...fleetList] },
() => { resolve(); }
);
});
this.setState({isFetched: true});
} catch (error) { this.setState({isFetched: true}); }
};
getFleetInstances = (fleet, gamelift) => {
return new Promise((resolve, reject) => {
gamelift.describeInstances(
{ FleetId: fleet.FleetId },
function(err, data) {
if (err) { reject("Fleet instances error"); }
else { resolve(data); }
}
);
});
};
getFleetGameSessions = (fleet, gamelift) => {
return new Promise((resolve, reject) => {
gamelift.describeGameSessions(
{ FleetId: fleet.FleetId },
function(err, data) {
if (err) { reject("Fleet game sessions error"); }
else { resolve(data); }
}
);
});
};
Now I have to get the Player Sessions. To that end, I added the following:
let playerSessionData; // Before the "Nesting"
try { playerSessionData = await Promise.all(playerSessionPromiseArr); }
catch (err) { throw new Error("Player session error"); }
getPlayersInSession = (gameSession, gamelift) => {
return new Promise((resolve, reject) => { // Function to get Player sessions, outside of requestGameLiftData
gamelift.describePlayerSessions(
{ GameSessionId: gameSession.GameSessionId },
function(err, data) {
if (err) { reject("Fleet player sessions error"); }
else { resolve(data); }
}
);
});
};
And then modified the Get all Game Sessions portion to the following:
try {
gameSessionData = await Promise.all(gameSessionPromiseArr);
for (const gameSessionItem of gameSessionData) {
for (const data of gameSessionItem.GameSessions) {
playerSessionPromiseArr.push(this.getPlayersInSession(data, gamelift, delay));
}
}
} catch (err) { throw new Error("Fleet game session error"); }
And nested it in:
fleetList.forEach((fleet, index) => { // Nesting game sessions and instances inside their respective fleets
fleet["Instances"] = instanceData[index].Instances;
for (const gameSession of gameSessionData[index].GameSessions) {
gameSession['PlayerSessions'] = [];
for (const playerSessions of playerSessionData) {
if (playerSessions.PlayerSessions.length > 0) {
for (const playerSessionItem of playerSessions.PlayerSessions) {
if (playerSessionItem.GameSessionId === gameSession.GameSessionId) {
gameSession['PlayerSessions'].push(playerSessionItem);
}
}
}
}
}
fleet["GameSessions"] = gameSessionData[index].GameSessions;
});
This works... sometimes. Most of the times, I get a ThrottlingException: Rate exceeded and 400 Bad Request. This doesn't happen in other regions with significantly less fleets, so I thought it was related to the sheer number of requests made at once (as of this writing, 8 for the fleets, 8 for GameSessions, and no less than 28 for the PlayerSessions). So I tried adding a delay:
for (const gameSessionItem of gameSessionData) {
let delay = 0;
for (const data of gameSessionItem.GameSessions) {
delay += 50;
playerSessionPromiseArr.push(this.getPlayersInSession(data, gamelift, delay));
}
}
getPlayersInSession = (gameSession, gamelift, delay) => {
return new Promise(resolve => setTimeout(resolve, delay)).then(() => {
return new Promise((resolve, reject) => {
gamelift.describePlayerSessions(
{ GameSessionId: gameSession.GameSessionId},
function(err, data) {
if (err) { reject("Fleet player sessions error"); }
else { resolve(data); }
}
);
});
});
};
Which didn't work, of course. Is there anything I'm missing? Or is there another approach to this, to get all the data in one sitting without making too many requests?
I'm doing a big loop once a day - which updating existing documents in the database (and also inserting new documents).
this loop get executed in a separate server ( prevents from the main server to be slow ), but the main problem is that all the find queries on the Data base (while the loop is executed) are very slow (the loop slows it down significantly).
This is a very big issue in my website ( this loop must be executed once a day ) and iv'e been trying to find a solution online - but i couldn't manage to find something.
Is there any way to prevent the find queries from being so slow while inserting/updating the database??
uploadProductsManually = async (name, products, map, valuesMap) => {
return new Promise(async function (resolve, reject) {
const company = await Company.findOne({ name }).exec();
if (!company) return reject(new errors.NotFound("Company not found"));
const rows = products;
const parsedRows = [];
const findCorrectKey = (key) => {
const correctKey = key.trim();
if (productFields[correctKey]) return productFields[correctKey];
const category = map.find((item) => {
return item.options.some((option) => {
return option.trim().toLowerCase() === correctKey.toLowerCase();
});
});
const categoryName = category && category.name;
return productFields[categoryName];
};
const hashProductValues = (product) => {
let valueToHash;
if (product.productId) {
valueToHash = product.productId;
} else if (product.certificateId) {
valueToHash = product.certificateId;
} else {
valueToHash = JSON.stringify(
product.size + product.color
);
}
return base64encode(valueToHash);
};
rows.forEach(function (row, i) {
var newProduct = {};
for (var key in row) {
var val = row[key];
if (val) {
let normalizedKey = findCorrectKey(key);
if (normalizedKey) {
newProduct[normalizedKey] = val;
}
let normalizedValue = normalizeValue(normalizedKey, val,valuesMap);
newProduct[normalizedKey] = normalizedValue;
}
}
newProduct.channels = [];
if (newProduct.productId) {
parsedRows.push(newProduct);
}
});
fetchProducts();
function fetchProducts() {
Product.find({ company: company._id }).exec(function (err, products) {
if (err) console.log(err);
var map = {};
if (products) {
products.forEach(function (product) {
const productIdentifier = hashProductValues(product);
map[productIdentifier] = product;
if (product.productStatus == "manual") {
// product.isAvailable = false;
// product.save();
} else {
product.status = "deleted";
product.save();
}
});
}
mergeData(map);
});
}
async function mergeData(map) {
let created = 0;
let updated = 0;
let manual = 0;
async.each(
parsedRows,
function (row, callback) {
const productIdentifier = hashProductValues(row);
let product = map[productIdentifier];
if (product) {
map[productIdentifier] = undefined;
Product.findByIdAndUpdate(id, { $set: updatedProduct }, function (
err,
updatedProd
) {
if (err) {
// errors.push(productIdentifier);
console.log("err is:", err);
}
updated++;
callback();
});
} else {
row = new Product(row);
row.save(function (err) {
if (err) {
// errors.push(productIdentifier);
console.log(err);
}
created++;
callback();
});
}
},
(err) => {
if (err) return reject(err);
Company.findByIdAndUpdate(
company._id,
{ lastUpdate: new Date() },
function (err, comp) {
if (err) console.log(err);
}
);
console.log(
`Created: ${created}\nUpdated: ${updated} \manual: ${manual}`
);
resolve({
created,
updated,
manual,
errors,
});
}
);
}
});
};
I would like to call use async waterfall function within a async series function. looks like parallel functions are executed but final callback function of async series does not seem to be executed. I get the results of the instances of the function which is called with different arguments in series function but cannot have the line executed for some reason.
➜ lib git:(jumpstart-compare) ✗ node aws-ecs.compare.js
Data 1f701a9754eb22ce8f0dcdb4c1b0b366a51ade9a
Data 4cc27bcc2a8482478ac2e5c0cf3ac1babe153374
var AWS = require('aws-sdk');
const async = require('async')
const _ = require('lodash');
AWS.config.update({
region: 'us-east-1'
});
const ecs = new AWS.ECS();
getClusterSha = (clustername,app,callback) => {
const ListServices = (callback) => {
let params = {
cluster: clustername
}
//console.log('list tasks executed')
ecs.listServices(params, (err, data) => {
if (err) {
callback(err)
} else {
let dataObj = {
data: data,
cluster: clustername
}
callback(null,dataObj)
}
})
}
const getService = (arg1, callback) => {
let appname = app
arg1.cluster
let finaldata = arg1.data.serviceArns.filter((elem)=>{
if(elem.indexOf(appname) != -1){
return elem
}
});
//console.log('finaldata: ',finaldata)
if(finaldata.length > 0){
callback(null,finaldata.toString().split('/')[1])
}else{
callback('No app with name: '+appname+' found!')
}
}
const describeService = (arg2, callback) => {
let params = {
services: [arg2],
cluster: clustername
}
ecs.describeServices(params, (err, data) => {
if (err) {
callback(err)
} else {
// console.log(data)
callback(null,data.services[0].taskDefinition.split('/')[1])
}
})
}
const describeTaskDef = (arg3, callback) => {
let params = {
taskDefinition: arg3
}
ecs.describeTaskDefinition(params, (err, data) => {
if (err) {
callback(err)
} else {
//console.log(data.taskDefinition.containerDefinitions[0].image.split('/')[1].split(':')[1])
finaldata = data.taskDefinition.containerDefinitions[0]
.image.split('/')[1]
.split(':')[1]
callback(null,finaldata)
}
})
}
// const githubCall = (arg4,callback) => {
// console.log('https://github.com/Jumpstart-Auto'+'/'+app+'/commit/'+arg4)
// callback(null,'https://github.com/Jumpstart-Auto'+'/'+app+'/commit/'+arg4)
// }
async.waterfall([
ListServices,
getService,
describeService,
describeTaskDef,
], (err, data) => {
if (err) {
console.log('Error', err)
callback(err)
} else {
console.log('Data', data)
callback(null,data)
}
})
}
compareSha = (clustername1,clustername2,app) => {
async.series([
getClusterSha(clustername1,app,(data)=>{return data}),
getClusterSha(clustername2,app,(data)=>{return data})
], (err,result)=>{
console.log(err,result)
})
}
compareSha('dev','staging','jamobot',function(err,data){
console.log(data)
})
//module.exports = getShaCluster
changing async.series to following fixed the problem.
async.waterfall([
ListServices,
getService,
describeService,
describeTaskDef,
], (err, data) => {
if (err) {
console.log('Error', err)
callback(err)
} else {
console.log('Data', data)
callback(null,data)
}
})
}
compareSha = (clustername1,clustername2,app,cb) => {
async.series([
function(callback){
getClusterSha(clustername1,app,callback)
},
function(callback){
getClusterSha(clustername2,app,callback)
},
], (err,result)=>{
if(err){
cb(err)
}else{
cb(null,result)
}
})