I'm trying to filter the results of a scan of a DynamoDB table using a FilterExpression in my query. I'm using a Lambda function deployed on API Gateway on AWS.
Function Querying the Endpoint
// Scan the table for entries and use data to populate HTML table.
async function scanTable(){
var query = {
TableName: "bookings",
ProjectionExpression: "id, username, start_booking, end_booking",
FilterExpression: "username = :u",
ExpressionAttributeValues: {":u":"human#gmail.com"}
};
try{
const response = await axios({ method: 'get', url: `${url}/bookings`, params: query });
}catch(err){
console.log(err);
}
}
Lambda Function
'use strict'
const AWS = require('aws-sdk');
exports.handler = async (event, context) => {
const documentClient = new AWS.DynamoDB.DocumentClient({convertEmptyValues: true});
let responseBody = "";
let statusCode = 0;
const params = (event.queryStringParameters);
// console.log(params);
try{
const data = await documentClient.scan(params).promise();
responseBody = JSON.stringify(data.Items);
statusCode = 200;
}catch(err){
responseBody = `Unable to get the bookings: ${err}`;
statusCode = 403;
}
console.log("Response Body: " + responseBody);
const response = {
statusCode: statusCode,
headers: {
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*"
},
body: responseBody
};
return response;
}
The complete error I receive is
ValidationException: ExpressionAttributeValues contains invalid key: Syntax error; key: "11"
where the key value at the end of the error (key:"11") changes somewhat dependent on the value assigned in ExpressionAttributeValues. I've tried to follow the FilterExpression example in the DynamoDB Docs for the scan function to no avail. Here is another example.
Here is a similar question from the AWS forums.
I have a lambda function invoked as a custom resource via a CloudFormation template. It Creates/Deletes AWS Connect instances. The API calls work fine but I cannot seem to terminate the custom resource invocation, so the last CF block remains CREATE_IN_PROGRESS. No matter what I return from the async function it just won't terminate the CF execution with a success.
I'm able to use a non-async handler successfully as in https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/walkthrough-custom-resources-lambda-lookup-amiids.html but I need to make multiple API calls and await completions, hence the need for async handler.
Below is the code in it's simplest form, though I've tried just about everything, including using callback and context (ie exports.handler = async function(event, context, callback) {...}), both of which should be unnecessary with an async handler. I've tried using cfn-response to directly send a response which seems to be ignored with async handlers. I've tried returning directly the promises with and without the await before them, tried returning variables containing various responseStatus and responseData, nothing seems to work.
Transform: 'AWS::Serverless-2016-10-31'
Parameters:
IdentityManagementType:
Description: The type of identity management for your Amazon Connect users.
Type: String
AllowedValues: ["SAML", "CONNECT_MANAGED", "EXISTING_DIRECTORY"]
Default: "SAML"
InboundCallsEnabled:
Description: Whether your contact center handles incoming contacts.
Type: String
AllowedValues: [true, false]
Default: true
InstanceAlias:
Description: The name for your instance.
Type: String
MaxLength: 62
OutboundCallsEnabled:
Description: Whether your contact center allows outbound calls.
Type: String
AllowedValues: [true, false]
Default: true
DirectoryId:
Description: Optional. The identifier for the directory, if using this type of Identity Management.
Type: String
ClientToken:
Description: Optional. The idempotency token. Used for concurrent deployments
Type: String
MaxLength: 500
Region:
Description: Region to place the AWS Connect Instance
Type: String
Default: us-east-1
#Handler for optional values
Conditions:
HasClientToken: !Not
- !Equals
- ""
- !Ref ClientToken
HasDirectoryId: !Not
- !Equals
- ""
- !Ref DirectoryId
Resources:
CreateConnectInstance:
Type: AWS::Serverless::Function
Properties:
FunctionName: !Sub "${AWS::StackName}-AWSConnectInstance"
Handler: index.handler
Runtime: nodejs12.x
Description: Invoke a function to create an AWS Connect instance.
MemorySize: 128
Timeout: 30
Role: !GetAtt LambdaExecutionRole.Arn
Layers:
- !Sub "arn:aws:lambda:us-east-1:${AWS::AccountId}:layer:node_sdk:1"
Environment:
Variables:
IdentityManagementType:
Ref: IdentityManagementType
InboundCallsEnabled:
Ref: InboundCallsEnabled
InstanceAlias:
Ref: InstanceAlias
OutboundCallsEnabled:
Ref: OutboundCallsEnabled
Region:
Ref: Region
#Optional Values
ClientToken: !If
- HasClientToken
- !Ref ClientToken
- !Ref "AWS::NoValue"
DirectoryId: !If
- HasClientToken
- !Ref ClientToken
- !Ref "AWS::NoValue"
InlineCode: |
var aws = require("aws-sdk");
exports.handler = async function(event) {
console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
var connect = new aws.Connect({region: event.ResourceProperties.Region});
var isInboundCallsEnabled = (process.env.InboundCallsEnabled == 'true');
var isOutboundCallsEnabled = (process.env.OutboundCallsEnabled == 'true');
var createInstanceParams = {
InboundCallsEnabled: isInboundCallsEnabled,
OutboundCallsEnabled: isOutboundCallsEnabled,
IdentityManagementType: process.env.IdentityManagementType,
ClientToken: process.env.ClientToken,
DirectoryId: process.env.DirectoryId,
InstanceAlias: process.env.InstanceAlias
};
// Create AWS Connect instance using specified parameters
if (event.RequestType == "Create") {
return await connect.createInstance(createInstanceParams).promise();
// I can store this in a variable and read the contents fine, but...
// returning the promise does not terminate execution
}
};
InvokeCreateConnectInstance:
Type: Custom::CreateConnectInstance
Properties:
ServiceToken: !GetAtt CreateConnectInstance.Arn
Region: !Ref "AWS::Region"
The documentaiton at https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html explicitly states that you should be able to return await apiCall.promise() directly from any async function, exactly what I'm trying to do, such as
const s3 = new AWS.S3()
exports.handler = async function(event) {
return s3.listBuckets().promise()
}
Why can't I return from my async function? Again the API calls are working, the Connect instances are created and deleted (though I've omitted the delete code for brevity), but CF just hangs hours and hours until eventually saying "Custom Resource failed to stabilize in expected time"
Here's the inline code by itself for readability:
exports.handler = async function(event) {
console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
var connect = new aws.Connect({region: event.ResourceProperties.Region});
var isInboundCallsEnabled = (process.env.InboundCallsEnabled == 'true');
var isOutboundCallsEnabled = (process.env.OutboundCallsEnabled == 'true');
var createInstanceParams = {
InboundCallsEnabled: isInboundCallsEnabled,
OutboundCallsEnabled: isOutboundCallsEnabled,
IdentityManagementType: process.env.IdentityManagementType,
ClientToken: process.env.ClientToken,
DirectoryId: process.env.DirectoryId,
InstanceAlias: process.env.InstanceAlias
};
// Create AWS Connect instance using specified parameters
if (event.RequestType == "Create") {
return await connect.createInstance(createInstanceParams).promise();
// I can store this in a variable and read the contents fine, but...
// returning the promise does not terminate CF execution
}
};
UPDATE: I've implemented the sendResponse method exactly as shown in the AMI lookup example (the first link) and am sending exactly the correct structure for the response, it even includes the newly created connect instance ID in the data field:
{
"Status": "SUCCESS",
"Reason": "See the details in CloudWatch Log Stream: 2020/12/23/[$LATEST]6fef3553870b4fba90479a37b4360cee",
"PhysicalResourceId": "2020/12/23/[$LATEST]6fef3553870b4fba90479a37b4360cee",
"StackId": "arn:aws:cloudformation:us-east-1:642608065726:stack/cr12/1105a290-4534-11eb-a6de-0a8534d05dcd",
"RequestId": "2f7c3d9e-941f-402c-b739-d2d965288cfe",
"LogicalResourceId": "InvokeCreateConnectInstance",
"Data": {
"InstanceId": "2ca7aa49-9b20-4feb-8073-5f23d63e4cbc"
}
}
And STILL the custom resource will just not close in CloudFormation. I just don't understand why this is happening when I am returning the above to the event.responseURL. It's like specifying an async handler completely breaks the custom resource handler and prevents it from closing.
UPDATE: When I manually CURL the above response directly to the event.responseUrl the CF resource registers a success! WTF... I'm sending the exact same response as the lambda function is sending, and it accepts it from the CURL but not from my lambda function.
UPDATE: latest code including sendResponse, etc
var aws = require("aws-sdk");
exports.handler = async function(event, context, callback) {
console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
var connect = new aws.Connect({region: event.ResourceProperties.Region});
var isInboundCallsEnabled = (process.env.InboundCallsEnabled == 'true');
var isOutboundCallsEnabled = (process.env.OutboundCallsEnabled == 'true');
var createInstanceParams = {
InboundCallsEnabled: isInboundCallsEnabled,
OutboundCallsEnabled: isOutboundCallsEnabled,
IdentityManagementType: process.env.IdentityManagementType,
ClientToken: process.env.ClientToken,
DirectoryId: process.env.DirectoryId,
InstanceAlias: process.env.InstanceAlias
};
var responseStatus;
var responseData = {};
// Create Connect instance
if (event.RequestType == "Create") {
try {
var createInstanceRequest = await connect.createInstance(createInstanceParams).promise();
responseStatus = "SUCCESS";
responseData = {"InstanceId": createInstanceRequest.Id};
} catch (err) {
responseStatus = "FAILED";
responseData = {Error: "CreateInstance failed"};
console.log(responseData.Error + ":\n", err);
}
sendResponse(event, context, responseStatus, responseData);
return;
}
// Look up the ID and call deleteInstance.
if (event.RequestType == "Delete") {
var instanceId;
var listInstanceRequest = await connect.listInstances({}).promise();
listInstanceRequest.InstanceSummaryList.forEach(instance => {
if (instance.InstanceAlias == createInstanceParams.InstanceAlias) {
instanceId = instance.Id;
}
});
if (instanceId !== undefined) {
try {
var deleteInstanceRequest = await connect.deleteInstance({"InstanceId": instanceId}).promise();
responseStatus = "SUCCESS";
responseData = {"InstanceId": instanceId};
} catch (err) {
responseStatus = "FAILED";
responseData = {Error: "DeleteInstance call failed"};
console.log(responseData.Error + ":\n", err);
}
} else {
responseStatus = "FAILED";
responseData = {Error: "DeleteInstance failed; no match found"};
console.log(responseData.Error);
}
sendResponse(event, context, responseStatus, responseData);
return;
}
};
// Send response to the pre-signed S3 URL
function sendResponse(event, context, responseStatus, responseData) {
var responseBody = JSON.stringify({
Status: responseStatus,
Reason: "CloudWatch Log Stream: " + context.logStreamName,
PhysicalResourceId: context.logStreamName,
StackId: event.StackId,
RequestId: event.RequestId,
LogicalResourceId: event.LogicalResourceId,
Data: responseData
});
console.log("RESPONSE BODY:\n", responseBody);
var https = require("https");
var url = require("url");
var parsedUrl = url.parse(event.ResponseURL);
var options = {
hostname: parsedUrl.hostname,
port: 443,
path: parsedUrl.path,
method: "PUT",
headers: {
"content-type": "",
"content-length": responseBody.length
}
};
console.log("SENDING RESPONSE...\n");
var request = https.request(options, function(response) {
console.log("STATUS: " + response.statusCode);
console.log("HEADERS: " + JSON.stringify(response.headers));
// Tell AWS Lambda that the function execution is done
context.done();
});
request.on("error", function(error) {
console.log("sendResponse Error:" + error);
// Tell AWS Lambda that the function execution is done
context.done();
});
// write data to request body
request.write(responseBody);
request.end();
}
Been at this for two days now :(
PS in the logs the "RESPONSE BODY" is shown as expected like I copied above, and log shows the "SENDING RESPONSE" but does not get to the the "STATUS: " and "HEADERS: " portion of the request.https() call, which makes me think something with async interferes with this call... IDK
This one was really tricky but finally have everything figured out. I had to make the sendResponse function asynchronous by adding a promise to it, awaiting that promise and returning it. This allowed me to ultimately call "return await sendResponse(event, context, responseStatus, responseData);" and finally everything is working, both create and delete operations are successful and the CloudFormation custom resource completes as expected. Phew. Posting code here in hopes that others will benefit from it.
var aws = require("aws-sdk");
exports.handler = async function(event, context, callback) {
console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
var connect = new aws.Connect({region: event.ResourceProperties.Region});
var isInboundCallsEnabled = (process.env.InboundCallsEnabled == 'true');
var isOutboundCallsEnabled = (process.env.OutboundCallsEnabled == 'true');
var createInstanceParams = {
InboundCallsEnabled: isInboundCallsEnabled,
OutboundCallsEnabled: isOutboundCallsEnabled,
IdentityManagementType: process.env.IdentityManagementType,
ClientToken: process.env.ClientToken,
DirectoryId: process.env.DirectoryId,
InstanceAlias: process.env.InstanceAlias
};
var responseStatus;
var responseData = {};
if (event.RequestType == "Create") {
try {
var createInstanceRequest = await connect.createInstance(createInstanceParams).promise();
responseStatus = "SUCCESS";
responseData = {"InstanceId": createInstanceRequest.Id};
} catch (err) {
responseStatus = "FAILED";
responseData = {Error: "CreateInstance failed"};
console.log(responseData.Error + ":\n", err);
}
return await sendResponse(event, context, responseStatus, responseData);
}
if (event.RequestType == "Delete") {
var instanceId;
var listInstanceRequest = await connect.listInstances({}).promise();
listInstanceRequest.InstanceSummaryList.forEach(instance => {
if (instance.InstanceAlias == createInstanceParams.InstanceAlias) {
instanceId = instance.Id;
}
});
if (instanceId !== undefined) {
try {
var deleteInstanceRequest = await connect.deleteInstance({"InstanceId": instanceId}).promise();
responseStatus = "SUCCESS";
responseData = {"InstanceId": instanceId};
} catch (err) {
responseStatus = "FAILED";
responseData = {Error: "DeleteInstance call failed"};
console.log(responseData.Error + ":\n", err);
}
} else {
responseStatus = "FAILED";
responseData = {Error: "DeleteInstance failed; no match found"};
console.log(responseData.Error);
}
return await sendResponse(event, context, responseStatus, responseData);
}
};
async function sendResponse(event, context, responseStatus, responseData) {
let responsePromise = new Promise((resolve, reject) => {
var responseBody = JSON.stringify({
Status: responseStatus,
Reason: "CloudWatch Log Stream: " + context.logStreamName,
PhysicalResourceId: context.logStreamName,
StackId: event.StackId,
RequestId: event.RequestId,
LogicalResourceId: event.LogicalResourceId,
Data: responseData
});
console.log("RESPONSE BODY:\n", responseBody);
var https = require("https");
var url = require("url");
var parsedUrl = url.parse(event.ResponseURL);
var options = {
hostname: parsedUrl.hostname,
port: 443,
path: parsedUrl.path,
method: "PUT",
headers: {
"content-type": "",
"content-length": responseBody.length
}
};
console.log("SENDING RESPONSE...\n");
var request = https.request(options, function(response) {
console.log("STATUS: " + response.statusCode);
console.log("HEADERS: " + JSON.stringify(response.headers));
resolve(JSON.parse(responseBody));
context.done();
});
request.on("error", function(error) {
console.log("sendResponse Error:" + error);
reject(error);
context.done();
});
request.write(responseBody);
request.end();
});
return await responsePromise;
}
This answer is a variant on the OP's answer for those using the "ZipFile" option in the "Code" property of an AWS::Lambda::Function resource in CloudFormation. The advantage of the ZipFile approach is that in addition permitting Lambda code inlined into the CF template, it also automatically bundles a "cfn-response.js" function very similar to the "async function sendResponse" in the OP's answer. With the insight gained from the OP's answer regarding a promised response (thank you, I was stuck and perplexed), this is how I incorporated the cfn-response function as an await-able Promise to signal CF after my asynchronous AWS API calls (omitted for brevity) were complete:
CreateSnapshotFunction:
Type: AWS::Lambda::Function
Properties:
Runtime: nodejs12.x
Handler: index.handler
Timeout: 900 # 15 mins
Code:
ZipFile: !Sub |
const resp = require('cfn-response');
const aws = require('aws-sdk');
const cf = new aws.CloudFormation({apiVersion: '2010-05-15'});
const rds = new aws.RDS({apiVersion: '2014-10-31'});
exports.handler = async function(evt, ctx) {
if (evt.RequestType == "Create") {
try {
// Query the given CF stack, determine its database
// identifier, create a snapshot of the database,
// and await an "available" status for the snapshot
let stack = await getStack(stackNameSrc);
let srcSnap = await createSnapshot(stack);
let pollFn = () => describeSnapshot(srcSnap.DBSnapshot.DBSnapshotIdentifier);
let continueFn = snap => snap.DBSnapshots[0].Status !== 'available';
await poll(pollFn, continueFn, 10, 89); // timeout after 14 min, 50 sec
// Send response to CF
await send(evt, ctx, resp.SUCCESS, {
SnapshotId: srcSnap.DBSnapshot.DBSnapshotIdentifier,
UpgradeRequired: upgradeRequired
});
} catch(err) {
await send(evt, ctx, resp.FAILED, { ErrorMessage: err } );
}
} else {
// Send success to CF for delete and update requests
await send(evt, ctx, resp.SUCCESS, {});
}
};
function send(evt, ctx, status, data) {
return new Promise(() => { resp.send(evt, ctx, status, data) });
}
I'm trying to develop a function in VS Code that takes an url as input and returns the response after processing is complete. However, when this function is run, it returns nothing. I tried testing similar code in Webstorm and and confirm that it console.logs the results just fine. I'm new to node and promises so not sure what I'm doing wrong.
Edit - added return keyword before driver.get as per the suggestion from #hellikiam. Also added a simple log statement to confirm that the results available yet not being returned in body.
var AxeBuilder = require('#axe-core/webdriverjs'),
WebDriver = require('selenium-webdriver');
const chromedriver = require('chromedriver');
const chrome = require("selenium-webdriver/chrome");
const screen = {
width: 640,
height: 480
};
chrome.setDefaultService(new chrome.ServiceBuilder(chromedriver.path).build());
var driver = new WebDriver.Builder()
.forBrowser('chrome')
.setChromeOptions(new chrome.Options().headless().windowSize(screen))
.build();
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
const url = (req.query.url || (req.body && req.body.url));
return driver.get(url).then(function () {
new AxeBuilder(driver).analyze(function (err, results) {
resultsJson = JSON.stringify(results);
console.log(resultsJson)
context.res = {
status: 200, /* Defaults to 200 */
body: resultsJson,
headers: {
'Content-Type': 'application/json'
}
};
if (err) {
// Handle error somehow
}
});
});
context.done();
}
you didn't returned anything from exported module.
return driver.get(url).then(function () {//add return statement in here.
new AxeBuilder(driver).analyze(function (err, results) {
resultsJson = JSON.stringify(results);
context.res = {
status: 200, /* Defaults to 200 */
body: resultsJson,
headers: {
'Content-Type': 'application/json'
}
};
if (err) {
// Handle error somehow
}
});
});
I found the other problem. What you want to return is inside callback function.
You should wrap the callback with promise to get the result outside of the callback. Try this out brother.
return new Promise((resolve, reject) =>{
driver.get(url).then(function () {//add return statement in here.
new AxeBuilder(driver).analyze(function (err, results) {
resultsJson = JSON.stringify(results);
context.res = {
status: 200, /* Defaults to 200 */
body: resultsJson,
headers: {
'Content-Type': 'application/json'
}
};
resolve(resultsJson)//Any value you want to return
if (err) reject(err)
});
});
})
Calling this function outside of module with await statement
const foo = require('./my/path/foo')
(async()=>{
const params = 'my param'
const bar = await foo(params)
console.log(bar)
})()
These results are for GET and POST requests response through API gateway and lambda. used same lambda function, but when i used post method of API gateway, response just shows me JSON. what should i have to do?
when i used post req
when i used get req
it's my lambda function
exports.handler = (event, context, callback) => {
const response = {
statusCode: 301,
headers: {
Location: 'https://google.com',
}
};
return callback(null, response);
}
thank you.
I usually use the new async/await pattern when defining my aws lambdas :
exports.handler = async (event) => {
// do stuff...
}
you usually don't need the context, unless you want to use some aws-related info regarding your lambda.
I have a helper function that is re-used a lot inside the code base
function proxyResponse(inBody, inStatusCode = null, headers = {}, event = null) {
if (!isApiGateway(event)) {
if (inBody instanceof Error) {
throw inBody;
}
return inBody;
}
let statusCode = inStatusCode;
let body;
if (inBody instanceof Error) {
statusCode = statusCode || INTERNAL_SERVER_ERROR;
body = JSON.stringify({
message: inBody.message,
code: statusCode,
});
} else if (inBody instanceof Object) {
body = JSON.stringify(inBody);
} else {
body = inBody;
}
const [origin] = event ? caseHandler(event.headers, 'origin') : [];
return {
statusCode: statusCode || 200,
body,
headers: Object.assign({
'Access-Control-Allow-Headers': 'Content-Type,Authorization,X-Amz-Date,X-Api-Key,X-Amz-Security-Token,x-api-key,Authorization',
'Access-Control-Allow-Origin': origin,
'Access-Control-Allow-Credentials': true,
'Access-Control-Allow-Methods': 'DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT',
}, headers),
};
}
and another one :
function caseHandler(mixedCaseObject, inputKey) {
if (!mixedCaseObject || !inputKey) {
return [];
}
return Object.keys(mixedCaseObject)
.filter((key => key.toLowerCase() === inputKey.toLowerCase()))
.map(key => mixedCaseObject[key]);
}
and this one :
function isApiGateway(event) {
return (!!event.httpMethod && !!event.headers && !!event.requestContext);
}
so inside the lambda whenever I want to return something I use this helper functions :
module.exports.handler = async (event) => {
try{
// do stuff...
return proxyResponse(YOUR_CUSTOM_BODY_OBJECT, Api.HTTP.OK, {}, event);
} catch(error) {
return proxyResponse(error, Api.HTTP.INTERNAL_SERVER_ERROR, {}, event);
}
}
The GET request will return any content provided in the response. In this case, it is the JSON object response. For POST requests the same response object is being returned back to the API Gateway which is then returning it back as-is to the client.
Perhaps the documentation here could be of help https://docs.aws.amazon.com/en_pv/apigateway/latest/developerguide/api-gateway-integration-settings-integration-response.html
So I have a React Native app, making calls to my own external api via the fetch() native API. I am encountering a strange issue with my requests not being processed by fetch() correctly in only one context.
The calls are supposed to POST to /api/token_login.
Here is how I make my calls in the app:
#/src/services/post.js
import headers from './headers';
const APIURL = '******/api';
export default async (path, data, callback) => {
try {
const postData = {
method: 'POST',
credentials: 'include',
headers: new Headers(headers),
body: JSON.stringify(data)
};
const requestURL = `${APIURL}/${path}`;
const request = new Request(requestURL, postData);
const response = await fetch(request);
const res = await response.text();
if (response.ok) {
callback(null, res);
} else {
const error = res;
callback(error, null);
}
} catch (error) {
console.log('caught error: ', error);
}
};
#/src/services/auth.js
import post from './post';
export function TokenLogin(data, callback) {
post('token_login', data, callback);
}
...other functions
#/src/containers/Login.js
const data = {
user: {
auth_token: token,
},
};
return TokenLogin(data, (err, response) => {
const res = JSON.parse(response);
... do stuff
};
This function gets called on componentDidMount under certain conditions, which is the expected behavior. However, every day or so it suddenly stops sending the request to the passed in path, and sends a GET to /api/login instead. Changing the path passed into the function changes the behavior back to the expected one (i.e., passing in flurgl sends a POST request to .../api/flurgl with the body passed in correctly. Here is some debugging info:
// the logs:
=> Request method: POST url: https://*****/api/token_auth
=> Response object:
=> Response {type: "default", status: 404, ok: false, statusText: undefined, headers: Headers, …}
bodyUsed: true
headers: Headers {map: {…}}
ok: false
status: 404
statusText: undefined
type: "default"
url: "https://*******/api/login"