I am looking to send CSV records coming from kinesis analytics to sagemaker endpoint and getting an inference through a lambda function and then passing it on to a firehose API to dump it into S3. But the data is not getting into sagemaker for some reason.
'use strict';
console.log('Loading function');
var AWS = require('aws-sdk');
var sagemakerruntime = new AWS.SageMakerRuntime({apiVersion: '2017-05-13'});
var firehose = new AWS.Firehose({apiVersion: '2015-08-04'});
exports.handler = (event, context, callback) => {
let success = 0;
let failure = 0;
const output = event.records.map((record) => {
/* Data is base64 encoded, so decode here */
const recordData = Buffer.from(record.data, 'base64');
try {
var params = {
Body: new Buffer('...') || recordData /* Strings will be Base-64 encoded on your behalf */, /* required */
EndpointName: 'String', /* required */
Accept: 'text/csv',
ContentType: 'text/csv'
};
sagemakerruntime.invokeEndpoint(params, function(err, data) {
var result1;
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
result1=data;
var params = {
DeliveryStreamName: 'String', /* required */
Record: { /* required */
Data: new Buffer('...') || result1 /* Strings will be Base-64 encoded on your behalf */ /* required */
}
};
firehose.putRecord(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
});
success++;
return {
recordId: record.recordId,
result: 'Ok',
};
} catch (err) {
failure++;
return {
recordId: record.recordId,
result: 'DeliveryFailed',
};
}
});
console.log(`Successful delivered records ${success}, Failed delivered records ${failure}.`);
callback(null, {
records: output,
});
};
Related
I am getting an error while running amazon ses code in next js. don't know where am wrong, please try to fix my error. If you have any question please free feel to ask.
sendmail.js
This is the sendmail.js file where i got error. here I am using amazon ses for sending mail.
var AWS = require('aws-sdk');
AWS.config.update({ region: process.env.AWS_REGION });
// var mail = '';
function sendMail(Email) {
var result;
// Create sendEmail params
var params = {
Destination: { /* required */
CcAddresses: [
Email,
/* more items */
],
ToAddresses: [
Email,
/* more items */
]
},
Message: { /* required */
Body: { /* required */
Html: {
Charset: "UTF-8",
Data: "HTML_FORMAT_BODY"
},
Text: {
Charset: "UTF-8",
Data: "TEXT_FORMAT_BODY"
}
},
Subject: {
Charset: 'UTF-8',
Data: 'Test email'
}
},
Source: 'abc#gmail.com', /* required */
ReplyToAddresses: [
'abc12#gmail.com',
/* more items */
],
};
// Create the promise and SES service object
var sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendEmail(params).promise();
// Handle promise's fulfilled/rejected states
sendPromise.then(
function (data) {
result = 'Success';
}).catch(
function (err) {
result = 'Failed';
});
}
export default sendMail;
dynamicid.js
This is the dynamic id .js file where i wrote my endpoint code
import { getDataFromSheets } from '../../../libs/sheets';
import sendmail from '../../../libs/ses/sendmail';
export default function handler(req, res) {
var data;
getDataFromSheets()
.then(sheet => {
data = sheet.length
for (var i = 1; i < data; i++) {
sendmail(sheet[i].Email)
}
})
.catch(err => console.log(err))
}
Each endpoint function must end the request-response cycle by sending a response ( res.send(), res.json(), res.end(), etc). So the solution would be:
export default function handler(req, res) {
var data;
getDataFromSheets()
.then(sheet => {
data = sheet.length
for (var i = 1; i < data; i++) {
sendmail(sheet[i].Email)
}
res.json({status: 'success', message: 'email has been sent'})
})
.catch(err => {
console.log(err)
res.json({status: 'fail', error: err})
})
}
Below is the piece of code i have written , to get the result but null in response
I am using selectObjectContent api to get the results with the simple SQL query
const bucket = 'myBucketname'
const key = 'file.json.gz'
const query = "SELECT * FROM s3object "
const params = {
Bucket: bucket,
Key: key,
ExpressionType: "SQL",
Expression: query,
InputSerialization: {
CompressionType: "GZIP",
JSON: {
Type: "LINES"
},
},
OutputSerialization: {
JSON: {
RecordDelimiter: ","
}
}
}
s3.selectObjectContent(params,(err, data) => {
if (err) {
console.log(data)
} else {
console.log(err)
}
})
I have found the solution to it. was logging error when getting successfull result/data , so corrected it below. Also i have found the way to read stream buffer data
s3.selectObjectContent(params,(err, data) => {
if (err) {
console.log(err)
} else {
console.log(data)
}
})
const eventStream = data.Payload;
// Read events as they are available
eventStream.on('data', (event) => {
if (event.Records) {
// event.Records.Payload is a buffer containing
// a single record, partial records, or multiple records
var records = event.Records.Payload.toString();
console.log( records )
} else if (event.Stats) {
console.log(`Processed ${event.Stats.Details.BytesProcessed} bytes`);
} else if (event.End) {
console.log('SelectObjectContent completed');
}
I am trying to execute AWS Endpoint using nodejs (aws-sdk). First, I am able to generate session token for Service Account which has access to execute the API.
var AWS = require('aws-sdk');
AWS.config.update({ "accessKeyId": "<>", "secretAccessKey": "<>", "region": "us-west" });
var sts = new AWS.STS();
var response = {};
sts.assumeRole({
RoleArn: 'arn:aws:iam::170000000000:role/service-account',
RoleSessionName: 'AssumtaseRole'
}, function(err, data) {
if (err) { // an error occurred
var error = {}
response.message = err.originalError.message,
response.errno = err.originalError.errno,
response.code = 404;
console.log(response);
} else { // successful response
response.code = 200,
response.accesskey = data.Credentials.AccessKeyId,
response.secretkey = data.Credentials.SecretAccessKey,
response.sessiontoken = data.Credentials.SessionToken,
console.log(response);
}
});
Now I am trying to execute the endpoint using the above session token. If test session token using postman, I am able to execute the API but not sure how to do it using (aws-sdk) or ('aws-api-gateway-client')
I tried to execute using simple HTPPS request but getting error: Here is the code:
var AWS = require('aws-sdk');
var apigClientFactory = require('aws-api-gateway-client').default;
AWS.config.update({ "accessKeyId": "<>", "secretAccessKey": "<>", "region": "us-west" });
var sts = new AWS.STS();
var response = {};
sts.assumeRole({
RoleArn: 'arn:aws:iam::170000000000:role/service_account',
RoleSessionName: 'AssumtaseRole'
}, function(err, data) {
if (err) { // an error occurred
var error = {}
response.message = err.originalError.message,
response.errno = err.originalError.errno,
response.code = 404;
console.log(response);
} else { // successful response
response.code = 200,
response.accesskey = data.Credentials.AccessKeyId,
response.secretkey = data.Credentials.SecretAccessKey,
response.sessiontoken = data.Credentials.SessionToken,
console.log(response);
var apigClient = apigClientFactory.newClient({
invokeUrl: "https://some-endpoint.com", // REQUIRED
accessKey: data.Credentials.AccessKeyId, // REQUIRED
secretKey: data.Credentials.SecretAccessKey, // REQUIRED
sessiontoken: data.Credentials.SessionToken,
region: "us-west", // REQUIRED: The region where the AapiKeyloyed.
retries: 4,
retryCondition: (err) => { // OPTIONAL: Callback to further control if request should be retried. Uses axon-retry plugin.
return err.response && err.response.status === 500;
}
});
var pathParams = "";
var pathTemplate = "/agent/registration"; // '/api/v1/sites'
var method = "post"; // 'POST';
var additionalParams = ""; //queryParams & Headers if any
var body = {
"agent_number": "1200",
"agent_name": "Test"
};
apigClient.invokeApi(pathParams, pathTemplate, method, additionalParams, body)
.then(function(result) {
console.log(result)
}).catch(function(error) {
console.log(error)
});
// console.log(output);
}
});
Here is the error:
data:
{ message: 'The security token included in the request is invalid.' } } }
Thanks in advance.
Thank You Kiran
Please change sessiontoken to sessionToken. that will fix your issue. I have tested the code on my machine.
When i tested with sessiontoken i also received the error The security token included in the request is invalid.. It worked when i changed it to the correct key which is sessionToken.
here is simplified code. When i tested, I have hard coded accessKey, secretKey and sessionToken.
var apigClientFactory = require('aws-api-gateway-client').default;
var apigClient = apigClientFactory.newClient({
invokeUrl:'https://api-url.com', // REQUIRED
accessKey: '', // REQUIRED
secretKey: '', // REQUIRED
sessionToken: '', //OPTIONAL: If you are using temporary credentials you must include the session token
region: 'ap-southeast-2', // REQUIRED: The region where the API is deployed.
systemClockOffset: 0, // OPTIONAL: An offset value in milliseconds to apply to signing time
retries: 4, // OPTIONAL: Number of times to retry before failing. Uses axon-retry plugin.
retryCondition: (err) => { // OPTIONAL: Callback to further control if request should be retried. Uses axon-retry plugin.
return err.response && err.response.status === 500;
}
});
(() => {
apigClient.invokeApi(null, `/hello`, 'GET')
.then(function(result){
console.log('result: ', result)
//This is where you would put a success callback
}).catch( function(result){
console.log('result: ', result)
//This is where you would put an error callback
});
})()
Working my way through tutorials for AWS...So ive created an S3 bucket which when a file is dropped into it calls my lambda 'testHelloWorld' which sends an email...this all works fine (see below)
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
exports.handler = function(event, context) {
console.log("Incoming: ", event);
// var output = querystring.parse(event);
var eParams = {
Destination: {
ToAddresses: ["johnb#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: "Hey! What is up?"
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnb#hotmail.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
but I want to extend the email to include data on the file that was uploaded to the bucket. I have found How to trigger my Lambda Function once the file is uploaded to s3 bucket which gives a node.js code snippet which should capture the data. I have tried to import this into my existing lambda
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });
exports.handler = function(event, context, exit){
console.log("Incoming: ", event);
// var output = querystring.parse(event);
// Get the object from the event and show its content type
// const bucket = event.Records[0].s3.bucket.name;
// const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: 'bucketName',
Key: 'keyName',
Source : 'SourceName',
Destination : 'DestinationName',
Message : 'MessageName'
};
s3.getObject(function(err, data){
if (err) {
console.log('ERROR ' + err);
// exit(err);
} else {
// the data has the content of the uploaded file
var eParams = {
Destination: {
ToAddresses: ["johnboy#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: data
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnboy#hotmail.com"
};
}
});
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
but this is failing on the params
message: 'There were 3 validation errors:
* MissingRequiredParameter: Missing required key \'Source\' in params
* MissingRequiredParameter: Missing required key \'Destination\' in params
* MissingRequiredParameter: Missing required key \'Message\' in params',
code: 'MultipleValidationErrors',
errors:
These source, destination and message are listed in the params, are they not correctly formatted and it isnt picking them up?
I cant find much online....any help appreciated
UPDATE
Ok iv got it working without failing...if i use the test function in the lambda with the following code...
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });
exports.handler = function(event, context) {
console.log("Incoming: ", event);
// var output = querystring.parse(event);
var testData = null;
// Get the object from the event and show its content type
// const bucket = event.Records[0].s3.bucket.name;
// const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: 'bucket',
Key: 'key',
};
s3.getObject(params, function(err, data){
if (err) {
console.log('ERROR ' + err);
exit(err);
} else {
testData = data;
}
});
var eParams = {
Destination: {
ToAddresses: ["jim#him.com"]
},
Message: {
Body: {
Text: { Data: 'testData2' + testData}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "jim#him.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
I get the email with the body- testData2null
So I tried uploading an image through the s3 bucket and I still get the email with the body testData2null
is there anyway to debug this further or does anyone kno who it is saying null. I never actually tested the code from the other post which passes the data over to the email I just assumed it would work. Does anyone else know who to obtain the data from the upload please? thanks
You are declaring the var eParams within the callback of s3.getObject, but then you run the ses.sendMail outside of the callback. I think that's why!
You also need to move the ses.sendEmail to inside the callback of s3.getObject if you want to send the data from your object inside the email.
Try this:
s3.getObject(function(err, objectData) {
if (err) {
console.log('Could not fetch object data: ', err);
} else {
console.log('Data was successfully fetched from object');
var eParams = {
Destination: {
ToAddresses: ["johnboy#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: objectData
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnboy#hotmail.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, emailResult) {
if (err) console.log('Error while sending email', err);
else {
console.log("===EMAIL SENT===");
console.log(objectData);
console.log("EMAIL CODE END");
console.log('EMAIL: ', emailResult);
context.succeed(event);
}
});
}
});
You need to read on how Nodejs works. It is event based and depends on callbacks and promises. You should do -
s3.getObject(params, function(err, data){
//This is your callback for s3 API call. DO stuff here
if (err) {
console.log('ERROR ' + err);
exit(err);
} else {
testData = data;
// Got your data. Send the mail here
}
});
I have added my comments in code above. Since Nodejs is single threaded it will make S3 api call and go ahead. When it is sending mail s3 api call is not complete so data is null. It is better to use promises here.
Anyway read up on callback and promises in nodejs and how it works. But hope this answers your logical error.
I have some codes from node.js project, it contains a post /uploadAvatar router, and a uploadAvatar function:
uploadAvatar: function(req, res) {
req.file('avatar').upload({
// don't allow the total upload size to exceed ~10MB
maxBytes: 10000000
},function whenDone(err, uploadedFiles) {
if (err) {
console.log(err);
return res.negotiate(err);
}
// If no files were uploaded, respond with an error.
if (uploadedFiles.length === 0){
return res.badRequest('No file was uploaded');
}
var avatarFd = uploadedFiles[0].fd;
var json = {fd: avatarFd};
console.log(json);
res.json(200, json);
});
}
Right now I want to using this api to upload image using AFNetworking. Current the swift code like below:
func uploadAvatar(avatar: NSData, success: JSONBlock, fail: ErrorBlock) -> Void {
sessionManager.POST("/uploadAvatar", parameters: nil, constructingBodyWithBlock: { (formData) in
formData.appendPartWithFormData(avatar, name: "avatar")
}, progress: { (_) in
}, success: { (_, responseObject) in
let json = self.jsonFromResponseObject(responseObject)
if json != nil {
success(json)
}
}) { (_, error) in
if let msgData = error.userInfo[AFNetworkingOperationFailingURLResponseDataErrorKey] as? NSData {
if let message = String(data: msgData, encoding: NSUTF8StringEncoding) {
print(message)
}
}
fail(error.localizedDescription);
}
}
The error response I get is: "No file was uploaded". It seems the node.js can't find my image data. What did I do wrong?