Develop Reference

Why can my code run in a standard Node.js file, but not in a AWS Lambda Function?

What I'm trying to do is create a lambda function where the function calls two commands on an ec2 instance. When I had trouble running this code in a lambda function, I removed the code from the exports.handler() method and ran the code in a standalone node.js file in the same ec2 instance and I was able to get the code to work. The command I ran was 'node app.js'.
exports.handler = async (event) => {
const AWS = require('aws-sdk')
AWS.config.update({region:'us-east-1'});
var ssm = new AWS.SSM();
var params = {
DocumentName: 'AWS-RunShellScript', /* required */
InstanceIds: ['i-xxxxxxxxxxxxxxxx'],
Parameters: {
'commands': [
'mkdir /home/ec2-user/testDirectory',
'php /home/ec2-user/helloWorld.php'
/* more items */
],
/* '<ParameterName>': ... */
}
};
ssm.sendCommand(params, function(err, data) {
if (err) {
console.log("ERROR!");
console.log(err, err.stack); // an error occurred
}
else {
console.log("SUCCESS!");
console.log(data);
} // successful response
});
const response = {
statusCode: 200,
ssm: ssm
};
return response;
};
I figured that it could have been a permissions related issue, but the lambda is apart of the same vpc that the ec2 instance is in.

You're trying to combine async/await with callbacks. That won't work in a lambda AWS Lambda Function Handler in Node.js. The reason it's working locally, or in a node server, is because the server is still running when the function exits, so the callback still happens. In a Lambda the node process is gone as soon as the lambda exits if you are using async (or Promises), so the callback is not able to be fired.

Solution based on Jason's Answer:
const AWS = require('aws-sdk');
const ssm = new AWS.SSM();
exports.handler = async (event,context) => {
AWS.config.update({region:'us-east-1'});
const params = {
DocumentName: 'AWS-RunShellScript', /* required */
InstanceIds: ['i-xxxxxxxxxxxxxx'],
Parameters: {
'commands': [
'mkdir /home/ec2-user/testDirectory',
'php /home/ec2-user/helloWorld.php'
/* more items */
],
/* '<ParameterName>': ... */
}
};
const ssmPromise = new Promise ((resolve, reject) => {
ssm.sendCommand(params, function(err, data) {
if (err) {
console.log("ERROR!");
console.log(err, err.stack); // an error occurred
context.fail(err);
}
else {
console.log("SUCCESS!");
console.log(data);
context.succeed("Process Complete!");
} // successful response
});
});
console.log(ssmPromise);
const response = {
statusCode: 200,
ssm: ssm
};
return response;
};

How to read json file from storage blob container with azure function using javascript?

I'm totally new in azure and I would like to create azure function, which will read the content from azure storage container file.json.
Folder structure :
Storage account name: storageaccounttest
Container name: test
File name: file.json
File.json:
[
{
"name":"Kate",
"age":"28"
},
{
"name":"John",
"age":"30"
}
]
Cors on storage account: get enabled.
Environemnts variable added: process.env.AZURE_STORAGE_NAME and process.env.AZURE_STORAGE_KEY and process.env.AZURE_CONNECTION_STRING
I'm using VisualStudioCode to deploy the function.
I installed locally the dependencies:
"dependencies": {
"azure-storage": "^2.10.3",
"dotenv": "^8.1.0"
}
I choose the javascript -> HttpTrigger fn-> anonymus options
I'm using getBlobToText fn.
My index.js:
var storage = require('azure-storage');
var blobService = storage.createBlobService();
var containerName = 'test';
var blobName = 'file.json';
module.exports = blobService.getBlobToText(
containerName,
blobName,
function(err, blobContent) {
if (err) {
console.error("Couldn't download blob");
console.error(err);
} else {
console.log("Sucessfully downloaded blob");
console.log(blobContent);
}
});
Fn is deployed successfully, but I'm not able to see results.
After start, fn is finished with status 500, Internal Server Errror, Console: No new trace in the past 1 min(s).
What I made wrong?

Just summarized for helping others who get the same issue.
I think you were using context.binding.response to pass the blobContent value to the output response as the offical document Azure Functions JavaScript developer guide said.
Here is my sample code with Promise feature to solve it.
var azure = require('azure-storage');
var blobService = azure.createBlobService();
var containerName = 'test';
var blobName = 'file.json';
async function getBlobContent(containerName, blobName) {
return new Promise((resolve, reject) => {
blobService.getBlobToText(containerName, blobName, function(err, blobContent) {
if (err) {
reject(err);
} else {
resolve(blobContent);
}
});
});
}
module.exports = async function (context, req) {
await getBlobContent(containerName, blobName).then(
function(content) {
context.res = {
headers: {"Content-Type": "application/json"},
body: content
}
}, function(error) {
context.res = {
status: 400,
body: error
}
}
);
};
It works as the figure below.

FTP in AWS Lambda - Issues Downloading Files (Async/Await)

I have been struggling with various FTP Node modules to try and get anything working in AWS Lambda. The best and most popular seems to be "Basic-FTP" that also supports async/await. But I just cannot get it to download files when any code is added beneath the FTP function.
I don't want to add the fs functions within the FTP async function as I need to solve what is causing the break when any code below is added and I also have other bits of code to add and work with the downloaded file and it's content later:
FTP SUCCESS - When the async function is used only with no fs code beneath it
FTP FAILURE - Adding the fs readdir/readFile functions or any other code below
ERROR Error: ENOENT: no such file or directory, open '/tmp/document.txt'
https://github.com/patrickjuchli/basic-ftp
const ftp = require("basic-ftp");
const fs = require("fs");
var FileNameWithExtension = "document.txt";
var ftpTXT;
exports.handler = async (event, context, callback) => {
example();
async function example() {
const client = new ftp.Client();
//client.ftp.verbose = true;
try {
await client.access({
host: host,
user: user,
password: password,
//secure: true
});
console.log(await client.list());
await client.download(fs.createWriteStream('/tmp/' + FileNameWithExtension), FileNameWithExtension);
}
catch (err) {
console.log(err);
}
client.close();
}
// Read the content from the /tmp/ directory to check FTP was succesful
fs.readdir("/tmp/", function (err, data) {
if (err) {
return console.error("There was an error listing the /tmp/ contents.");
}
console.log('Contents of AWS Lambda /tmp/ directory: ', data);
});
// Read TXT file and convert into string format
fs.readFile('/tmp/' + FileNameWithExtension, 'utf8', function (err, data) {
if (err) throw err;
ftpTXT = data;
console.log(ftpTXT);
});
// Do other Node.js coding with the downloaded txt file and it's contents
};

The problem is that you are getting lost when creating an async function inside your handler. Since example() is async, it returns a Promise. But you don't await on it, so the way it has been coded, it's kind of a fire and forget thing. Also, your Lambda is being terminated before your callbacks are triggered, so even if it got to download you would not be able to see it.
I suggest you wrap your callbacks in Promises so you can easily await on them from your handler function.
I have managed to make it work: I have used https://dlptest.com/ftp-test/ for testing, so change it accordingly. Furthermore, see that I have uploaded the file myself. So if you want to replicate this example, just create a readme.txt on the root of your project and upload it. If you already have this readme.txt file on your FTP server, just delete the line where it uploads the file.
Here's a working example:
const ftp = require("basic-ftp");
const fs = require("fs");
const FileNameWithExtension = "readme.txt";
module.exports.hello = async (event) => {
const client = new ftp.Client();
try {
await client.access({
host: 'ftp.dlptest.com',
user: 'dlpuser#dlptest.com',
password: 'puTeT3Yei1IJ4UYT7q0r'
});
console.log(await client.list());
await client.upload(fs.createReadStream(FileNameWithExtension), FileNameWithExtension)
await client.download(fs.createWriteStream('/tmp/' + FileNameWithExtension), FileNameWithExtension);
}
catch (err) {
console.log('logging err')
console.log(err);
}
client.close();
console.log(await readdir('/tmp/'))
console.log(await readfile('/tmp/', FileNameWithExtension))
return {
statusCode: 200,
body: JSON.stringify({message: 'File downloaded successfully'})
}
};
const readdir = dir => {
return new Promise((res, rej) => {
fs.readdir(dir, function (err, data) {
if (err) {
return rej(err);
}
return res(data)
});
})
}
const readfile = (dir, filename) => {
return new Promise((res, rej) => {
fs.readFile(dir + filename, 'utf8', function (err, data) {
if (err) {
return rej(err);
}
return res(data)
})
})
}
Here is the output of the Lambda function:
And here are the complete CloudWatch logs:
My file contains nothing but a 'hello' inside it. You can see it on the logs.
Do keep in mind that, in Lambda Functions, you have a 512MB limit when downloading anything to /tmp. You can see the limits in the docs

Download file via FTP, write to /tmp/ and output .txt contents to the console with AWS Lambda

I am using just a single Node package, basic-ftp to try and download a TXT file and write the contents to the console. Further down the line I will be editing the text so will need to use fs. Just struggling to work with the output from createWriteStream from within the FTP program.
Can anyone help me write a TXT file to the /tmp/ file within AWS Lambda and then the correct syntax to open and edit the file after createWriteStream has been used?
var fs = require('fs');
const ftp = require("basic-ftp")
var path = require('path');
exports.handler = (event, context, callback) => {
var fullPath = "/home/example/public_html/_uploads/15_1_5c653e6f6780f.txt"; // File Name FULL PATH -------
const extension = path.extname(fullPath); // Used to calculate filenames below
const wooFileName = path.basename(fullPath, extension); // Uploaded filename with no path or extension eg. filename
const myFileNameWithExtension = path.basename(fullPath); // Uploaded filename with the file extension eg. filename.txt
const FileNameWithExtension = path.basename(fullPath); // Uploaded filename with the file extension eg. filename.txt
example()
async function example() {
const client = new ftp.Client()
client.ftp.verbose = true
try {
await client.access({
host: "XXXX",
user: "XXXX",
password: "XXXX",
//secure: true
})
await client.download(fs.createWriteStream('./tmp/' + myFileNameWithExtension), myFileNameWithExtension)
}
catch(err) {
console.log(err)
}
client.close()
}
//Read the content from the /tmp directory to check it's empty
fs.readdir("/tmp/", function (err, data) {
console.log(data);
console.log('Contents of AWS Lambda /tmp/ directory');
});
/*
downloadedFile = fs.readFile('./tmp/' + myFileNameWithExtension)
console.log(downloadedFile)
console.log("Raw text:\n" + downloadedFile.Body.toString('ascii'));
*/
}

Pretty sure your fs.createWriteStream() has to use an absolute path to /tmp in Lambdas. Your actual working directory is var/task not /.
Also, if you're using fs.createWriteStream() you'll need to wait for the finish event before reading from the file. Somethign like this...
async function example() {
var finalData = '';
const client = new ftp.Client()
client.ftp.verbose = true
try {
await client.access({
host: "XXXX",
user: "XXXX",
password: "XXXX",
//secure: true
})
let writeStream = fs.createWriteStream('/tmp/' + myFileNameWithExtension);
await client.download(writeStream, myFileNameWithExtension)
await finalData = (()=>{
return new Promise((resolve, reject)=> {
writeStream
.on('finish', ()=>{
fs.readFile("/tmp/"+myFileNameWithExtension, function (err, data) {
if (err) {
reject(err)
} else {
console.log('Contents of AWS Lambda /tmp/ directory', data);
resolve(data);
}
});
})
.on('error', (err)=> {
console.log(err);
reject(err);
})
})
})();
}
catch(err) {
console.log(err)
}
client.close();
return finalData;
}
You'll also need to access the file using fs.readFile(). What you were using fs.readdir() gives you a list of files in the directory, not the file's contents.
If you want to used readdir() you could do it like this, but as you can see it is redundant in your case. To handle errors I would suggest just handling the error event in the initial createWriteStream() instead of adding this extra overhead (added to previous example)...
writeStream
.on('finish', ()=>{
fs.readdir('/tmp',(err, files)=> {
let saved = files.find(file => file === myFileNameWithExtension);
fs.readFile("/tmp/"+saved, function (err, data) {
if (err) throw new Error();
console.log(data);
console.log('Contents of AWS Lambda /tmp/ directory');
});
})
})
.on('error', (err)=> {
console.log(err);
throw new Error();
})
NOTE: Please log out the result of saved, I can't remember if the files array is absolute of relative paths.

Trying to upload to S3 bucket. After 5-10 uploads it hangs

I've created a promise that uses the aws-sdk to upload to my S3 bucket. My application is a simple command line script to add images to S3 and update the database. I have anywhere between 300-1000 images to upload every run.
The problem I am having is it uploads 5-10 images but then seems to hang. I've confirmed this by placing a console.log(data) after the error checking in the promise below.
The first 5 images upload quickly, the sixth takes about a minute, the seventh a lot longer and at which point it just hangs.
s3-upload-promise.js
'use strict'
let AWS = require('aws-sdk')
let s3 = new AWS.S3()
module.exports = function(params) {
return new Promise(function(resolve, reject) {
s3.upload(params, function(err, data) {
if(err) return reject(Error(err))
resolve(data)
})
})
}
And here is the code that calls the promise:
let s3UploadPromise = require('../src/s3-upload-promise/s3-upload-promise')
// Get all PNG files from given path and upload to bucket
listpng(process.argv[2]).then(function(files) {
let promises = []
files.forEach(function(el, i) {
promises.push(el, s3UploadPromise({
Bucket: process.env.S3_BUCKET,
Key: 'templates/' + randomstring.generate() + '.png',
Body: fs.createReadStream(process.argv[2] + el),
ACL: 'public-read'
}))
});
Promise.all(promises).then(function(values) {
return console.log(values)
})
})
Any ideas what I'm doing wrong? Does this have anything to do with me not closing createReadStream?
Edit
I've tried closing the stream within the the s3.upload callback. It didn't make any difference and it still hung:
s3.upload(params, function(err, data) {
params.Body.close()
if(err) return reject(Error(err))
resolve(data)
})
Edit 2
I added some error checking and I am getting the following error:
{
message: 'Your socket connection to the server was not read from or written to within the timeout period. Idle connections w
ill be closed.',
code: 'RequestTimeout',
region: null,
time: 2016-06-19T13:14:39.223Z,
requestId: 'F7E64E8F99E774F3',
extendedRequestId: 'PW/mPy6t3w9U1uJc8xYKhUGi/KiSY+6yK6nq0RB21Ke1KqRmTWjjm3KXEp0qAEPDadypw+kiwCEP3upER1uecEP4Sl9Tk/lt',
cfId: undefined,
statusCode: 400,
retryable: true
}
A comment on an issue on Github mentions:
I've been noticing this a lot when using concurrency > 1, on several systems. Most of the time an uploaded folder will begin to get 400's due to timeouts after the first 10 or so requests.
Maybe adding a Content-Length somewhere will help. Will try.
Edit 3
I decided to try a different library called knox. I get the same problem! This is crazy. It must surely an Amazon issue if two different libraries are facing the same problem?
s3-upload-promise.js
'use strict'
let knox = require('knox')
let process = require('process')
var client = knox.createClient({
key: process.env.AWS_KEY,
secret: process.env.AWS_SECRET,
bucket: process.env.S3_BUCKET,
});
module.exports = function(params) {
return new Promise(function(resolve, reject) {
let headers = {
'Content-Length': params.contentLength,
'Content-Type': params.contentType,
'x-amz-acl': params.permissions
}
client.putBuffer(params.buffer, params.key, headers, function(err, res){
if(err) return reject(err)
resolve(res)
});
})
}
Calling code ...
// Get all PNG files from given path
listpng(process.argv[2]).then(function(files) {
let promises = []
files.forEach(function(el, i) {
let file = process.argv[2] + el;
fs.readFile(file, function(err, data) {
if(err) return console.log(err)
fs.stat(process.argv[2] + el, function(err, stats) {
if(err) return console.log(err)
let key = process.env.S3_TEMPLATES + '/'
let buffer = fs.createReadStream(process.argv[2] + el)
let params = {
buffer: data,
key: key + randomstring.generate() + '.png',
contentLength: stats.size,
contentType: 'image/png',
permissions: 'public-read',
}
promises.push(s3Uploader(params))
Promise.all(promises).then(function(values) {
return console.log(values)
})
})
})
})
})
Not sure what else to do now.