Develop Reference

Google Cloud Function frozen for over minute

have a strange thing happening running a Google cloud function. The function starts and logs the user id and job id as expected. Then it calls firestore db and basically sits there for 1 minute, sometimes 2 before it executes the first call... It was even timing out on 240 seconds.
const AWS = require('aws-sdk');
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
exports.run = functions.https.onCall((data, context) => {
var id = data.id;
var userid = data.uid;
var retry = data.retry;
var project;
var db = admin.firestore();
var storage = admin.storage();
console.log("Starting Collect");
console.log("UID: " + userid);
console.log("id ID: " + id);
// Times out on this call
db.collection("users").doc(userid).collection("ids").doc(id).get().then(function(doc) {
console.log("Loaded DB");
project = doc.data();
createexport();
}).catch(function(err) {
console.log(err);
error('Loading DB Error, ' + err, false);
});
function createexport() {
db.collection("exports").doc(id).set({
status: 'Collecting',
stage: 'Export Checker',
percent: 0,
id: id,
}).then(function() {
console.log("Creating Export");
setdb();
}).catch(function(err) {
error("Error creating export in database :" + err, true)
});
}
function setdb() {
db.collection("users").doc(userid).collection("ids").doc(id).update({
status: 'Analyzing Files',
stage: 'Collecting'
}).then(function() {
getaudio();
}).catch(function(err) {
error("Error updating users id in database :" + err, true)
});
}
function getaudio() {
const from = userid + '/projects/' + project.originalproject.id + '/audio.' + project.originalproject.extension;
const to = userid + '/' + id + '/audio.' + project.originalproject.extension;
storage.bucket('---------').file(from).copy(storage.bucket('---------').file(to)).then(function() {
console.log("Collecting files");
copy2();
}).catch(function(err) {
error('Collecting Audio Error, ' + err, true);
});
}
function copy2() {
const from = userid + '/projects/' + project.originalproject.id + '/overlay.png';
const to = userid + '/' + id + '/overlay.png';
storage.bucket('--------.appspot.com').file(from).copy(storage.bucket('---------').file(to)).then(function() {
updateexport();
}).catch(function(err) {
error('Collecting Overlay Error, ' + err, true);
});
}
function updateexport() {
db.collection("exports").doc(id).update({ status: "Waiting" }).then(function() {
console.log("All files collected");
return { status: 'Success' };
}).catch(function(err) {
error("Error creating export entry in database :" + err, true)
});
}
function error(evt, evt2) {
AWS.config.update({ region: "us-east-1" });
var html;
var sub = 'Error with id ' + id;
console.log(evt);
if (evt2) {
db.collection('users').doc(userid).collection('ids').doc(id).update({
status: 'Error'
}).catch(function(err) {
console.log(err);
});
db.collection("exports").doc(id).update({
status: 'Error',
stage: 'Collecting',
error: evt,
}).catch(function(err) {
console.log(err);
});
html = `
Username: ${project.username} <br>
UserID: ${userid} <br>
Email: ${project.email} <br>
id: ${id}
`
} else {
html = `id: ${id}<br>
UserID: ${userid} <br>
Message: Error logged was: ${evt}
`
}
var params = {
Destination: {
ToAddresses: [
'errors#mail.com'
]
},
Message: {
Body: {
Html: {
Charset: "UTF-8",
Data: html
},
},
Subject: {
Charset: 'UTF-8',
Data: sub
}
},
Source: 'errors#mail.com',
ReplyToAddresses: [
project.email
],
};
var sendPromise = new AWS.SES({
apiVersion: "2010-12-01",
"accessKeyId": "-----------",
"secretAccessKey": "------------------------",
"region": "--------",
}).sendEmail(params).promise();
sendPromise.then(function(data) {
return { data: data };
}).catch(function(err) {
return { err: err };
});
}
});
Seems to me to be way too long for a database call of only a few kb. I will attach the cloud log to show time difference. After this initial slump it then performs as expected.
Cloud log image
Anyone got any ideas as to why this could be happening? Many thanks...

Your function is appearing to hang because it isn't handling promises correctly. Also, it doesn't appear to be sending a specific response to the client app. The main point of callable functions is to send a response.
I suggest reviewing the documentation, where you will learn that callable functions are required to return a promise that resolves with an object to send to the client app, after all the async work is complete.
Minimally, it will take a form like this:
return db.collection("users").doc(userid).collection("files").doc(id).get().then(function(doc) {
console.log("Loaded DB");
project = doc.data();
return { "data": "to send to the client" };
}
Note that the promise chain is being returned, and the promise itself resolves to an object to send to the client.

Node JS Imap-simple disconnect error after some time

I fetch unread messages in a middle ware that triggers my function every five minutes. It is successfully completing the request but eventually It triggers and crashes my nodejs process. With an error of:
Error: This socket has been ended by the other party
I have tried adding all these catches and error handling but It seams it eventually times out my connection to my inbox? Am I not reconnecting on every request to access my inbox?
Here is my code (also I have tried with authTimeout: 3000 in the config before anyone suggests that is the issue):
imap: {
user: 'myemail#gmail.com',
password: 'myPW',
host: 'imap.gmail.com',
port: 993,
tls: true,
keepalive: true
}
imaps.connect(settings.emailConfig).then(function (connection) {
return connection.openBox('INBOX').then(function () {
var searchCriteria = ['UNSEEN'];//only grab unread messages in LTS email
var fetchOptions = {
bodies: ['HEADER', 'TEXT', ''],
markSeen: true//mark as read
};
return connection.search(searchCriteria, fetchOptions).then(function (messages) {
console.log('looking for unread messages')
if (messages.length > 0) {
console.log('unread messages to save')
console.log('looking at messages : ' + util.inspect(messages, { depth: null }))
var saveUnreadEmais = messages.map(function (item) {//resolve each promise of unread message
var all = _.find(item.parts, { "which": "" })
var id = item.attributes.uid;
var idHeader = "Imap-Id: " + id + "\r\n";
simpleParser(idHeader + all.body, (err, mail) => {
var BarCode = "";
var subject = mail.subject;
var to = mail.to.value[0].address;
var from = mail.from.value[0].address;
var text = mail.text;
if (mail.subject.includes('Barcode :')) {//if the message to the system includes BarCode in the subject line grab it
var split = mail.subject.split(':');
BarCode = split[1];
}
if (!subject) {
subject = "LTS Email Recieved - Barcode : " + BarCode;
}
if (!text) {
text = "LTS auto email"
}
var newrec = {};
newrec.MessageDate = new Date();
newrec.MessageFor = to;
newrec.MessageFrom = from;
newrec.Message = text;
newrec.MessageAbout = BarCode;
newrec.MessageSubject = subject;
newrec.MessageStatusCode = 1;//1 sent, 2 read, 3 saved, -1 deleted
//console.log('saving message with details of : ' + util.inspect(newrec, { depth: null }))
return db.knex('dbo.PersonnelMessages').insert(newrec).then((data) => {
return "sent";
});
});
});
return Promise.all(saveUnreadEmais).then(() => {
//do anything you want after saving all unred emails to our db.
}).catch((err) => { console.log("error : " + err) })
}
else {
console.log('no new emails')
}
}).catch((err) => { console.log("error : " + err) })
}).catch((err) => { console.log("opening inbox error : " + err) })
}).then(function () {
console.log('In the INBOX');
}).catch(function (e) {
console.log('error : ' + e);
});

How to get all the emails sent to and received from an email Id?

I'm trying to implement node-imap and I need to get all the emails received from and sent to an email id.
I've been able to use https://developers.google.com/oauthplayground to generate a refresh token but it expires in an hour so I need a method to generate it directly from the code and In the search request response I'm not getting any message subject or anything.
let Imap = require("imap"),
inspect = require("util").inspect,
config = require('lt-config'),
xoauth2 = require("xoauth2"),
xoauth2gen,imap;
xoauth2gen = xoauth2.createXOAuth2Generator({
user: "mymailid",
clientId: config.MAIL_GOOGLE_CLIENT_ID,
clientSecret: config.MAIL_GOOGLE_CLIENT_SECRET,
refreshToken: config.MAIL_REFRESH_TOKEN,
accessToken: config.MAIL_ACCESS_TOKEN
});
function getAuthorizedImap() {
return new Promise((resolve, reject) => {
xoauth2gen.getToken(function(err, token) {
if (err) {
return console.log(err);
}
console.log("AUTH XOAUTH2 " + token);
config.xoauth2 = token;
imap = new Imap({
user: "test#example.com",
xoauth2: token,
host: "imap.gmail.com",
port: 993,
tls: true
});
imap.once("ready", function() {
resolve(imap);
});
imap.once("error", function(err) {
console.log(err);
});
imap.once("end", function() {
console.log("Connection ended");
});
imap.connect();
});
setTimeout(() => reject('Timeout get token process'), 6000);
})
}
function filterMails(emailId) {
return new Promise(async resolve => {
try {
imap = await getAuthorizedImap();
openInbox(function(err, box) {
if (err) throw err;
imap.search([ ['HEADER', 'FROM', emailId] ], function(err, results) {
if (err) throw err;
console.log("results", results);
var f = imap.fetch(results, { bodies: "HEADER.FIELDS (FROM TO SUBJECT DATE)" });
f.on("message", function(msg, seqno) {
console.log("Message #%d", seqno);
var prefix = "(#" + seqno + ") ";
msg.on("body", function(stream, info) {
console.log(prefix + "Body");
console.log("Body received", info);
});
msg.once("attributes", function(attrs) {
console.log(
prefix + "Attributes: %s",
inspect(attrs, false, 8)
);
});
msg.once("end", function() {
console.log(prefix + "Finished");
});
});
f.once("error", function(err) {
console.log("Fetch error: " + err);
});
f.once("end", function() {
console.log("Done fetching all messages!");
imap.end();
});
});
});
} catch(e) {
console.log("Errror while getting emails from GMAIL ====>>", e);
}
});
}
function openInbox(cb) {
imap.openBox("INBOX", true, cb);
}
module.exports = filterMails;
I want a JSON response from search and to generate a refresh token dynamically. Any help will be appericiated.

aws upload object to S3 bucket and pass details of data to lambda

Working my way through tutorials for AWS...So ive created an S3 bucket which when a file is dropped into it calls my lambda 'testHelloWorld' which sends an email...this all works fine (see below)
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
exports.handler = function(event, context) {
console.log("Incoming: ", event);
// var output = querystring.parse(event);
var eParams = {
Destination: {
ToAddresses: ["johnb#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: "Hey! What is up?"
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnb#hotmail.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
but I want to extend the email to include data on the file that was uploaded to the bucket. I have found How to trigger my Lambda Function once the file is uploaded to s3 bucket which gives a node.js code snippet which should capture the data. I have tried to import this into my existing lambda
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });
exports.handler = function(event, context, exit){
console.log("Incoming: ", event);
// var output = querystring.parse(event);
// Get the object from the event and show its content type
// const bucket = event.Records[0].s3.bucket.name;
// const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: 'bucketName',
Key: 'keyName',
Source : 'SourceName',
Destination : 'DestinationName',
Message : 'MessageName'
};
s3.getObject(function(err, data){
if (err) {
console.log('ERROR ' + err);
// exit(err);
} else {
// the data has the content of the uploaded file
var eParams = {
Destination: {
ToAddresses: ["johnboy#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: data
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnboy#hotmail.com"
};
}
});
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
but this is failing on the params
message: 'There were 3 validation errors:
* MissingRequiredParameter: Missing required key \'Source\' in params
* MissingRequiredParameter: Missing required key \'Destination\' in params
* MissingRequiredParameter: Missing required key \'Message\' in params',
code: 'MultipleValidationErrors',
errors:
These source, destination and message are listed in the params, are they not correctly formatted and it isnt picking them up?
I cant find much online....any help appreciated
UPDATE
Ok iv got it working without failing...if i use the test function in the lambda with the following code...
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });
exports.handler = function(event, context) {
console.log("Incoming: ", event);
// var output = querystring.parse(event);
var testData = null;
// Get the object from the event and show its content type
// const bucket = event.Records[0].s3.bucket.name;
// const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: 'bucket',
Key: 'key',
};
s3.getObject(params, function(err, data){
if (err) {
console.log('ERROR ' + err);
exit(err);
} else {
testData = data;
}
});
var eParams = {
Destination: {
ToAddresses: ["jim#him.com"]
},
Message: {
Body: {
Text: { Data: 'testData2' + testData}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "jim#him.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
I get the email with the body- testData2null
So I tried uploading an image through the s3 bucket and I still get the email with the body testData2null
is there anyway to debug this further or does anyone kno who it is saying null. I never actually tested the code from the other post which passes the data over to the email I just assumed it would work. Does anyone else know who to obtain the data from the upload please? thanks

You are declaring the var eParams within the callback of s3.getObject, but then you run the ses.sendMail outside of the callback. I think that's why!
You also need to move the ses.sendEmail to inside the callback of s3.getObject if you want to send the data from your object inside the email.
Try this:
s3.getObject(function(err, objectData) {
if (err) {
console.log('Could not fetch object data: ', err);
} else {
console.log('Data was successfully fetched from object');
var eParams = {
Destination: {
ToAddresses: ["johnboy#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: objectData
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnboy#hotmail.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, emailResult) {
if (err) console.log('Error while sending email', err);
else {
console.log("===EMAIL SENT===");
console.log(objectData);
console.log("EMAIL CODE END");
console.log('EMAIL: ', emailResult);
context.succeed(event);
}
});
}
});

You need to read on how Nodejs works. It is event based and depends on callbacks and promises. You should do -
s3.getObject(params, function(err, data){
//This is your callback for s3 API call. DO stuff here
if (err) {
console.log('ERROR ' + err);
exit(err);
} else {
testData = data;
// Got your data. Send the mail here
}
});
I have added my comments in code above. Since Nodejs is single threaded it will make S3 api call and go ahead. When it is sending mail s3 api call is not complete so data is null. It is better to use promises here.
Anyway read up on callback and promises in nodejs and how it works. But hope this answers your logical error.

Express promise throw error

I am using Express with denodeify module. I`m getting error when i try to render email template asynchronously.
.then(function () {
return denodeify(res.render)(path.resolve('verify-email'), {
name: user.displayName,
appName: config.app.title,
mail: user.email,
url: 'http://' + req.headers.host
});
})
.then(function (emailHTML) {
// code which is not executed
})
.catch(function (err) {
// [TypeError: Cannot read property 'req' of undefined]
});
I tryed use res.render without callback and it worked fine, but i need to output email template in variable.
Also i cant understand how i can debug this part of code. Thanks for help!
Whole code
function sendVerificationEmail(user, req, res) {
return denodeify(crypto.randomBytes)(20)
.then(function (buffer) {
user.verificationToken = buffer.toString('hex');
return user.save();
})
.then(function () {
user.password = undefined;
user.salt = undefined;
delete req.session.redirect_to;
return denodeify(res.render)(path.resolve('modules/users/server/templates/verify-email'), {
name: user.displayName,
appName: config.app.title,
mail: user.email,
url: 'http://' + req.headers.host + '/api/auth/verify/' + user.verificationToken + "/" + user.email
});
})
.then(function (emailHTML) {
const mailOptions = {
to: user.email,
from: config.mailer.from,
subject: 'Verify Email',
html: emailHTML
};
return denodeify(smtpTransport.sendMail)(mailOptions);
});
}
exports.updateAuthData = function (req, res, next) {
const userId = req.session.userId;
delete req.session.userId;
if (!userId) {
return res.status(401).send({
message: 'No active session'
});
}
let user;
User.findById(userId)
.then()
.then((_user) => {
user = _user;
const fields = Object.keys(req.body);
const neededFields = user.getMissingAuthFields();
const missingFields = _.difference(fields, neededFields);
if (missingFields.length !== 0) {
throw new CustomError('Missing fields: ' + missingFields.join(', '))
}
for (let field of User.requiredAuthorizationFields) {
if (!!user[field]) {
throw new CustomError('User already has ' + field);
}
}
User.requiredAuthorizationFields.forEach((field) => user[field] = req.body[field]);
return true;
})
.then(function () {
return sendVerificationEmail(user, req, res);
})
.then(() => {
res.send({
message: 'An email has been sent to the provided email, please verify your account.'
});
})
.catch(function (err) {
res.status(400).send({
message: 'Failure sending email'
});
})
};

I found the solution, we have to bind "this" variable like this
return denodeify(res.render.bind(res))(path.resolve('modules/users/server/templates/verify-email'), {
name: user.displayName,
appName: config.app.title,
mail: user.email,
url: 'http://' + req.headers.host + '/api/auth/verify/' + user.verificationToken + "/" + user.email
});
The reason of error was the following code in express "res.render" function
var app = this.req.app;
var done = callback;
var opts = options || {};
var req = this.req;
var self = this;
the right way is to use:
denodeify(res.render.bind(res))(path, options)
instead of
denodeify(res.render)(path, options)
Thanks to all for replies!