Google Cloud Function frozen for over minute - javascript

have a strange thing happening running a Google cloud function. The function starts and logs the user id and job id as expected. Then it calls firestore db and basically sits there for 1 minute, sometimes 2 before it executes the first call... It was even timing out on 240 seconds.
const AWS = require('aws-sdk');
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
exports.run = functions.https.onCall((data, context) => {
var id = data.id;
var userid = data.uid;
var retry = data.retry;
var project;
var db = admin.firestore();
var storage = admin.storage();
console.log("Starting Collect");
console.log("UID: " + userid);
console.log("id ID: " + id);
// Times out on this call
db.collection("users").doc(userid).collection("ids").doc(id).get().then(function(doc) {
console.log("Loaded DB");
project = doc.data();
createexport();
}).catch(function(err) {
console.log(err);
error('Loading DB Error, ' + err, false);
});
function createexport() {
db.collection("exports").doc(id).set({
status: 'Collecting',
stage: 'Export Checker',
percent: 0,
id: id,
}).then(function() {
console.log("Creating Export");
setdb();
}).catch(function(err) {
error("Error creating export in database :" + err, true)
});
}
function setdb() {
db.collection("users").doc(userid).collection("ids").doc(id).update({
status: 'Analyzing Files',
stage: 'Collecting'
}).then(function() {
getaudio();
}).catch(function(err) {
error("Error updating users id in database :" + err, true)
});
}
function getaudio() {
const from = userid + '/projects/' + project.originalproject.id + '/audio.' + project.originalproject.extension;
const to = userid + '/' + id + '/audio.' + project.originalproject.extension;
storage.bucket('---------').file(from).copy(storage.bucket('---------').file(to)).then(function() {
console.log("Collecting files");
copy2();
}).catch(function(err) {
error('Collecting Audio Error, ' + err, true);
});
}
function copy2() {
const from = userid + '/projects/' + project.originalproject.id + '/overlay.png';
const to = userid + '/' + id + '/overlay.png';
storage.bucket('--------.appspot.com').file(from).copy(storage.bucket('---------').file(to)).then(function() {
updateexport();
}).catch(function(err) {
error('Collecting Overlay Error, ' + err, true);
});
}
function updateexport() {
db.collection("exports").doc(id).update({ status: "Waiting" }).then(function() {
console.log("All files collected");
return { status: 'Success' };
}).catch(function(err) {
error("Error creating export entry in database :" + err, true)
});
}
function error(evt, evt2) {
AWS.config.update({ region: "us-east-1" });
var html;
var sub = 'Error with id ' + id;
console.log(evt);
if (evt2) {
db.collection('users').doc(userid).collection('ids').doc(id).update({
status: 'Error'
}).catch(function(err) {
console.log(err);
});
db.collection("exports").doc(id).update({
status: 'Error',
stage: 'Collecting',
error: evt,
}).catch(function(err) {
console.log(err);
});
html = `
Username: ${project.username} <br>
UserID: ${userid} <br>
Email: ${project.email} <br>
id: ${id}
`
} else {
html = `id: ${id}<br>
UserID: ${userid} <br>
Message: Error logged was: ${evt}
`
}
var params = {
Destination: {
ToAddresses: [
'errors#mail.com'
]
},
Message: {
Body: {
Html: {
Charset: "UTF-8",
Data: html
},
},
Subject: {
Charset: 'UTF-8',
Data: sub
}
},
Source: 'errors#mail.com',
ReplyToAddresses: [
project.email
],
};
var sendPromise = new AWS.SES({
apiVersion: "2010-12-01",
"accessKeyId": "-----------",
"secretAccessKey": "------------------------",
"region": "--------",
}).sendEmail(params).promise();
sendPromise.then(function(data) {
return { data: data };
}).catch(function(err) {
return { err: err };
});
}
});
Seems to me to be way too long for a database call of only a few kb. I will attach the cloud log to show time difference. After this initial slump it then performs as expected.
Cloud log image
Anyone got any ideas as to why this could be happening? Many thanks...

Your function is appearing to hang because it isn't handling promises correctly. Also, it doesn't appear to be sending a specific response to the client app. The main point of callable functions is to send a response.
I suggest reviewing the documentation, where you will learn that callable functions are required to return a promise that resolves with an object to send to the client app, after all the async work is complete.
Minimally, it will take a form like this:
return db.collection("users").doc(userid).collection("files").doc(id).get().then(function(doc) {
console.log("Loaded DB");
project = doc.data();
return { "data": "to send to the client" };
}
Note that the promise chain is being returned, and the promise itself resolves to an object to send to the client.

Related

Missing required key 'Message- AWS SNS

I want to send SMS from a aws sns function were we provide the mobile number and text message and use SNS to deliver that message but I had an error
{"Error":{"message":"Missing required key 'Message' in
params","code":"MissingRequiredParameter","time":"2022-04-11T16:14:53.306Z"}}
Can any one suggest a solution?
router.post('/send-text', (req, res) => {
var dynamoDBConfiguration = {
"accessKeyId": "X...",
"secretAccessKey": "l...",
"region": "eu-west-3" };
AWS.config.update(dynamoDBConfiguration);
console.log("Message = " + req.body.message);
console.log("Number = " + req.body.number);
console.log("Subject = " + req.body.subject);
var params = {
Message: req.query.message,
PhoneNumber: '+' + req.query.number,
MessageAttributes: {
'AWS.SNS.SMS.SenderID': {
'DataType': 'String',
'StringValue': req.query.subject
}
}
};
var publishTextPromise = new AWS.SNS({ apiVersion: '2010-03-31' }).publish(params).promise();
publishTextPromise.then(
function (data) {
res.end(JSON.stringify({ MessageID: data.MessageId }));
}).catch(
function (err) {
res.end(JSON.stringify({ Error: err }));
});

Node JS Imap-simple disconnect error after some time

I fetch unread messages in a middle ware that triggers my function every five minutes. It is successfully completing the request but eventually It triggers and crashes my nodejs process. With an error of:
Error: This socket has been ended by the other party
I have tried adding all these catches and error handling but It seams it eventually times out my connection to my inbox? Am I not reconnecting on every request to access my inbox?
Here is my code (also I have tried with authTimeout: 3000 in the config before anyone suggests that is the issue):
imap: {
user: 'myemail#gmail.com',
password: 'myPW',
host: 'imap.gmail.com',
port: 993,
tls: true,
keepalive: true
}
imaps.connect(settings.emailConfig).then(function (connection) {
return connection.openBox('INBOX').then(function () {
var searchCriteria = ['UNSEEN'];//only grab unread messages in LTS email
var fetchOptions = {
bodies: ['HEADER', 'TEXT', ''],
markSeen: true//mark as read
};
return connection.search(searchCriteria, fetchOptions).then(function (messages) {
console.log('looking for unread messages')
if (messages.length > 0) {
console.log('unread messages to save')
console.log('looking at messages : ' + util.inspect(messages, { depth: null }))
var saveUnreadEmais = messages.map(function (item) {//resolve each promise of unread message
var all = _.find(item.parts, { "which": "" })
var id = item.attributes.uid;
var idHeader = "Imap-Id: " + id + "\r\n";
simpleParser(idHeader + all.body, (err, mail) => {
var BarCode = "";
var subject = mail.subject;
var to = mail.to.value[0].address;
var from = mail.from.value[0].address;
var text = mail.text;
if (mail.subject.includes('Barcode :')) {//if the message to the system includes BarCode in the subject line grab it
var split = mail.subject.split(':');
BarCode = split[1];
}
if (!subject) {
subject = "LTS Email Recieved - Barcode : " + BarCode;
}
if (!text) {
text = "LTS auto email"
}
var newrec = {};
newrec.MessageDate = new Date();
newrec.MessageFor = to;
newrec.MessageFrom = from;
newrec.Message = text;
newrec.MessageAbout = BarCode;
newrec.MessageSubject = subject;
newrec.MessageStatusCode = 1;//1 sent, 2 read, 3 saved, -1 deleted
//console.log('saving message with details of : ' + util.inspect(newrec, { depth: null }))
return db.knex('dbo.PersonnelMessages').insert(newrec).then((data) => {
return "sent";
});
});
});
return Promise.all(saveUnreadEmais).then(() => {
//do anything you want after saving all unred emails to our db.
}).catch((err) => { console.log("error : " + err) })
}
else {
console.log('no new emails')
}
}).catch((err) => { console.log("error : " + err) })
}).catch((err) => { console.log("opening inbox error : " + err) })
}).then(function () {
console.log('In the INBOX');
}).catch(function (e) {
console.log('error : ' + e);
});

aws upload object to S3 bucket and pass details of data to lambda

Working my way through tutorials for AWS...So ive created an S3 bucket which when a file is dropped into it calls my lambda 'testHelloWorld' which sends an email...this all works fine (see below)
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
exports.handler = function(event, context) {
console.log("Incoming: ", event);
// var output = querystring.parse(event);
var eParams = {
Destination: {
ToAddresses: ["johnb#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: "Hey! What is up?"
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnb#hotmail.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
but I want to extend the email to include data on the file that was uploaded to the bucket. I have found How to trigger my Lambda Function once the file is uploaded to s3 bucket which gives a node.js code snippet which should capture the data. I have tried to import this into my existing lambda
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });
exports.handler = function(event, context, exit){
console.log("Incoming: ", event);
// var output = querystring.parse(event);
// Get the object from the event and show its content type
// const bucket = event.Records[0].s3.bucket.name;
// const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: 'bucketName',
Key: 'keyName',
Source : 'SourceName',
Destination : 'DestinationName',
Message : 'MessageName'
};
s3.getObject(function(err, data){
if (err) {
console.log('ERROR ' + err);
// exit(err);
} else {
// the data has the content of the uploaded file
var eParams = {
Destination: {
ToAddresses: ["johnboy#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: data
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnboy#hotmail.com"
};
}
});
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
but this is failing on the params
message: 'There were 3 validation errors:
* MissingRequiredParameter: Missing required key \'Source\' in params
* MissingRequiredParameter: Missing required key \'Destination\' in params
* MissingRequiredParameter: Missing required key \'Message\' in params',
code: 'MultipleValidationErrors',
errors:
These source, destination and message are listed in the params, are they not correctly formatted and it isnt picking them up?
I cant find much online....any help appreciated
UPDATE
Ok iv got it working without failing...if i use the test function in the lambda with the following code...
'use strict';
console.log('Loading function');
var aws = require('aws-sdk');
var ses = new aws.SES({
region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });
exports.handler = function(event, context) {
console.log("Incoming: ", event);
// var output = querystring.parse(event);
var testData = null;
// Get the object from the event and show its content type
// const bucket = event.Records[0].s3.bucket.name;
// const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: 'bucket',
Key: 'key',
};
s3.getObject(params, function(err, data){
if (err) {
console.log('ERROR ' + err);
exit(err);
} else {
testData = data;
}
});
var eParams = {
Destination: {
ToAddresses: ["jim#him.com"]
},
Message: {
Body: {
Text: { Data: 'testData2' + testData}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "jim#him.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, data){
if(err) console.log(err);
else {
console.log("===EMAIL SENT===");
console.log(data);
console.log("EMAIL CODE END");
console.log('EMAIL: ', email);
context.succeed(event);
}
});
};
I get the email with the body- testData2null
So I tried uploading an image through the s3 bucket and I still get the email with the body testData2null
is there anyway to debug this further or does anyone kno who it is saying null. I never actually tested the code from the other post which passes the data over to the email I just assumed it would work. Does anyone else know who to obtain the data from the upload please? thanks
You are declaring the var eParams within the callback of s3.getObject, but then you run the ses.sendMail outside of the callback. I think that's why!
You also need to move the ses.sendEmail to inside the callback of s3.getObject if you want to send the data from your object inside the email.
Try this:
s3.getObject(function(err, objectData) {
if (err) {
console.log('Could not fetch object data: ', err);
} else {
console.log('Data was successfully fetched from object');
var eParams = {
Destination: {
ToAddresses: ["johnboy#hotmail.com"]
},
Message: {
Body: {
Text: {
Data: objectData
}
},
Subject: {
Data: "Email Subject!!!"
}
},
Source: "johnboy#hotmail.com"
};
console.log('===SENDING EMAIL===');
var email = ses.sendEmail(eParams, function(err, emailResult) {
if (err) console.log('Error while sending email', err);
else {
console.log("===EMAIL SENT===");
console.log(objectData);
console.log("EMAIL CODE END");
console.log('EMAIL: ', emailResult);
context.succeed(event);
}
});
}
});
You need to read on how Nodejs works. It is event based and depends on callbacks and promises. You should do -
s3.getObject(params, function(err, data){
//This is your callback for s3 API call. DO stuff here
if (err) {
console.log('ERROR ' + err);
exit(err);
} else {
testData = data;
// Got your data. Send the mail here
}
});
I have added my comments in code above. Since Nodejs is single threaded it will make S3 api call and go ahead. When it is sending mail s3 api call is not complete so data is null. It is better to use promises here.
Anyway read up on callback and promises in nodejs and how it works. But hope this answers your logical error.

Error promise after publish data to MQTT broker in My Alexa Lambda node js

I have problem with my Lambda, actually in promise nodejs. I have wrote code like this in my Lambda:
'use strict'
const Alexa = require('alexa-sdk');
const mqtt = require('mqtt');
const APP_ID = undefined;
const WELCOME_MESSAGE = 'Welcome to the lamp control mode';
const WELCOME_REPROMT = 'If you new please say help'
const HELP_MESSAGE = 'In this skill you can controlling lamp to turn off or on, dim the lamp, change the lamp color and schedule the lamp';
const STOP_MESSAGE = 'Thanks for using this skill, Goodbye!';
const OFF_RESPONSE = 'Turning off the lamp';
const ON_RESPONSE = 'Turning on the lamp';
const DIM_RESPONSE = 'Dimming the lamp';
const CHANGE_RESPONSE = 'Changing the lamp color';
const AFTER_RESPONSE = 'Wanna control something again ?';
const handlers = {
'LaunchRequest': function () {
this.emit(':ask', WELCOME_MESSAGE, WELCOME_REPROMT);
},
'OnOffIntent' : function () {
var status = this.event.request.intent.slots.status.value;
var location = this.event.request.intent.slots.location.value;
console.log(status);
console.log(location);
if (status == 'on') {
// Promise Start
var mqttPromise = new Promise(function(resolve, reject) {
var options = {
port: '1883',
clientId: 'mqttjs_' + Math.random().toString(16).substr(2, 8),
username: 'username',
password: 'password',
};
var client = mqtt.connect('mqtt://broker-address', options)
client.on('connect', function() {
client.publish("lamp/status", status + ' ' + location, function() {
console.log("Message is published");
client.end();
resolve('Done Sending');
});
});
});
mqttPromise.then(
function(data) {
console.log('Function called succesfully', data);
this.emit(':ask', ON_RESPONSE, AFTER_RESPONSE);
}, function(err) {
console.log('An error occurred: ', err);
}
);
// Promise END
// this.emit(':ask', ON_RESPONSE, AFTER_RESPONSE);
// client.publish("lamp/status", status + ' ' + location);
} else if (status == 'off') {
this.emit(':ask', OFF_RESPONSE, AFTER_RESPONSE);
// client.publish("lamp/status", status + ' ' + location);
}
},
'DimIntent' : function () {
// to do here
},
'ChangeColorIntent' : function () {
// to do here
},
'ShceduleIntent' : function () {
// to do here
},
'AMAZON.HelpIntent': function () {
this.emit(':ask', HELP_MESSAGE, 'Wanna control something ?');
},
'AMAZON.StopIntent': function () {
this.emit(':tell', STOP_MESSAGE);
}
};
exports.handler = function (event, context, callback) {
const alexa = Alexa.handler(event, context, callback);
alexa.APP_ID = APP_ID;
alexa.registerHandlers(handlers);
alexa.execute();
}
I test my code with Service Simulator in Alexa Developer and get this result :
Result Image
So I checked output in Lambda and I got this error report :
Error in Lamda
Can anyone please help me? I have no idea with this because this is my first trial :)
The crux of your error seems to be this specific line in the log:
Cannot read property 'emit' of undefined
And after following the flow of your program, it's likely ocurring here:
mqttPromise.then(
function(data) {
console.log('Function called succesfully', data);
// It's probably ocurring in this line below
this.emit(':ask', ON_RESPONSE, AFTER_RESPONSE);
}, function(err) {
console.log('An error occurred: ', err);
}
)
The log is saying that you tried using this, it's undefined and doesn't have an emit property. Thats ocurring because of how this works in Js. You could workaround this problem by saving a reference to this
var that = this;
var mqttPromise = new Promise(function(resolve, reject) {
var options = {
port: '1883',
clientId: 'mqttjs_' + Math.random().toString(16).substr(2, 8),
username: 'username',
password: 'password',
};
var client = mqtt.connect('mqtt://broker-address', options)
client.on('connect', function() {
client.publish("lamp/status", status + ' ' + location, function() {
console.log("Message is published");
client.end();
resolve('Done Sending');
});
});
});
mqttPromise.then(
function(data) {
console.log('Function called succesfully', data);
that.emit(':ask', ON_RESPONSE, AFTER_RESPONSE);
}, function(err) {
console.log('An error occurred: ', err);
}
);
I would also recommend reading up a bit on "How 'this' works in Javascript"
MDN
Stack Overflow - "how does 'this' work"

return resolve error in node function

Why wont usernametoid function return the acual id? cause im trying to send the result of the userdata as the return. In this case, i want to only send the userdata`s _id attribute. but it seems like it wont work.
console.log(userdata._id); // works
return resolve(userdata._id); // wont work.
output of variable userdata:
{
cash: 7300002,
bank: 0,
xp: 0,
rank: 1,
points: 1,
location: 1,
health: 100,
protection: 1,
attack: 1,
family: '0',
password: 'jKa4qC7pRCgE5jvzD9Vv1pRUNxFlQEM7Jpq/IoJ/sUWOAv1Wx1RI/j/Vu6Zf8zyNkCFcg3QBtdfAC+lmPS8KIA==',
profileImageURL: 'modules/users/client/img/profile/default.png',
roles: [ 'user' ],
created: Sat Aug 27 2016 12:33:55 GMT-0400 (EDT),
__v: 0,
username: 'signature',
provider: 'local',
salt: '4ySlrr9ggESxBB3dR5bx4Q==',
_id: 57c1c0f3b6b20c011242bf22 }
when i do: `return resolve(userdata._id) it would get this error:
/server/factory/user_factory.js:52
return resolve(userdata._id);
^
TypeError: Cannot read property '_id' of null
node.js call:
var articles = require('../controllers/articles.server.controller'),
path = require('path'),
mongoose = require('mongoose'),
Article = mongoose.model('Article'),
Users = mongoose.model('User'),
errorHandler = require(path.resolve('./modules/core/server/controllers/errors.server.controller'));
var userFunc = require('../factory/user_factory.js');
app.post('/api/kill', function (req, res) {
console.log("starting");
var username = "signature";//req.query.username;
var result = ["test service"];
var data = req.user;
userFunc.usernametoid(username).then( function (otherplayerid) {
if (!(otherplayerid)) {
console.log("other player is acually " + otherplayerid);
result.push("denne brukeren finnes ikke! " + otherplayerid);
} else {
userFunc.usernametoid(otherplayerid).then( function (otherplayer) {
if (data.location != otherplayer.location) {
result.push("Du er ikke i samme lokasjon som " + username);
result.push(data.location + " vs " + otherplayer.location);
} else {
userFunc.addCash(req.user._id,100000);
result.push("starter lokasjonisering");
}
});
}
res.json(result);
});
});
user factory:
var articles = require('../controllers/articles.server.controller'),
path = require('path'),
mongoose = require('mongoose'),
Article = mongoose.model('Article'),
Users = mongoose.model('User'),
errorHandler = require(path.resolve('./modules/core/server/controllers/errors.server.controller'));
exports.usernametoid = usernametoid;
function usernametoid(id) {
return new Promise( function (resolve, reject) {
var query = Users.findOne( { username : id } );
// var query = Users.find({_id:id});
query.exec(function(err, userdata) {
if (err){
return reject({err : 'Error while getting user info'});
}
console.log(userdata._id);
return resolve(userdata);
});
}, function (){
return reject({err : 'error while fetching cash'});
});
}
Because you are not passing correctly the fetched user to the query.exec.
You need to do:
var Users = require('../models/users-model.js');
function usernametoid(id) {
return new Promise( function (resolve, reject) {
Users.findOne({ username : id }).then( function(user){
//If you use lodash you can do _.isNull(user)
if(user == null){
return reject({error : 'User not found'});
}
user.exec(function(userdata, error) {
if(userdata){
return resolve(userdata);
}
if(error){
return reject({error : 'Error while executing query'});
}
});
});
});
}
I don't really get why you are importing Users Model like that. I do not think Node will be able to fetch it like that.
And, you should require mongoose in your server.js
To catch the rejection you need the following code:
UserFactory.userNameToId(id).then( function(response){
if(response.error){
console.log('error '+response.error);
}
if(response){
console.log('Got response '+response);
}
});

Categories

Resources