Develop Reference

AWS Lambda function processes same dynamodb stream multiple times. What am I missing?

I have written a node.js lambda function that triggers based on a dynamodb stream when new records are inserted into a particular table.
The function receives only new events, filters for inserted records, and then for each record, uses a couple of fields to retrieve data from other tables. Using this combined data a message is composed and sent via SNS to specific target ARN.
The function performs correctly. All the relevant data is retrieved, and a push notification is sent out.
However, for some reason the function appears to be called several times for the same stream, and processes the newly inserted records several times. The result is the target device receiving the same push notification several times.
Should I be placing the callback in a different place, or am I not calling on the context correctly?
This is the function:
'use strict';
var AWS = require("aws-sdk");
var dynamodb = new AWS.DynamoDB();
var sns = new AWS.SNS();
console.log('Loading function');
exports.handler = (event, context, callback) => {
console.log('Received event:', JSON.stringify(event, null, 2));
event.Records.forEach((record) => {
console.log(record.eventID);
console.log(record.eventName);
console.log('DynamoDB Record: %j', record.dynamodb);
if (record.eventName == 'INSERT') {
var matchId = record.dynamodb.NewImage.eventId.S;
var match_params = {
Key: {
"eventId": {
S: matchId
}
},
TableName: "xxxxxxxxxxx-mobilehub-xxxxxxx-Event"
};
//retrieve the match information from Event table
dynamodb.getItem(match_params, function(err, data) {
var match_description = "";
if (err) {
console.log(err, err.stack);
context.fail('No match event record found in Event table');
} else {
match_description = data.Item.description.S;
var uId = record.dynamodb.NewImage.participantUserId.S; //participantUserId
var user_params = {
Key: {
"userId": {
S: uId
}
},
TableName: "xxxxxxxxxxx-mobilehub-xxxxxxxxx-User"
};
//retrieve the user record from User table
dynamodb.getItem(user_params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
context.fail('Error occurred. See log.');
} else {
console.log(data); // successful response
if (data.length === 0) {
console.log("No User Record Found.");
context.fail('No user found for participantUserId.');
} else {
var deviceARN = data.Item.device_arn.S;
if (deviceARN <= 1) {
console.log("User has not registered their device for push notifications.");
context.fail('User has not registered for notifications');
} else {
var json_message = JSON.stringify({
APNS_SANDBOX: JSON.stringify({
aps: {
alert: "You are playing in an upcoming match " + match_description,
badge: 1,
sound: 'default'
}
})
});
var snsparams = {
Message: json_message,
MessageStructure: 'json',
TargetArn: deviceARN
};
sns.publish(snsparams, function(err, data) {
if (err) {
console.log(err); // an error occurred
context.fail('SNS send failed. See log.');
} else {
console.log(data); // successful response
context.success('Push notification sent to user.');
}
});
}
}
}
});
}
});
}
});
callback(null, `Successfully processed ${event.Records.length} records.`);
};

In my case, I added the same event source multiple times.
Quote from the conversation with an AWS support engineer:
Using my internal tools, I noticed that the Lambda function xxxxxx has
the event source:
arn:aws:events:my_region:my_acct_id:rule/my_event_target
configured twice as push event source. This means that this might be the cause
why you are seeing two invokes at every minute. Would you please
confirm on your side if this event is configured twice for the $LATEST
version of your lambda and also confirm if it's intended?
I hope this could save someelse :)

In your lambda page at the bottom, try tweaking "Concurrency" Unreserved account concurrency to 1 and "Asynchronous invocation" Retry attempts to 0 . As a test try these and observe the behaviour. Might help.

node.js: Return from function not acting as expected

I'm very new to javascript/node.js and I'm having trouble with the following code. This is the handler for API an call. The 2nd code segment is just like the 1st, except there is an additional database lookup Merchant.findOne(...), and therefor the 'newTransaction.save()' function is nested one level deeper.
Both code segments return the 'output' variable value correctly. However, the second code segment does NOT also properly save the 'newTransaction' to the Mongo database.
I'm pretty sure the issue has to do with how/when the code returning from newTransaction.save(function (err, transaction){..} but I can't seem to get it straightened out.
I have been looking all over the internet trying to understand and fix this, with no success. Any help is appreciated...
Here is the older, simpler code that works as expected:
handler : function(request, reply) {
var output = {
"success": true,
"operations": [],
"epoch": Date.now()
};
Terminal.findById(request.payload.deviceNumber, function (err, terminal) {
if (err) {
return reply(Boom.internal('Error looking up terminal.', err));
}
if (terminal) {
ticket.quote("bitstamp", "USD", 1, function (err, exchangeRate) {
if (err) {
console.error(err);
return reply(Boom.internal('Error obtaining ticket quote.', err));
}
var newTransaction = new Transaction({
terminal: request.payload.deviceNumber,
merchant: terminal.merchant,
ccExchangeRate: exchangeRate.buy,
fiatAmtDue: request.payload.transactionValue,
ccAmtDue: ccAmtDueTruncated
});
newTransaction.save(function (err, transaction){
if (err) {
return reply(Boom.internal('Error creating new transaction.', err));
}
output.operations.push(
{
"control": "KeyPairGenControl",
"rand": cc.pseudoRandomBytes(32).toString('hex'),
"follow": {
"url": "/pos/v1/AddressAndEncKey",
"post": {
"transactionId": transaction.transactionId
}
}
}
);
return reply(output);
});
});
} else {
return reply(Boom.internal('Error looking up terminal.', err));
}
});
}
Here is the new code that does NOT save the newTransaction data into the Mongo DB.
handler : function(request, reply) {
var output = {
"success": true,
"operations": [],
"epoch": Date.now()
};
Terminal.findById(request.payload.deviceNumber, function (err, terminal) {
if (err) {
return reply(Boom.internal('Error looking up terminal.', err));
}
if (terminal) {
Merchant.findOne({merchantId: terminal.merchant}, function(err, merchant) {
if (err) {
console.log('Cannot find merchant');
return reply(output);
}
var processor = merchant.backendPaymentProcessor.name;
var localCurrency = merchant.localFiatCurrency;
//###################
ticket.quote(processor, localCurrency, 1, function (err, exchangeRate) {
if (err) {
console.error(err);
return reply(Boom.internal('Error obtaining ticket quote.', err));
}
var newTransaction = new Transaction({
terminal: request.payload.deviceNumber,
merchant: terminal.merchant,
ccExchangeRate: exchangeRate.buy,
fiatAmtDue: request.payload.transactionValue,
ccAmtDue: ccAmtDueTruncated
});
newTransaction.save(function (err, transaction){
if (err) {
return reply(Boom.internal('Error creating new transaction.', err));
}
output.operations.push(
{
"control": "KeyPairGenControl",
"rand": cc.pseudoRandomBytes(32).toString('hex'),
"follow": {
"url": "/pos/v1/AddressAndEncKey",
"post": {
"transactionId": transaction.transactionId
}
}
}
);
return reply(output);
});
//return reply(output);
});
//###################
});
} else {
return reply(Boom.internal('Error looking up terminal.', err));
}
});
}

I did a diff of your 2 version:
Check 1
ticket.quote
Callback are identical for both version
processor, localCurrency are different
Is exchangeRate pass into callback correct?
Check 2
newTransaction.save
newTransaction and callback for .save are setup identical
Check(console.log()) the values used in setting up new Transaction({...})
Check transaction object received by callback
Check/debug the code of Transaction.save().
I don't think the issue is with the code you posted. Both version reached return reply(output); inside newTransaction.save's callback. Very likely issue is inside Transaction class or Transaction.save() logic.
One scenario I can think of is when a transaction failed:
Transaction object is available (even for failed transaction)
Transaction Class / Transaction.save() does not write to db because transaction failed
Transaction.save() pass transaction object to callback, but NOT setting err, even when it should.

Mongoose having a feature to specify the collection name under the schema, or as the third argument when declaring the model. Otherwise it will use the pluralized version given by the name you map to the model.
Mongoose official doc having following statement:
Mongoose by default produces a collection name by passing the model name to the utils.toCollectionName method. This method pluralizes the name. Set this option if you need a different name for your collection.
schema-mapped:
new Schema({ <key>: <value>},
{ collection : '<collection name>' }); // collection name
model-mapped:
mongoose.model('<Model name>',
new Schema({ <key>: <value>}),
'<collection name>'); // collection name
You may also find same here

nodejs and MongoDB's collection.find() does not respond

I have about 30,000 documents in a MongoDB collection. And have been stuck in developing a node.js script to retrieve only the records with a specific string key-value pair.
this query on MongoDB server returns me the exact results I've been looking for:
db.getCollection('posts').find({authorName: "Ashwin-kumar"})
Returns me about 33 documents instantly. Likewise I've about 40 authors with different names.
Here's my node.js script to retrieve posts by authorName (Yes, it is based on Name, a string, as there is no ID for these authors :( ):
var fs = require('fs'),
request = require('request'),
async = require("async"),
assert = require('assert');
_ = require('lodash'),
MongoClient = require('mongodb').MongoClient;
var db, postsCollection, postCol;
async.series([dbConnect, checkCollection, createMeta, dbClose], function(){
console.log("Executed all calls in series.");
process.exit(0);
});
function dbConnect(callback){
MongoClient.connect("mongodb://localhost:27017/jPosts", function(pErr, pDb) {
if(pErr) {
console.dir(pDb);
return 0;
}
db = pDb;
callback();
});
}
function dbClose(callback){
db.close(true, function (err) {
if (err) console.error(err);
else console.log("close complete");
callback();
});
}
function checkCollection(callback) {
db.collection('posts', function(err, collection) {});
postsCollection = db.collection('posts');
postCol = db.collection('posts');
callback();
}
function createMeta(callback){
var meta = [];
postsCollection.aggregate([
{
$group : {_id : "$authorName"}
}]).toArray(function(err, result) {
assert.equal(err, null);
async.forEachLimit(result, 1, function(pPost, callback) {
getPosts(pPost._id, callback);
}, function(err) {
console.log(err);
callback();
});
});
}
function getPosts(pAuthor, callback){
var cursor = postCol.find({ "authorName": pAuthor});
cursor.toArray(function(err,items){
if(err)
callback(err);
else
callback(null, items);
});
}
This does not seem to work for me. cursor.toArray() does nothing but wait forever. Is it because of too many fields in each document?
I tried to get the count of the documents the cursor fetched and it works well.
function getPosts(pAuthor, callback){
var cursor = postCol.find({ "authourName": pAuthor});
cursor.count().then(function(items_count) {
console.log(items_count);
callback();
});
}
Also, I tried the cursor's .each method to iterate the documents fetched. But no luck yet.
function getPosts(pAuthor, callback){
var cursor = postCol.find({ "authourName": pAuthor});
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
console.dir(doc);
} else {
console.log(err);
}
});
}
Am I missing something here? What else can be done to make this work? Is there any issues with the way I'm using async?
P.S: The idea here is to query the dump and generate the PDF's for authours in the jPost collection.
P.S 2: Here's a sample document
{
"_id" : ObjectId("571d36b55672f713fe346a66"),
"id" : 56517,
"authorName" : "Ashwin-kumar",
"comment_count" : 380,
"tagline" : "... Opinions you don't really need",
"vote_count" : 5152,
"exclusive" : null,
"post": [
],
"post_comments" : [
//comment_count objects
],
"date" : "2016-03-27"
}
(I've omitted post & post_comments parts for brevity.)

try this:
var collection = db.collection("collection_name");
collection.find({authourName: "Ashwin-kumar"}).toArray(function (err,items) {
if (err) {
console.dir(err);
} else {
//do something with items array
console.dir(items);
}
});

Did you check what is the value of pAuthor in getPosts? Because when you do aggregation, you receive a collection of objects with _id field (not authourName), so you should do:
// not sure why you need meta array, at least it's not used in the code you provided
meta.push({
author: pPost._id
});
getPosts(pPost._id, callback);

Mongoose - Create document if not exists, otherwise, update- return document in either case

I'm looking for a way to refactor part of my code to be shorter and simpler, but I don't know Mongoose very well and I'm not sure how to proceed.
I am trying to check a collection for the existence of a document and, if it doesn't exist, create it. If it does exist, I need to update it. In either case I need to access the document's contents afterward.
What I've managed to do so far is query the collection for a specific document and, if it's not found, create a new document. If it is found, I update it (currently using dates as dummy data for this). From there I can access either the found document from my initial find operation or the newly saved document and this works, but there must be a better way to accomplish what I'm after.
Here's my working code, sans distracting extras.
var query = Model.find({
/* query */
}).lean().limit(1);
// Find the document
query.exec(function(error, result) {
if (error) { throw error; }
// If the document doesn't exist
if (!result.length) {
// Create a new one
var model = new Model(); //use the defaults in the schema
model.save(function(error) {
if (error) { throw error; }
// do something with the document here
});
}
// If the document does exist
else {
// Update it
var query = { /* query */ },
update = {},
options = {};
Model.update(query, update, options, function(error) {
if (error) { throw error; }
// do the same something with the document here
// in this case, using result[0] from the topmost query
});
}
});
I've looked into findOneAndUpdate and other related methods but I'm not sure if they fit my use case or if I understand how to use them correctly. Can anyone point me in the right direction?
(Probably) Related questions:
How to check if that data already exist in the database during update (Mongoose And Express)
Mongoose.js: how to implement create or update?
NodeJS + Mongo: Insert if not exists, otherwise - update
Return updated collection with Mongoose
Edit
I didn't come across the question pointed out to me in my searching, but after reviewing the answers there I've come up with this. It's certainly prettier, in my opinion, and it works, so unless I'm doing something horribly wrong I think my question can probably be closed.
I would appreciate any additional input on my solution.
// Setup stuff
var query = { /* query */ },
update = { expire: new Date() },
options = { upsert: true };
// Find the document
Model.findOneAndUpdate(query, update, options, function(error, result) {
if (!error) {
// If the document doesn't exist
if (!result) {
// Create it
result = new Model();
}
// Save the document
result.save(function(error) {
if (!error) {
// Do something with the document
} else {
throw error;
}
});
}
});

You are looking for the new option parameter. The new option returns the newly created document(if a new document is created). Use it like this:
var query = {},
update = { expire: new Date() },
options = { upsert: true, new: true, setDefaultsOnInsert: true };
// Find the document
Model.findOneAndUpdate(query, update, options, function(error, result) {
if (error) return;
// do something with the document
});
Since upsert creates a document if not finds a document, you don't need to create another one manually.

Since you wish to refactor parts of your code to be shorter and simpler,
Use async / await
Use .findOneAndUpdate() as suggested in this answer
let query = { /* query */ };
let update = {expire: new Date()};
let options = {upsert: true, new: true, setDefaultsOnInsert: true};
let model = await Model.findOneAndUpdate(query, update, options);

///This is simple example explaining findByIDAndUpdate from my code added with try catch block to catch errors
try{
const options = {
upsert: true,
new: true,
setDefaultsOnInsert: true
};
const query = {
$set: {
description: req.body.description,
title: req.body.title
}
};
const survey = await Survey.findByIdAndUpdate(
req.params.id,
query,
options
).populate("questions");
}catch(e){
console.log(e)
}

Here is an example I am using. I have to return custom responses for UI updates etc. This can be even shorter. User is
const UserScheme = mongoose.Schema({
_id: String,
name: String,
city: String,
address: String,
},{timestamps: true});
const User = mongoose.model('Users', UserScheme);
async function userUpdateAdd(data){
var resp = '{"status": "error"}';
if(data){
var resp = await User.updateOne({ _id: data._id }, data).then(function(err, res){
console.log("database.userUpdateAdd -> Update data saved in database!");
if(err){
var errMessage = err.matchedCount == 0 ? "User Record does not exist, will create new..." : "Record not updated";
// If no match, create new
if(err.matchedCount == 0){
const create_user = new User(data);
resp = create_user.save().then(function(){
console.log("database.userUpdateAdd -> Data saved to database!");
return '{"status":"success", "message": "New User added successfully"}';
});
return resp;
}
// Exists, return success update message
if(err.matchedCount == 1){
return '{"status": "success", "message" : "Update saved successfully"}';
} else {
return '{"status": "error", "code": "' + err.modifiedCount + '", "message": "' + errMessage + '"}';
}
}
})
.catch((error) => {
//When there are errors We handle them here
console.log("database.userUpdateAdd -> Error, data not saved! Server error");
return '{"status": "error", "code": "400", "message": "Server error!"}';
});
}
return resp;
}

Here's an example:
const mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/rsvp', {useNewUrlParser: true, useUnifiedTopology: true});
const db = mongoose.connection;
db.on('error', () => {
console.log('mongoose connection error');
});
db.once('open', () => {
console.log('mongoose connected successfully');
});
const rsvpSchema = mongoose.Schema({
firstName: String,
lastName: String,
email: String,
guests: Number
});
const Rsvp = mongoose.model('Rsvp', rsvpSchema);
// This is the part you will need... In this example, if first and last name match, update email and guest number. Otherwise, create a new document. The key is to learn to put "upsert" as the "options" for the argument.
const findRsvpAndUpdate = (result, callback) => {
Rsvp.findOneAndUpdate({firstName: result.firstName, lastName: result.lastName}, result, { upsert: true }, (err, results) => {
if (err) {
callback(err);
} else {
callback(null, results);
}
})
};
// From your server index.js file, call this...
app.post('/rsvps', (req, res) => {
findRsvpAndUpdate(req.body, (error, result) => {
if (error) {
res.status(500).send(error);
} else {
res.status(200).send(result);
}
})
});

What am I doing wrong on my update function on nodejs/mongodb?

Hi I am currently new to nodejs and mongodb what I want to do is make a function to update my win,lose,draw record from my userschema.
My Schema:
UserSchema = new mongoose.Schema({
username:'string',
password:'string',
email:'string',
//Change Made
win:{ type: Number, default: 0 },
lose:{ type: Number, default: 0 },
draw:{ type: Number, default: 0 }
});
var db = mongoose.createConnection(app.get('MONGODB_CONN')),
User = db.model('users', UserSchema);
My Function for updating:
app.post('/user/updateScores',function(req, res){
try{
var query = req.body.username;
User.findOneAndUpdate(query, { win : req.body.win, lose : req.body.lose, draw : req.body.draw }, function (err,user){
if (err) res.json(err) ;
req.session.loggedIn = true;
res.redirect('/user/' + user.username);
});
}
catch(e){
console.log(e)
}
});
The problem is when I try updating, it updates the current data BUT goes to a blank page and throws an exception saying:
ReferenceError: win is not defined
at eval (eval at <anonymous> (C:\Users\ryan-utb\Desktop\RockScissorsPaper\node_modules\underscore\underscore.js:1176:16), <anonymous>:5:9)
at template (C:\Users\ryan-utb\Desktop\RockScissorsPaper\node_modules\underscore\underscore.js:1184:21)
at Function.exports.underscore.render (C:\Users\ryan-utb\Desktop\RockScissorsPaper\node_modules\consolidate\lib\consolidate.js:410:14)
at C:\Users\ryan-utb\Desktop\RockScissorsPaper\node_modules\consolidate\lib\consolidate.js:106:23
at C:\Users\ryan-utb\Desktop\RockScissorsPaper\node_modules\consolidate\lib\consolidate.js:90:5
at fs.js:266:14
at Object.oncomplete (fs.js:107:15)
but I already defined win properly, what seems to be the problem?

User.update(
{username:req.body.username},
{ win : req.body.win, lose : req.body.lose, draw : req.body.draw },
function (err, data) { //look - no argument name "user" - just "data"
//2 mistakes here that you should learn to never do
//1. Don't ignore the `err` argument
//2. Don't assume the query returns something. Check that data is not null.
console.log(data);
//The next line must be INSIDE this function in order to access "data"
res.redirect('/user/' + data.username);
});
//ACK don't try to access "data" (or "user") here. this is asynchronous code!
//this code executes EARLIER IN TIME than the User.update callback code
update after your snippet v2
your find call is simply not matching any documents, so user is null
FYI You can do a find and update at the same time with a single findOneAndUpdate operation

User.update({
username: req.body.username
},{
$set: {
win : req.body.name,
loose : req.body.loose
}
}, function(err, result) {
if (err) {
console.log(err);
} else if (result === 0) {
console.log("user is not updated");
} else {
console.log("user is updated");
}
});
I hope you can understand your issue and can update your User collection.