I am building a multi tenant app with only a few connections (max 3 or 4) on the same mongo host. What I am really doing is establishing mongoose connections at server startup and store it in a context object.
// For each tenants
tenantConnection(name, uri) => new Promise((resolve, reject) => {
const connection = mongoose.createConnection(uri, {
useCreateIndex: true,
useNewUrlParser: true,
useFindAndModify: false,
retryWrites: false,
useUnifiedTopology: true
})
connection.on('connected', async () => {
const MessageModel = connection.model('Message', Message) // <- Message is a classic mongoose Schema
...
return resolve({ connection, models: { Message: MessageModel } })
})
})
All works well except when I am trying to use the prototype .save() (same with Model.create({...}). When I try to create a new record, the function stuck, no callback are triggered nor any error.
const { models: { Message } } = tenant
const messageRecord = new Message({ content }
await messageRecord.save() // -> Stuck here, nothing happens
At this moment, the only way I have found is to use UpdateOne({}, {...}, {upsert: true}) to create records but I'd rather like to use native mongoose prototype .save() to trigger the setters from my schema.
Does anyone has any idea of what I am doing wrong ?
Related
I have created the following user schema, including two methods:
getSnapshot()
getLastTweetId()
user.js
const mongoose = require('mongoose')
const getLastTweetId = require('../utilities/getLastTweetId')
const getFollowers = require('../utilities/getFollowers')
const userSchema = new mongoose.Schema({
twitterId: {
type: String,
required: true
},
screenName: {
type: String
},
snapshots: {
type: [snapshotSchema],
default: []
},
createdAt: {
type: Date
},
})
userSchema.method('getSnapshot', async function () {
const { user, snapshot } = await getFollowers({user: this})
await user.save()
return snapshot
})
userSchema.method('getLastTweetId', async function () {
const tweetId = await getLastTweetId({user: this})
return tweetId
})
const User = mongoose.model('User', userSchema)
module.exports = User
When I define a user instance in passport.js, I can call getSnapshot() on user with no problems. (see below)
passport.js
const passport = require('passport')
const mongoose = require('mongoose')
const needle = require('needle')
const { DateTime } = require('luxon')
const User = mongoose.model('User')
// Setup Twitter Strategy
passport.use(new TwitterStrategy({
consumerKey: process.env.TWITTER_CONSUMER_API_KEY,
consumerSecret: process.env.TWITTER_CONSUMER_API_SECRET_KEY,
callbackURL: process.env.CALLBACK_URL,
proxy: trustProxy
},
async (token, tokenSecret, profile, cb) => {
const twitterId = profile.id
const screenName = profile.screen_name
const existingUser = await User.findOne({ twitterId })
if (existingUser) {
// Track if this is a new login from an existing user
if (existingUser.screenName !== screenName) {
existingUser.screenName = screenName
await existingUser.save()
}
// we already have a record with the given profile ID
cb(undefined, existingUser)
} else {
// we don't have a user record with this ID, make a new record
const user = await new User ({
twitterId ,
screenName,
}).save()
**user.getSnapshot()**
cb(undefined, user)
}
}
)
However, when I call getLastTweetId() on a user instance in tweet.js, I receive the following error in my terminal:
TypeError: user.getLastTweetId is not a function
Then my app crashes.
tweets.js
const express = require('express')
const mongoose = require('mongoose')
const User = mongoose.model('User')
const Tweet = mongoose.model('Tweet')
const { DateTime } = require('luxon')
const auth = require('../middleware/auth')
const requestTweets = require('../utilities/requestTweets')
const router = new express.Router()
const getRecentTweets = async (req, res) => {
const twitterId = req.user.twitterId
const user = await User.find({twitterId})
*const sinceId = user.getLastTweetId()*
let params = {
'start_time': `${DateTime.now().plus({ month: -2 }).toISO({ includeOffset: false })}Z`,
'end_time': `${DateTime.now().toISO({ includeOffset: false })}Z`,
'max_results': 100,
'tweet.fields': "created_at,entities"
}
if (sinceId) {
params.since_id = sinceId
}
let options = {
headers: {
'Authorization': `Bearer ${process.env.TWITTER_BEARER_TOKEN}`
}
}
const content = await requestTweets(twitterId, params, options)
const data = content.data
const tweets = data.map((tweet) => (
new Tweet({
twitterId,
tweetId: tweet.id,
text: tweet.text,
})
))
tweets.forEach(async (tweet) => await tweet.save())
}
// Get all tweets of one user either since last retrieved tweet or for specified month
router.get('/tweets/user/recent', auth, getRecentTweets)
module.exports = router
I would really appreciate some support to figure out what is going on here.
Thank you for bearing with me!
My first guess was that the user instance is not created properly in tweets.js, but then I verified via log messages that the user instance is what I expect it to be in both passport.js as well as tweets.js
My second guess was that the problem is that the user instance in the database was created before I added the new method to the schema, but deleting and reinstantiating the entire collection in the db changed nothing.
Next I went about checking if the issue is related to instantiating the schema itself or just importing it and it seems to be the latter, since when I call getLastTweetId in passport.js it also works, when I call getSnapshot() in tweets.js it also fails.
This is where I'm stuck, because as far as I can tell, I am requiring the User model exactly the same way in both files.
Even when I print User.schema.methods in either file, it shows the following:
[0] {
[0] getSnapshot: [AsyncFunction (anonymous)],
[0] getLastTweetId: [AsyncFunction (anonymous)]
[0] }
It looks like my first guess regarding what was wrong was on point, and I was just sloppy in verifying that I'm instantiating the user correctly.
const user = await User.find({twitterId})
The above line was returning an array of users.
Instead, I should have called:
const user = await User.findOne({twitterId})
I did not detect the bug at first, because logging an array that contains only one object looks nearly the same as just logging the object itself, I simply overlooked the square brackets.
Changing that single line fixed it.
We are creating a discord bot for task management based on teams. We're using a collection as a team. In code to find a team and add tasks to it we use: const Team = await mongoose.connection.collection(teamName, {strict: true}); and strict is supposed to make it so that the collection isn't created when it's not found but instead this happens:
I've tried with no luck as the mongoose has no documentation on the options and everything I've tried doesn't work.
How do I make mongoose.connection.collection(teamName) return an error if teamName isn't a collection name/isn't found?
You are right, strict: true should do it, but doesn't.
Mongoose doesn't mention the options it accepts, though usually it takes the ones used by the underlying mongo client.
But it does mention in the documentation that it makes missing collections:
Retrieves a collection, creating it if not cached.
I looked into the repo, and the collection method always makes a missing collection
Connection.prototype.collection = function(name, options) {
const defaultOptions = {
autoIndex: this.config.autoIndex != null ? this.config.autoIndex : >this.base.options.autoIndex,
autoCreate: this.config.autoCreate != null ? this.config.autoCreate : >this.base.options.autoCreate
};
options = Object.assign({}, defaultOptions, options ? utils.clone(options) : {});
options.$wasForceClosed = this.$wasForceClosed;
if (!(name in this.collections)) {
this.collections[name] = new Collection(name, this, options);
}
return this.collections[name];
};
The mongo-client collection does take a strict parameter.
You can access it from mongoose.connection.client.db()
Update
Here is how you can call it:
mongoose.connection.client.db().collection(teamName, {strict: true}, function (error, collection) {
console.log('error', error);
console.log('collection', collection);
})
Example
>
> mongoose.connection.client.db().collection('nonExistantCollectionName', {strict: true}, function () { console.log(arguments) })
undefined
> [Arguments] {
'0': MongoError: Collection nonExistantCollectionName does not exist. Currently in strict mode.
at Function.create (./node_modules/mongodb/lib/core/error.js:57:12)
at toError (./node_modules/mongodb/lib/utils.js:130:22)
at ./node_modules/mongodb/lib/db.js:482:9
at ./node_modules/mongodb/lib/utils.js:704:5
at handleCallback (./node_modules/mongodb/lib/utils.js:109:55)
at ./node_modules/mongodb/lib/cursor.js:840:66
at ./node_modules/mongodb/lib/utils.js:704:5
at ./node_modules/mongodb/lib/cursor.js:925:9
at CommandCursor._endSession (./node_modules/mongodb/lib/core/cursor.js:397:7)
at ./node_modules/mongodb/lib/cursor.js:923:12
at maybePromise (./node_modules/mongodb/lib/utils.js:692:3)
at CommandCursor.close (./node_modules/mongodb/lib/cursor.js:916:12)
at ./node_modules/mongodb/lib/cursor.js:840:27
at ./node_modules/mongodb/lib/core/cursor.js:739:9
at handleCallback (./node_modules/mongodb/lib/core/cursor.js:32:5)
at ./node_modules/mongodb/lib/core/cursor.js:683:38
at _setCursorNotifiedImpl (./node_modules/mongodb/lib/core/cursor.js:696:10)
at setCursorNotified (./node_modules/mongodb/lib/core/cursor.js:683:3)
at done (./node_modules/mongodb/lib/core/cursor.js:458:16)
at queryCallback (./node_modules/mongodb/lib/core/cursor.js:503:20)
at ./node_modules/mongodb/lib/core/cursor.js:548:9
at ./node_modules/mongodb/lib/utils.js:704:5
at executeCallback (./node_modules/mongodb/lib/operations/execute_operation.js:65:7)
at callbackWithRetry (./node_modules/mongodb/lib/operations/execute_operation.js:112:14)
at ./node_modules/mongodb/lib/operations/command_v2.js:102:9
at ./node_modules/mongodb/lib/core/connection/pool.js:405:18
at processTicksAndRejections (internal/process/task_queues.js:79:11) {
driver: true
},
'1': null
}
You can use this code to check if a collection exists or not in a specific database - and then perform actions as needed.
const mongoose = require('mongoose');
const uri = 'mongodb://127.0.0.1:27017/';
const opts = {
useNewUrlParser: true,
useUnifiedTopology: true,
dbName: 'test'
};
(async () => {
const conn = await mongoose.createConnection(uri, opts);
let collNames = await conn.db.listCollections().toArray();
collNames = collNames.map(e => e.name);
console.log(collNames);
console.log(collNames.includes('existing_coll')); // true
console.log(collNames.includes('non_existing_coll')); // false
await conn.close();
})(uri, opts);
I have a method that connects to the MongoDB, but I can't figure out if this connection ends after a call was made or not.
This is the method:
import { Db, MongoClient } from "mongodb";
let cachedConnection: { client: MongoClient; db: Db } | null = null;
export async function connectToDatabase(mongoUri?: string, database?: string) {
if (!mongoUri) {
throw new Error(
"Please define the MONGO_URI environment variable inside .env.local"
);
}
if (!database) {
throw new Error(
"Please define the DATABASE environment variable inside .env.local"
);
}
if (cachedConnection) return cachedConnection;
cachedConnection = await MongoClient.connect(mongoUri, {
useNewUrlParser: true,
useUnifiedTopology: true,
}).then((client) => ({
client,
db: client.db(database),
}));
return cachedConnection!;
}
I use this with Next.js and I am afraid that the app I am actually doing goes down if there will be too many connections. Intuitively, I think that the mongoDB connection ends after a call, but I am not sure.
In my code, I am sending a query to my Mongo database. The method findUser() shall return the response of this query. The query works fine, tested with console.log(users).
The problem is the function returns null, it doesn't wait till the query got a response to return the var foundUser.
How could I use await/async in this case in order to wait for the query response before returning anything ?
function findUser(username) {
foundUser = null
const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology : true});
client.connect(err => {
const collection = client.db("YourCV").collection("Accounts");
result = collection.findOne({username : username }, function(err, user) {
console.log(user)
if(user){
foundUser = user
}
});
});
return foundUser
};
console.log(user) outputs :
{
_id: 601695084b28102500ae0015,
username: 'jetlime',
password: '$2b$10$EN5k/YKOMBgibqy62s0hGOX9MffHZtXkfsw0Du0j8QVS7mGab5FLi'
}
Many thanks
Update the code to the following:
async function findUser(username) {
const client = await MongoClient.connect(url, { useNewUrlParser: true })
.catch(err => { console.log(err); });
if (!client) {
return;
}
const collection = client.db("YourCV").collection("Accounts");
const user = await collection.findOne({username : username });
client.close(); // -> To be under finally clause.
return user;
};
And call the function with await findUser(username);
Caution: The above of connecting to DB is not recommended. You are establishing a DB connection for every function call. This quickly leads to running out of connections on the DB side when you have a large number of requests.
Move the DB connection establishing part to a commonplace and re-use the connection.
See whatever you do, any operation you perform with your database, it returns promise, so write async keyword before the name of your function in line 1 and then await keyword before your query in line 6, it will then behave like synchronous call and every line will execute accordingly
As you said, you need to use async/await operators for asynchron methods.
You have two choices:
1)Implement callback method on findUser
async function findUser(username, onUsersReady) {
const client = new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true
}).connect();
const collection = await client.db("YourCV").collection("Accounts");
await collection.findOne({
username: username
}, function(err, user) {
console.log(user)
if (user) {
foundUser = user
onUsersReady(user);
}
});
};
2)Use function to return results directly
async function findUser(username) {
const client = await new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true
}).connect();
const collection = await client.db("YourCV").collection("Accounts");
const foundUser = await collection.findOne({
username
});
return foundUser;
}
I'm building a website for my sister so that she can sell her art. I am using Next.js to set everything up. The website renders the artwork by grabbing an array from a database and mapping through it.
Two Example objects
{
id: 8,
path: "images/IMG_0008.jpg",
size: "9x12x.75",
price: "55",
sold: false
}
{
id: 9,
path: "images/IMG_0009.jpg",
size: "9x12x.75",
price: "55",
sold: false
}
pages/Shop.js
import Card from "../Components/Card";
import fetch from 'node-fetch'
import Layout from "../components/Layout";
function createCard(work) {
return (
<Card
key={work.id}
id={work.id}
path={work.path}
size={work.size}
price={work.price}
sold={work.sold}
/>
);
}
export default function Shop({artwork}) {
return (
<Layout title="Shop">
<p>This is the Shop page</p>
{artwork.map(createCard)}
</Layout>
);
}
export async function getStaticProps() {
const res = await fetch('http://localhost:3000/api/get-artwork')
const artwork = await res.json()
return {
props: {
artwork,
},
}
}
The problem I am running into is that when I try to use mongoose in the api/get-artwork. It will only render the page once and once it is refreshed it will break I believe do to the fact the Schema and Model get redone.
pages/api/get-artwork.js/
const mongoose = require("mongoose");
mongoose.connect('mongodb://localhost:27017/ArtDB', {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false
});
const itemsSchema = {
id: String,
description: String,
path: String,
size: String,
price: Number,
sold: Boolean
};
const Art = mongoose.model("Art", itemsSchema);
export default (req, res) => {
Art.find({sold: false}, (err, foundItems)=> {
if (err) {
console.log(err);
} else {
console.log(foundItems);
res.status(200).json(foundItems);
}
});
};
So to try to fix this I decided to use the native MongoDB driver. Like this.
/pages/api/get-artwork/
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
// Connection URL
const url = 'mongodb://localhost:27017';
// Database Name
const dbName = 'ArtDB';
// Create a new MongoClient
const client = new MongoClient(url, {useUnifiedTopology: true});
let foundDocuments = ["Please Refresh"];
const findDocuments = function(db, callback) {
// Get the documents collection
const collection = db.collection('arts');
// Find some documents
collection.find({}).toArray(function(err, arts) {
assert.equal(err, null);
foundDocuments = arts;
callback(arts);
});
}
// Use connect method to connect to the Server
client.connect(function(err) {
assert.equal(null, err);
const db = client.db(dbName);
findDocuments(db, function() {
client.close();
});
});
export default (req, res) => {
res.send(foundDocuments);
};
This works for the most part but occasionally the array will not be returned. I think this is because the page is loading before the mongodb part finishes? So I guess my question is how do I make 100% sure that it loads the art correctly every time whether that be using mongoose or the native driver.
Thanks!
The Next.js team has a good set of example code, which they add to regularly, one of them being Next.js with MongoDB and Mongoose. Check it out, https://github.com/vercel/next.js/tree/canary/examples/with-mongodb-mongoose, and hope this helps if you're still searching for solutions.
A little more complete answer might be helpful here. Here's what's working for us.
I would suggest using the next-connect library to make this a little easier and not so redundant.
Noticed unnecessary reconnects in dev so I bind to the global this property in Node. Perhaps this isn't required but that's what I've noticed. Likely tied to hot reloads during development.
This is a lengthy post but not nearly as complicated as it seems, comment if you have questions.
Create Middleware Helper:
import mongoose from 'mongoose';
// Get your connection string from .env.local
const MONGODB_CONN_STR = process.env.MONGODB_CONN_STR;
const databaseMiddleware = async (req, res, next) => {
try {
if (!global.mongoose) {
global.mongoose = await mongoose.connect(MONGODB_CONN_STR, {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
});
}
}
catch (ex) {
console.error(ex);
}
// You could extend the NextRequest interface
// with the mongoose instance as well if you wish.
// req.mongoose = global.mongoose;
return next();
};
export default databaseMiddleware;
Create Model:
Typically the path here might be src/models/app.ts.
import mongoose, { Schema } from 'mongoose';
const MODEL_NAME = 'App';
const schema = new Schema({
name: String
});
const Model = mongoose.models[MODEL_NAME] || mongoose.model(MODEL_NAME, schema);
export default Model;
Implement Next Connect:
Typically I'll put this in a path like src/middleware/index.ts (or index.js if not using Typescript).
Note: the ...middleware here just allows you, see below, to pass in additional middleware on the fly when the handler here is created.
This is quite useful as our handler creator here can have things like logging and other useful middleware so it's not so redundant in each page/api file.
export function createHandler(...middleware) {
return nextConnect().use(databaseMiddleware, ...middleware);
}
Use in Api Route:
Putting it together, we can now use our App Model with ease
import createHandler from 'path/to/above/createHandler';
import App from 'path/to/above/model/app';
// again you can pass in middleware here
// maybe you have some permissions middleware???
const handler = createHandler();
handler.get(async (req, res) => {
// Do something with App
const apps = await App.find().exec();
res.json(apps);
});
export default handler;
In pages/api/get-artwork.js/
const mongoose = require("mongoose");
mongoose.connect("mongodb://localhost:27017/ArtDB", {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
});
const itemsSchema = {
id: String,
description: String,
path: String,
size: String,
price: Number,
sold: Boolean,
};
let Art;
try {
// Trying to get the existing model to avoid OverwriteModelError
Art = mongoose.model("Art");
} catch {
Art = mongoose.model("Art", itemsSchema);
}
export default (req, res) => {
Art.find({ sold: false }, (err, foundItems) => {
if (err) {
console.log(err);
} else {
console.log(foundItems);
res.status(200).json(foundItems);
}
});
};
It works just fine for me.
When we use mongoose with express.js, we connect to mongodb and run the code once. But in next.js everytime we need to connect to mongodb, we have to run connection code.
Connection to mongodb
import mongoose from "mongoose";
const dbConnect = () => {
if (mongoose.connection.readyState >= 1) {
// if it is not ready yet return
return;
}
mongoose
.connect(process.env.DB_LOCAL_URI, {
//****** since mongoose 6, we dont need those******
// useNewUrlParser: true,
// useUnifiedTopology: true,
// useFindAndModify: false,
// useCreateIndex: true,
})
.catch((err) => console.log(err))
.then((con) => console.log("connected to db"));
};
export default dbConnect;
Use it inside next api
Before running handler code, you need to connect first
import dbConnect from "../../../config/dbConnect";
dbConnect();
... then write handler code
A concise OO approach
Inspired by #Blujedis answer this is what I ended up with.
/* /api/util/handler.js */
import nextConnect from "next-connect";
import mongoose from "mongoose";
export class Handler {
static dbConnection = null
static dbMiddleware = async (req, res, next) => {
try {
Handler.dbConnection ||= await mongoose.connect(process.env.MONGO_URL)
next()
} catch (err) {
console.error(err);
next(err)
}
}
constructor(...middleware) {
return nextConnect().use(Handler.dbMiddleware, ...middleware);
}
}
Example handler:
/* /api/people.js */
import { Handler } from "./util/handler"
import { Person } from "./models/person"
export default
new Handler()
.get(async (req, res) => {
const people = await Person.find({})
return res.status(200).json(people)
})