Develop Reference

Javascript - Optimize algorithm (complex data strcuture)

Introduction
I am implementing a method which inserts posts to the respective users posts lists in my map, sorted by date (recent posts first).
This is how I am structuring my data:
state = {
userId: {
posts: [
{ // object returned from my feeds algorithm in the server side
id,
userData: {
id,
},
date,
},
... more posts ...
],
},
... more users ...
}
In my algorithm, I just need to insert all the posts that are inside a given list
[
{ id: "post1", { userData: { id: "alex" }, date },
{ id: "post2", { userData: { id: "sara" }, date }
]
in the posts list of each respective user.
Problem
I also need to avoid inserting posts that already exists in my state, and I can't find a simple way to do it optimally.
Current code
This is my current implementation. I feel that this can be done easier and faster. Any help?
/*
Algorithm
*/
function addContents(state, contents, contentType, cached) {
const newState = state;
contents.forEach((content) => {
const { userData: { id: userId } } = content;
const prevUserState = state.get(userId);
const prevContents = prevUserState?.[contentType] ?? [];
const newContents = prevContents;
// TODO - Avoid inserting if already exists in prevContents! (check by **id**)
let inserted = false;
for (const [index, prevContent] of prevContents.entries()) {
// Replace
if (content.id === prevContent.id) {
newContents[index] = content;
inserted = true;
break;
}
// Insert in the correct order
if(content.date >= prevContent.date) {
newContents.splice(index, 0, content);
inserted = true;
break;
}
}
if (!inserted) {
newContents.push(content);
}
newState.set([
userId,
{
...prevUserState,
[contentType]: newContents
}
]);
});
// if(isEqual(state, newState)) return state; (deep compare to avoid re-renderizations because of state update)
return new Map([...newState]);
}
/*
Test
*/
(() => {
// State
const state = new Map([]);
// User ALEX
const userId1 = "alex";
const userPosts1 = [ // already sorted by date
{
id: "78q78w0w0",
userData: {
id: userId1,
},
date: new Date("10/26/1999 00:00:01")
},
{
id: "92uwdq092",
userData: {
id: userId1,
},
date: new Date("10/26/1999 00:00:00")
}
];
state.set(userId1, { posts: userPosts1 });
// User SARA
const userId2 = "sara";
const userPosts2 = [ // already sorted by date
{
id: "iipzxx115",
userData: {
id: userId2,
},
date: new Date("12/25/2003 03:30:10")
},
{
id: "Wxrr22232",
userData: {
id: userId2,
},
date: new Date("01/01/2000 17:44:41")
}
];
state.set(userId2, { posts: userPosts2 });
const newPosts = [
{
id: "OLDEST FOR ALEX!",
userData: {
id: userId1
},
date: new Date("10/25/1999 23:59:59")
},
{
id: "NEWEST FOR SARA!",
userData: {
id: userId2
},
date: new Date("01/05/2010 22:22:22")
},
{
id: "OLDEST FOR SARA!",
userData: {
id: userId2
},
date: new Date("10/25/1999 23:59:59")
}
]
addContents(state, newPosts, "posts");
console.log(state.get(userId1))
console.log(state.get(userId2))
})();
Note: As this method is implemented in a React's reducer, to manage complex states, I am returning a new Map, after deep comparing the previous and the new state, to produce UI re-renderizations.
UPDATE
I have implemented another version where I do what I need, but maybe, it can be more optimized.
function addContents(state, contents, contentType, cached) {
const newState = state;
const exists = {}; // optimization
for (const content of contents) {
const {
userData: { id: userId },
} = content;
const prevUserState = state.get(userId);
const prevContents = prevUserState?.[contentType] ?? [];
const newContents = prevContents;
if (cached) {
if (!exists[userId]) {
exists[userId] = prevContents.reduce((map, content) => {
map[content.id] = true;
return map;
}, {});
}
// Avoid inserting if necessary
if (exists[userId][content.id]) {
break;
}
}
// Insert the new content in the user's content list
console.log(`Inserting ${content.id}`);
let inserted = false;
for (const [index, prevContent] of prevContents.entries()) {
// Replace
if (content.id === prevContent.id) {
newContents[index] = content;
inserted = true;
break;
}
// Insert in the correct order
if(content.date >= prevContent.date) {
newContents.splice(index, 0, content);
inserted = true;
break;
}
}
if (!inserted) {
newContents.push(content);
}
newState.set([
userId,
{
...prevUserState,
[contentType]: newContents
}
]);
}
// if (isEqual(state, newState)) return state;
return new Map([...newState]);
}
/*
Test
*/
(() => {
// State
let state = new Map([]);
// User ALEX
const userId1 = "alex";
const userPosts1 = [ // already sorted by date
{
id: "78q78w0w0",
userData: {
id: userId1,
},
date: new Date("10/26/1999 00:00:01")
},
{
id: "92uwdq092",
userData: {
id: userId1,
},
date: new Date("10/26/1999 00:00:00")
}
];
state.set(userId1, { posts: userPosts1 });
// User SARA
const userId2 = "sara";
const userPosts2 = [ // already sorted by date
{
id: "iipzxx115",
userData: {
id: userId2,
},
date: new Date("12/25/2003 03:30:10")
},
{
id: "Wxrr22232",
userData: {
id: userId2,
},
date: new Date("01/01/2000 17:44:41")
}
];
state.set(userId2, { posts: userPosts2 });
const newPosts = [
{
id: "OLDEST FOR ALEX!",
userData: {
id: userId1
},
date: new Date("10/25/1999 23:59:59")
},
{
id: "NEWEST FOR SARA!",
userData: {
id: userId2
},
date: new Date("01/05/2010 22:22:22")
},
{
id: "OLDEST FOR SARA!",
userData: {
id: userId2
},
date: new Date("10/25/1999 23:59:59")
}
]
state = addContents(state, newPosts, "posts");
console.log(state.get(userId1))
console.log(state.get(userId2))
/*
Insert again!
*/
state = addContents(state, newPosts, "posts", true);
})();

use an object instead of an array:
This is the same concept of the normalizr library for redux: https://github.com/paularmstrong/normalizr
state = {
[user1Id]: {
posts: {
[post1Id]: {
id,
userData: {
id,
},
date,
},
[post2Id]: {
id,
userData: {
id,
},
date,
},
... more posts ...
},
},
... more users ...
}
This way you can easily access the object you want by its Id and check whether it exists or not just doing: if(state[23].posts[12])
if you need to iterate the users or a user posts use
object.keys(state).map(userId => ...)
or
object.keys(state[23].posts).map(postId => ...)
INSERT/UPDATE:
state[23].posts[newId]: { ...newPost}

I'm not able to follow what you are doing but I think this is what you are after.
You can do it to a oneline very easy.
newdata = [{ id: "post1", { userData: { id: "alex" }, date }]
if(!oldstates.find(d =>
d.id === newdata.id &&
d.userData.id === newdata.userData.id &&
d.date === newdata.date
)) {
oldstates.push(newdata)
}
// oneliner
if(!oldstates.find(d => d.id === newdata.id && d.userData.id === newdata.userData.id && d.date === newdata.date )) oldstates.push(newdata)

How to check if item exists in MonoDB array?

I have a MongoDB model:
const userSchema = new Schema = ({
name: { type: String },
company: [
companyId: {
type: String,
},
movies: [
{
genre: {
type: String,
enum: [
'horror',
'comedy',
'action',
'romance',
],
},
ratings: { type: String }
},
]
],
})
In my query, I have an endpoint that pushes a genre to the movies array but I want to check if there is an existing genre with the name already, if it exists, I want to show a message that says it already exists, otherwise, push the new items to the movies array
const result = await UserProfile.updateOne(
{
_id: id,
'company.companyId{ $eq: req.params.companyId},
'company.movies.$.genre': {
$eq: { genre: req.body.genre},
},
}
},
{
$push: {
'company.$.movies': {
...model,
},
},
},
{ new: true, runValidators: true }
).catch((err) => handleErrorThrow(err));
if (result.nModified === 0)
throw new CustomError(409, 'Movie exists already');
And if I want to remove the array based on another endpoint, I tried the same thing it doesn't work
const result = await UserProfile.updateOne(
{
_id: id
}
},
{
$pull: {
company: {
$elemMatch: {
companyId: req.params.companyId,
"movies.genre": {
$ne: req.body.genre
}
}
}
},
{ new: true, runValidators: true }
)
.catch((err) => handleErrorThrow(err));
if (result.nModified === 0)
throw new CustomError(409, 'Not exist');
It returned Not exist'

use $elemMatch for nested array condition, and $ne for genre should not exists before push into movies,
const result = await UserProfile.updateOne(
{
_id: id,
company: {
$elemMatch: {
companyId: req.params.companyId,
"movies.genre": {
$ne: req.body.genre
}
}
}
},
{
$push: {
"company.$.movies": model
}
},
{ runValidators: true }
)
.catch((err) => handleErrorThrow(err));
if (result.nModified === 0) {
throw new CustomError(409, 'Movie exists already');
}
And if I want to remove the array based on another endpoint
const result = await UserProfile.updateOne(
{
_id: id,
company: {
$elemMatch: {
companyId: req.params.companyId,
"movies.genre": req.body.genre
}
}
},
{
$pull: {
"company.$.movies": {
genre: req.body.genre
}
}
},
{ runValidators: true }
).catch((err) => handleErrorThrow(err));
if (result.nModified === 0) {
throw new CustomError(409, 'Not exist');
}

Implement feed with retweets in MongoDB

I want to implement retweet feature in my app. I use Mongoose and have User and Message models, and I store retweets as array of objects of type {userId, createdAt} where createdAt is time when retweet occurred. Message model has it's own createdAt field.
I need to create feed of original and retweeted messages merged together based on createdAt fields. I am stuck with merging, whether to do it in a single query or separate and do the merge in JavaScript. Can I do it all in Mongoose with a single query? If not how to find merge insertion points and index of the last message?
So far I just have fetching of original messages.
My Message model:
const messageSchema = new mongoose.Schema(
{
fileId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'File',
required: true,
},
userId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
required: true,
},
likesIds: [{ type: mongoose.Schema.Types.ObjectId, ref: 'User' }],
reposts: [
{
reposterId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
},
createdAt: { type: Date, default: Date.now },
},
],
},
{
timestamps: true,
},
);
Edit: Now I have this but pagination is broken. I am trying to use newCreatedAt field for cursor, that doesn't seem to work. It returns empty array in second call when newCreatedAt is passed from the frontend.
messages: async (
parent,
{ cursor, limit = 100, username },
{ models },
) => {
const user = username
? await models.User.findOne({
username,
})
: null;
const options = {
...(cursor && {
newCreatedAt: {
$lt: new Date(fromCursorHash(cursor)),
},
}),
...(username && {
userId: mongoose.Types.ObjectId(user.id),
}),
};
console.log(options);
const aMessages = await models.Message.aggregate([
{
$addFields: {
newReposts: {
$concatArrays: [
[{ createdAt: '$createdAt', original: true }],
'$reposts',
],
},
},
},
{
$unwind: '$newReposts',
},
{
$addFields: {
newCreatedAt: '$newReposts.createdAt',
original: '$newReposts.original',
},
},
{ $match: options },
{
$sort: {
newCreatedAt: -1,
},
},
{
$limit: limit + 1,
},
]);
const messages = aMessages.map(m => {
m.id = m._id.toString();
return m;
});
//console.log(messages);
const hasNextPage = messages.length > limit;
const edges = hasNextPage ? messages.slice(0, -1) : messages;
return {
edges,
pageInfo: {
hasNextPage,
endCursor: toCursorHash(
edges[edges.length - 1].newCreatedAt.toString(),
),
},
};
},
Here are the queries. The working one:
Mongoose: messages.aggregate([{
'$match': {
createdAt: {
'$lt': 2020 - 02 - 02 T19: 48: 54.000 Z
}
}
}, {
'$sort': {
createdAt: -1
}
}, {
'$limit': 3
}], {})
And the non working one:
Mongoose: messages.aggregate([{
'$match': {
newCreatedAt: {
'$lt': 2020 - 02 - 02 T19: 51: 39.000 Z
}
}
}, {
'$addFields': {
newReposts: {
'$concatArrays': [
[{
createdAt: '$createdAt',
original: true
}], '$reposts'
]
}
}
}, {
'$unwind': '$newReposts'
}, {
'$addFields': {
newCreatedAt: '$newReposts.createdAt',
original: '$newReposts.original'
}
}, {
'$sort': {
newCreatedAt: -1
}
}, {
'$limit': 3
}], {})

This can be done in one query, although its a little hack-ish:
db.collection.aggregate([
{
$addFields: {
reposts: {
$concatArrays: [[{createdAt: "$createdAt", original: true}],"$reports"]
}
}
},
{
$unwind: "$reposts"
},
{
$addFields: {
createdAt: "$reposts.createdAt",
original: "$reposts.original"
}
},
{
$sort: {
createdAt: -1
}
}
]);
You can add any other logic you want to the query using the original field, documents with original: true are the original posts while the others are retweets.

mongodb after updating document, returns old values

router.delete('/deletepost', (req, res) => {
// console.log(req.query.postid)
if (req.query.category === 'forsale') {
ForSalePosts.findById(req.query.postid)
// .then(post => console.log(post))
.deleteOne()
.catch(err => console.log(err))
AllPosts.updateOne({ user: req.query.userid },
{ $pull: { posts: { postid: req.query.postid } } })
.catch(err => console.log(err))
AllPosts.aggregate(
[
{ $match: { user: ObjectId(req.query.userid) } },
{ $unwind: '$posts' },
{ $sort: { 'posts.date': -1 } }
]
)
.then(posts => {
// console.log(posts)
res.json(posts)
})
.catch(err => res.status(404).json({ nopostfound: 'There is no posts' }))
}
})
this is my route. i am trying to delete an item in my document. the item is being deleted however it returns old values. for example :
Allposts has an array with posts:[postid:{type:String}, ...]
I am trying to delete a specific postid by using $pull,
postid is being deleted however when I aggregate the same model, .then(posts=> console.log(posts)) returns old values on first call, doesnt update the component.
EDIT: just realized sometimes it returns the right values but sometimes it returns the old values as well. does anyone know why and what can i do to solve it ?
const mongoose = require('mongoose')
const Schema = mongoose.Schema;
const AllPostsSchema = new Schema({
user: {
type: Schema.Types.ObjectId,
ref: 'users'
},
posts: [{
postid: {
type: String
},
title: {
type: String
},
category: {
type: String
},
subcategory: {
type: String
}, category: {
type: String
},
description: {
type: String
},
name: {
type: String
},
price: {
type: Number
},
email: {
type: String
},
phonenumber: {
type: Number
},
language: {
type: String
},
make: {
type: String
},
model: {
type: Number
},
odometer: {
type: Number
},
condition: {
type: String
},
state: {
type: String
},
town: {
type: String
},
city: {
type: String
},
links: [{ type: String }],
date: {
type: Date,
default: Date.now
}
}]
})
module.exports = AllPosts = mongoose.model('allposts', AllPostsSchema)
REACT FUNCTION CALL :
deletePost = (category, postid) => {
const postinfo = {
category: category.toLowerCase(),
postid: postid,
userid: this.props.auth.user.id
}
this.props.deletePost(postinfo)
}

You need to add options parameter to delete like:
AllPosts.updateOne({ user: req.query.userid },
{
$pull: { posts: { postid: req.query.postid } }
},
{ new: true }
);
This will return the new object after performing the operation. Hope this works for you.

All the mongo queries return partial promise. You have to use .then in order to resolve each promises.
Here you are running all the queries in series without using .then or async-await. So whenever you $pull from AllPosts after that immediately you call the AllPosts aggregate query which sometimes get executed and sometimes it doesn't.
So in order to make it run one by one you have to use either .then or async-await.
router.delete("/deletepost", (req, res) => {
if (req.query.category === "forsale") {
ForSalePosts.findById(req.query.postid)
.deleteOne()
.then(() => {
AllPosts.updateOne(
{ "user": req.query.userid },
{ "$pull": { "posts": { "postid": req.query.postid } } }
)
.then(() => {
AllPosts.aggregate([
{ "$match": { "user": ObjectId(req.query.userid) } },
{ "$unwind": "$posts" },
{ "$sort": { "posts.date": -1 } }
]).then(posts => {
// console.log(posts)
res.json(posts);
});
})
.catch(err =>
res.status(404).json({ "nopostfound": "There is no posts" })
);
});
}
})

Can't POST to nested array with express js

It's my first post here so please let me know if there's anything incomplete about my question, or if there's anything else that is missing :)
I'm trying to make a POST request to an array in my data structure called features:
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const CategorySchema = new Schema({
user: {
type: Schema.Types.ObjectId,
ref: 'users'
},
categoryname: {
type: String,
required: true
},
items: [
{
user: {
type: Schema.Types.ObjectId,
ref: 'users'
},
itemname: {
type: String,
required: true
},
features: [
{
user: {
type: Schema.Types.ObjectId,
ref: 'users'
},
firstfeature: {
type: String
},
date: {
type: Date,
default: Date.now
}
},
{
user: {
type: Schema.Types.ObjectId,
ref: 'users'
},
secondfeature: {
type: String
},
date: {
type: Date,
default: Date.now
}
}
],
date: {
type: Date,
default: Date.now
}
}
],
date: {
type: Date,
default: Date.now
}
});
module.exports = Category = mongoose.model('category', CategorySchema);
I don't have any issues with posting to the items array with the following code:
router.post(
'/item/:id',
passport.authenticate('jwt', { session: false }),
(req, res) => {
const { errors, isValid } = validateItemInput(req.body);
// Check Validation
if (!isValid) {
// if any errors, send 400 with erros object
return res.status(400).json(errors);
}
Category.findById(req.params.id)
.then(category => {
const newItem = {
itemname: req.body.itemname,
user: req.user.id
};
// Add to item array
category.items.unshift(newItem);
// Save
category.save().then(category => res.json(category));
})
.catch(err =>
res.status(404).json({ categorynotfound: 'No category found' })
);
}
);
But I can't figure out what I need to change here in order to add data to the features array:
router.post(
'/feature/:id/:item_id',
passport.authenticate('jwt', { session: false }),
(req, res) => {
Category.findById(req.params.id)
.then(category => {
const newFeature = {
firstfeature: req.body.firstfeature,
secondfeature: req.body.secondfeature,
user: req.user.id
};
// Add to item array
category.items.features.unshift(newFeature);
// Save
category.save().then(category => res.json(category));
})
.catch(err => res.status(404).json({ itemnotfound: 'Item not found'
}));
}
);

Issue solved with the following data structure:
features: [
{
user: {
type: Schema.Types.ObjectId,
ref: 'users'
},
price: {
type: String
},
size: {
type: String
},
date: {
type: Date,
default: Date.now
}
}
]
And then simply make a post request for one feature at a time.