I would like to delete data that is older than two hours. Currently, on the client-side, I loop through all the data and run a delete on the outdated data. When I do this, the db.on('value') function is invoked every time something is deleted. Also, things will only be deleted when a client connects, and what might happen if two clients connect at once?
Where can I set up something that deletes old data? I have a timestamp inside each object created by a JavaScript Date.now().
Firebase does not support queries with a dynamic parameter, such as "two hours ago". It can however execute a query for a specific value, such as "after August 14 2015, 7:27:32 AM".
That means that you can run a snippet of code periodically to clean up items that are older than 2 hours at that time:
var ref = firebase.database().ref('/path/to/items/');
var now = Date.now();
var cutoff = now - 2 * 60 * 60 * 1000;
var old = ref.orderByChild('timestamp').endAt(cutoff).limitToLast(1);
var listener = old.on('child_added', function(snapshot) {
snapshot.ref.remove();
});
As you'll note I use child_added instead of value, and I limitToLast(1). As I delete each child, Firebase will fire a child_added for the new "last" item until there are no more items after the cutoff point.
Update: if you want to run this code in Cloud Functions for Firebase:
exports.deleteOldItems = functions.database.ref('/path/to/items/{pushId}')
.onWrite((change, context) => {
var ref = change.after.ref.parent; // reference to the items
var now = Date.now();
var cutoff = now - 2 * 60 * 60 * 1000;
var oldItemsQuery = ref.orderByChild('timestamp').endAt(cutoff);
return oldItemsQuery.once('value', function(snapshot) {
// create a map with all children that need to be removed
var updates = {};
snapshot.forEach(function(child) {
updates[child.key] = null
});
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});
});
This function triggers whenever data is written under /path/to/items, so child nodes will only be deleted when data is being modified.
This code is now also available in the functions-samples repo.
I have a http triggered cloud function that deletes nodes, depending on when they were created and their expiration date.
When I add a node to the database, it needs two fields: timestamp to know when it was created, and duration to know when the offer must expire.
Then, I have this http triggered cloud function:
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
/**
* #function HTTP trigger that, when triggered by a request, checks every message of the database to delete the expired ones.
* #type {HttpsFunction}
*/
exports.removeOldMessages = functions.https.onRequest((req, res) => {
const timeNow = Date.now();
const messagesRef = admin.database().ref('/messages');
messagesRef.once('value', (snapshot) => {
snapshot.forEach((child) => {
if ((Number(child.val()['timestamp']) + Number(child.val()['duration'])) <= timeNow) {
child.ref.set(null);
}
});
});
return res.status(200).end();
});
You can create a cron job that every X minutes makes a request to the URL of that function: https://cron-job.org/en/
But I prefer to run my own script, that makes a request every 10 seconds:
watch -n10 curl -X GET https://(your-zone)-(your-project-id).cloudfunctions.net/removeOldMessages
In the latest version of Firebase API, ref() is changed to ref
var ref = new Firebase('https://yours.firebaseio.com/path/to/items/');
var now = Date.now();
var cutoff = now - 2 * 60 * 60 * 1000;
var old = ref.orderByChild('timestamp').endAt(cutoff).limitToLast(1);
var listener = old.on('child_added', function(snapshot) {
snapshot.ref.remove();
});
If someone will have the same problem, but in Firestore. I did a little script that at first read documents to console.log and then delete documents from a collection messages older than 24h. Using https://cron-job.org/en/ to refresh website every 24h and that's it. Code is below.
var yesterday = firebase.firestore.Timestamp.now();
yesterday.seconds = yesterday.seconds - (24 * 60 * 60);
console.log("Test");
db.collection("messages").where("date",">",yesterday)
.get().then(function(querySnapshote) {
querySnapshote.forEach(function(doc) {
console.log(doc.id," => ",doc.data());
});
})
.catch(function(error) {
console.log("Error getting documents: ", error);
});
db.collection("messages").where("date","<",yesterday)
.get().then(function(querySnapshote) {
querySnapshote.forEach(element => {
element.ref.delete();
});
})
You could look into Scheduling Firebase Functions with Cron Jobs. That link shows you how to schedule a Firebase Cloud Function to run at a fixed rate. In the scheduled Firebase Function you could use the other answers in this thread to query for old data and remove it.
Related
I've already done this on the Firebase Realtime Database and it's worked perfectly before, but I decided to switch to Firestore and am trying to have the same functionality with the cloud functions.
My goal: Have a cloud function that deletes messages after 24 hours from Firestore
I've tried to use the same function that I had and convert it over to Firestore, but the syntax isn't correct and I'm not very familiar with Javascript.
This is the code for the cloud function which worked form my RTDB:
'use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 24 Hours in seconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomID}/{messageId}').onWrite(function(change) {
var ref = change.after.ref.parent;
var now = Date.now()
var cutoff = (now - CUT_OFF_TIME) / 1000;
var oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
return oldItemsQuery.once('value').then(function(snapshot) {
var updates = {};
snapshot.forEach(function(child) {
updates[child.key] = null;
console.log(cutoff)
});
return ref.update(updates);
});
});
Path: My path in the Firestore is different from the RTDB. Collection('Message').Document('userId' (the actual userId of that user)).Collection('chatRoomId' (the actual chatRoomId of that chat)).Document('messageId') and then one of the fields is Seconds: 1587334120
I would greatly appreciate it if someone could help me try to convert/have the same functionality as my previous Cloud Function, but works for Firestore and incorporates the new path. All help is greatly appreciated
Edit: This is what I was able to try and figure out on my own. I understand that I have to query the collection based off of the seconds, and from there see if it's past 24 hours, then delete the document, but I'm unsure on the syntax on how to do it. In the compiler I get the error "Unexpected token query aslant [28, 25]"
exports.deleteOldFirestoreMessages = functions.firestore.document('Message/{userId}/{chatRoomId}/{messageId}').onWrite((change, context) => {
var now = Date().now
var cutoff = (now - CUT_OFF_TIME) / 1000;
const query = admin.firestore().collection('Message/{userId}/{chatRoomId}').where('seconds', '>=', cutoff)
const snapshots = await query.get();
const batch = firestore.batch();
snapshots.forEach(v => batch.delete(v.ref));
await batch.commit();
});
These are the resources I've tried looking through:
Firebase Cloud Functions - Delete Entries where timestamp field is over a month old and run daily
Firebase - Cloud Functions - make queries on collection
Extend Cloud Firestore with Cloud Functions
Delete firebase data older than 2 hours
I have a firebase function that deletes old messages after 24 hours as in my old question here. I now have just the messageIds stored in an array under the user such that the path is: /User/objectId/myMessages and then an array of all the messageIds under myMessages. All of the messages get deleted after 24 hours, but the iDs under the user's profile stay there. Is there a way to continue the function so that it also deletes the messageIds from the array under the user's account?
I'm new to Firebase functions and javascript so I'm not sure how to do this. All help is appreciated!
Building upon #frank-van-puffelen's accepted answer on the old question, this will now delete the message IDs from their sender's user data as part of the same atomic delete operation without firing off a Cloud Function for every message deleted.
Method 1: Restructure for concurrency
Before being able to use this method, you must restructure how you store entries in /User/someUserId/myMessages to follow best practices for concurrent arrays to the following:
{
"/User/someUserId/myMessages": {
"-Lfq460_5tm6x7dchhOn": true,
"-Lfq483gGzmpB_Jt6Wg5": true,
...
}
}
This allows you to modify the previous function to:
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 2 Hours in milliseconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomId}').onWrite(async (change) => {
const rootRef = admin.database().ref(); // needed top level reference for multi-path update
const now = Date.now();
const cutoff = (now - CUT_OFF_TIME) / 1000; // convert to seconds
const oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
const updates = {};
snapshot.forEach(messageSnapshot => {
let senderId = messageSnapshot.child('senderId').val();
updates['Message/' + messageSnapshot.key] = null; // to delete message
updates['User/' + senderId + '/myMessages/' + messageSnapshot.key] = null; // to delete entry in user data
});
// execute all updates in one go and return the result to end the function
return rootRef.update(updates);
});
Method 2: Use an array
Warning: This method falls prey to concurrency issues. If a user was to post a new message during the delete operation, it's ID could be removed while evaluating the deletion. Use method 1 where possible to avoid this.
This method assumes your /User/someUserId/myMessages object looks like this (a plain array):
{
"/User/someUserId/myMessages": {
"0": "-Lfq460_5tm6x7dchhOn",
"1": "-Lfq483gGzmpB_Jt6Wg5",
...
}
}
The leanest, most cost-effective, anti-collision function I can come up for this data structure is the following:
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 2 Hours in milliseconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomId}').onWrite(async (change) => {
const rootRef = admin.database().ref(); // needed top level reference for multi-path update
const now = Date.now();
const cutoff = (now - CUT_OFF_TIME) / 1000; // convert to seconds
const oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
const updates = {};
const messagesByUser = {};
snapshot.forEach(messageSnapshot => {
updates['Message/' + messageSnapshot.key] = null; // to delete message
// cache message IDs by user for next step
let senderId = messageSnapshot.child('senderId').val();
if (!messagesByUser[senderId]) { messagesByUser[senderId] = []; }
messagesByUser[senderId].push(messageSnapshot.key);
});
// Get each user's list of message IDs and remove those that were deleted.
let pendingOperations = [];
for (let [senderId, messageIdsToRemove] of Object.entries(messagesByUser)) {
pendingOperations.push(admin.database.ref('User/' + senderId + '/myMessages').once('value')
.then((messageArraySnapshot) => {
let messageIds = messageArraySnapshot.val();
messageIds.filter((id) => !messageIdsToRemove.includes(id));
updates['User/' + senderId + '/myMessages'] = messageIds; // to update array with non-deleted values
}));
}
// wait for each user's new /myMessages value to be added to the pending updates
await Promise.all(pendingOperations);
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});
Update: DO NOT USE THIS ANSWER (I will leave it as it may still be handy for detecting a delete operation for some other need, but do not use for the purpose of cleaning up an array in another document)
Thanks to #samthecodingman for providing an atomic and concurrency safe answer.
If using Firebase Realtime Database you can add an onChange event listener:
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
exports.onDeletedMessage = functions.database.ref('Message/{messageId}').onChange(async event => {
// Exit if this item exists... if so it was not deleted!
if (event.data.exists()) {
return;
}
const userId = event.data.userId; //hopefully you have this in the message document
const messageId = event.data.messageId;
//once('value') useful for data that only needs to be loaded once and isn't expected to change frequently or require active listening
const myMessages = await functions.database.ref('/users/' + userId).once('value').snapshot.val().myMessages;
if(!myMessages || !myMessages.length) {
//nothing to do, myMessages array is undefined or empty
return;
}
var index = myMessages.indexOf(messageId);
if (index === -1) {
//nothing to delete, messageId is not in myMessages
return;
}
//removeAt returns the element removed which we do not need
myMessages.removeAt(index);
const vals = {
'myMessages': myMessages;
}
await admin.database.ref('/users/' + userId).update(vals);
});
If using Cloud Firestore can add an event listener on the document being deleted to handle cleanup in your user document:
exports.onDeletedMessage = functions.firestore.document('Message/{messageId}').onDelete(async event => {
const data = event.data();
if (!data) {
return;
}
const userId = data.userId; //hopefully you have this in the message document
const messageId = data.messageId;
//now you can do clean up for the /user/{userId} document like removing the messageId from myMessages property
const userSnapShot = await admin.firestore().collection('users').doc(userId).get().data();
if(!userSnapShot.myMessages || !userSnapShot.myMessages.length) {
//nothing to do, myMessages array is undefined or empty
return;
}
var index = userSnapShot.myMessages.indexOf(messageId);
if (index === -1) {
//nothing to delete, messageId is not in myMessages
return;
}
//removeAt returns the element removed which we do not need
userSnapShot.myMessages.removeAt(index);
const vals = {
'myMessages': userSnapShot.myMessages;
}
//To update some fields of a document without overwriting the entire document, use the update() method
await admin.firestore().collection('users').doc(userId).update(vals);
});
I'm trying to delete multiple nodes on my database that are older than 12hrs. I"m using a pub/sub function to trigger this event. I don't know if my code is actually looping through all nodes as I'm not using the onWrite, onCreate database triggers on specific. Here is the image sample of the database
this is the pub/sub code
exports.deletejob = functions.pubsub.topic('Oldtask').onPublish(() => {
deleteOldItem();
})
and the deleteOldItem function
function deleteOldItem(){
const CUT_OFF_TIME = 12 * 60 * 1000; // 12 Hours in milliseconds.
//var ref = admin.database().ref(`/articles/${id}`);
const ref = admin.database().ref(`/articles`);
const updates = {};
ref.orderByChild('id').limitToLast(100).on('value', function (response) {
var index = 0;
response.forEach(function (child) {
var element = child.val();
const datetime = element.timestamp;
const now = Date.now();
const cutoff = now - datetime;
if (CUT_OFF_TIME < cutoff){
updates[element.key] = null;
}
});
//This is supposed to be the returened promise
return ref.child(response.key).update(updates);
});
If there's something I'm doing wrong, I'll like to know. The pub/sub is triggered with a JobScheduler already setup on google cloud scheduler
You had several problems in your code that were giving you trouble.
The handling of promises wasn't correct. In particular, your top level function never actually returned a promise, it just called deleteOldItems().
You should use the promise form of once() instead of calling on() with a callback since you don't want to install a listener in this case, you just need the result a single time, and you want to handle it as part of a promise chain.
To delete nodes, you should call remove() on a reference to that node. It also generates a promise for you to use here.
You didn't calculate 12 hours in milliseconds properly, you calculated 12 minutes in milliseconds :)
Here's what I came up with. It uses an http function instead of a pubsub function as well as adding a log statement for my testing, but the modification you need should be trivial/obvious (just change the prototype and remove the response after deleteOldItems, but do make sure you keep returning the result of deleteOldItems()):
const functions = require('firebase-functions');
const admin = require('firebase-admin');
function deleteOldItems() {
const CUT_OFF_TIME = 12 * 60 * 60 * 1000; // 12 Hours in milliseconds.
const ref = admin.database().ref('/articles');
return ref.orderByChild('id').limitToLast(100).once('value')
.then((response) => {
const updatePromises = [];
const now = Date.now();
response.forEach((child) => {
const datetime = child.val().timestamp;
const cutoff = now - datetime;
console.log(`processing ${datetime} my cutoff is ${CUT_OFF_TIME} and ${cutoff}`);
if (CUT_OFF_TIME < cutoff){
updatePromises.push(child.ref.remove())
}
});
return Promise.all(updatePromises);
});
}
exports.doIt = functions.https.onRequest((request, response) => {
return deleteOldItems().then(() => { return response.send('ok') });
}
While I have not tested it, I'm pretty sure this will work to include inside your original function call for cloud scheduler:
exports.deletejob = functions.pubsub.topic('Oldtask').onPublish(() => {
return deleteOldItems();
})
Of course, this is still more complicated than you need, since ordering by id doesn't really gain you anything here. Instead, why not just use the query to return the earliest items before the cut off time (e.g. exactly the ones you want to remove)? I've also switched to limitToFirst to ensure the earliest entries get thrown out, which seems more natural and ensures fairness:
function deleteOldItems() {
const cutOffTime = Date.now() - (12 * 60 * 60 * 1000); // 12 Hours earlier in milliseconds.
const ref = admin.database().ref('/articles');
return ref.orderByChild('timestamp').endAt(cutOffTime).limitToFirst(100).once('value')
.then((response) => {
const updatePromises = [];
response.forEach((child) => {
updatePromises.push(child.ref.remove())
});
return Promise.all(updatePromises);
});
}
If you do this on more than a few items, of course, you probably want to add an index on the timestamp field so the range query is more efficient.
i have bunch of comments in Firebase database and i want to do some updates to the comments via Cloud Function ( this is simplified example, i will be doing some logic which does require Cloud Function ).
What i need to do is go through all the comments in the database, adjust its rating node and then update the database with adjusted comments.
I spent a lot of time researching this, but i am completely new to Cloud Functions, so i have realy hard time figuring this out.
I am assuming i want to store all the changes to all the comments (there can be thousands of them) in the array or object and then do the update at one time instead of for each comment separately ?
Btw this code is not working, i am assuming the array and return is completely wrong.
exports.increaseRating = functions.database.ref('/comments/')
.onUpdate((snapshot) => {
var updates = [];
snapshot.before.forEach((element) => {
var comment = element.val();
comment.rating += 1000;
updates.push(comment);
});
return updates;
})
Code i am using to update one entry. I need to do the same thing for all the comments at one time.
exports.increaseRating = functions.database.ref('/comments/{commentId}')
.onUpdate((snapshot, context) => {
const comment = snapshot.before.val();
const newRating = comment.rating += 1000;
const now = new Date().getTime();
if (comment.lastUpdate) {
if (comment.lastUpdate > now - (30 * 1000)) {
return null;
}
}
return admin.database().ref(`/comments/${context.params.commentId}`).update({
"rating": newRating,
"lastUpdate": now
})
})
If you want to update all child nodes, you can do something like this:
var ref = firebase.database().ref("comments"); // or admin.database().ref("comments")
ref.once("value").then((snapshot) => {
var updates = {};
snapshot.forEach((commentSnapshot => {
var comment = commentSnapshot.val();
var newRating = comment.rating + 1000;
updates[commentSnapshot.key+"/rating"] = newRating;
});
ref.update(updates);
})
This performs a single multi-location update for all comments. Note that the performance benefit over performing separate updates is quite small, since Firebase pipelines the multiple requests over a single connection.
Also note that you should not put this in a Cloud Functions trigger on /comments, since that will lead to an endless loop: every time the comments get written, your function triggers, which updates the comments, which triggers the function again.
If you need this in Cloud Functions, you'll want to use a HTTP-triggered function, which is triggered by HTTP calls instead of database writes.
exports.updateCommentRatings = functions.https.onRequest((req, res) => {
var ref = admin.database().ref("comments")
ref.once("value").then((snapshot) => {
var updates = {};
snapshot.forEach((commentSnapshot => {
var comment = commentSnapshot.val();
var newRating = comment.rating + 1000;
updates[commentSnapshot.key+"/rating"] = newRating;
});
ref.update(updates).then(() => {
res.status(200).send("Comment ratings updated");
});
})
})
You can then periodically call this URL/function with a service like cron-job.org. For more on this see Cloud Functions for Firebase trigger on time?.
I'm trying Firebase Cloud Functions in my app using this code to remove data after 2 hours of created.
exports.deleteOldItems = functions.database.ref('/Rooms/{pushId}')
.onWrite(event => {
var ref = event.data.ref.parent; // reference to the items
var now = Date.now();
var cutoff = now - 2 * 60 * 60 * 1000;
var oldItemsQuery = ref.orderByChild('timestampCreated/timestamp').endAt(cutoff);
return oldItemsQuery.once('value', function(snapshot) {
// create a map with all children that need to be removed
var updates = {};
snapshot.forEach(function(child) {
updates[child.key] = null
});
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});
});
This works. Now I want to write in another ref (for example: /Users/{userID}/) every time data is deleted. Regards
Depending on whether you want the update to run as the current user or as an administrator, you can use event.data.ref or event.data.adminRef and work from there:
exports.deleteOldItems = functions.database.ref('/Rooms/{pushId}')
.onWrite(event => {
...
var ref = event.data.ref.root;
return ref.child("/Users/123").set("New value");
});
things have changed on version 1.0, adminRef is deprecated, you should use just ref for admin access and event has been substituted by snapshot and context, see here: cloud functions documentation 1.0 API changes
Frank's example becomes:
exports.deleteOldItems = functions.database.ref('/Rooms/{pushId}')
.onWrite((snapshot,context) => {
...
var ref = snapshot.ref.root;
return ref.child("/Users/123").set("New value");
});