Develop Reference

Delete same value from multiple locations Firebase Functions

I have a firebase function that deletes old messages after 24 hours as in my old question here. I now have just the messageIds stored in an array under the user such that the path is: /User/objectId/myMessages and then an array of all the messageIds under myMessages. All of the messages get deleted after 24 hours, but the iDs under the user's profile stay there. Is there a way to continue the function so that it also deletes the messageIds from the array under the user's account?
I'm new to Firebase functions and javascript so I'm not sure how to do this. All help is appreciated!

Building upon #frank-van-puffelen's accepted answer on the old question, this will now delete the message IDs from their sender's user data as part of the same atomic delete operation without firing off a Cloud Function for every message deleted.
Method 1: Restructure for concurrency
Before being able to use this method, you must restructure how you store entries in /User/someUserId/myMessages to follow best practices for concurrent arrays to the following:
{
"/User/someUserId/myMessages": {
"-Lfq460_5tm6x7dchhOn": true,
"-Lfq483gGzmpB_Jt6Wg5": true,
...
}
}
This allows you to modify the previous function to:
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 2 Hours in milliseconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomId}').onWrite(async (change) => {
const rootRef = admin.database().ref(); // needed top level reference for multi-path update
const now = Date.now();
const cutoff = (now - CUT_OFF_TIME) / 1000; // convert to seconds
const oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
const updates = {};
snapshot.forEach(messageSnapshot => {
let senderId = messageSnapshot.child('senderId').val();
updates['Message/' + messageSnapshot.key] = null; // to delete message
updates['User/' + senderId + '/myMessages/' + messageSnapshot.key] = null; // to delete entry in user data
});
// execute all updates in one go and return the result to end the function
return rootRef.update(updates);
});
Method 2: Use an array
Warning: This method falls prey to concurrency issues. If a user was to post a new message during the delete operation, it's ID could be removed while evaluating the deletion. Use method 1 where possible to avoid this.
This method assumes your /User/someUserId/myMessages object looks like this (a plain array):
{
"/User/someUserId/myMessages": {
"0": "-Lfq460_5tm6x7dchhOn",
"1": "-Lfq483gGzmpB_Jt6Wg5",
...
}
}
The leanest, most cost-effective, anti-collision function I can come up for this data structure is the following:
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 2 Hours in milliseconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomId}').onWrite(async (change) => {
const rootRef = admin.database().ref(); // needed top level reference for multi-path update
const now = Date.now();
const cutoff = (now - CUT_OFF_TIME) / 1000; // convert to seconds
const oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
const updates = {};
const messagesByUser = {};
snapshot.forEach(messageSnapshot => {
updates['Message/' + messageSnapshot.key] = null; // to delete message
// cache message IDs by user for next step
let senderId = messageSnapshot.child('senderId').val();
if (!messagesByUser[senderId]) { messagesByUser[senderId] = []; }
messagesByUser[senderId].push(messageSnapshot.key);
});
// Get each user's list of message IDs and remove those that were deleted.
let pendingOperations = [];
for (let [senderId, messageIdsToRemove] of Object.entries(messagesByUser)) {
pendingOperations.push(admin.database.ref('User/' + senderId + '/myMessages').once('value')
.then((messageArraySnapshot) => {
let messageIds = messageArraySnapshot.val();
messageIds.filter((id) => !messageIdsToRemove.includes(id));
updates['User/' + senderId + '/myMessages'] = messageIds; // to update array with non-deleted values
}));
}
// wait for each user's new /myMessages value to be added to the pending updates
await Promise.all(pendingOperations);
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});

Update: DO NOT USE THIS ANSWER (I will leave it as it may still be handy for detecting a delete operation for some other need, but do not use for the purpose of cleaning up an array in another document)
Thanks to #samthecodingman for providing an atomic and concurrency safe answer.
If using Firebase Realtime Database you can add an onChange event listener:
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
exports.onDeletedMessage = functions.database.ref('Message/{messageId}').onChange(async event => {
// Exit if this item exists... if so it was not deleted!
if (event.data.exists()) {
return;
}
const userId = event.data.userId; //hopefully you have this in the message document
const messageId = event.data.messageId;
//once('value') useful for data that only needs to be loaded once and isn't expected to change frequently or require active listening
const myMessages = await functions.database.ref('/users/' + userId).once('value').snapshot.val().myMessages;
if(!myMessages || !myMessages.length) {
//nothing to do, myMessages array is undefined or empty
return;
}
var index = myMessages.indexOf(messageId);
if (index === -1) {
//nothing to delete, messageId is not in myMessages
return;
}
//removeAt returns the element removed which we do not need
myMessages.removeAt(index);
const vals = {
'myMessages': myMessages;
}
await admin.database.ref('/users/' + userId).update(vals);
});
If using Cloud Firestore can add an event listener on the document being deleted to handle cleanup in your user document:
exports.onDeletedMessage = functions.firestore.document('Message/{messageId}').onDelete(async event => {
const data = event.data();
if (!data) {
return;
}
const userId = data.userId; //hopefully you have this in the message document
const messageId = data.messageId;
//now you can do clean up for the /user/{userId} document like removing the messageId from myMessages property
const userSnapShot = await admin.firestore().collection('users').doc(userId).get().data();
if(!userSnapShot.myMessages || !userSnapShot.myMessages.length) {
//nothing to do, myMessages array is undefined or empty
return;
}
var index = userSnapShot.myMessages.indexOf(messageId);
if (index === -1) {
//nothing to delete, messageId is not in myMessages
return;
}
//removeAt returns the element removed which we do not need
userSnapShot.myMessages.removeAt(index);
const vals = {
'myMessages': userSnapShot.myMessages;
}
//To update some fields of a document without overwriting the entire document, use the update() method
await admin.firestore().collection('users').doc(userId).update(vals);
});

How to increase your limit of Github API uses per hour in Javascript

I'm trying to work pull requests, issues, and commits with repos and I have the following code:
const axios = require('axios');
var gitPullApiLink = "https://api.github.com/repos/elixir-lang/elixir/pulls";
var listOfCommits = [];
var listOfSHAs = [];
var mapOfInfoObjects = new Map();
var mapPullRequestNumberToCommits = new Map();
var mapPRNumbersToCommitObjects = new Map();
var listOfPrObjects = [];
var setOfFileObjects = new Set();
var listOfNumbersOfTargetedIssues = [];
var mapPRnumberToCloseOpenDateObjects = new Map();
class PullRequestParser {
async getListOfPullRequests(pullrequestLink) {
const message = await axios.get(pullrequestLink);
//console.log(message);
listOfPrObjects = message['data'];
}
async getCommitsForEachPullRequestAndPRinformation() {
var listOfPrNumbers = [];
var k;
// this loop will just make a list of Pull Request Numbers
for (k = 0; k < listOfPrObjects.length; k++){
var currPrNumber = listOfPrObjects[k]['number'];
listOfPrNumbers.push(currPrNumber);
}
// I created a separate list just because... I did it this way because on the github API website it seems
// like the pull request has the same number as the issue it affects. I explain how you can see this down below
listOfNumbersOfTargetedIssues = listOfPrNumbers;
// next loop will make objects that contain information about each pull request.
var n;
for (n = 0; n < listOfPrNumbers; n++){
var ApiLinkForEachPullRequest = gitPullApiLink + "/" + listOfPrNumbers[n];
const mes = await axios.get(ApiLinkForEachPullRequest);
var temp = {OpeningDate: mes['data']['created_at'],
ClosingDate: mes['data']['closed_at'],
IssueLink: mes['data']['_links']['issue']['href']};
//mapPRnumberToCloseOpenDateObjects will be a map where the key is the pull request number and the value
// is the object that stores the open date, close date, and issue link for that pull request. The reason
// why I said I think the pull request number is the same as the number of the issue it affects is because
// if you take any object from the map, say you do mapPRnumberToCloseOpenDateObjects.get(10). You'll
// get an object with a pull request number 10. Now if you take this object and look at it's "IssueLink"
// field, the very last part of the link will have the number 10, and if you look at the github API
// it says for a single issue, you do: /repos/:owner/:repo/issues/:issue_number <---- As you can see,
// the IssueLink field will have this structure and in place of the issue_number, the field will be 10
// for our example object.
mapPRnumberToCloseOpenDateObjects.set(listOfPrNumbers[n], temp);
}
//up to this point, we have the pull request numbers. we will now start getting the commits associated with
//each pull request
var j;
for (j = 0; j < listOfPrNumbers.length; j++){
var currentApiLink = gitPullApiLink + "/" + listOfPrNumbers[j] + "/commits";
const res = await axios.get(currentApiLink);
//here we map a single pull request to the information containing the commits. I'll just warn you in
// advance: there's another object called mapPRNumbersToCommitObjects. THIS MAP IS DIFFERENT! I know it's
// subtle, but I hope the language can make the distinction: mapPullRequestNumberToCommits will just
// map a pull request number to some data about the commits it's linked to. In contrast,
// mapPRNumbersToCommitObjects will be the map that actually maps pull request numbers to objects
// containing information about the commits a pull request is associated with!
mapPullRequestNumberToCommits.set(listOfPrNumbers[j], res['data']);
}
// console.log("hewoihoiewa");
}
async createCommitObjects(){
var x;
// the initial loop using x will loop over all pull requests and get the associated commits
for (x = 0; x < listOfPrObjects.length; x++){
//here we will get the commits
var currCommitObjects = mapPullRequestNumberToCommits.get(listOfPrObjects[x]['number']);
//console.log('dhsiu');
// the loop using y will iterate over all commits that we get from a single pull request
var y;
for (y = 0; y < currCommitObjects.length; y++){
var currentSHA = currCommitObjects[y]['sha'];
listOfSHAs.push(currentSHA);
var currApiLink = "https://api.github.com/repos/elixir-lang/elixir/commits/" + currentSHA;
const response = await axios.get(currApiLink,);
//console.log("up to here");
// here we start extracting some information from a single commit
var currentAuthorName = response['data']['commit']['committer']['name'];
var currentDate = response['data']['commit']['committer']['date'];
var currentFiles = response['data']['files'];
// this loop will iterate over all changed files for a single commit. Remember, every commit has a list
// of changed files, so this loop will iterate over all those files, get the necessary information
// from those files.
var z;
// we create this temporary list of file objects because for every file, we want to make an object
// that will store the necessary information for that one file. after we store all the objects for
// each file, we will add this list of file objects as a field for our bigger commit object (see down below)
var tempListOfFileObjects = [];
for (z = 0; z < currentFiles.length; z++){
var fileInConsideration = currentFiles[z];
var nameOfFile = fileInConsideration['filename'];
var numberOfAdditions = fileInConsideration['additions'];
var numberOfDeletions = fileInConsideration['deletions'];
var totalNumberOfChangesToFile = fileInConsideration['changes'];
//console.log("with file");
var tempFileObject = {fileName: nameOfFile, totalAdditions: numberOfAdditions,
totalDeletions: numberOfDeletions, numberOfChanges: totalNumberOfChangesToFile};
// we add the same file objects to both a temporary, local list and a global set. Don't be tripped
// up by this; they're doing the same thing!
setOfFileObjects.add(tempFileObject);
tempListOfFileObjects.push(tempFileObject);
}
// here we make an object that stores information for a single commit. sha, authorName, date are single
// values, but files will be a list of file objects and these file objects will store further information
// for each file.
var tempObj = {sha: currentSHA, authorName: currentAuthorName, date: currentDate, files: tempListOfFileObjects};
var currPrNumber = listOfPrObjects[x]['number'];
console.log(currPrNumber);
// here we will make a single pull request number to an object that will contain all the information for
// every single commit associated with that pull request. So for every pull request, it will map to a list
// of objects where each object stores information about a commit associated with the pull request.
mapPRNumbersToCommitObjects.set(currPrNumber, tempObj);
}
}
return mapPRNumbersToCommitObjects;
}
async startParsingPullRequests() {
this.getListOfPullRequests(gitPullApiLink + "?state=all").then(() => {
this.getCommitsForEachPullRequestAndPRinformation().then(() => {
this.createCommitObjects().then((response) => {
console.log("functions were successful");
return new mapPRNumbersToCommitObjects;
//return mapPRNumbersToCommitObjects;
}).catch((error) => {
console.log("printing first error");
console.log(error);
})
}).catch((error2) => {
console.log("printing the second error");
console.log(error2);
})
}).catch((error3) => {
console.log("printing the third error");
console.log(error3);
});
}
//adding some getter methods so they can be used to work with whatever information people may need.
//I start all of them with the this.startParsingPullRequests() method because by calling that method
it gets all
// the information for the global variables.
async getSetOfFileObjects(){
var dummyMap = await this.startParsingPullRequests();
return {files: setOfFileObjects, prMap: mapPRnumberToCloseOpenDateObjects};
}
async OpenCloseDateObjects(){
var dummyMap = await this.startParsingPullRequests();
return mapPRnumberToCloseOpenDateObjects;
}
async getNumbersOfTargetedIssues(){
var dummyMap = await this.startParsingPullRequests();
return listOfNumbersOfTargetedIssues;
}
}
var dummy = new PullRequestParser();
var dummyMap = dummy.startParsingPullRequests().then((message) => {
console.log("dummyMap is defined! :)");
console.log(dummyMap);
});
module.exports = PullRequestParser;
Whenever I run the code on the webstorm terminal though, with:
node PullRequestParser.js
I get a 403 error, followed by a bunch of error output, with the following statement:
data: {
message: "API rate limit exceeded for 138.186.17.173. (But here's the good news: Authenticated
requests get a higher rate limit. Check out the documentation for more details.)"
I looked up the documentation for this and found out that without authentication, I can make 60 requests per hour to a repo. In order to get authentication, however, the only example provided is an example they provide by using the command line. I don't think this would be enough though because I want to do some further analysis with the results I get. Does anybody know how I can increase the number of requests I can make? Where in the code would I need to make changes and what kind of changes would I need to make? Thanks!

The first line of the documentation says everything you need to know.
For API requests using Basic Authentication or OAuth, you can make up
to 5000 requests per hour.
Using Basic Authentication is pretty simple, so that may be the easiest thing to get you up and running. OAuth is more complicated, but more desirable in production.
The axios library supports basic auth requests out of the box.
async getListOfPullRequests(pullrequestLink) {
const message = await axios.get(pullrequestLink, {
auth: {
username: 'username',
password: 'password'
}
});
//console.log(message);
listOfPrObjects = message['data'];
}
You just need to supply the correct username and password information.

How to pass Google/sheetsAPI data to website?

Sooooo,
I want to get data from a google sheet to a website which appends the passed data dynamically to the website so the amount of itme's on the website is equal to the amount of rows. I pretty much got everything done separately but I just cant seem to figure out how to combine them.
I set up a node.js project which uses my Api key to retrieve the data, manipulate and push it into an array.
also maybe there's a more efficient way to push the data but i couldn't figure anything else out but saving it to a file and then reading it back. I then pumped that into a AWS api gateway+Lambda funtion which didn't really work because I couldn't get Lambda to require google/api but i think i could sort that out.
I also already did all the HTML / JS on the website to push the data to the page I just cant seem to figure out how to get the data from my AWS-http-link into my website. No matter if I use async or promise or none of both It just can't get the data.
'ISSUES: using google/api inside Lambda, getting data from Api to array, maybe diffrent way to get the data from google api'
Thank you in advance!
AWS-link: https://po7bu16g2i.execute-api.eu-central-1.amazonaws.com/live/celldata
//JS code on website
var receivearray =[];
function createEntry(data){
for(i=0;i<data.length;i++){
var outdt = document.createElement('div');
outdt.className='outwrap';
var pers_name = document.createElement('div');
pers_name.className='name';
var pers_name_txt= document.createElement('h3');
pers_name_txt.className='name_head';
pers_name_txt.innerHTML=data[i][0];
outdt.append(pers_name);
pers_name.append(pers_name_txt);
for(k=1;k<data[i].length;k++){
var top =document.createElement('div');
top.className='text_wrap';
var q = document.createElement('p');
q.className='frage';
q.innerHTML=data[i][k][0];
var a= document.createElement('p');
a.innerHTML=data[i][k][1];
outdt.append(top);
top.append(q,a);
}
document.body.appendChild(outdt);
}
};
createEntry(receivearray);
//Node.js Code
//requirements
var GoogleSpreadsheet = require('google-spreadsheet');
var crds = require('./credt.json');
const fs = require('fs');
//updates the data-tfile
function update(sheetid){
//google api shapens
var doc = new GoogleSpreadsheet(sheetid);
doc.useServiceAccountAuth(crds,
function (err) {
doc.getRows(1,
function (err, rows) {
//saves all retrieved data into textfile
fs.writeFile('ting.txt', JSON.stringify(rows),
function (err) {
//checks if file was saved succesfully
if (err) throw err;
console.log('updated')
}
)
}
)
}
)
}
function prepdata(){
//define variables
var text = fs.readFileSync("ting.txt", "utf8");
var findata =[];
var hold=[];
//singleout all row-elements
for(x=0;x<text.length;x++){
if(text.slice(x,x+7)=='"name":'){
hold.push(text.slice(x+7,x+120));
}
}
//split data by , into sets
for(x=0;x<hold.length;x++){
findata.push(hold[x].split(','));
}
//push sets into arrays
for (var k = 0; k < findata.length; k++){
for (var i = 0; i < findata[k].length; i++){
findata[k][i] = findata[k][i].replace(/"/g, " ");
findata[k][i] = findata[k][i].split(':');
}
}
return findata;
}
update("1W6tVuj0krrwI7PyTRJha2ZOX72kGGfFAI8eqXOirWHo");
console.log(prepdata());

How can I `put` new values in an already existing objectStore in an already existing indexedDB?

I am tying myself up in knots trying to update a series of four entries in an objectStore in an indexedDB.
This is what I want to achieve (in pseudo-code):
let myDatabase = indexedDB('myDatabase', 1);
let myObjectStore = myDatabase.myObjectStore;
myObjectStore.entry1 = 'newValue1';
myObjectStore.entry2 = 'newValue2';
myObjectStore.entry3 = 'newValue3';
myObjectStore.entry4 = 'newValue4';
But of course, it isn't anything like that straightforward.
I understand I need to use put. But, despite numerous attempted approaches, I can't get further than that.
I have got as far as successfully setting up and populating the objectStore in the first place when the indexedDB is first created:
// SET UP VALUES OBJECT
let valuesObject = {
entry1 : 'a',
entry2 : 'b',
entry3 : 'c',
entry4 : 'd'
};
// SET UP INDEXED DATABASE
const setUpIndexedDatabase = (valuesObject) => {
let database
const databaseVersion = 1;
const databaseName = \'myDatabase\';
const databaseOpenRequest = indexedDB.open(databaseName, databaseVersion);
databaseOpenRequest.onupgradeneeded = () => {
database = databaseOpenRequest.result;
let myObjectStore = database.createObjectStore('myObjectStore');
myObjectStore.transaction.oncomplete = () => {
let objectStoreValues = database.transaction('Values', 'readwrite').objectStore('Values');
const valuesEntries = Object.entries(valuesObject);
for (let i = 0; i < valuesEntries.length; i++) {
objectStoreValues.add(valuesEntries[i][1], valuesEntries[i][0]);
}
}
}
databaseOpenRequest.onsuccess = () => {
database = databaseOpenRequest.result;
// >>> THIS IS THE BIT THAT I NEED TO WRITE <<<
database.close();
}
}
setUpIndexedDatabase(valuesObject);
So far, so good. The code above fires the onupgradeneeded event if no database exists yet, which creates myObjectStore and populates it with four key-value pairs.
But if the database does exist and already contains myObjectStore, then every variation of code I have written using put fails to update the values for the keys and returns various errors - and quite often no errors at all.
All I want to do is update values in the database.
I think the problem is that I don't know how to use put properly when the Database Version remains unchanged and onupgradeneeded doesn't fire.

If you want to update an already existing value in the database, you can do so with the following code (as example, I am updating the entry1 entry):
databaseOpenRequest.onsuccess = function(event) {
db = event.target.result;
const objectStore = db.transaction('myObjectStore', 'readwrite').objectStore('myObjectStore');
const request = objectStore.put('e', 'entry1');
request.onerror = function(event) {
// There was an error while updating.
};
request.onsuccess = function(event) {
// The update was successful.
};
}

asychronous function in loop javascript nodejs

I need to scan the trip array and calculate the travel time between the current trip with each trip in the array and select the shortest one. For calculation i need to send google maps api call.
I am very confused about the asynchronous callback function .
Can anyone help me on this how to send api call within for loop and check the results and continue?
Thank you.
The trips are in my array list;
Array :
array=[trip1,trip2, trip3,....];
JS :
function assigntrips(array){
var triplist = [];
for(var i=0; i< array.length; i++){
var fstnode = array[i];
for(var j=i+1; j<array.length; j++){
//here i want to get the response from google api and decide if i want to choose the trip.
if not the for loop continues and send another api call.
}
}
}
function apicall(inputi, cb){
var destination_lat = 40.689648;
var destination_long = -73.981440;
var origin_lat = array[inputi].des_lat;
var origin_long = array[inputi].des_long;
var departure_time = 'now';
var options = {
host: 'maps.googleapis.com',
path: '/maps/api/distancematrix/json?origins='+ origin_lat +','+origin_long+ '&destinations=' + office_lat + ',' + office_long + '&mode=TRANSIT&departure_time=1399399424&language=en-US&sensor=false'
}
http.get(options).on('response',function(response){
var data = '';
response.on('data',function(chunk){
data += chunk;
});
response.on('end',function(){
var json = JSON.parse(data);
console.log(json);
var ttltimereturnoffice = json.rows[0].elements[0].duration.text;
//var node = new Node(array[i],null, triptime,0,ttltimereturnoffice,false);
//tripbylvtime.push(node);
cb(ttltimereturnoffice + '\t' + inputi);
});
});
}

You cannot check the results in the loop. The loop is in the past, the callbacks happen in the future - you can't change that. There are only two things you can do, and one is an abstraction of the other:
1) You can create your callback in such a manner that it will collect the results and compare them when all are present.
2) You can use promises to do the same thing.
The #1 approach would look something like this (while modifying the cb call in your code appropriately):
var results = [];
function cb(index, ttltimereturnoffice) {
results.push([index, ttltimereturnoffice]);
if (results.length == array.length) {
// we have all the results; find the best one, display, do whatever
}
}
I don't quite know what library you are using, and if it supports promises, but if http.get returns a promise, you can do #2 by collecting the promises into an array, then using the promise library's all or when or similar to attach a callback on all gets being done.