Develop Reference

How can i speed up firebase functions in an express app which are taking too long to load taking more than 8 seconds most times

We have an application that uses a nodejs and express backend powered by firebase but one problem that we're facing is very long load times even for simple queries. We refractored our endpoints into different files so that they all don't run all at once and we also have cron jobs which i suspected would help with cold starts in which i'm convinced and doubt we experiencing as all the requests including subsquent req are just slow and they really take at most times more than 8 seconds which is a poor perfomance for users to experience.
in our package.json this what we have in order for you to see the firebase versions we using
{
"name": "functions",
"description": "Cloud Functions for Firebase",
"scripts": {
"serve": "firebase serve --only functions",
"shell": "firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log"
},
"engines": {
"node": "14"
},
"dependencies": {
"#google-cloud/storage": "^5.8.2",
"#sendgrid/mail": "^7.2.1",
"algoliasearch": "^4.3.0",
"bcrypt": "^5.1.0",
"busboy": "^0.3.1",
"cookie-parser": "^1.4.5",
"cors": "^2.8.5",
"dayjs": "^1.10.4",
"dotenv": "^8.2.0",
"easy-soap-request": "^4.1.3",
"express": "^4.17.1",
"firebase": "^7.15.5",
"firebase-admin": "^8.6.0",
"firebase-functions": "^3.23.0",
"fs-extra": "^9.0.1",
"jwt-decode": "^2.2.0",
"moment": "^2.29.1",
"request": "^2.88.2",
"sharp": "^0.25.4",
"sib-api-v3-sdk": "^8.4.2",
"uuid": "^8.2.0",
"xml-js": "^1.6.11"
},
"devDependencies": {
"firebase-functions-test": "^0.1.6"
},
"private": true
}
below is the index.js file on how we set up everything.
require("dotenv").config();
const functions = require("firebase-functions");
const express = require("express");
const app = express();
const cookieParser = require("cookie-parser");
const cors = require("cors");
app.use(cors());
app.use(cookieParser());
app.use(express.json());
const dashboardRoutes = require("./routes/dashboardRoutes");
const userRoutes = require("./routes/userRoutes");
const pagesRoutes = require("./routes/pagesRoutes");
const orderRoutes = require("./routes/orderRoutes");
const cartRoutes = require("./routes/cartRoutes");
const wishlistRoutes = require("./routes/wishlistRoutes");
const authRoutes = require("./routes/authRoutes");
const storeRoutes = require("./routes/storeRoutes");
const createSellerRoutes = require("./routes/createSellerRoutes");
app.use("/", pagesRoutes);
app.use("/user", userRoutes);
app.use("/dash", dashboardRoutes);
app.use("/order", orderRoutes);
app.use("/cart", cartRoutes);
app.use("/wishlist", wishlistRoutes);
app.use("/auth", authRoutes);
app.use("/s", storeRoutes);
app.use("/cr", createSellerRoutes);
const {
cron_job1,
cron_job2,
cron_job3,
cron_job4,
} = require("./triggers/search_triggers_and_cron_jobs"); <-- not the name of the actual file
const { **other cron jobs** } = require("./cron-jobs");
const {
update_exchange_currency_rates,
} = require("./cron-jobs/currency_exchange_rates");
const { reset_product_visits } = require("./triggers/products");
const { Home } = require("./handlers/pages");
const { db } = require("./util/admin");
const { product_basic_obj } = require("./util/product_basic_obj");
exports.apis = functions.https.onRequest(app);
// this functionality below is the sample for the kind of executions we perfom in the endpoints in which we also experience slow execution times ven for a function in this file
app.get("/test-home", (req, res) => {
let content = {};
db.collection("products")
.where("status", "==", "active")
.orderBy("visited", "desc")
.limit(20)
.get()
.then((data) => {
content.popular_today = [];
data.forEach((x) => {
content.popular_today.push(product_basic_obj(x.data()));
});
return db
.collection("products")
.where("status", "==", "active")
.orderBy("todaysSales", "desc")
.limit(20)
.get();
})
.then((data) => {
content.hot_today = [];
data.forEach((x) => {
content.hot_today.push(product_basic_obj(x.data()));
});
return db.collection("departments").get();
})
.then((data) => {
content.departments = [];
data.forEach((x) => {
content.departments.push(x.data());
});
return db
.collection("departments")
.orderBy("products_sold_today", "desc")
.limit(6)
.get();
})
.then((data) => {
content.top_departments = [];
data.forEach((x) => {
content.top_departments.push(x.data());
});
return res.status(200).json(content);
});
});
//cron jobs
exports.cron_job1 = cron_job1;
exports.cron_job2 = cron_job2;
exports.cron_job3 = cron_job3;
exports.cron_job4 = cron_job4;
Upon executing an end point this is what shows in the consol and the in a deployed development we experience the same slow execution times which seems to be the average for all our executions
i functions: Beginning execution of "us-central1-apis"
⚠ Google API requested!
- URL: "https://www.googleapis.com/oauth2/v4/token"
- Be careful, this may be a production service.
i functions: Finished "us-central1-apis" in ~9s
i functions: Finished "us-central1-apis" in ~8s
i functions: Beginning execution of "us-central1-apis"
i functions: Finished "us-central1-apis" in ~7s
i functions: Beginning execution of "us-central1-apis"
i functions: Finished "us-central1-apis" in ~7s
i functions: Beginning execution of "us-central1-apis"
i functions: Finished "us-central1-apis" in ~7s
i functions: Beginning execution of "us-central1-apis"
i functions: Finished "us-central1-apis" in ~6s
i functions: Beginning execution of "us-central1-apis"
i functions: Finished "us-central1-apis" in ~7s
i functions: Beginning execution of "us-central1-apis"
i functions: Finished "us-central1-apis" in ~7s
How can i speed up the execution using the information above.
We tried breaking code into smaller files which couldn't work as we expected to get more faster excecutions and we also removed most of our 3rd part libraries but we failed to make a change. What could we do to bring down executions times futher down.

Your data loading strategy is sequential, Load 1 then Load2 then Load3, and in case none of the following Load depends on the result of previous Load - that aproach is not really effective and useful.
Instead - you can try to utilize Promise.all() to fire all that promises "in parallel".
Next issue - you are loading departments from firebase twice, actual departments and top_departments, and there is no need to load top_departments again due to all the data that is needed is already in departments, you only need to .sort and slice them (or its shallow copy [...departments]).
I'd try this approach:
// popular_today
function getPopularProductsByVisitedAsync() {
return db
.collection("products")
.where("status", "==", "active")
.orderBy("visited", "desc")
.limit(20)
.get()
.then((data) => {
return data.docs.map((x) => product_basic_obj(x.data()));
});
}
// hot_today
function getPopularProductsByTodaySalesAsync() {
return db
.collection("products")
.where("status", "==", "active")
.orderBy("todaysSales", "desc")
.limit(20)
.get()
.then((data) => {
return data.docs.map((x) => product_basic_obj(x.data()));
});
}
function getAllDepartmentsAsync() {
return db
.collection("departments")
.get()
.then((data) => data.docs.map((x) => x.data()));
}
app.get("/test-home", async (req, res) => {
const [popular_today, hot_today, departments] = await Promise.all([
getPopularProductsByVisitedAsync(),
getPopularProductsByTodaySalesAsync(),
getAllDepartmentsAsync()
]);
// TODO: Check asc/desc, im not sure, was not testing
const top_departments = [...departments]
.sort((a, b) => a.products_sold_today - b.products_sold_today)
.slice(0, 6);
const content = {
popular_today: popular_today,
hot_today: hot_today,
departments: departments,
top_departments: top_departments
};
return res.status(200).json(content);
});

Try to execute your requests in parallel on the index.js.
This optimization will provide some gains on the network request timings.

Firebase functions crashes while accessing firestore

Could you please find an error in following code?
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
exports.GetShort = functions.https.onRequest((request, response) => {
response.header("Access-Control-Allow-Origin", "*");
longURL = request.query.long
functions.logger.info("url is - " ,longURL)
SaveToDB(longURL)
})
function SaveToDB(link){
functions.logger.info("here")
admin.firestore().collection("url").where("urlNames","array_contains",link).get().then(
function(querySnapshot){
functions.logger.info("snap, " ,querySnapshot)
querySnapshot.forEach(function(doc) {
functions.logger.info("things : " ,doc.id, " => ", doc.data())
// doc.data() is never undefined for query doc snapshots
console.log(doc.id, " => ", doc.data());
});
}
) .catch(function(error) {
functions.logger.info("Error getting documents: ", error);
});
}
After hitting above function, firebase-functions logs displays logs till "here". After that it crashes without any more logs/stacktrace.
below is the contents of packages.json from functions directory.
{
"name": "functions",
"description": "Cloud Functions for Firebase",
"scripts": {
"lint": "eslint .",
"serve": "firebase emulators:start --only functions",
"shell": "firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log"
},
"engines": {
"node": "14"
},
"main": "index.js",
"dependencies": {
"firebase-admin": "^9.8.0",
"firebase-functions": "^3.14.1"
},
"devDependencies": {
"eslint": "^7.6.0",
"eslint-config-google": "^0.14.0",
"firebase-functions-test": "^0.2.0"
},
"private": true
}

I would kindly suggest you watch the 3 videos about "JavaScript Promises" from the Firebase video series to see how to manage the life cycle of a Cloud Function and the way to deal with calls to asynchronous methods.
In particular, for an HTTPS Cloud Function, you need to end it with send(), redirect(), or end().
So your code could be adapted as follows:
exports.GetShort = functions.https.onRequest((request, response) => {
response.header("Access-Control-Allow-Origin", "*");
const longURL = request.query.long;
functions.logger.info("url is - ", longURL)
SaveToDB(longURL)
.then(() => {
response.status(200).send('Saved to DB');
})
.catch(error => {
// See video series
response.status(500).send(error);
})
})
function SaveToDB(link) {
functions.logger.info("here")
return admin.firestore().collection("url").where("urlNames", "array_contains", link).get()
.then(querySnapshot => {
functions.logger.info("snap, ", querySnapshot)
querySnapshot.forEach(function (doc) {
functions.logger.info("things : ", doc.id, " => ", doc.data())
// doc.data() is never undefined for query doc snapshots
console.log(doc.id, " => ", doc.data());
// => Here, depending on your real functional requirements, you may need to use Promise.all()
});
return null;
}
).catch(function (error) {
functions.logger.info("Error getting documents: ", error);
// Throw an error
});
}

Well, I found the problem. Sometimes silly things make most of the noise around.
Instead of "array-contains", I wrote "array_contains".

Dialogflow fulfilment : agent.add() not returning response to webhook (twilio), works good with dialogflow console and google assistant

I'm creating a chatbot using dialogflow, firebase and twilio on Whatsapp. The interaction works well using the responses from the Intents and using the inline editor function, writes data to the firebase database too. But when I try to return the response from the fulfilment inline editor using the agent.add() method the responses are shown only in the dialogflow console interaction (and the google assistant as well), but not in the Twilio-whatsapp webhook integration. The logs in the GCP show that the data is being read when a request is made from using a text from twilio-whatsapp number.
I just want that the agent.add() text to be returned to twilio message body.
Or if there is some other possible way to do it, so that I can generate dynamic responses based on the data being read from the database.
Response in Dialogflow console
The code in my inline editor.
index.js
// See https://github.com/dialogflow/dialogflow-fulfillment-nodejs
// for Dialogflow fulfillment library docs, samples, and to report issues
"use strict";
const functions = require("firebase-functions");
const admin = require("firebase-admin");
const { WebhookClient } = require("dialogflow-fulfillment");
process.env.DEBUG = "dialogflow:debug"; // enables lib debugging statements
admin.initializeApp(functions.config().firebase);
const db = admin.firestore();
exports.dialogflowFirebaseFulfillment = functions.https.onRequest(
(request, response) => {
const agent = new WebhookClient({
request,
response,
});
function welcome(agent) {
agent.add(`Welcome to my agent!`);
}
function saveUserDataHandler(agent) {
const firstname = agent.parameters.firstname;
const lastname = agent.parameters.lastname;
const phoneno = agent.parameters.phonenumber;
const dialogflowAgentRef = db.collection("users").doc();
return db
.runTransaction((t) => {
t.set(dialogflowAgentRef, {
firstname: firstname,
lastname: lastname,
phoneno: phoneno,
});
return Promise.resolve("Write complete");
})
.then((doc) => {
agent.add(
` Wrote "${firstname} ${lastname}, ${phoneno}" to the Firestore database.`
);
console.log("wrote to db", firstname, lastname, phoneno);
})
.catch((err) => {
console.log(`Error writing to firestore : ${err}`);
agent.add(
` Failed to write "${firstname} ${lastname}, ${phoneno}" to the Firestore database.`
);
});
}
function readUserDataHandler(agent) {
const dialogflowAgentDoc = db.collection("users");
return dialogflowAgentDoc
.get()
.then((snapshot) => {
snapshot.forEach((doc) => {
if (!doc.exists) {
agent.add("No Data found in the database");
} else {
agent.add(doc.data().firstname);
agent.add(doc.data().lastname);
agent.add(doc.data().phoneno);
console.log(doc.data());
}
return Promise.resolve("Read complete");
});
})
.catch((err) => {
agent.add("Error reading entry from the Firestore database.");
console.log(err);
});
}
function fallback(agent) {
agent.add(`I didn't understand`);
agent.add(`I'm sorry, can you try again?`);
}
// Run the proper function handler based on the matched Dialogflow intent name
let intentMap = new Map();
intentMap.set("Default Welcome Intent", welcome);
intentMap.set("Default Fallback Intent", fallback);
intentMap.set("Get User Data", saveUserDataHandler);
intentMap.set("Read User Data", readUserDataHandler);
agent.handleRequest(intentMap);
}
);
package.json
{
"name": "dialogflowFirebaseFulfillment",
"description": "This is the default fulfillment for a Dialogflow agents using Cloud Functions for Firebase",
"version": "0.0.1",
"private": true,
"license": "Apache Version 2.0",
"author": "Google Inc.",
"engines": {
"node": "8"
},
"scripts": {
"start": "firebase serve --only functions:dialogflowFirebaseFulfillment",
"deploy": "firebase deploy --only functions:dialogflowFirebaseFulfillment"
},
"dependencies": {
"actions-on-google": "^2.5.0",
"dialogflow": "^4.0.3",
"dialogflow-fulfillment": "^0.6.1",
"firebase-admin": "^6.4.0",
"firebase-functions": "^2.1.0"
}
}
Links referred : fulfillment-firestore-nodejs, updating dependencies
In the twilio debugger I get a response that the message body is invalid.
MESSAGE
Message body must be specified
Invalid Body
Warning - 14103
Message Invalid Body
The Message body does not contain valid text or a media URL
Something mentioned here too, but the comments there didn't help.
Am I missing something, or is it a bug as others are facing it too??
Thanks in advance.

Unit testing of HttpsCallable cloud functions - online mode

I am designing a backend api (for android and iOS apps) around HttpsCallable cloud functions. It becomes quite cumbersome to test them through the app, so I wish to switch to unit testing the functions (before production deployment) using the firebase-functions-test tool. I have been following this unit testing tutorial.
I am having some issues running the unit tests in online mode. Let me provide some details.
Here is my `package.json' content:
{
"name": "functions",
"description": "Cloud Functions for Firebase",
"scripts": {
"lint": "eslint .",
"serve": "firebase serve --only functions",
"shell": "firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log",
"test": "mocha --reporter spec"
},
"engines": {
"node": "8"
},
"dependencies": {
"#google-cloud/tasks": "^1.9.0",
"#googlemaps/google-maps-services-js": "^2.0.2",
"chai": "^4.2.0",
"firebase-admin": "^8.6.0",
"firebase-functions": "^3.3.0",
},
"devDependencies": {
"eslint": "^5.12.0",
"eslint-plugin-promise": "^4.0.1",
"firebase-functions-test": "^0.1.7",
"mocha": "^7.1.1"
},
"private": true
}
I am using google APIs (Directions, Geocoding etc) from firebase backend, therefore to be able to access them while running my tests, I configured my tests located at test/index.test.js as recommended in the Unit testing tutorial as follows:
const firebaseConfig = {
apiKey: ...,
authDomain: ...,
databaseURL: "https://my-project.firebaseio.com",
projectId: "my-project",
storageBucket: "my-project.appspot.com",
messagingSenderId: ********,
appId: *********
};
const test = require('firebase-functions-test')(firebaseConfig
, 'path_to_json_keyfile/myproject.json');
Below is the sample test code. Note that all my callable functions return HttpsError, and the test below, simply checks for the code field of HttpsError, if its ok, the test passes. The functions I am testing reside in a separate rides.js file, which the index.js exports as exports.rides = require(./rides.js) (not shown below)
const chai = require('chai');
const assert = chai.assert;
describe('Cloud functions', () => {
let rideFunctions;
before(() => {
rideFunctions = require('../index.js');
});
after(() => {
test.cleanup();
});
describe('getRideEstimate', () => {
it('should return ok', async () => {
data = {
riderType: 1,
pickup_lat: 37.0,
pickup_lng: -122,
dropoff_lat: 37.01,
dropoff_lng: -122.01,
scheduledTime: 1585939000,
ts: 1585939000,
tz_id: "uslax"
}
context = {
auth: {
uid: AUTH_ID_FOR_USER_ACCOUNT
}
};
const wrappedGetRideEstimate = test.wrap(rideFunctions.rides.getRideEstimate);
let httpsError = await wrappedGetRideEstimate(data, context);
return assert.equal(httpsError.code, 'ok');
});
})
describe('testCallable', () => {
it('should return ok', async () => {
data = {}
context = {}
const wrappedTestCallable = test.wrap(rideFunctions.rides.testCallable);
let httpsError = await wrappedTestCallable(data, context);
return assert.equal(httpsError.code, 'ok');
})
})
})
Problem
My simple testCallable function of the form
exports.testCallable = functions.https.onCall((data, context) => {
console.log('testCallable');
return new functions.https.HttpsError('ok', '', '');
})
passes the test (as expected) but inexplicably, it seems it is running in the offline mode, as there is no record of it on cloud function logs in Firebase Console. Also, if I disable connectivity of my laptop, the test result remains the same suggesting that somehow this function is running in the offline mode.
My getRideEstimate function which calls the google Directions API, returns a lengthy 400 error indicating some issue with forming the request to Directions API. I don't know if this error is related to the first problem, but since the Directions API call is embedded deeper inside the getRideEstimate function, it does suggest that the function is getting called, but somehow the API call is failing.
Is there any other way to test HttpsCallable functions in online mode?

For me this works:
import * as firebaseSetup from 'firebase-functions-test';
export const testEnvironment = firebaseSetup({
databaseURL: "https://project-id.firebaseio.com",
projectId: 'project-id'
}, './service-account-project-id-firebase-adminsdk.json');
You can find a full YouTube tutorial under this: https://youtu.be/UDMDpdu5-rE?t=1122

React-Native app doesn't work if i use await

This is the first time I'm using react-native to develop an Android app.
I'm trying to use async/await when I retrieve some data from firestore, but when I add the word await the app crashes and it won't start anymore.
I use react-native-firebase library as you can see in my package.json file.
This is the part of my component that doesn't work:
componentDidMount() {
this.getAccountFromFirestore(this.props.user);
}
getAccountFromFirestore = async user => {
const accounts = await firebase
.firestore()
.collection('accounts')
.get();
this.setState({ loading: false });
};
This is my package.json, if it's usefull
{
"name": "myapp",
"version": "0.0.1",
"private": true,
"scripts": {
"start": "node node_modules/react-native/local-cli/cli.js start",
"test": "jest",
"ios": "react-native run-ios",
"android": "react-native run-android"
},
"dependencies": {
"native-base": "^2.8.1",
"react": "16.6.1",
"react-native": "0.57.6",
"react-native-firebase": "^5.1.1",
"react-native-navigation": "^1.1.493",
"react-native-vector-icons": "^6.1.0",
"react-redux": "^5.1.1",
"redux": "^4.0.1"
},
"devDependencies": {
"babel-jest": "23.6.0",
"jest": "23.6.0",
"metro-react-native-babel-preset": "0.49.2",
"react-test-renderer": "16.6.1"
},
"jest": {
"preset": "react-native"
}
}

You can implement with Promise like :
firebase
.firestore()
.collection('accounts')
.get()
.then(accounts => {
console.log ('Use accounts variable as per your requirement,'accounts)
})
.catch(error => {console.error(error)})

I don't know whether it's your React Native or if you're using Node 6. But for what I know, async/await is only supported on Node 8 or above. So, your best bet is still using Promises with this problem. For example:
componentDidMount() {
this.getAccountFromFirestore(this.props.user);
}
getAccountFromFirestore = user => {
firebase
.firestore()
.collection('accounts')
.get()
.then(accounts => {
let data = accounts.docs.map(account => account.data());
console.log(data);
this.setState({ loading: false });
})
.catch(err => console.log(err));
};
Also when using async/await, don't forget the try-catch blocks:
componentDidMount() {
this.getAccountFromFirestore(this.props.user);
}
getAccountFromFirestore = async user => {
try {
const accounts = await firebase
.firestore()
.collection('accounts')
.get();
const data = accounts.docs.map(a => a.data());
console.log(data);
this.setState({ loading: false });
} catch (err) {
console.error(err);
}
};

Option a) keep using promises
Option b) Migrate that RN, if you can use async/await your code is prob too old to go in the stores anyway.
Option c) Babel...