add a function and script for cleaning old roots

Author: scytacki

.vscode/launch.json

@@ -14,6 +14,15 @@
       "console": "integratedTerminal",
       "program": "${workspaceFolder}/node_modules/.bin/jest",
       "args": ["${fileBasenameNoExtension}"]
+    },
+    {
+      "name": "Debug functions-v2 test",
+      "request": "launch",
+      "type": "node",
+      "console": "integratedTerminal",
+      "program": "${workspaceFolder}/functions-v2/node_modules/.bin/jest",
+      "args": ["${fileBasenameNoExtension}"],
+      "cwd": "${workspaceFolder}/functions-v2"
     }
   ]
 }

functions-v2/jest.config.js

@@ -2,6 +2,15 @@ module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
   testPathIgnorePatterns: ['lib/', 'node_modules/'],
+  moduleNameMapper: {
+    // These are necessary so code imported from ../shared/ will use the same version of
+    // firebase-admin that the local code does.
+    // The explicit `^` and `$` are needed so this only matches what we are importing.
+    // Otherwise it breaks the internal firebase admin code's imports
+    "^firebase-admin$": "<rootDir>/node_modules/firebase-admin",
+    "^firebase-admin/firestore$": "<rootDir>/node_modules/firebase-admin/lib/firestore",
+    "^firebase-admin/app$": "<rootDir>/node_modules/firebase-admin/lib/app",
+  }
 };
 
 // This is configured here because the clearFirebaseData function from

functions-v2/package-lock.json

@@ -5,8 +5,9 @@
     "build": "tsc",
     "build:watch": "tsc --watch",
     "emulator": "firebase emulators:start --project demo-test",
+    "emulator:online": "firebase emulators:start",
     "serve": "npm run build && firebase emulators:start --only functions",
-    "shell": "npm run build && firebase functions:shell",
+    "shell": "npm run build && firebase functions:shell --project demo-test",
     "start": "npm run shell",
     "test": "jest",
     "test:emulator": "firebase emulators:start --project demo-test --only firestore,database",
@@ -16,10 +17,10 @@
   "engines": {
     "node": "20"
   },
-  "main": "lib/src/index.js",
+  "main": "lib/functions-v2/src/index.js",
   "dependencies": {
     "firebase-admin": "^12.1.0",
-    "firebase-functions": "^5.0.0"
+    "firebase-functions": "^5.1.1"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

functions-v2/package.json

@@ -0,0 +1,29 @@
+import {onSchedule} from "firebase-functions/v2/scheduler";
+import * as logger from "firebase-functions/logger";
+
+// NOTE: in order for this import from shared to work it is necessary
+// to alias "firebase-admin" in tsconfig.json. Otherwise Typescript will
+// read the types from the parent node_modules. The parent directory
+// has a different version of the firebase dependencies, which cause
+// type errors.
+import {cleanFirebaseRoots} from "../../shared/clean-firebase-roots";
+
+export const atMidnight = onSchedule("0 0 * * *", runAtMidnight);
+
+// This function is split out so it can be tested by Jest. The
+// firebase-functions-test library doesn't support wrapping onSchedule.
+export async function runAtMidnight() {
+  await cleanFirebaseRoots({
+    appMode: "qa",
+    // hoursAgo: 6,
+    hoursAgo: 90.7,
+    logger,
+    dryRun: true,
+  });
+
+  // When cleanFirebaseRoots is called from a NodeJS script it is
+  // necessary to call Firebase's deleteApp so no threads are left running.
+  // Inside of a firebase function according to
+  // https://stackoverflow.com/a/72933644/3195497
+  // it isn't necessary to call deleteApp when the function is done.
+}

functions-v2/src/at-midnight.ts

@@ -1,4 +1,5 @@
 import * as admin from "firebase-admin";
 export {onUserDocWritten} from "./on-user-doc-written";
+export {atMidnight} from "./at-midnight";
 
 admin.initializeApp();

functions-v2/src/index.ts

@@ -0,0 +1,39 @@
+import initializeFFT from "firebase-functions-test";
+import {
+  clearFirestoreData,
+} from "firebase-functions-test/lib/providers/firestore";
+import {initializeApp} from "firebase-admin/app";
+// import {getFirestore} from "firebase-admin/firestore";
+import {runAtMidnight} from "../src/at-midnight";
+
+process.env["FIRESTORE_EMULATOR_HOST"]="127.0.0.1:8088";
+const projectConfig = {projectId: "demo-test"};
+const fft = initializeFFT(projectConfig);
+
+// When the function is running in the cloud initializeApp is called by index.ts
+// Here we are importing the function's module directly so we can call
+// initializeApp ourselves. This is beneficial since initializeApp needs to
+// be called after initializeFFT above.
+initializeApp();
+
+// type CollectionRef = admin.firestore.CollectionReference<
+//   admin.firestore.DocumentData, admin.firestore.DocumentData
+// >;
+
+describe("atMidnight", () => {
+  beforeEach(async () => {
+    await clearFirestoreData(projectConfig);
+  });
+
+  test("clean up firestore roots", async () => {
+    // The wrapper doesn't support onSchedule. The Typescript types don't allow it
+    // and at run time it doesn't pass the right event:
+    // https://github.com/firebase/firebase-functions-test/issues/210
+    // const wrapped = fft.wrap(atMidnight);
+    await runAtMidnight();
+  });
+
+  afterAll(() => {
+    fft.cleanup();
+  });
+});

functions-v2/test/at-midnight.test.ts

@@ -11,6 +11,14 @@
     // This prevents typescript from trying to include @types from the parent folders.
     // The types in the parent folders conflict so they break the build.
     "typeRoots": ["./node_modules/@types"],
+    "paths": {
+      // These are necessary so code imported from ../shared/ will use the same version of
+      // firebase-admin that the local code does.
+      "firebase-admin": ["./node_modules/firebase-admin/lib"],
+      "firebase-admin/firestore": ["./node_modules/firebase-admin/lib/firestore"],
+      "firebase-admin/app": ["./node_modules/firebase-admin/lib/app"],
+    },
+
   },
   "compileOnSave": true,
   "include": [

functions-v2/tsconfig.json

@@ -0,0 +1,33 @@
+#!/usr/bin/node
+
+// This script finds documents without metadata in the realtime database.
+// If the deleteTypes array is uncommented, it will delete these documents.
+
+// to run this script type the following in the terminal
+// cf. https://stackoverflow.com/a/66626333/16328462
+// $ cd scripts/ai
+// $ npx tsx clean-firebase-roots.ts
+
+import admin from "firebase-admin";
+import { deleteApp } from "firebase-admin/app";
+import {cleanFirebaseRoots} from "../shared/clean-firebase-roots.js";
+import { getScriptRootFilePath } from "./lib/script-utils.js";
+
+const databaseURL = "https://collaborative-learning-ec215.firebaseio.com";
+
+const serviceAccountFile = getScriptRootFilePath("serviceAccountKey.json");
+const credential = admin.credential.cert(serviceAccountFile);
+// Initialize the app with a service account, granting admin privileges
+const fbApp = admin.initializeApp({
+  credential,
+  databaseURL
+});
+
+await cleanFirebaseRoots({
+  appMode: "qa",
+  hoursAgo: 90.7,
+  logger: console,
+  dryRun: true
+});
+
+await deleteApp(fbApp);

scripts/clean-firebase-roots.ts

@@ -0,0 +1,51 @@
+// This requires the modern firebase-admin, so it can't be used by functions-v1
+import {Timestamp, getFirestore} from "firebase-admin/firestore";
+import {getDatabase} from "firebase-admin/database";
+
+const HOUR = 1000 * 60 * 60;
+
+interface Logger {
+  info(...args: any[]): void;
+}
+
+interface Params {
+  appMode: "qa" | "dev";
+  hoursAgo: number;
+  logger: Logger;
+  dryRun?: boolean;
+}
+
+export async function cleanFirebaseRoots(
+  { appMode, hoursAgo, logger, dryRun }: Params
+) {
+
+  // Be extra careful so we don't delete production data
+  if (!["qa", "dev"].includes(appMode)) {
+    throw new Error(`Invalid appMode ${appMode}`);
+  }
+
+  // Clean up Firestore and Realtime database roots that haven't been updated in a 6 hours
+  const cutOffMillis = Date.now() - hoursAgo*HOUR;
+  const qaRootsResult = await getFirestore()
+    .collection(appMode)
+    .where("lastLaunchTime", "<", Timestamp.fromMillis(cutOffMillis))
+    .get();
+
+  logger.info(`Found ${qaRootsResult.size} roots to delete`);
+
+  // Need to be careful to clean up the root in the realtime database
+  // first. The record in Firestore is our only way to figure out which
+  // roots in the realtime database need to be deleted.
+  for (const root of qaRootsResult.docs) {
+    // The Realtime database root is deleted first incase it fails.
+    // This way the root in firestore will remain so we can find it
+    // and try again later.
+    const databasePath = `/${appMode}/${root.id}`;
+    logger.info(`Deleting Realtime Database root: ${databasePath} ...`);
+    // TODO: what if the ref doesn't exist?
+    if (!dryRun) await getDatabase().ref(`/${appMode}/${root.id}`).remove();
+    logger.info(`Deleting Firestore root: ${root.ref.path} ...`);
+    if (!dryRun) await getFirestore().recursiveDelete(root.ref);
+  }
+
+}