chore(smoke-test): Arm "test installers" GHA workflow (#6718)

* Add e2e tests as dep of smoke-tests

* Arm workflow

* Print debug messages

* Fix and allow downgrade in auto-update-to tests

* Cleanup on exit

* Wait for update server to close

* Add more debug messages and fix check

Author: kraenhansen

.github/workflows/test-installers.yml

@@ -21,8 +21,111 @@ on:
 
 jobs:
   test:
-    name: Dummy action
-    runs-on: ubuntu-latest
+    name: ${{ matrix.package }} test ${{ matrix.test }} (${{ matrix.hadron-distribution }})
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - time-to-first-query
+          - auto-update-from
+          - auto-update-to
+        package:
+          - osx_dmg
+          - osx_zip
+          - windows_zip
+          - windows_msi
+          - windows_setup
+          - linux_deb
+          # TODO: Re-enable (see https://github.com/mongodb-js/compass/actions/runs/13281152689/job/37079619474)
+          # - linux_tar
+          # TODO: Enable (needs a docker container)
+          # - linux_rpm
+        hadron-distribution:
+          - compass
+          - compass-readonly
+          - compass-isolated
+        include:
+          # MacOS
+          - package: osx_dmg
+            runs-on: macos-13
+            arch: x64
+            hadron-platform: darwin
+          - package: osx_zip
+            runs-on: macos-latest
+            arch: arm64
+            hadron-platform: darwin
+
+          # Windows
+          - package: windows_zip
+            runs-on: windows-latest
+            arch: x64
+            hadron-platform: windows
+          - package: windows_msi
+            runs-on: windows-latest
+            arch: x64
+            hadron-platform: windows
+          - package: windows_setup
+            runs-on: windows-latest
+            arch: x64
+            hadron-platform: windows
+
+          # Linux
+          - package: linux_deb
+            runs-on: ubuntu-latest
+            arch: x64
+            hadron-platform: linux
+          # - package: linux_tar
+          #   runs-on: ubuntu-latest
+          #   arch: x64
+          #   hadron-platform: linux
+          # - package: linux_rpm
+          #   runs-on: ubuntu-latest
+          #   arch: x64
+          #   hadron-platform: linux
+
+          # Install the update server for auto-update tests
+          - test: auto-update-from
+            install-update-server: true
+          - test: auto-update-to
+            install-update-server: true
+
+        exclude:
+          # TODO: See https://github.com/mongodb-js/compass/actions/runs/13281152689/job/37079620322
+          - package: osx_dmg
+            test: auto-update-from
+          # TODO: See https://github.com/mongodb-js/compass/actions/runs/13281152689/job/37079621700
+          - package: windows_setup
+            test: auto-update-from
+          # Skip auto-update tests for isolated and readonly distributions for now
+          # This is not supported by the update server we're using in the test harness
+          - hadron-distribution: compass-isolated
+            test: auto-update-from
+          - hadron-distribution: compass-isolated
+            test: auto-update-to
+          - hadron-distribution: compass-readonly
+            test: auto-update-from
+          - hadron-distribution: compass-readonly
+            test: auto-update-to
+          # Temporary skip failing auto-update-to tests
+          - test: auto-update-to
+            package: osx_dmg
+          - test: auto-update-to
+            package: windows_zip
+          - test: auto-update-to
+            package: windows_msi
+          - test: auto-update-to
+            package: windows_setup
+          - test: auto-update-to
+            package: linux_deb
+
+          # Skip time-to-first-query tests for readonly because it doesn't have editable documents
+          # See https://github.com/mongodb-js/compass/actions/runs/13286945911/job/37097791601
+          - hadron-distribution: compass-readonly
+            test: time-to-first-query
+
+    runs-on: ${{ matrix.runs-on }}
+    env:
+      DEBUG: compass:smoketests:*
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -31,6 +134,47 @@ jobs:
         with:
           node-version: 20
           cache: "npm"
-      - name: Install dependencies
+      - name: Cache downloads
+        uses: actions/cache@v4
+        with:
+          key: smoke-tests-downloads-${{ inputs.version }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.package }}
+          path: packages/compass-smoke-tests/.downloads
+      - name: Install dependencies and build packages
         run: npm ci
-        
+      - name: Bootstrap packages
+        run: npx lerna run bootstrap --scope @mongodb-js/compass-smoke-tests --include-dependencies
+
+      - name: Create GitHub App Token
+        if: matrix.install-update-server
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEVTOOLS_BOT_APP_ID }}
+          private-key: ${{ secrets.DEVTOOLS_BOT_PRIVATE_KEY }}
+          owner: 10gen
+          repositories: compass-mongodb-com
+      - name: Checkout Compass Update server
+        if: matrix.install-update-server
+        uses: actions/checkout@v4
+        with:
+          repository: 10gen/compass-mongodb-com
+          token: ${{ steps.app-token.outputs.token }}
+          # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config
+          persist-credentials: false
+          path: 'compass-mongodb-com'
+      - name: Install Compass Update server
+        if: matrix.install-update-server
+        run: npm install --no-save --workspace packages/compass-smoke-tests ${{ github.workspace }}/compass-mongodb-com
+
+      - name: Run tests
+        env:
+          EVERGREEN_BUCKET_NAME: ${{ inputs.bucket_name }}
+          EVERGREEN_BUCKET_KEY_PREFIX: ${{ inputs.bucket_key_prefix }}
+          DEV_VERSION_IDENTIFIER: ${{ inputs.version }}
+          HADRON_DISTRIBUTION: ${{ matrix.hadron-distribution }}
+          PLATFORM: ${{ matrix.hadron-platform }}
+          ARCH: ${{ matrix.arch }}
+          # Exposing token to prevent update server from being rate limited
+          GITHUB_TOKEN: ${{ github.token }}
+        working-directory: packages/compass-smoke-tests
+        run: npm start -- --package ${{ matrix.package }} --tests ${{ matrix.test }}

package-lock.json

@@ -2,3 +2,5 @@ ignores:
  - '@mongodb-js/prettier-config-compass'
  - '@mongodb-js/tsconfig-compass'
  - compass-mongodb-com
+ # Calling into the e2e tests via child processes
+ - compass-e2e-tests

packages/compass-smoke-tests/.depcheckrc

@@ -34,6 +34,7 @@
     "@mongodb-js/eslint-config-compass": "^1.3.1",
     "@mongodb-js/prettier-config-compass": "^1.2.1",
     "@mongodb-js/tsconfig-compass": "^1.2.1",
+    "compass-e2e-tests": "^1.29.0",
     "depcheck": "^1.4.1",
     "debug": "^4.3.4",
     "eslint": "^7.25.0",

packages/compass-smoke-tests/package.json

@@ -9,6 +9,7 @@ import { SUPPORTED_PACKAGES } from './packages';
 import { testTimeToFirstQuery } from './tests/time-to-first-query';
 import { testAutoUpdateFrom } from './tests/auto-update-from';
 import { testAutoUpdateTo } from './tests/auto-update-to';
+import { deleteSandboxesDirectory } from './directories';
 
 const debug = createDebug('compass:smoketests');
 
@@ -92,7 +93,7 @@ const argv = yargs(hideBin(process.argv))
   })
   .option('skipCleanup', {
     type: 'boolean',
-    description: 'Do not delete the sandbox after a run',
+    description: 'Do not delete the sandboxes after a run',
     default: false,
   })
   .option('tests', {
@@ -106,6 +107,27 @@ const argv = yargs(hideBin(process.argv))
 async function run() {
   const context: SmokeTestsContext = argv.parseSync();
 
+  function cleanupMaybe() {
+    if (context.skipCleanup) {
+      console.log('Skipped cleanup of sandboxes');
+    } else {
+      debug('Cleaning up sandboxes');
+      try {
+        deleteSandboxesDirectory();
+      } catch (err) {
+        if (err instanceof Error) {
+          console.warn(`Failed cleaning sandboxes: ${err.message}`);
+        } else {
+          throw err;
+        }
+      }
+    }
+  }
+
+  process.once('SIGTERM', cleanupMaybe);
+  process.once('SIGINT', cleanupMaybe);
+  process.once('exit', cleanupMaybe);
+
   debug(`Running tests`);
 
   debug(

packages/compass-smoke-tests/src/cli.ts

@@ -3,12 +3,21 @@ import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
+const SANDBOXES_PATH = path.resolve(__dirname, '../.sandboxes');
+
+export function deleteSandboxesDirectory() {
+  fs.rmSync(SANDBOXES_PATH, { recursive: true, force: true });
+}
+
+export function sandboxesDirectoryExists() {
+  return fs.existsSync(SANDBOXES_PATH);
+}
+
 function ensureSandboxesDirectory() {
-  const sandboxesPath = path.resolve(__dirname, '../.sandboxes');
-  if (!fs.existsSync(sandboxesPath)) {
-    fs.mkdirSync(sandboxesPath, { recursive: true });
+  if (!sandboxesDirectoryExists()) {
+    fs.mkdirSync(SANDBOXES_PATH, { recursive: true });
   }
-  return sandboxesPath;
+  return SANDBOXES_PATH;
 }
 
 export function createSandbox() {

packages/compass-smoke-tests/src/directories.ts

@@ -19,6 +19,7 @@ export async function downloadFile({
   targetFilename,
   clearCache,
 }: DownloadFileOptions): Promise<string> {
+  debug('Fetching', { url });
   const response = await fetch(url);
 
   assert(
@@ -40,7 +41,7 @@ export async function downloadFile({
     if (clearCache) {
       fs.rmSync(cacheDirectoryPath, { recursive: true, force: true });
     } else {
-      debug('Skipped downloading', url, '(cache existed)');
+      debug('Skipped downloading (cache existed)', { url });
       return outputPath;
     }
   }
@@ -51,11 +52,11 @@ export async function downloadFile({
 
   // Write the response to file
   assert(response.body, 'Expected a response body');
-  debug('Downloading', url);
+  debug('Downloading', { url, outputPath });
   await stream.promises.pipeline(
     response.body,
     fs.createWriteStream(outputPath)
   );
-
+  debug('Download completed', { url, outputPath });
   return outputPath;
 }

packages/compass-smoke-tests/src/downloads.ts

@@ -1,11 +1,14 @@
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import path from 'node:path';
+import createDebug from 'debug';
 
 import type { InstalledAppInfo, InstallablePackage } from './types';
 import { execute } from '../execute';
 import * as apt from './apt';
 
+const debug = createDebug('compass:smoketests:linux-deb');
+
 export function installLinuxDeb({
   kind,
   filepath,
@@ -18,6 +21,7 @@ export function installLinuxDeb({
   const appPath = path.resolve(installPath, appName);
 
   function uninstall() {
+    debug('Uninstalling %s', filepath);
     execute('sudo', ['apt', 'remove', '--yes', '--purge', packageName]);
   }
 

packages/compass-smoke-tests/src/installers/linux-deb.ts

@@ -2,9 +2,13 @@ import assert from 'node:assert/strict';
 import cp from 'node:child_process';
 import path from 'node:path';
 import fs from 'node:fs';
+import createDebug from 'debug';
 
 import type { InstalledAppInfo, InstallablePackage } from './types';
 import { execute } from '../execute';
+
+const debug = createDebug('compass:smoketests:linux-rpm');
+
 /**
  * Call dnf to get the package name
  */
@@ -48,6 +52,7 @@ export function installLinuxRpm({
   const appPath = path.resolve(installPath, appName);
 
   function uninstall() {
+    debug('Uninstalling %s', filepath);
     execute('sudo', ['dnf', 'remove', '-y', packageName]);
   }
 

packages/compass-smoke-tests/src/installers/linux-rpm.ts

@@ -21,6 +21,7 @@ export function installWindowsMSI({
   const appPath = path.resolve(installDirectory, `${appName}.exe`);
 
   function uninstall() {
+    debug('Uninstalling %s', filepath);
     execute('msiexec', ['/uninstall', filepath, '/passive']);
   }