ci(replay): Overhead measurement (#6611)

Add overhead measurements to measure and compare the performance impact of the Sentry Browser SDK and Replay:
- Add three measurement scenarios:
  - Plain app without Sentry
  - App with Sentry+Tracing
  - App with Sentry+Tracing+Replay
- Measure heap memory, CPU and web vitals impact
- Add GHA job to add comment with results if a label is added to a PR

Author: vaind

.github/workflows/build.yml

@@ -815,3 +815,51 @@ jobs:
         if: contains(needs.*.result, 'failure')
         run: |
           echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1
+
+  replay_metrics:
+    name: Replay Metrics
+    needs: [job_get_metadata, job_build]
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    if: contains(github.event.pull_request.labels.*.name, 'ci-overhead-measurements')
+    steps:
+      - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+      - name: Set up Node
+        uses: volta-cli/action@v4
+      - name: Check dependency cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CACHED_DEPENDENCY_PATHS }}
+          key: ${{ needs.job_build.outputs.dependency_cache_key }}
+      - name: Check build cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CACHED_BUILD_PATHS }}
+          key: ${{ env.BUILD_CACHE_KEY }}
+
+      - name: Setup
+        run: yarn install
+        working-directory: packages/replay/metrics
+
+      - name: Collect
+        run: yarn ci:collect
+        working-directory: packages/replay/metrics
+
+      - name: Process
+        id: process
+        run: yarn ci:process
+        working-directory: packages/replay/metrics
+        # Don't run on forks - the PR comment cannot be added.
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Upload results
+        uses: actions/upload-artifact@v3
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        with:
+          name: ${{ steps.process.outputs.artifactName }}
+          path: ${{ steps.process.outputs.artifactPath }}

.gitignore

@@ -47,4 +47,4 @@ tmp.js
 
 # eslint
 .eslintcache
-eslintcache/*
+**/eslintcache/*

.vscode/launch.json

@@ -37,6 +37,22 @@
       "internalConsoleOptions": "openOnSessionStart",
       "outputCapture": "std"
     },
+    {
+        "type": "node",
+        "name": "Debug replay metrics collection script",
+        "request": "launch",
+        "cwd": "${workspaceFolder}/packages/replay/metrics/",
+        "program": "${workspaceFolder}/packages/replay/metrics/configs/dev/collect.ts",
+        "preLaunchTask": "Build Replay metrics script",
+    },
+    {
+        "type": "node",
+        "name": "Debug replay metrics processing script",
+        "request": "launch",
+        "cwd": "${workspaceFolder}/packages/replay/metrics/",
+        "program": "${workspaceFolder}/packages/replay/metrics/configs/dev/process.ts",
+        "preLaunchTask": "Build Replay metrics script",
+    },
     // Run rollup using the config file which is in the currently active tab.
     {
       "name": "Debug rollup (config from open file)",

.vscode/tasks.json

@@ -7,7 +7,13 @@
 			"type": "npm",
 			"script": "predebug",
 			"path": "packages/nextjs/test/integration/",
-			"detail": "Link the SDK (if not already linked) and build test app"
+			"detail": "Link the SDK (if not already linked) and build test app",
+		},
+		{
+			"label": "Build Replay metrics script",
+			"type": "npm",
+			"script": "build",
+			"path": "packages/replay/metrics",
 		}
 	]
 }

packages/replay/.eslintignore

@@ -3,3 +3,4 @@ build/
 demo/build/
 # TODO: Check if we can re-introduce linting in demo
 demo
+metrics

packages/replay/metrics/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  extends: ['../.eslintrc.js'],
+  ignorePatterns: ['test-apps'],
+  overrides: [
+    {
+      files: ['*.ts'],
+      rules: {
+        'no-console': 'off',
+        '@typescript-eslint/no-non-null-assertion': 'off',
+        'import/no-unresolved': 'off',
+      },
+    },
+  ],
+};

packages/replay/metrics/.gitignore

@@ -0,0 +1 @@
+out

packages/replay/metrics/README.md

@@ -0,0 +1,11 @@
+# Replay performance metrics
+
+Evaluates Replay impact on website performance by running a web app in Chromium via Playwright and collecting various metrics.
+
+The general idea is to run a web app without Sentry Replay and then run the same app again with Sentry and another one with Sentry+Replay included.
+For the three scenarios, we collect some metrics (CPU, memory, vitals) and later compare them and post as a comment in a PR.
+Changes in the metrics, compared to previous runs from the main branch, should be evaluated on case-by-case basis when preparing and reviewing the PR.
+
+## Resources
+
+* https://github.com/addyosmani/puppeteer-webperf

packages/replay/metrics/configs/README.md

@@ -0,0 +1,4 @@
+# Replay metrics configuration & entrypoints (scripts)
+
+* [dev](dev) contains scripts launched during local development
+* [ci](ci) contains scripts launched in CI

packages/replay/metrics/configs/ci/collect.ts

@@ -0,0 +1,54 @@
+import { Metrics, MetricsCollector } from '../../src/collector.js';
+import { MetricsStats, NumberProvider } from '../../src/results/metrics-stats.js';
+import { JankTestScenario } from '../../src/scenarios.js';
+import { printStats } from '../../src/util/console.js';
+import { latestResultFile } from './env.js';
+
+function checkStdDev(results: Metrics[], name: string, provider: NumberProvider, max: number): boolean {
+  const value = MetricsStats.stddev(results, provider);
+  if (value == undefined) {
+    console.warn(`✗ | Discarding results because StandardDeviation(${name}) is undefined`);
+    return false;
+  } else if (value > max) {
+    console.warn(`✗ | Discarding results because StandardDeviation(${name}) is larger than ${max}. Actual value: ${value}`);
+    return false;
+  } else {
+    console.log(`✓ | StandardDeviation(${name}) is ${value} (<= ${max})`)
+  }
+  return true;
+}
+
+const collector = new MetricsCollector({ headless: true, cpuThrottling: 2 });
+const result = await collector.execute({
+  name: 'jank',
+  scenarios: [
+    new JankTestScenario('index.html'),
+    new JankTestScenario('with-sentry.html'),
+    new JankTestScenario('with-replay.html'),
+  ],
+  runs: 10,
+  tries: 10,
+  async shouldAccept(results: Metrics[]): Promise<boolean> {
+    await printStats(results);
+
+    if (!checkStdDev(results, 'lcp', MetricsStats.lcp, 50)
+      || !checkStdDev(results, 'cls', MetricsStats.cls, 0.1)
+      || !checkStdDev(results, 'cpu', MetricsStats.cpu, 1)
+      || !checkStdDev(results, 'memory-mean', MetricsStats.memoryMean, 1000 * 1024)
+      || !checkStdDev(results, 'memory-max', MetricsStats.memoryMax, 1000 * 1024)) {
+      return false;
+    }
+
+    const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu)!;
+    if (cpuUsage > 0.85) {
+      // Note: complexity on the "JankTest" is defined by the `minimum = ...,` setting in app.js - specifying the number of animated elements.
+      console.warn(`✗ | Discarding results because CPU usage is too high and may be inaccurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+        'Consider simplifying the scenario or changing the CPU throttling factor.');
+      return false;
+    }
+
+    return true;
+  },
+});
+
+result.writeToFile(latestResultFile);