uploads Playwright html report for e2e Playwright tests (#1010)

Co-authored-by: Marlon <[email protected]>

.github/workflows/build-lint-test.yml

@@ -167,6 +167,13 @@ jobs:
         if: runner.os != 'Windows'
       - name: Run e2e playwright tests
         run: npm run test:e2e
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report-${{ matrix.os }}
+          path: frontend/playwright-report/
+          retention-days: 30
 
   deploy:
     name: Deploy to abacus-test.nl

.github/workflows/release.yml

@@ -59,14 +59,12 @@ jobs:
           path: ${{ matrix.target.binary }}
 
   playwright-e2e:
-    name: Playwright e2e tests (${{ matrix.os }}, ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
+    name: Playwright e2e tests (${{ matrix.os }}
     needs:
       - build
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        shardIndex: [1, 2, 3]
-        shardTotal: [3]
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -90,9 +88,16 @@ jobs:
         run: chmod a+x ../builds/backend/abacus
         if: runner.os != 'Windows'
       - name: Run Playwright e2e tests
-        run: npm run test:e2e -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: npm run test:e2e
         env:
           BACKEND_BUILD: release
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report-release-${{ matrix.os }}
+          path: frontend/playwright-report/
+          retention-days: 30
 
   release:
     name: Release

frontend/.gitignore

@@ -29,6 +29,7 @@ test-results
 coverage
 mock
 test-report.junit.xml
+playwright-report/
 
 # Automatically created: https://github.com/mswjs/msw/discussions/1015#discussioncomment-1747585
 mockServiceWorker.js

frontend/README.md

@@ -60,6 +60,9 @@ npm run test:ladle
 npm run test:e2e
 # run tests, expect builds and database to be available:
 npm run test:e2e-dev
+
+# view reports and traces, e.g. the ones saved by our pipeline:
+npx playwright show-report <path-to-unzipped-report-folder>
 ```
 
 ### UI Component development

frontend/playwright.common.config.ts

@@ -12,11 +12,10 @@ const commonConfig: PlaywrightTestConfig = defineConfig({
   workers: process.env.CI ? "100%" : undefined,
   // Increase the test timeout on CI, which is usually slower
   timeout: process.env.CI ? 30_000 : 10_000,
-  // Use the list reporter even on CI, to get immediate feedback
-  reporter: "list",
   fullyParallel: true,
   use: {
-    trace: "retain-on-failure",
+    // Local runs don't have retries, so we have a trace of each failure. On CI we do have retries, so keeping the trace of the first failure allows us to investigate flaky tests.
+    trace: "retain-on-first-failure",
     testIdAttribute: "id",
   },
   projects: [

frontend/playwright.e2e.config.ts

@@ -20,6 +20,7 @@ function returnWebserverCommand(): string {
 
 const config: PlaywrightTestConfig = defineConfig({
   ...commonConfig,
+  reporter: process.env.CI ? [["list"], ["html", { open: "never" }]] : "list",
   testDir: "./e2e-tests",
   outputDir: "./test-results/e2e-tests",
   testMatch: /\.e2e\.ts/,

frontend/playwright.ladle.config.ts

@@ -4,6 +4,8 @@ import commonConfig from "./playwright.common.config";
 
 const config: PlaywrightTestConfig = defineConfig({
   ...commonConfig,
+  // Use the list reporter even on CI, to get immediate feedback
+  reporter: "list",
   testDir: "./lib/ui",
   outputDir: "./test-results/ladle",
   testMatch: /\.e2e\.ts/,