From 58184fd645d04b9435b090180e02280ff3e67dec Mon Sep 17 00:00:00 2001
From: Joep Schuurkes <145749778+jschuurk-kr@users.noreply.github.com>
Date: Tue, 11 Feb 2025 13:14:14 +0100
Subject: [PATCH] uploads Playwright html report for e2e Playwright tests
 (#1010)

Co-authored-by: Marlon <marlonbaeten@users.noreply.github.com>
---
 .github/workflows/build-lint-test.yml |  7 +++++++
 .github/workflows/release.yml         | 13 +++++++++----
 frontend/.gitignore                   |  1 +
 frontend/README.md                    |  3 +++
 frontend/playwright.common.config.ts  |  5 ++---
 frontend/playwright.e2e.config.ts     |  1 +
 frontend/playwright.ladle.config.ts   |  2 ++
 7 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-lint-test.yml b/.github/workflows/build-lint-test.yml
index c8af6282b..b850d436c 100644
--- a/.github/workflows/build-lint-test.yml
+++ b/.github/workflows/build-lint-test.yml
@@ -167,6 +167,13 @@ jobs:
         if: runner.os != 'Windows'
       - name: Run e2e playwright tests
         run: npm run test:e2e
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report-${{ matrix.os }}
+          path: frontend/playwright-report/
+          retention-days: 30
 
   deploy:
     name: Deploy to abacus-test.nl
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 4d97d3562..70c42c45a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -59,14 +59,12 @@ jobs:
           path: ${{ matrix.target.binary }}
 
   playwright-e2e:
-    name: Playwright e2e tests (${{ matrix.os }}, ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
+    name: Playwright e2e tests (${{ matrix.os }}
     needs:
       - build
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        shardIndex: [1, 2, 3]
-        shardTotal: [3]
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -90,9 +88,16 @@ jobs:
         run: chmod a+x ../builds/backend/abacus
         if: runner.os != 'Windows'
       - name: Run Playwright e2e tests
-        run: npm run test:e2e -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: npm run test:e2e
         env:
           BACKEND_BUILD: release
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report-release-${{ matrix.os }}
+          path: frontend/playwright-report/
+          retention-days: 30
 
   release:
     name: Release
diff --git a/frontend/.gitignore b/frontend/.gitignore
index 8e24e898a..20de1624d 100644
--- a/frontend/.gitignore
+++ b/frontend/.gitignore
@@ -29,6 +29,7 @@ test-results
 coverage
 mock
 test-report.junit.xml
+playwright-report/
 
 # Automatically created: https://github.com/mswjs/msw/discussions/1015#discussioncomment-1747585
 mockServiceWorker.js
diff --git a/frontend/README.md b/frontend/README.md
index bb9111c5a..c556ca8ab 100644
--- a/frontend/README.md
+++ b/frontend/README.md
@@ -60,6 +60,9 @@ npm run test:ladle
 npm run test:e2e
 # run tests, expect builds and database to be available:
 npm run test:e2e-dev
+
+# view reports and traces, e.g. the ones saved by our pipeline:
+npx playwright show-report <path-to-unzipped-report-folder>
 ```
 
 ### UI Component development
diff --git a/frontend/playwright.common.config.ts b/frontend/playwright.common.config.ts
index bb31c415b..966dfc381 100644
--- a/frontend/playwright.common.config.ts
+++ b/frontend/playwright.common.config.ts
@@ -12,11 +12,10 @@ const commonConfig: PlaywrightTestConfig = defineConfig({
   workers: process.env.CI ? "100%" : undefined,
   // Increase the test timeout on CI, which is usually slower
   timeout: process.env.CI ? 30_000 : 10_000,
-  // Use the list reporter even on CI, to get immediate feedback
-  reporter: "list",
   fullyParallel: true,
   use: {
-    trace: "retain-on-failure",
+    // Local runs don't have retries, so we have a trace of each failure. On CI we do have retries, so keeping the trace of the first failure allows us to investigate flaky tests.
+    trace: "retain-on-first-failure",
     testIdAttribute: "id",
   },
   projects: [
diff --git a/frontend/playwright.e2e.config.ts b/frontend/playwright.e2e.config.ts
index f4199dc19..61e147520 100644
--- a/frontend/playwright.e2e.config.ts
+++ b/frontend/playwright.e2e.config.ts
@@ -20,6 +20,7 @@ function returnWebserverCommand(): string {
 
 const config: PlaywrightTestConfig = defineConfig({
   ...commonConfig,
+  reporter: process.env.CI ? [["list"], ["html", { open: "never" }]] : "list",
   testDir: "./e2e-tests",
   outputDir: "./test-results/e2e-tests",
   testMatch: /\.e2e\.ts/,
diff --git a/frontend/playwright.ladle.config.ts b/frontend/playwright.ladle.config.ts
index 0e224122f..2780dc4e8 100644
--- a/frontend/playwright.ladle.config.ts
+++ b/frontend/playwright.ladle.config.ts
@@ -4,6 +4,8 @@ import commonConfig from "./playwright.common.config";
 
 const config: PlaywrightTestConfig = defineConfig({
   ...commonConfig,
+  // Use the list reporter even on CI, to get immediate feedback
+  reporter: "list",
   testDir: "./lib/ui",
   outputDir: "./test-results/ladle",
   testMatch: /\.e2e\.ts/,