e2e tweaks to help debug. (#1098)

Aligns with other Foundation projects.

Author: microbit-matt-hillsdon

package-lock.json

@@ -61,6 +61,7 @@
     "@testing-library/user-event": "^12.8.3",
     "@types/file-saver": "^2.0.3",
     "@types/jest": "^26.0.24",
+    "cross-env": "^7.0.3",
     "pptr-testing-library": "^0.7.0",
     "prettier": "2.3.2",
     "puppeteer": "^13.4.0",
@@ -84,8 +85,8 @@
     "serve": "npx serve --no-clipboard -l 3000 -- build/",
     "typecheck": "tsc --noEmit",
     "test:all": "craco test --testTimeout 15000",
-    "test:e2e": "craco test --testPathPattern e2e -w 1 --testTimeout 15000",
-    "test:e2e:headless": "CI=true npm run test:e2e",
+    "test:e2e": "cross-env E2E_HEADLESS=0 craco test --testPathPattern e2e -w 1 --testTimeout 15000",
+    "test:e2e:headless": "cross-env E2E_HEADLESS=1 craco test --testPathPattern e2e -w 1 --testTimeout 15000",
     "ci:update-version": "update-ci-version",
     "deploy": "website-deploy-aws",
     "invalidate": "aws cloudfront create-invalidation --distribution-id $(printenv ${STAGE}_CLOUDFRONT_DISTRIBUTION_ID) --paths \"/*\"",

package.json

@@ -5,7 +5,7 @@
  */
 import { App } from "./app";
 
-describe("Browser - accessibility", () => {
+describe("accessibility", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/accessibility.test.ts

@@ -75,7 +75,11 @@ export class App {
 
   constructor(options: Options = {}) {
     this.url = this.optionsToURL(options);
-    this.browser = puppeteer.launch();
+    this.browser = puppeteer.launch({
+      headless: process.env.E2E_HEADLESS !== "0",
+      // Needs to be large enough to display Reference + Simulator or tests need to show/hide them.
+      defaultViewport: { width: 1920, height: 1440 },
+    });
     this.page = this.createPage();
   }
 
@@ -116,8 +120,6 @@ export class App {
     ]);
 
     const page = await context.newPage();
-    // Needs to be large enough to display Reference + Simulator or tests need to show/hide them.
-    await page.setViewport({ width: 1920, height: 1440 });
     await page.setCookie({
       // See corresponding code in App.tsx.
       name: "mockDevice",
@@ -1095,7 +1097,7 @@ export class App {
     tabName: "Project" | "API" | "Reference" | "Ideas"
   ): Promise<ElementHandle<Element>> {
     const document = await this.document();
-    const tab = await document.getByRole("tab", {
+    const tab = await document.findByRole("tab", {
       name: tabName,
     });
     await tab.click();

src/e2e/app.ts

@@ -8,7 +8,7 @@ import { App } from "./app";
 const showFullSignature =
   "show(image, delay=400, wait=True, loop=False, clear=False)";
 
-describe("Browser - autocomplete and signature help tests", () => {
+describe("autocomplete", () => {
   // Enable flags to allow testing the toolkit interactions.
   const app = new App();
   beforeEach(app.reset.bind(app));

src/e2e/autocomplete.test.ts

@@ -10,7 +10,7 @@ const traceback = `Traceback (most recent call last):
 SyntaxError: invalid syntax
 `; // Needs trailing newline!
 
-describe("Browser - WebUSB (mocked)", () => {
+describe("connect", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/connect.test.ts

@@ -5,7 +5,7 @@
  */
 import { App } from "./app";
 
-describe("Browser - toolkit tabs", () => {
+describe("documentaion", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/documentation.test.ts

@@ -5,7 +5,7 @@
  */
 import { App } from "./app";
 
-describe("Browser - edits", () => {
+describe("edits", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   // We intentionally close the page so can't screenshot here.

src/e2e/edits.test.ts

@@ -11,7 +11,7 @@ const heartMigrationFragment =
 const sunlightSensorMigrationFragment =
   "#project:XQAAgAByAQAAAAAAAAA9iImmlGSt1R++5LD+ZJ36cRz46B+lhYtNRoWF0nijpaVyZlK7ACfSpeoQpgfk21st4ty06R4PEOW6kOsIEMK7SL0Qco7jgsHFKZXfjv/XcHWvXG9qyz1a/a3NUulFDj/FDJxVAIV+WZLpRoo4E6MbW70FOgIfBPWP2hDVsojpoLc7ZfKI8SHxv54FSfB5bkbzaAKO+8CO73t6Odtv691JGjJ9MExFighY6GxyM/DoNInDDpAjFeaqCWrYdwENX7ZVM3we8f4swI71tL28N7sg588aB//A78AA";
 
-describe("Browser - migration", () => {
+describe("migration", () => {
   const app = new App({
     fragment: heartMigrationFragment,
   });

src/e2e/migration.test.ts

@@ -5,7 +5,7 @@
  */
 import { App, LoadDialogType } from "./app";
 
-describe("Browser - multiple and missing file cases", () => {
+describe("multiple-files", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/multiple-files.test.ts

@@ -5,7 +5,7 @@
  */
 import { App, LoadDialogType } from "./app";
 
-describe("Browser - open", () => {
+describe("open", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/open.test.ts

@@ -3,9 +3,9 @@
  *
  * SPDX-License-Identifier: MIT
  */
-import { App, LoadDialogType } from "./app";
+import { App } from "./app";
 
-describe("Browser - reset", () => {
+describe("reset", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/reset.test.ts

@@ -5,7 +5,7 @@
  */
 import { App, LoadDialogType } from "./app";
 
-describe("Browser - save", () => {
+describe("save", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));

src/e2e/save.test.ts

@@ -5,7 +5,7 @@
  */
 import { App } from "./app";
 
-describe("Browser - settings", () => {
+describe("settings", () => {
   const app = new App();
   beforeEach(() => {
     app.setOptions({});

src/e2e/settings.test.ts

@@ -16,7 +16,7 @@ const gestureTest =
 const sliderTest =
   "from microbit import *\nwhile True:\nif temperature() == -5:\ndisplay.show(Image.NO)";
 
-describe("Browser - simulator", () => {
+describe("simulator", () => {
   const app = new App();
   beforeEach(app.reset.bind(app));
   afterEach(app.screenshot.bind(app));