Automated UI testing using Galata (jupyterlab#10331)

* UI testing using Galata

* add toolbar capture

* notebook edit tests

* use galata 3.0.7-4

* add TOC test

* use galata 3.0.7-5, add helper scripts

* add notebook toolbar test

* run markdown cells, remove unnecessary steps

* fixes for CI failures

* fix typedoc failure

* use jlpm

* add ui-tests workflow

* full build

* launch jlab in empty directory

* delete jlab_root after tests finish

* compare menu and sidebar captures

* install ipywidgets

* refactor

* documentation and repeated run script

* fix lint error

* use galata 3.0.7-6, simplify jlab build step

* wait for export menu

* fix lint errors

* use --dev-mode, remove ipywidgets dependency

* capture only tags panel in property inspector

* remove wait for export menu and file menu capture

* enable visual regression, add reference images

* Update ui-tests/package.json

Co-authored-by: Jeremy Tuloup <[email protected]>

Co-authored-by: Steven Silvester <[email protected]>
Co-authored-by: Jeremy Tuloup <[email protected]>

Author: 3 people

.github/workflows/ui-tests.yml

@@ -0,0 +1,86 @@
+name: UI Tests
+
+on: [push, pull_request]
+
+jobs:
+  ui-test:
+    name: Visual Regression
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        group: [ui_test]
+        python: [3.8]
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Set up Node
+      uses: actions/setup-node@v1
+      with:
+        node-version: '12.x'
+
+    - name: Cache pip on Linux
+      uses: actions/cache@v2
+      if: startsWith(runner.os, 'Linux')
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ hashFiles('**/requirements.txt', 'setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python }}
+
+    - name: Get yarn cache directory path
+      id: yarn-cache-dir-path
+      run: echo "::set-output name=dir::$(yarn cache dir)"
+    - name: Cache yarn
+      uses: actions/cache@v2
+      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    - name: Install dependencies
+      env:
+        GROUP: ${{ matrix.group }}
+      run: |
+        bash ./scripts/ci_install.sh
+        jlpm
+        jlpm build
+        jupyter lab build
+
+    - name: Install Galata
+      run: |
+        cd ui-tests
+        jlpm install --frozen-lockfile
+    
+    - name: Launch JupyterLab
+      run: |
+        cd ui-tests
+        mkdir jlab_root
+        jlpm run start-jlab:detached
+    
+    - name: Wait for JupyterLab
+      uses: ifaxity/wait-on-action@v1
+      with:
+        resource: http-get://localhost:8888/api
+        timeout: 20000
+    
+    - name: Run UI Tests
+      run: |
+        cd ui-tests
+        jlpm run test
+        rm -rf jlab_root
+
+    - name: Upload UI Test artifacts
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: ui-test-output
+        path: |
+          ui-tests/test-output

.gitignore

@@ -111,3 +111,5 @@ junit.xml
 *.code-workspace
 .history
 .vscode
+
+ui-tests/test-output

MANIFEST.in

@@ -31,3 +31,5 @@ prune jupyterlab/staging/build
 
 recursive-exclude jupyterlab *.pyc
 recursive-exclude jupyterlab *.js.map
+
+prune ui-tests

tsconfig.eslint.json

@@ -20,6 +20,7 @@
     "scripts/*",
     "jupyterlab/*",
     "jupyterlab/staging/*",
-    "testutils/**/*"
+    "testutils/**/*",
+    "ui-tests/**/*"
   ]
 }

tsconfigdoc.json

@@ -3,7 +3,8 @@
   "extends": "./tsconfigbase",
   "exclude": [
     "**/test/**",
-    "**/testutils/**"
+    "**/testutils/**",
+    "**/ui-tests/**"
   ],
   "compilerOptions": {
     "paths": {

typedoc.js

@@ -103,7 +103,8 @@ const entryPoints = packages
 const exclude =
   packages.flatMap(p => [`packages/${p}/test`]) +
   [
-    'packages/application-extension/src/index.tsx'
+    'packages/application-extension/src/index.tsx',
+    'ui-tests/**/*'
     //'packages/*/test/*.spec.ts',
   ];
 

ui-tests/README.md

@@ -0,0 +1,3 @@
+{
+  "testId": "test"
+}

ui-tests/galata-config.json

@@ -0,0 +1,9 @@
+c.ServerApp.port = 8888
+c.ServerApp.token = ""
+c.ServerApp.password = ""
+c.ServerApp.disable_check_xsrf = True
+c.ServerApp.open_browser = False
+c.ServerApp.root_dir = 'jlab_root'
+c.LabApp.dev_mode = True
+c.LabApp.open_browser = False
+c.LabApp.expose_app_in_browser = True

ui-tests/jupyter_server_config.py

@@ -0,0 +1,19 @@
+{
+  "name": "@jupyterlab/ui-tests",
+  "version": "1.0.0",
+  "description": "JupyterLab UI Tests",
+  "private": true,
+  "scripts": {
+    "start-jlab": "jupyter lab --config ./jupyter_server_config.py",
+    "start-jlab:detached": "jlpm run start-jlab&",
+    "test:create-references": "galata --skip-visual-regression --skip-html-regression",
+    "test:debug": "galata --no-headless --no-discard-matched-captures --slow-mo 200 --result-server --include notebook-edit",
+    "test:launch-last-report": "galata --launch-result-server",
+    "test": "galata"
+  },
+  "author": "Project Jupyter",
+  "license": "BSD-3-Clause",
+  "dependencies": {
+    "@jupyterlab/galata": "3.0.7-6"
+  }
+}

ui-tests/package.json

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+i=0
+
+while true; do 
+    i=$((i+1))
+    jlpm run test
+    if [ $? -eq 0 ]
+    then
+        echo "Test run succeeded $i times..."
+    else
+        echo "Test run failed! Run 'jlpm run test:launch-last-report' to inspect test results."
+        exit 0
+    fi
+done

ui-tests/reference-output/screenshots/general_dark_theme.png

@@ -0,0 +1,89 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { galata, describe, test } from '@jupyterlab/galata';
+import { runMenuOpenTest, runSidebarOpenTest } from './util';
+
+jest.setTimeout(60000);
+
+describe('General Tests', () => {
+  beforeAll(async () => {
+    await galata.resetUI();
+    galata.context.capturePrefix = 'general';
+  });
+
+  afterAll(async () => {
+    galata.context.capturePrefix = '';
+  });
+
+  test('Launch Screen', async () => {
+    const imageName = 'launch';
+    await galata.capture.screenshot(imageName);
+    expect(await galata.capture.compareScreenshot(imageName)).toBe('same');
+  });
+
+  runSidebarOpenTest();
+
+  test('Enter Simple Mode', async () => {
+    await galata.toggleSimpleMode(true);
+    expect(await galata.isInSimpleMode()).toBeTruthy();
+
+    const imageName = 'simple-mode';
+    await galata.capture.screenshot(imageName);
+    expect(await galata.capture.compareScreenshot(imageName)).toBe('same');
+  });
+
+  test('Leave Simple Mode', async () => {
+    await galata.toggleSimpleMode(false);
+    expect(await galata.isInSimpleMode()).toBeFalsy();
+  });
+
+  test('Toggle Dark theme', async () => {
+    await galata.theme.setDarkTheme();
+    const imageName = 'dark-theme';
+    await galata.capture.screenshot(imageName);
+    expect(await galata.capture.compareScreenshot(imageName)).toBe('same');
+  });
+
+  test('Toggle Light theme', async () => {
+    await galata.theme.setLightTheme();
+  });
+
+  test('Move File Browser to right', async () => {
+    await galata.sidebar.moveTabToRight('filebrowser');
+    expect(await galata.sidebar.getTabPosition('filebrowser')).toBe('right');
+  });
+
+  test('Open File Browser on right', async () => {
+    await galata.sidebar.openTab('filebrowser');
+    expect(await galata.sidebar.isTabOpen('filebrowser')).toBeTruthy();
+  });
+
+  test('Close Sidebar on right', async () => {
+    await galata.sidebar.close('right');
+    expect(await galata.sidebar.isOpen('right')).toBeFalsy();
+  });
+
+  test('Open Sidebar on right', async () => {
+    await galata.sidebar.open('right');
+    expect(await galata.sidebar.isOpen('right')).toBeTruthy();
+  });
+
+  test('Capture File Browser on right', async () => {
+    let imageName = 'filebrowser-right';
+    await galata.capture.screenshot(imageName);
+    expect(await galata.capture.compareScreenshot(imageName)).toBe('same');
+  });
+
+  test('Move File Browser to left', async () => {
+    await galata.sidebar.moveTabToLeft('filebrowser');
+    expect(await galata.sidebar.getTabPosition('filebrowser')).toBe('left');
+  });
+
+  test('Open File Browser on left', async () => {
+    await galata.sidebar.openTab('filebrowser');
+    expect(await galata.sidebar.isTabOpen('filebrowser')).toBeTruthy();
+  });
+
+  runMenuOpenTest();
+});

ui-tests/reference-output/screenshots/general_filebrowser_right.png

@@ -0,0 +1,82 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { galata, describe, test } from '@jupyterlab/galata';
+import { runMenuOpenTest } from './util';
+
+const fileName = 'notebook.ipynb';
+
+jest.setTimeout(60000);
+
+describe('Notebook Create', () => {
+  beforeAll(async () => {
+    await galata.resetUI();
+    galata.context.capturePrefix = 'notebook-create';
+  });
+
+  afterAll(() => {
+    galata.context.capturePrefix = '';
+  });
+
+  test('Create new Notebook', async () => {
+    await galata.notebook.createNew(fileName);
+  });
+
+  test('Create a Raw cell', async () => {
+    await galata.notebook.setCell(0, 'raw', 'Just a raw cell');
+    expect(await galata.notebook.getCellCount()).toBe(1);
+    expect(await galata.notebook.getCellType(0)).toBe('raw');
+  });
+
+  test('Create a Markdown cell', async () => {
+    await galata.notebook.addCell(
+      'markdown',
+      '## This is **bold** and *italic* [link to jupyter.org!](http://jupyter.org)'
+    );
+    await galata.notebook.runCell(1, true);
+    expect(await galata.notebook.getCellCount()).toBe(2);
+    expect(await galata.notebook.getCellType(1)).toBe('markdown');
+  });
+
+  test('Create a Code cell', async () => {
+    await galata.notebook.addCell('code', '2 ** 3');
+    expect(await galata.notebook.getCellCount()).toBe(3);
+    expect(await galata.notebook.getCellType(2)).toBe('code');
+  });
+
+  test('Save Notebook', async () => {
+    await galata.notebook.save();
+    expect(await galata.contents.fileExists(fileName)).toBeTruthy();
+  });
+
+  runMenuOpenTest();
+
+  test('Run cells', async () => {
+    await galata.notebook.run();
+    await galata.notebook.save();
+    const imageName = 'run-cells';
+
+    expect((await galata.notebook.getCellTextOutput(2))[0]).toBe('8');
+
+    const nbPanel = await galata.notebook.getNotebookInPanel();
+    await galata.capture.screenshot(imageName, nbPanel);
+    expect(await galata.capture.compareScreenshot(imageName)).toBe('same');
+  });
+
+  test('Toggle Dark theme', async () => {
+    await galata.theme.setDarkTheme();
+    const nbPanel = await galata.notebook.getNotebookInPanel();
+    const imageName = 'dark-theme';
+    await galata.capture.screenshot(imageName, nbPanel);
+    expect(await galata.capture.compareScreenshot(imageName)).toBe('same');
+  });
+
+  test('Toggle Light theme', async () => {
+    await galata.theme.setLightTheme();
+  });
+
+  test('Delete Notebook', async () => {
+    await galata.contents.deleteFile(fileName);
+    expect(await galata.contents.fileExists(fileName)).toBeFalsy();
+  });
+});