Experimental separate CircleCI pipeline

.circleci/extra.yml

@@ -0,0 +1,223 @@
+version: 2.1
+
+orbs:
+  win: circleci/[email protected]
+
+executors:
+  linux-node:
+    docker:
+      - image: cimg/node:20.15.1
+  linux-python:
+    docker:
+      - image: cimg/python:3.10.7
+  focal:
+    docker:
+      - image: emscripten/emscripten-ci:focal
+    environment:
+      LANG: "C.UTF-8"
+      EMCC_CORES: "4"
+      EMSDK_NOTTY: "1"
+      EMTEST_WASI_SYSROOT: "~/wasi-sdk/wasi-sysroot"
+      EMTEST_BUILD_VERBOSE: "2"
+      EMTEST_DETECT_TEMPFILE_LEAKS: "1"
+  mac-arm64:
+    environment:
+      EMSDK_NOTTY: "1"
+    macos:
+      xcode: "16.2.0"
+    resource_class: macos.m1.medium.gen1
+
+commands:
+  emsdk-env:
+    description: "emsdk_env.sh"
+    steps:
+      - run:
+          name: emsdk_env.sh
+          command: |
+            EMSDK_BASH=1 ~/emsdk/emsdk construct_env >> $BASH_ENV
+            # In order make our external version of emscripten use the emsdk
+            # config we need to explicitly set EM_CONFIG here.
+            echo "export EM_CONFIG=~/emsdk/.emscripten" >> $BASH_ENV
+  bootstrap:
+    description: "bootstrap"
+    steps:
+      - run: ./bootstrap
+  pip-install:
+    description: "pip install"
+    parameters:
+      python:
+        description: "Python executable to use"
+        type: string
+        default: python3
+    steps:
+      - run:
+          name: pip install
+          command: << parameters.python >> -m pip install -r requirements-dev.txt
+  install-emsdk:
+    description: "Install emsdk"
+    steps:
+      - run:
+          name: install emsdk
+          command: |
+            curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz
+            tar -C ~ -xf ~/emsdk-main.tar.gz
+            mv ~/emsdk-main ~/emsdk
+            cd ~/emsdk
+            ./emsdk install tot
+            ./emsdk activate tot
+            # Remove the emsdk version of emscripten to save space in the
+            # persistent workspace and to avoid any confusion with the version
+            # we are trying to test.
+            rm -Rf emscripten
+            echo "import os" >> .emscripten
+            # We use an out-of-tree cache directory so it can be part of the
+            # persisted workspace (see below).
+            echo "CACHE = os.path.expanduser('~/cache')" >> .emscripten
+            # Refer to commit 0bc3640 if we need to pin V8 version.
+            echo "V8_ENGINE = [os.path.expanduser('~/.jsvu/bin/v8')]" >> .emscripten
+            echo "JS_ENGINES = [NODE_JS]" >> .emscripten
+            echo "if os.path.exists(V8_ENGINE[0]): JS_ENGINES.append(V8_ENGINE)" >> .emscripten
+            echo "WASM_ENGINES = []" >> .emscripten
+            test -f ~/vms/wasmtime && echo "WASMTIME = os.path.expanduser(os.path.join('~', 'vms', 'wasmtime')) ; WASM_ENGINES.append(WASMTIME)" >> .emscripten || true
+            test -f ~/vms/wasmer && echo "WASMER = os.path.expanduser(os.path.join('~', 'vms', 'wasmer')) ; WASM_ENGINES.append(WASMER)" >> .emscripten || true
+            test -d ~/wasi-sdk && cp -a ~/wasi-sdk/lib/ $(~/emsdk/upstream/bin/clang -print-resource-dir)
+            cd -
+            echo "final .emscripten:"
+            cat ~/emsdk/.emscripten
+      - emsdk-env
+      - bootstrap
+  upload-test-results:
+    description: "Upload test results"
+    steps:
+      - store_test_results:
+          path: out/test-results.xml
+  run-tests:
+    description: "Runs emscripten tests"
+    parameters:
+      test_targets:
+        description: "Test suites to run"
+        type: string
+      title:
+        description: "Name of given test suite"
+        type: string
+        default: ""
+      extra-cflags:
+        description: "Extra EMCC_CFLAGS"
+        type: string
+        default: ""
+    steps:
+      - when:
+          # We only set EMTEST_RETRY_FLAKY on pull requests.  When we run
+          # normal CI jobs on branches like main we still want to be able to
+          # detect flakyness.
+          condition: ${CIRCLE_PULL_REQUEST}
+          steps:
+            - set-retry-flaky-tests
+      - run:
+          name: run tests (<< parameters.title >>)
+          environment:
+            EMCC_CFLAGS: << parameters.extra-cflags >>
+          command: |
+            env
+            ./test/runner << parameters.test_targets >>
+            $EMSDK_PYTHON ./test/check_clean.py
+  freeze-cache:
+    description: "Freeze emscripten cache"
+    steps:
+      - run:
+          name: Add EM_FROZEN_CACHE to bash env
+          command: echo "export EM_FROZEN_CACHE=1" >> $BASH_ENV
+  set-retry-flaky-tests:
+    description: "Set EMTEST_RETRY_FLAKY"
+    steps:
+      - run:
+          name: Add EMTEST_RETRY_FLAKY to bash env
+          command: echo "export EMTEST_RETRY_FLAKY=2" >> $BASH_ENV
+
+  setup-macos:
+    steps:
+      - run:
+          name: Install brew package dependencies
+          environment:
+            HOMEBREW_NO_AUTO_UPDATE: "1"
+          # Use --force-bottle to force homebrew use binary packages avoiding
+          # costly build times.
+          command: brew install --force-bottle cmake ninja pkg-config
+      - checkout
+      - run:
+          name: submodule update
+          command: git submodule update --init
+
+jobs:
+  # windows and mac do not have separate build and test jobs, as they only run
+  # a limited set of tests; it is simpler and faster to do it all in one job.
+  test-windows:
+    working_directory: "~/path with spaces"
+    executor:
+      name: win/server-2019
+      shell: bash.exe -eo pipefail
+    environment:
+      PYTHONUNBUFFERED: "1"
+      EMSDK_NOTTY: "1"
+      # clang can compile but not link in the current setup, see
+      # https://github.com/emscripten-core/emscripten/pull/11382#pullrequestreview-428902638
+      EMTEST_LACKS_NATIVE_CLANG: "1"
+      EMTEST_SKIP_V8: "1"
+      EMTEST_SKIP_EH: "1"
+      EMTEST_SKIP_WASM64: "1"
+      EMTEST_SKIP_SCONS: "1"
+      EMTEST_SKIP_RUST: "1"
+      EMTEST_SKIP_NODE_CANARY: "1"
+      EMTEST_BROWSER: "0"
+    steps:
+      - checkout
+      - run:
+          name: Install packages
+          command: |
+            choco install -y cmake.portable ninja pkgconfiglite
+      - run:
+          name: Add python to bash path
+          command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV
+      # note we do *not* build all libraries and freeze the cache; as we run
+      # only limited tests here, it's more efficient to build on demand
+      - install-emsdk
+      - pip-install:
+          python: "$EMSDK_PYTHON"
+      - run-tests:
+          title: "crossplatform tests"
+          test_targets: "--crossplatform-only"
+      - upload-test-results
+      # Run a single websockify-based test to ensure it works on windows.
+      - run-tests:
+          title: "sockets.test_nodejs_sockets_echo*"
+          test_targets: "sockets.test_nodejs_sockets_echo*"
+      - upload-test-results
+
+  test-mac-arm64:
+    executor: mac-arm64
+    environment:
+      # We don't install d8 or modern node on the mac runner so we skip any
+      # tests that depend on those.
+      EMTEST_SKIP_V8: "1"
+      EMTEST_SKIP_EH: "1"
+      EMTEST_SKIP_WASM64: "1"
+      EMTEST_SKIP_SCONS: "1"
+      EMTEST_SKIP_RUST: "1"
+      # Some native clang tests assume x86 clang (e.g. -sse2)
+      EMTEST_LACKS_NATIVE_CLANG: "1"
+      EMCC_SKIP_SANITY_CHECK: "1"
+    steps:
+      - setup-macos
+      - install-emsdk
+      # TODO: We can't currently do pip install here since numpy and other packages
+      # are currently missing arm64 macos binaries.
+      - run-tests:
+          title: "crossplatform tests"
+          test_targets: "--crossplatform-only"
+      - upload-test-results
+
+workflows:
+  extra-test:
+    jobs:
+      - test-windows
+      - test-mac-arm64