Run tests on CI with cargo-nextest

Author: messense

.config/nextest.toml

@@ -0,0 +1,64 @@
+[store]
+# The directory under the workspace root at which nextest-related files are
+# written. Profile-specific storage is currently written to dir/<profile-name>.
+dir = "target/nextest"
+
+# This section defines the default nextest profile. Custom profiles are layered
+# on top of the default profile.
+[profile.default]
+# "retries" defines the number of times a test should be retried. If set to a
+# non-zero value, tests that succeed on a subsequent attempt will be marked as
+# non-flaky. Can be overridden through the `--retries` option.
+retries = 2
+
+# Show these test statuses in the output.
+#
+# The possible values this can take are:
+# * none: no output
+# * fail: show failed (including exec-failed) tests
+# * retry: show flaky and retried tests
+# * slow: show slow tests
+# * pass: show passed tests
+# * skip: show skipped tests (most useful for CI)
+# * all: all of the above
+#
+# Each value includes all the values above it; for example, "slow" includes
+# failed and retried tests.
+#
+# Can be overridden through the `--status-level` flag.
+status-level = "pass"
+
+# "failure-output" defines when test failures are output to standard output.
+# Accepted values are
+# * "immediate": output failures as soon as they happen
+# * "final": output failures at the end of the test run
+# * "immediate-final": output failures as soon as they happen and at the end of
+#   the test run
+# * "never": don't output failures at all
+#
+# For large test suites and CI it is generally useful to use "immediate-final".
+#
+# Can be overridden through the `--failure-output` option.
+failure-output = "immediate"
+
+# "success-output" controls output on success. This should generally be set to
+# "never".
+success-output = "never"
+
+# Cancel the test run on the first failure. For CI runs, consider setting this
+# to false.
+fail-fast = false
+
+# Treat a test that takes longer than this as slow, and print a message.
+slow-timeout = "60s"
+
+[profile.default.junit]
+# Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
+# If unspecified, JUnit is not written out.
+
+# path = "junit.xml"
+
+# The name of the top-level "report" element in JUnit report. If aggregating
+# reports across different test runs, it may be useful to provide separate names
+# for each report.
+report-name = "nextest-run"

.github/workflows/ci.yml

@@ -163,6 +163,9 @@ jobs:
           # needed to correctly format errors, see #1865
           components: rust-src
 
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
       - uses: Swatinem/rust-cache@v1
         with:
           key: cargo-${{ matrix.platform.python-architecture }}-${{ matrix.platform.os }}-${{ matrix.msrv }}
@@ -193,13 +196,13 @@ jobs:
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (no features)
-        run: cargo test --no-default-features --lib --tests
+        run: cargo nextest run --no-default-features --lib --tests
 
       # --no-default-features when used with `cargo build/test -p` doesn't seem to work!
       - name: Test pyo3-build-config (no features)
         run: |
           cd pyo3-build-config
-          cargo test --no-default-features
+          cargo nextest run --no-default-features
 
       - name: Build (all additive features)
         run: cargo build --lib --tests --no-default-features --features "full ${{ matrix.extra_features }}"
@@ -211,23 +214,23 @@ jobs:
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test
-        run: cargo test --no-default-features --features "full ${{ matrix.extra_features }}"
+        run: cargo nextest run --no-default-features --features "full ${{ matrix.extra_features }}"
 
       # Run tests again, but in abi3 mode
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3 full ${{ matrix.extra_features }}"
+        run: cargo nextest run --no-default-features --features "abi3 full ${{ matrix.extra_features }}"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
       - if: ${{ (!startsWith(matrix.python-version, 'pypy')) && (matrix.python-version != '3.7') }}
         name: Test (abi3-py37)
-        run: cargo test --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}"
+        run: cargo nextest run --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}"
 
       - name: Test proc-macro code
-        run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml
+        run: cargo nextest run --manifest-path=pyo3-macros-backend/Cargo.toml
 
       - name: Test build config
-        run: cargo test --manifest-path=pyo3-build-config/Cargo.toml
+        run: cargo nextest run --manifest-path=pyo3-build-config/Cargo.toml
 
       - name: Test python examples and tests
         shell: bash