chore: Try out nextest in CI (#11692)

* chore: Try out nextest in CI

Starting with flakey test retries.

Signed-off-by: Jesse Szwedko <[email protected]>

* Split up benches and non-benches tests

`nextest` doesn't work with criterion benches:

nextest-rs/nextest#96

Signed-off-by: Jesse Szwedko <[email protected]>

* Avoid testing benches with nextest

Since criterion doesn't support the flags mentioned on

https://nexte.st/book/custom-test-harnesses.html

Signed-off-by: Jesse Szwedko <[email protected]>

* Fix cargo nextest installation

Signed-off-by: Jesse Szwedko <[email protected]>

* Run docs tests too since nextest doesn't support them

Signed-off-by: Jesse Szwedko <[email protected]>

* Fix docs test

Signed-off-by: Jesse Szwedko <[email protected]>

* Don't test benches since there isn't a way to test just them

Signed-off-by: Jesse Szwedko <[email protected]>

* Install nextest for Windows

Signed-off-by: Jesse Szwedko <[email protected]>

* Fix CLI test running

Signed-off-by: Jesse Szwedko <[email protected]>

Author: jszwedko

.config/nextest.toml

@@ -0,0 +1,2 @@
+[profile.default]
+retries = 3

.github/workflows/test.yml

@@ -157,6 +157,7 @@ jobs:
       - run: make test-cli
       - run: make test-behavior
       - run: make check-examples
+      - run: make test-docs
       - name: Stop sccache
         run: sccache --stop-server
 

Cargo.toml

@@ -818,44 +818,37 @@ enrichment-tables-benches = ["enrichment-tables-file"]
 [[bench]]
 name = "default"
 harness = false
-test = true
 required-features = ["benches"]
 
 [[bench]]
 name = "dnstap"
 path = "benches/dnstap/mod.rs"
 harness = false
-test = true
 required-features = ["dnstap-benches"]
 
 [[bench]]
 name = "remap"
 harness = false
-test = true
 required-features = ["remap-benches"]
 
 [[bench]]
 name = "enrichment_tables_file"
 harness = false
-test = true
 required-features = ["enrichment-tables-benches"]
 
 [[bench]]
 name = "languages"
 harness = false
-test = true
 required-features = ["language-benches"]
 
 [[bench]]
 name = "loki"
 harness = false
-test = true
 required-features = ["loki-benches"]
 
 [[bench]]
 name = "distribution_statistic"
 harness = false
-test = true
 required-features = ["statistic-benches"]
 
 [[bench]]
@@ -869,5 +862,4 @@ required-features = ["transform-benches"]
 name = "codecs"
 path = "benches/codecs/main.rs"
 harness = false
-test = false
 required-features = ["codecs-benches"]

Makefile

@@ -283,12 +283,27 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS
 
 ##@ Testing (Supports `ENVIRONMENT=true`)
 
+# nextest doesn't support running doc tests yet so this is split out as
+# `test-docs`
+# https://github.com/nextest-rs/nextest/issues/16
+#
+# criterion doesn't support the flags needed by nextest to run so these are left
+# out for now
+# https://github.com/bheisler/criterion.rs/issues/562
+#
+# `cargo test` lacks support for testing _just_ benches otherwise we'd have
+# a target for that
+# https://github.com/rust-lang/cargo/issues/6454
 .PHONY: test
 test: ## Run the unit test suite
-	${MAYBE_ENVIRONMENT_EXEC} cargo test --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES} metrics-benches codecs-benches language-benches remap-benches statistic-benches ${DNSTAP_BENCHES} benches" ${SCOPE}
+	${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE}
+
+.PHONY: test-docs
+test-docs: ## Run the docs test suite
+	${MAYBE_ENVIRONMENT_EXEC} cargo test --doc --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE}
 
 .PHONY: test-all
-test-all: test test-behavior test-integration ## Runs all tests, unit, behaviorial, and integration.
+test-all: test test-docs test-behavior test-integration ## Runs all tests: unit, docs, behaviorial, and integration.
 
 .PHONY: test-x86_64-unknown-linux-gnu
 test-x86_64-unknown-linux-gnu: cross-test-x86_64-unknown-linux-gnu ## Runs unit tests on the x86_64-unknown-linux-gnu triple
@@ -330,7 +345,7 @@ ifeq ($(AUTOSPAWN), true)
 	@scripts/setup_integration_env.sh nats start
 	sleep 10 # Many services are very slow... Give them a sec..
 endif
-	${MAYBE_ENVIRONMENT_EXEC} cargo test --no-fail-fast --no-default-features --features nats-integration-tests --lib ::nats::
+	${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features nats-integration-tests --lib ::nats::
 ifeq ($(AUTODESPAWN), true)
 	@scripts/setup_integration_env.sh nats stop
 endif
@@ -369,7 +384,7 @@ test-shutdown-cleanup:
 
 .PHONY: test-cli
 test-cli: ## Runs cli tests
-	${MAYBE_ENVIRONMENT_EXEC} cargo test --no-fail-fast --no-default-features --features cli-tests --test cli -- --test-threads 4
+	${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features cli-tests --test cli --test-threads 4
 
 ##@ Benching (Supports `ENVIRONMENT=true`)
 

lib/file-source/Cargo.toml

@@ -79,4 +79,3 @@ pretty_assertions = "1.1.0"
 [[bench]]
 name = "buffer"
 harness = false
-test = true

lib/tracing-limit/Cargo.toml

@@ -21,4 +21,3 @@ tracing-subscriber = { version = "0.3.9", default-features = false, features = [
 [[bench]]
 name = "limit"
 harness = false
-test = true

lib/vrl/compiler/Cargo.toml

@@ -35,7 +35,6 @@ vector_common = { path = "../../vector-common", default-features = false, featur
 [[bench]]
 name = "kind"
 harness = false
-test = true
 
 [features]
 test = []

lib/vrl/stdlib/Cargo.toml

@@ -303,5 +303,4 @@ bench = false
 [[bench]]
 name = "benches"
 harness = false
-test = true
 required-features = ["default"]

lib/vrl/vrl/Cargo.toml

@@ -25,4 +25,3 @@ vrl-stdlib = { path = "../stdlib" }
 [[bench]]
 name = "vm"
 harness = false
-test = true

scripts/environment/bootstrap-windows-2019.ps1

@@ -5,3 +5,5 @@ if ($env:CI -ne $null) {
 } else {
     $env:Path += ";$HOME\.cargo\bin"
 }
+
+rustup run stable cargo install cargo-nextest --version 0.9.8

scripts/environment/prepare.sh

@@ -5,6 +5,7 @@ rustup show # causes installation of version from rust-toolchain.toml
 rustup default "$(rustup show active-toolchain | awk '{print $1;}')"
 rustup run stable cargo install cargo-deb --version 1.29.2
 rustup run stable cargo install cross --version 0.2.1
+rustup run stable cargo install cargo-nextest --version 0.9.8
 
 cd scripts
 bundle install