Check all cargo features of Substrate on CI

[joao-paulo-parity]

Problem: feature checks are hardcoded at the moment

substrate/scripts/ci/gitlab/pipeline/test.yml

Line 159 in eb1a2a8

- time cargo check --locked --features try-runtime

substrate/scripts/ci/gitlab/pipeline/test.yml

Line 173 in eb1a2a8

- time cargo check --locked --features wasmer-sandbox

The current approach has at least two problems

• We're not testing all the features
• When a new feature is added, removed or renamed, we'd have to manually edit the pipeline

Solution: come up with a way of automatically testing all the features provided by Substrate.

The general idea of #12709 could be used for reference.

[niklasad1]

@joao-paulo-parity

I suggest trying cargo-hack we are using it in jsonrpsee for instance but I reckon that it's quite slow and not sure if that is "acceptable" in substrate.

https://github.com/paritytech/jsonrpsee/blob/master/.github/workflows/ci.yml#L68-#L92

[joao-paulo-parity]

I suggest trying cargo-hack we are using it in jsonrpsee for instance

Thanks for the hint. I also found a similar one at https://github.com/frewsxcv/cargo-all-features#usage.

I reckon that it's quite slow and not sure if that is "acceptable" in substrate.

I think we could use something that just prints the crates and their features, then process that output to generate the commands for a given execution strategy. Perhaps contributing a --print feature to those projects for this purpose would be worthwhile.

I've played a bit with a script for that idea

cargo metadata --quiet --format-version=1 | jq -r '
  . as $in |
  paths | select(
    .[0]=="packages" and
    .[-1]=="source" and
    (. | length == 3) and
    . as $p | $in | getpath($p)==null # .source == null for workspace crates
  ) as $crate_path |
  del($crate_path[-1]) as $crate_path |
  $in | getpath($crate_path) |
  .features | if type == "array" then
    $in | getpath($crate_path) | .name, [$in | getpath($crate_path + ["features"]) | .[]]
  else
    $in | getpath($crate_path) | .name, [$in | getpath($crate_path + ["features"]) | keys | .[]]
  end
'

Output

beefy-gadget
[]
beefy-gadget-rpc
[]
beefy-merkle-tree
[
  "array-bytes",
  "debug",
  "default",
  "log",
  "std"
]
beefy-primitives
[
  "default",
  "serde",
  "std"
]
chain-spec-builder
[]
fork-tree
[]
frame-benchmarking
[
  "default",
  "linregress",
  "runtime-benchmarks",
  "serde",
  "std"
]
frame-benchmarking-cli
[
  "default",
  "rocksdb",
  "runtime-benchmarks"
]
frame-election-provider-solution-type
[]
frame-election-provider-support
[
  "default",
  "fuzz",
  "runtime-benchmarks",
  "std"
]
frame-election-solution-type-fuzzer
[]
frame-executive
[
  "default",
  "frame-try-runtime",
  "std",
  "try-runtime",
  "with-tracing"
]
frame-support
[
  "default",
  "full-metadata-docs",
  "no-metadata-docs",
  "once_cell",
  "runtime-benchmarks",
  "serde",
  "sp-state-machine",
  "std",
  "try-runtime",
  "tuples-128",
  "tuples-96"
]
frame-support-procedural
[
  "default",
  "no-metadata-docs",
  "std"
]
frame-support-procedural-tools
[]
frame-support-procedural-tools-derive
[]
frame-support-test
[
  "default",
  "disable-ui-tests",
  "frame-feature-testing",
  "frame-feature-testing-2",
  "no-metadata-docs",
  "sp-state-machine",
  "std",
  "try-runtime"
]
frame-support-test-compile-pass
[
  "default",
  "std"
]
frame-support-test-pallet
[
  "default",
  "std"
]
frame-system
[
  "default",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
frame-system-benchmarking
[
  "default",
  "runtime-benchmarks",
  "std"
]
frame-system-rpc-runtime-api
[
  "default",
  "std"
]
frame-try-runtime
[
  "default",
  "std",
  "try-runtime"
]
generate-bags
[]
kitchensink-runtime
[
  "contracts-unstable-interface",
  "default",
  "frame-benchmarking",
  "frame-system-benchmarking",
  "frame-try-runtime",
  "pallet-election-provider-support-benchmarking",
  "pallet-nomination-pools-benchmarking",
  "pallet-offences-benchmarking",
  "pallet-session-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime",
  "wasmer-sandbox",
  "with-tracing"
]
node-bench
[]
node-cli
[
  "clap",
  "clap_complete",
  "cli",
  "default",
  "frame-benchmarking-cli",
  "node-inspect",
  "runtime-benchmarks",
  "sc-cli",
  "substrate-build-script-utils",
  "substrate-frame-cli",
  "try-runtime",
  "try-runtime-cli"
]
node-executor
[
  "stress-test"
]
node-inspect
[]
node-primitives
[
  "default",
  "std"
]
node-rpc
[]
node-runtime-generate-bags
[]
node-template
[
  "default",
  "runtime-benchmarks",
  "try-runtime",
  "try-runtime-cli"
]
node-template-runtime
[
  "default",
  "frame-benchmarking",
  "frame-system-benchmarking",
  "frame-try-runtime",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
node-testing
[]
pallet-alliance
[
  "array-bytes",
  "default",
  "frame-benchmarking",
  "pallet-collective",
  "runtime-benchmarks",
  "sha2",
  "std",
  "try-runtime"
]
pallet-asset-tx-payment
[
  "default",
  "serde",
  "std",
  "try-runtime"
]
pallet-assets
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-atomic-swap
[
  "default",
  "std",
  "try-runtime"
]
pallet-aura
[
  "default",
  "std",
  "try-runtime"
]
pallet-authority-discovery
[
  "default",
  "std",
  "try-runtime"
]
pallet-authorship
[
  "default",
  "std",
  "try-runtime"
]
pallet-babe
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-bags-list
[
  "default",
  "frame-benchmarking",
  "fuzz",
  "pallet-balances",
  "runtime-benchmarks",
  "sp-core",
  "sp-io",
  "sp-tracing",
  "std",
  "try-runtime"
]
pallet-bags-list-fuzzer
[]
pallet-bags-list-remote-tests
[]
pallet-balances
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-beefy
[
  "default",
  "serde",
  "std",
  "try-runtime"
]
pallet-beefy-mmr
[
  "array-bytes",
  "default",
  "serde",
  "std",
  "try-runtime"
]
pallet-bounties
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-child-bounties
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-collective
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-contracts
[
  "default",
  "frame-benchmarking",
  "rand",
  "rand_pcg",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime",
  "unstable-interface"
]
pallet-contracts-primitives
[
  "default",
  "std"
]
pallet-contracts-proc-macro
[
  "full"
]
pallet-conviction-voting
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-democracy
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-election-provider-multi-phase
[
  "default",
  "frame-benchmarking",
  "pallet-election-provider-support-benchmarking",
  "rand",
  "runtime-benchmarks",
  "std",
  "strum",
  "try-runtime"
]
pallet-election-provider-support-benchmarking
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std"
]
pallet-elections-phragmen
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-example-basic
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-example-offchain-worker
[
  "default",
  "sp-keystore",
  "std",
  "try-runtime"
]
pallet-fast-unstake
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-gilt
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-grandpa
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-identity
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-im-online
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-indices
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "sp-keyring",
  "std",
  "try-runtime"
]
pallet-lottery
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-membership
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-mmr
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-mmr-rpc
[]
pallet-multisig
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-nicks
[
  "default",
  "std",
  "try-runtime"
]
pallet-node-authorization
[
  "default",
  "std",
  "try-runtime"
]
pallet-nomination-pools
[
  "default",
  "fuzzing",
  "pallet-balances",
  "runtime-benchmarks",
  "sp-tracing",
  "std",
  "try-runtime"
]
pallet-nomination-pools-benchmarking
[
  "default",
  "runtime-benchmarks",
  "std"
]
pallet-nomination-pools-fuzzer
[]
pallet-nomination-pools-runtime-api
[
  "default",
  "std"
]
pallet-nomination-pools-test-staking
[]
pallet-offences
[
  "default",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-offences-benchmarking
[
  "default",
  "runtime-benchmarks",
  "std"
]
pallet-preimage
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "sp-core",
  "std",
  "try-runtime"
]
pallet-proxy
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-randomness-collective-flip
[
  "default",
  "std",
  "try-runtime"
]
pallet-ranked-collective
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-recovery
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-referenda
[
  "assert_matches",
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-remark
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-root-offences
[
  "default",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-root-testing
[
  "default",
  "std",
  "try-runtime"
]
pallet-scheduler
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-scored-pool
[
  "default",
  "std",
  "try-runtime"
]
pallet-session
[
  "default",
  "historical",
  "sp-trie",
  "std",
  "try-runtime"
]
pallet-session-benchmarking
[
  "default",
  "runtime-benchmarks",
  "std"
]
pallet-society
[
  "default",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-staking
[
  "default",
  "frame-benchmarking",
  "fuzz",
  "rand_chacha",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-staking-reward-curve
[]
pallet-staking-reward-fn
[
  "default",
  "std"
]
pallet-state-trie-migration
[
  "default",
  "frame-benchmarking",
  "remote-externalities",
  "remote-test",
  "runtime-benchmarks",
  "serde",
  "std",
  "substrate-state-trie-migration-rpc",
  "thousands",
  "try-runtime",
  "zstd"
]
pallet-sudo
[
  "default",
  "std",
  "try-runtime"
]
pallet-template
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-timestamp
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "sp-io",
  "std",
  "try-runtime"
]
pallet-tips
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-transaction-payment
[
  "default",
  "serde",
  "std",
  "try-runtime"
]
pallet-transaction-payment-rpc
[]
pallet-transaction-payment-rpc-runtime-api
[
  "default",
  "std"
]
pallet-transaction-storage
[
  "array-bytes",
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-treasury
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "serde",
  "std",
  "try-runtime"
]
pallet-uniques
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-utility
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-vesting
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
pallet-whitelist
[
  "default",
  "frame-benchmarking",
  "runtime-benchmarks",
  "std",
  "try-runtime"
]
remote-externalities
[
  "frame-support",
  "remote-test"
]
sc-allocator
[]
sc-authority-discovery
[]
sc-basic-authorship
[]
sc-block-builder
[]
sc-chain-spec
[]
sc-chain-spec-derive
[]
sc-cli
[
  "default",
  "rocksdb"
]
sc-client-api
[]
sc-client-db
[
  "default",
  "kvdb-rocksdb",
  "rocksdb",
  "runtime-benchmarks",
  "test-helpers"
]
sc-consensus
[]
sc-consensus-aura
[]
sc-consensus-babe
[]
sc-consensus-babe-rpc
[]
sc-consensus-epochs
[]
sc-consensus-manual-seal
[]
sc-consensus-pow
[]
sc-consensus-slots
[]
sc-consensus-uncles
[]
sc-executor
[
  "default",
  "std",
  "wasm-extern-trace",
  "wasmer-sandbox"
]
sc-executor-common
[
  "default",
  "wasmer",
  "wasmer-sandbox"
]
sc-executor-wasmi
[]
sc-executor-wasmtime
[]
sc-finality-grandpa
[]
sc-finality-grandpa-rpc
[]
sc-informant
[]
sc-keystore
[]
sc-network
[
  "default"
]
sc-network-bitswap
[]
sc-network-common
[]
sc-network-gossip
[]
sc-network-light
[]
sc-network-sync
[]
sc-network-test
[]
sc-network-transactions
[]
sc-offchain
[
  "default"
]
sc-peerset
[]
sc-proposer-metrics
[]
sc-rpc
[
  "lazy_static",
  "test-helpers",
  "tokio"
]
sc-rpc-api
[]
sc-rpc-server
[]
sc-rpc-spec-v2
[]
sc-runtime-test
[
  "default",
  "std"
]
sc-service
[
  "default",
  "rocksdb",
  "runtime-benchmarks",
  "test-helpers"
]
sc-service-test
[]
sc-state-db
[]
sc-sync-state-rpc
[]
sc-sysinfo
[]
sc-telemetry
[]
sc-tracing
[]
sc-tracing-proc-macro
[]
sc-transaction-pool
[]
sc-transaction-pool-api
[]
sc-utils
[
  "default",
  "metered"
]
sp-api
[
  "default",
  "disable-logging",
  "hash-db",
  "sp-state-machine",
  "sp-trie",
  "std",
  "thiserror"
]
sp-api-proc-macro
[
  "default",
  "std"
]
sp-api-test
[
  "default",
  "std"
]
sp-application-crypto
[
  "default",
  "full_crypto",
  "serde",
  "std"
]
sp-application-crypto-test
[]
sp-arithmetic
[
  "default",
  "serde",
  "std"
]
sp-arithmetic-fuzzer
[]
sp-authority-discovery
[
  "default",
  "std"
]
sp-authorship
[
  "async-trait",
  "default",
  "std"
]
sp-block-builder
[
  "default",
  "std"
]
sp-blockchain
[]
sp-consensus
[
  "default"
]
sp-consensus-aura
[
  "async-trait",
  "default",
  "sp-consensus",
  "std"
]
sp-consensus-babe
[
  "async-trait",
  "default",
  "serde",
  "sp-consensus",
  "sp-keystore",
  "sp-timestamp",
  "std"
]
sp-consensus-pow
[
  "default",
  "std"
]
sp-consensus-slots
[
  "default",
  "serde",
  "std"
]
sp-consensus-vrf
[
  "default",
  "std"
]
sp-core
[
  "array-bytes",
  "base58",
  "blake2",
  "default",
  "dyn-clonable",
  "ed25519-zebra",
  "full_crypto",
  "futures",
  "impl-serde",
  "lazy_static",
  "libsecp256k1",
  "merlin",
  "parking_lot",
  "rand",
  "regex",
  "schnorrkel",
  "secp256k1",
  "serde",
  "sp-core-hashing",
  "sp-externalities",
  "std",
  "substrate-bip39",
  "thiserror",
  "tiny-bip39",
  "wasmi"
]
sp-core-hashing
[
  "default",
  "std"
]
sp-core-hashing-proc-macro
[]
sp-database
[]
sp-debug-derive
[
  "default",
  "force-debug",
  "std"
]
sp-externalities
[
  "default",
  "std"
]
sp-finality-grandpa
[
  "default",
  "log",
  "serde",
  "sp-keystore",
  "std"
]
sp-inherents
[
  "async-trait",
  "default",
  "sp-runtime",
  "std",
  "thiserror"
]
sp-io
[
  "default",
  "disable_allocator",
  "disable_oom",
  "disable_panic_handler",
  "futures",
  "improved_panic_error_reporting",
  "libsecp256k1",
  "log",
  "parking_lot",
  "secp256k1",
  "sp-keystore",
  "sp-state-machine",
  "sp-trie",
  "std",
  "with-tracing"
]
sp-keyring
[]
sp-keystore
[
  "default",
  "serde",
  "std"
]
sp-maybe-compressed-blob
[]
sp-mmr-primitives
[
  "default",
  "serde",
  "std"
]
sp-npos-elections
[
  "bench",
  "default",
  "serde",
  "std"
]
sp-npos-elections-fuzzer
[]
sp-offchain
[
  "default",
  "std"
]
sp-panic-handler
[]
sp-rpc
[]
sp-runtime
[
  "default",
  "parity-util-mem",
  "rand",
  "runtime-benchmarks",
  "serde",
  "std"
]
sp-runtime-interface
[
  "default",
  "disable_target_static_assertions",
  "std"
]
sp-runtime-interface-proc-macro
[]
sp-runtime-interface-test
[]
sp-runtime-interface-test-wasm
[
  "default",
  "std"
]
sp-runtime-interface-test-wasm-deprecated
[
  "default",
  "std"
]
sp-sandbox
[
  "default",
  "std",
  "strict",
  "wasmer-sandbox"
]
sp-serializer
[]
sp-session
[
  "default",
  "sp-runtime",
  "std"
]
sp-staking
[
  "default",
  "runtime-benchmarks",
  "std"
]
sp-state-machine
[
  "default",
  "log",
  "parking_lot",
  "rand",
  "sp-panic-handler",
  "std",
  "thiserror",
  "tracing"
]
sp-std
[
  "default",
  "std"
]
sp-storage
[
  "default",
  "impl-serde",
  "serde",
  "std"
]
sp-test-primitives
[
  "default",
  "serde",
  "std"
]
sp-timestamp
[
  "async-trait",
  "default",
  "futures-timer",
  "log",
  "std",
  "thiserror"
]
sp-tracing
[
  "default",
  "std",
  "tracing-subscriber",
  "with-tracing"
]
sp-transaction-pool
[
  "default",
  "std"
]
sp-transaction-storage-proof
[
  "async-trait",
  "default",
  "log",
  "sp-core",
  "sp-trie",
  "std"
]
sp-trie
[
  "ahash",
  "default",
  "hashbrown",
  "lazy_static",
  "lru",
  "memory-tracker",
  "nohash-hasher",
  "parking_lot",
  "std",
  "thiserror",
  "tracing"
]
sp-version
[
  "default",
  "impl-serde",
  "parity-wasm",
  "serde",
  "std",
  "thiserror"
]
sp-version-proc-macro
[]
sp-wasm-interface
[
  "default",
  "log",
  "std",
  "wasmi",
  "wasmtime"
]
sp-weights
[
  "default",
  "full-metadata-docs",
  "serde",
  "std"
]
subkey
[]
substrate-build-script-utils
[]
substrate-frame-cli
[
  "default"
]
substrate-frame-rpc-support
[]
substrate-frame-rpc-system
[]
substrate-prometheus-endpoint
[]
substrate-rpc-client
[]
substrate-state-trie-migration-rpc
[]
substrate-test-client
[]
substrate-test-runtime
[
  "default",
  "disable-logging",
  "sc-service",
  "serde",
  "sp-keyring",
  "std"
]
substrate-test-runtime-client
[]
substrate-test-runtime-transaction-pool
[]
substrate-test-utils
[]
substrate-test-utils-derive
[]
substrate-test-utils-test-crate
[]
substrate-wasm-builder
[]
try-runtime-cli
[
  "frame-try-runtime",
  "try-runtime"
]

I don't know if that is enough (looks like there's some special handling in https://github.com/taiki-e/cargo-hack/blob/4ccbb89b4f91e8d79a999280d3b0c22c3d30df24/src/features.rs#L46-L60) or if it's correct, but it could be a starting point.

[niklasad1]

I don't know if that is enough (looks like there's some special handling in https://github.com/taiki-e/cargo-hack/blob/4ccbb89b4f91e8d79a999280d3b0c22c3d30df24/src/features.rs#L46-L60) or if it's correct, but it could be a starting point.

oh didn't know that then it makes sense not to use it for substrate where not all crates are published on crates.io