Optimize some CI stuff (#1880)

* CI Optimizations

Author: jackcmay

ci/buildkite.yml

@@ -1,13 +1,25 @@
 steps:
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable-perf"
+    timeout_in_minutes: 20
+    agents:
+      - "queue=cuda"
+  - command: "ci/test-bench.sh"
+    name: "bench [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "nightly"
+    timeout_in_minutes: 30
   - command: "ci/docker-run.sh solanalabs/rust:1.30.1 ci/test-stable.sh"
     name: "stable [public]"
     env:
       CARGO_TARGET_CACHE_NAME: "stable"
     timeout_in_minutes: 30
-  - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-11-12 ci/test-bench.sh"
-    name: "bench [public]"
+  - command: "ci/docker-run.sh solanalabs/rust:1.30.1 ci/test-checks.sh"
+    name: "checks [public]"
     env:
-      CARGO_TARGET_CACHE_NAME: "nightly"
+      CARGO_TARGET_CACHE_NAME: "checks"
     timeout_in_minutes: 30
   - command: "ci/shellcheck.sh"
     name: "shellcheck [public]"
@@ -17,13 +29,6 @@ steps:
     env:
       CARGO_TARGET_CACHE_NAME: "nightly"
     timeout_in_minutes: 30
-  - command: "ci/test-stable-perf.sh"
-    name: "stable-perf [public]"
-    env:
-      CARGO_TARGET_CACHE_NAME: "stable-perf"
-    timeout_in_minutes: 20
-    agents:
-      - "queue=cuda"
   # TODO: Fix and re-enable test-large-network.sh
   # - command: "ci/test-large-network.sh || true"
   #   name: "large-network [public] [ignored]"

ci/test-bench.sh

@@ -9,14 +9,21 @@ source ci/upload_ci_artifact.sh
 eval "$(ci/channel-info.sh)"
 
 ci/version-check.sh nightly
-export RUST_BACKTRACE=1
+if ! ci/version-check.sh nightly; then
+  # This job doesn't run within a container, try once to upgrade tooling on a
+  # version check failure
+  rustup install nightly
+  rustup default nightly
+  ci/version-check.sh nightly
+fi
 
 _() {
   echo "--- $*"
   "$@"
 }
 
 set -o pipefail
+export RUST_BACKTRACE=1
 
 UPLOAD_METRICS=""
 TARGET_BRANCH=$BUILDKITE_BRANCH
@@ -29,6 +36,15 @@ fi
 BENCH_FILE=bench_output.log
 BENCH_ARTIFACT=current_bench_results.log
 _ cargo bench --features=unstable --verbose -- -Z unstable-options --format=json | tee "$BENCH_FILE"
-_ cargo run --release --bin solana-upload-perf -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" >"$BENCH_ARTIFACT"
+
+# Run bpf_loader bench with bpf_c feature enabled
+(
+  set -x
+  cd "programs/native/bpf_loader"
+  echo --- program/native/bpf_loader bench --features=bpf_c
+  cargo bench --verbose --features="bpf_c" -- -Z unstable-options --format=json --nocapture | tee -a ../../../"$BENCH_FILE"
+)
+
+_ cargo run --release --bin solana-upload-perf -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" > "$BENCH_ARTIFACT"
 
 upload_ci_artifact "$BENCH_ARTIFACT"

ci/test-checks.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+ci/version-check.sh stable
+export RUST_BACKTRACE=1
+export RUSTFLAGS="-D warnings"
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo fmt -- --check
+_ cargo clippy -- --version
+_ cargo clippy -- --deny=warnings
+
+_ ci/audit.sh

ci/test-nightly.sh

@@ -27,14 +27,6 @@ maybe_cargo_install() {
   done
 }
 
-# Run program/native/bpf_loader's bench with bpf_c feature
-(
-  set -x
-  cd "programs/native/bpf_loader"
-  echo --- program/native/bpf_loader bench --features=bpf_c
-  cargo bench --verbose --features="bpf_c" -- --nocapture
-)
-
 maybe_cargo_install cov
 
 # Generate coverage data and report via unit-test suite.

ci/test-stable-perf.sh

@@ -10,35 +10,45 @@ if ! ci/version-check.sh stable; then
   # This job doesn't run within a container, try once to upgrade tooling on a
   # version check failure
   rustup install stable
+  rustup default stable
   ci/version-check.sh stable
 fi
+
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 
-./fetch-perf-libs.sh
-# shellcheck source=/dev/null
-source ./target/perf-libs/env.sh
-
 _() {
   echo "--- $*"
   "$@"
 }
 
-FEATURES=cuda,erasure,chacha
+./fetch-perf-libs.sh
+# shellcheck source=/dev/null
+source ./target/perf-libs/env.sh
+
+FEATURES=bpf_c,cuda,erasure,chacha
 _ cargo build --all --verbose --features="$FEATURES"
 _ cargo test --verbose --features="$FEATURES" --lib
 
 # Run integration tests serially
 for test in tests/*.rs; do
   test=${test##*/} # basename x
   test=${test%.rs} # basename x .rs
-  _ cargo test --verbose --jobs=1 --features="$FEATURES" --test="$test"
+  _ cargo test --verbose --features="$FEATURES" --test="$test" -- --test-threads=1
 done
 
+# Run bpf_loader test with bpf_c features enabled
+(
+  set -x
+  cd "programs/native/bpf_loader"
+  echo --- program/native/bpf_loader test --features=bpf_c
+  cargo test --verbose --features="bpf_c"
+)
+
 echo --- ci/localnet-sanity.sh
 (
   set -x
   # Assume |cargo build| has populated target/debug/ successfully.
   export PATH=$PWD/target/debug:$PATH
   USE_INSTALL=1 ci/localnet-sanity.sh
-)
+)

ci/test-stable.sh

@@ -24,20 +24,17 @@ maybe_install() {
   done
 }
 
-_ cargo fmt -- --check
-_ cargo clippy -- --version
-_ cargo clippy -- --deny=warnings
 _ cargo build --all --verbose
 _ cargo test --verbose --lib
 
 # Run integration tests serially
 for test in tests/*.rs; do
   test=${test##*/} # basename x
   test=${test%.rs} # basename x .rs
-  _ cargo test --verbose --jobs=1 --test="$test" --features="bpf_c"
+  _ cargo test --verbose --test="$test" -- --test-threads=1
 done
 
-# Run native program's tests
+# Run native program tests
 for program in programs/native/*; do
   echo --- "$program"
   (
@@ -47,14 +44,6 @@ for program in programs/native/*; do
   )
 done
 
-# Run program/native/bpf_loader's test with bpf_c feature
-(
-  set -x
-  cd "programs/native/bpf_loader"
-  echo --- program/native/bpf_loader test --features=bpf_c
-  cargo test --verbose --features="bpf_c"
-)
-
 # Build the HTML
 export PATH=$CARGO_HOME/bin:$PATH
 maybe_install mdbook
@@ -68,5 +57,3 @@ echo --- ci/localnet-sanity.sh
   export PATH=$PWD/target/debug:$PATH
   USE_INSTALL=1 ci/localnet-sanity.sh
 )
-
-_ ci/audit.sh

programs/native/bpf_loader/benches/bpf_loader.rs

@@ -94,6 +94,7 @@ fn bench_program_alu(bencher: &mut Bencher) {
     assert!(0f64 != summary.median);
     let mips = (instructions * (ns_per_s / summary.median as u64)) / one_million;
     println!("  {:?} MIPS", mips);
+    println!("{{ \"type\": \"bench\", \"name\": \"bench_program_alu_interpreted_mips\", \"median\": {:?}, \"deviation\": 0 }}", mips);
 
     println!("JIT to native:");
     vm.jit_compile().unwrap();
@@ -112,4 +113,5 @@ fn bench_program_alu(bencher: &mut Bencher) {
     assert!(0f64 != summary.median);
     let mips = (instructions * (ns_per_s / summary.median as u64)) / one_million;
     println!("  {:?} MIPS", mips);
+    println!("{{ \"type\": \"bench\", \"name\": \"bench_program_alu_jit_to_native_mips\", \"median\": {:?}, \"deviation\": 0 }}", mips);
 }

src/bin/upload-perf.rs

@@ -100,16 +100,20 @@ fn main() {
         println!("bench_name, median, last_median, deviation, last_deviation");
         for (entry, values) in results {
             println!(
-                "{}, {}, {}, {}, {}",
-                entry, values.0, values.2, values.1, values.3
+                "{:#10?}, {:#10?}, {:#10?}, {:#10?}, {}",
+                values.0,
+                values.2.parse::<i32>().unwrap(),
+                values.1,
+                values.3.parse::<i32>().unwrap(),
+                entry,
             );
         }
     } else {
         println!("No previous results found for {} branch", branch);
         println!("hash: {}", trimmed_hash);
         println!("bench_name, median, deviation");
         for (entry, values) in results {
-            println!("{}, {}, {}", entry, values.0, values.1);
+            println!("{:10?}, {:10?}, {}", values.0, values.1, entry);
         }
     }
     solana_metrics::flush();