feat: Add initial benchmarks, integrate them into CI & add getters/settters for Scripts resource (#381)

# Summary
- Moves tests to run centrally in the workspace tests directory
- Refactors test utilities to share a lot of the test discovery and
running code between languages and benchmarks
- Adds criterion benchmarks
- Adds CI for snapshoting performance from main to `bencher.dev`

Author: makspll

.github/workflows/snapshot_benchmark_main.yml

@@ -0,0 +1,32 @@
+on:
+  push:
+    branches: main
+
+jobs:
+  benchmark_base_branch:
+    name: Continuous Benchmarking with Bencher
+    permissions:
+      checks: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Xtask initializer
+        run: |
+          cargo xtask init
+      - uses: bencherdev/bencher@main
+      - name: Track base branch benchmarks with Bencher
+        run: |
+          bencher run \
+          --project bms \
+          --token '${{ secrets.BENCHER_API_TOKEN }}' \
+          --branch main \
+          --testbed ubuntu-latest \
+          --threshold-measure latency \
+          --threshold-test t_test \
+          --threshold-max-sample-size 64 \
+          --threshold-upper-boundary 0.99 \
+          --thresholds-reset \
+          --err \
+          --adapter json \
+          --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+          bencher run --adapter rust_criterion "cargo bench --features lua54"

Cargo.toml

@@ -22,8 +22,10 @@ features = ["lua54", "rhai"]
 [features]
 default = ["core_functions", "bevy_bindings"]
 
-## lua
-lua = ["bevy_mod_scripting_lua", "bevy_mod_scripting_functions/lua_bindings"]
+lua = [
+    "bevy_mod_scripting_lua",
+    "bevy_mod_scripting_functions/lua_bindings",
+] ## lua
 # one of these must be selected
 lua51 = ["bevy_mod_scripting_lua/lua51", "lua"]
 lua52 = ["bevy_mod_scripting_lua/lua52", "lua"]
@@ -76,8 +78,12 @@ clap = { version = "4.1", features = ["derive"] }
 rand = "0.8.5"
 bevy_console = "0.13"
 # rhai-rand = "0.1"
+criterion = { version = "0.5" }
 ansi-parser = "0.9"
 ladfile_builder = { path = "crates/ladfile_builder", version = "0.2.6" }
+script_integration_test_harness = { workspace = true }
+test_utils = { workspace = true }
+libtest-mimic = "0.8"
 
 [workspace]
 members = [
@@ -149,3 +155,11 @@ todo = "deny"
 
 [workspace.lints.rust]
 missing_docs = "deny"
+
+[[bench]]
+name = "benchmarks"
+harness = false
+
+[[test]]
+name = "script_tests"
+harness = false

assets/benchmarks/component/access.lua

@@ -0,0 +1,5 @@
+local entity_with_component = world._get_entity_with_test_component("TestComponent")
+
+function bench()
+    local strings = world.get_component(entity_with_component, types.TestComponent).strings
+end

assets/benchmarks/component/access.rhai

@@ -0,0 +1,5 @@
+let entity_with_component = world._get_entity_with_test_component.call("TestComponent");
+
+fn bench(){
+    let strings = world.get_component.call(entity_with_component, types.TestComponent).strings;
+}

assets/benchmarks/component/get.lua

@@ -0,0 +1,5 @@
+local entity_with_component = world._get_entity_with_test_component("TestComponent")
+
+function bench()
+    world.get_component(entity_with_component, types.TestComponent)
+end

assets/benchmarks/component/get.rhai

@@ -0,0 +1,5 @@
+let entity_with_component = world._get_entity_with_test_component.call("TestComponent");
+
+fn bench(){
+    world.get_component.call(entity_with_component, types.TestComponent);
+}

benches/benchmarks.rs

@@ -0,0 +1,95 @@
+use std::path::PathBuf;
+
+use bevy::utils::HashMap;
+use criterion::{
+    criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, Criterion,
+};
+use script_integration_test_harness::{run_lua_benchmark, run_rhai_benchmark};
+use test_utils::{discover_all_tests, Test};
+
+extern crate bevy_mod_scripting;
+extern crate script_integration_test_harness;
+extern crate test_utils;
+
+pub trait BenchmarkExecutor {
+    fn benchmark_group(&self) -> String;
+    fn benchmark_name(&self) -> String;
+    fn execute<M: Measurement>(&self, criterion: &mut BenchmarkGroup<M>);
+}
+
+impl BenchmarkExecutor for Test {
+    fn benchmark_group(&self) -> String {
+        // we want to use OS agnostic paths
+        // use the file path from `benchmarks` onwards using folders as groupings
+        // replace file separators with `/`
+        // replace _ with spaces
+        let path = self.path.to_string_lossy();
+        let path = path.split("benchmarks").collect::<Vec<&str>>()[1]
+            .replace(std::path::MAIN_SEPARATOR, "/");
+        let first_folder = path.split("/").collect::<Vec<&str>>()[1];
+        first_folder.replace("_", " ")
+    }
+
+    fn benchmark_name(&self) -> String {
+        // use just the file stem
+        let name = self
+            .path
+            .file_stem()
+            .unwrap()
+            .to_string_lossy()
+            .to_string()
+            .replace("_", " ");
+
+        let language = self.kind.to_string();
+
+        format!("{name} {language}")
+    }
+
+    fn execute<M: Measurement>(&self, criterion: &mut BenchmarkGroup<M>) {
+        match self.kind {
+            test_utils::TestKind::Lua => run_lua_benchmark(
+                &self.path.to_string_lossy(),
+                &self.benchmark_name(),
+                criterion,
+            )
+            .expect("Benchmark failed"),
+            test_utils::TestKind::Rhai => run_rhai_benchmark(
+                &self.path.to_string_lossy(),
+                &self.benchmark_name(),
+                criterion,
+            )
+            .expect("benchmark failed"),
+        }
+    }
+}
+
+fn script_benchmarks(criterion: &mut Criterion) {
+    // find manifest dir
+    let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let tests = discover_all_tests(manifest_dir, |p| p.starts_with("benchmarks"));
+
+    // group by benchmark group
+    let mut grouped: HashMap<String, Vec<Test>> =
+        tests.into_iter().fold(HashMap::default(), |mut acc, t| {
+            acc.entry(t.benchmark_group()).or_default().push(t);
+            acc
+        });
+
+    // sort within groups by benchmark name
+    for (_, tests) in grouped.iter_mut() {
+        tests.sort_by_key(|a| a.benchmark_name());
+    }
+
+    for (group, tests) in grouped {
+        let mut benchmark_group = criterion.benchmark_group(group);
+
+        for t in tests {
+            t.execute(&mut benchmark_group);
+        }
+
+        benchmark_group.finish();
+    }
+}
+
+criterion_group!(benches, script_benchmarks);
+criterion_main!(benches);

crates/bevy_mod_scripting_core/src/script.rs

@@ -39,6 +39,44 @@ pub struct Scripts<P: IntoScriptPluginParams> {
     pub(crate) scripts: HashMap<ScriptId, Script<P>>,
 }
 
+impl<P: IntoScriptPluginParams> Scripts<P> {
+    /// Inserts a script into the collection
+    pub fn insert(&mut self, script: Script<P>) {
+        self.scripts.insert(script.id.clone(), script);
+    }
+
+    /// Removes a script from the collection, returning `true` if the script was in the collection, `false` otherwise
+    pub fn remove<S: Into<ScriptId>>(&mut self, script: S) -> bool {
+        self.scripts.remove(&script.into()).is_some()
+    }
+
+    /// Checks if a script is in the collection
+    /// Returns `true` if the script is in the collection, `false` otherwise
+    pub fn contains<S: Into<ScriptId>>(&self, script: S) -> bool {
+        self.scripts.contains_key(&script.into())
+    }
+
+    /// Returns a reference to the script with the given id
+    pub fn get<S: Into<ScriptId>>(&self, script: S) -> Option<&Script<P>> {
+        self.scripts.get(&script.into())
+    }
+
+    /// Returns a mutable reference to the script with the given id
+    pub fn get_mut<S: Into<ScriptId>>(&mut self, script: S) -> Option<&mut Script<P>> {
+        self.scripts.get_mut(&script.into())
+    }
+
+    /// Returns an iterator over the scripts
+    pub fn iter(&self) -> impl Iterator<Item = &Script<P>> {
+        self.scripts.values()
+    }
+
+    /// Returns a mutable iterator over the scripts
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Script<P>> {
+        self.scripts.values_mut()
+    }
+}
+
 impl<P: IntoScriptPluginParams> Default for Scripts<P> {
     fn default() -> Self {
         Self {

crates/languages/bevy_mod_scripting_lua/Cargo.toml

@@ -46,14 +46,5 @@ smol_str = "0.2.2"
 smallvec = "1.13"
 profiling = { workspace = true }
 
-[dev-dependencies]
-script_integration_test_harness = { workspace = true }
-libtest-mimic = "0.8"
-regex = "1.11"
-
-[[test]]
-name = "lua_tests"
-harness = false
-
 [lints]
 workspace = true