Merge pull request #59 from biaslab/bench

Add simple benchmark suite

.gitignore

@@ -5,3 +5,4 @@
 /docs/Manifest.toml
 
 .DS_Store
+benchmark*.md

Makefile

@@ -8,4 +8,7 @@ doc_init:
 	julia --project=docs -e 'ENV["PYTHON"]=""; using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate();'
 
 docs: doc_init ## Generate documentation
-	julia --project=docs/ docs/make.jl
+	julia --project=docs/ docs/make.jl
+
+bench: ## Run benchmark, use `make bench branch=...` to test against a specific branch
+	julia --startup-file=no --project=scripts/ scripts/bench.jl $(branch)

benchmark/benchmarks.jl

@@ -0,0 +1,26 @@
+using BenchmarkTools
+
+const SUITE = BenchmarkGroup()
+
+include("mockenv.jl")
+
+function discrete_benchmarks()
+    SUITE = BenchmarkGroup(["Discrete Environment"])
+    env = RxEnvironment(MockEnvironment; is_discrete=true)
+    agent = add!(env, MockEntity())
+    SUITE["send to agent"] = @benchmarkable send!($agent, $env, 10.0)
+    SUITE["send to environment"] = @benchmarkable send!($env, $agent, 10.0)
+    return SUITE
+end
+
+function continuous_benchmarks()
+    SUITE = BenchmarkGroup(["Continuous Environment"])
+    env = RxEnvironment(MockEnvironment; is_discrete=false)
+    agent = add!(env, MockEntity())
+    SUITE["send to agent"] = @benchmarkable send!($agent, $env, 10.0)
+    SUITE["send to environment"] = @benchmarkable send!($env, $agent, 10.0)
+    return SUITE
+end
+
+SUITE["Discrete Environment"] = discrete_benchmarks()
+SUITE["Continuous Environment"] = continuous_benchmarks()

benchmark/mockenv.jl

@@ -0,0 +1,30 @@
+using RxEnvironments
+
+struct MockEntity end
+
+struct MockEnvironment end
+
+RxEnvironments.update!(subject::MockEnvironment, elapsed_time) = nothing
+RxEnvironments.what_to_send(
+    recipient::MockEntity,
+    emitter::MockEnvironment,
+    observation::Float64,
+) = 1.0
+
+RxEnvironments.what_to_send(
+    recipient::MockEnvironment,
+    emitter::MockEntity,
+    observation::Float64,
+) = 1.0
+
+RxEnvironments.emits(
+    subject::MockEntity,
+    listener::MockEnvironment,
+    action::Float64,
+) = false
+
+RxEnvironments.emits(
+    subject::MockEnvironment,
+    listener::MockEntity,
+    action::Float64,
+) = true

benchmark/tune.json

@@ -0,0 +1 @@
+[{"Julia":"1.10.2","BenchmarkTools":"1.5.0"},[["BenchmarkGroup",{"data":{"Discrete Environment":["BenchmarkGroup",{"data":{"send to environment":["Parameters",{"gctrial":true,"time_tolerance":0.05,"evals_set":false,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"send to agent":["Parameters",{"gctrial":true,"time_tolerance":0.05,"evals_set":false,"samples":10000,"evals":5,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["Discrete Environment"]}],"Continuous Environment":["BenchmarkGroup",{"data":{"send to environment":["Parameters",{"gctrial":true,"time_tolerance":0.05,"evals_set":false,"samples":10000,"evals":9,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"send to agent":["Parameters",{"gctrial":true,"time_tolerance":0.05,"evals_set":false,"samples":10000,"evals":152,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["Continuous Environment"]}]},"tags":[]}]]]

scripts/bench.jl

@@ -0,0 +1,16 @@
+using RxEnvironments
+using BenchmarkTools
+using PkgBenchmark
+
+result, name = if ARGS == []
+    PkgBenchmark.benchmarkpkg(RxEnvironments), "current"
+else
+    BenchmarkTools.judge(
+        RxEnvironments,
+        ARGS[1];
+        judgekwargs = Dict(:time_tolerance => 0.1, :memory_tolerance => 0.05),
+    ),
+    ARGS[1]
+end
+
+export_markdown("benchmark_vs_$(name)_result.md", result)