diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000000..c2dc0fea8b4 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,48 @@ +name: Benchmarks +on: + workflow_dispatch: + schedule: + - cron: '0 8 * * *' + push: + tags: + - 'v*' + +concurrency: benchmarks + +env: + ROBUST_BENCHMARKS_ENABLE_SQL: 1 + ROBUST_BENCHMARKS_SQL_ADDRESS: ${{ secrets.BENCHMARKS_WRITE_ADDRESS }} + ROBUST_BENCHMARKS_SQL_PORT: ${{ secrets.BENCHMARKS_WRITE_PORT }} + ROBUST_BENCHMARKS_SQL_USER: ${{ secrets.BENCHMARKS_WRITE_USER }} + ROBUST_BENCHMARKS_SQL_PASSWORD: ${{ secrets.BENCHMARKS_WRITE_PASSWORD }} + ROBUST_BENCHMARKS_SQL_DATABASE: content_benchmarks + +jobs: + benchmark: + name: Run Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Get Engine version + run: | + cd RobustToolbox + git fetch --depth=1 + echo "::set-output name=out::$(git rev-parse HEAD)" + id: engine_version + - name: Run script on centcomm + uses: appleboy/ssh-action@master + with: + host: centcomm.spacestation14.io + username: robust-benchmark-runner + key: ${{ secrets.CENTCOMM_ROBUST_BENCHMARK_RUNNER_KEY }} + command_timeout: 100000m + script: | + mkdir benchmark_run_${{ github.sha }} + cd benchmark_run_${{ github.sha }} + wget https://raw.githubusercontent.com/space-wizards/RobustToolbox/${{ steps.engine_version.outputs.out }}/Tools/run_benchmarks.py + python3 run_benchmarks.py "${{ secrets.BENCHMARKS_WRITE_ADDRESS }}" "${{ secrets.BENCHMARKS_WRITE_PORT }}" "${{ secrets.BENCHMARKS_WRITE_USER }}" "${{ secrets.BENCHMARKS_WRITE_PASSWORD }}" "${{ github.sha }}" + rm run_benchmarks.py + cd .. + rmdir benchmark_run_${{ github.sha }} diff --git a/Content.Benchmarks/ColorInterpolateBenchmark.cs b/Content.Benchmarks/ColorInterpolateBenchmark.cs index 03ee12de5f3..2243bb4819e 100644 --- a/Content.Benchmarks/ColorInterpolateBenchmark.cs +++ b/Content.Benchmarks/ColorInterpolateBenchmark.cs @@ -5,6 +5,7 @@ using System; using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; +using Robust.Shared.Analyzers; using Robust.Shared.Maths; using Robust.Shared.Random; using SysVector4 = System.Numerics.Vector4; @@ -12,7 +13,8 @@ namespace Content.Benchmarks { [DisassemblyDiagnoser] - public sealed class ColorInterpolateBenchmark + [Virtual] + public class ColorInterpolateBenchmark { #if NETCOREAPP private const MethodImplOptions AggressiveOpt = MethodImplOptions.AggressiveOptimization; diff --git a/Content.Benchmarks/ComponentFetchBenchmark.cs b/Content.Benchmarks/ComponentFetchBenchmark.cs index c29df7aa6e4..e0601f71eef 100644 --- a/Content.Benchmarks/ComponentFetchBenchmark.cs +++ b/Content.Benchmarks/ComponentFetchBenchmark.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Attributes; +using Robust.Shared.Analyzers; using Robust.Shared.Utility; namespace Content.Benchmarks { [SimpleJob] - public sealed class ComponentFetchBenchmark + [Virtual] + public class ComponentFetchBenchmark { [Params(5000)] public int NEnt { get; set; } diff --git a/Content.Benchmarks/Content.Benchmarks.csproj b/Content.Benchmarks/Content.Benchmarks.csproj index eec2f80d655..5c4a0521683 100644 --- a/Content.Benchmarks/Content.Benchmarks.csproj +++ b/Content.Benchmarks/Content.Benchmarks.csproj @@ -19,6 +19,7 @@ + diff --git a/Content.Benchmarks/DependencyInjectBenchmark.cs b/Content.Benchmarks/DependencyInjectBenchmark.cs index aed00ec70b6..06bd45fe3f2 100644 --- a/Content.Benchmarks/DependencyInjectBenchmark.cs +++ b/Content.Benchmarks/DependencyInjectBenchmark.cs @@ -6,7 +6,8 @@ namespace Content.Benchmarks { // To actually run this benchmark you'll have to make DependencyCollection public so it's accessible. - public sealed class DependencyInjectBenchmark + [Virtual] + public class DependencyInjectBenchmark { [Params(InjectMode.Reflection, InjectMode.DynamicMethod)] public InjectMode Mode { get; set; } diff --git a/Content.Benchmarks/DynamicTreeBenchmark.cs b/Content.Benchmarks/DynamicTreeBenchmark.cs index 056982c8475..ab489966b78 100644 --- a/Content.Benchmarks/DynamicTreeBenchmark.cs +++ b/Content.Benchmarks/DynamicTreeBenchmark.cs @@ -1,11 +1,13 @@ using BenchmarkDotNet.Attributes; +using Robust.Shared.Analyzers; using Robust.Shared.Maths; using Robust.Shared.Physics; namespace Content.Benchmarks { [SimpleJob, MemoryDiagnoser] - public sealed class DynamicTreeBenchmark + [Virtual] + public class DynamicTreeBenchmark { private static readonly Box2[] _aabbs1 = { diff --git a/Content.Benchmarks/EntityFetchBenchmark.cs b/Content.Benchmarks/EntityFetchBenchmark.cs index 165f0e9fb84..d65b7faeddc 100644 --- a/Content.Benchmarks/EntityFetchBenchmark.cs +++ b/Content.Benchmarks/EntityFetchBenchmark.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Attributes; +using Robust.Shared.Analyzers; using Robust.Shared.Utility; namespace Content.Benchmarks { [SimpleJob] - public sealed class EntityFetchBenchmark + [Virtual] + public class EntityFetchBenchmark { [Params(1000)] public int N { get; set; } diff --git a/Content.Benchmarks/EntityManagerGetAllComponents.cs b/Content.Benchmarks/EntityManagerGetAllComponents.cs index e0ff948e0dd..4266b9d708d 100644 --- a/Content.Benchmarks/EntityManagerGetAllComponents.cs +++ b/Content.Benchmarks/EntityManagerGetAllComponents.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Attributes; using Moq; +using Robust.Shared.Analyzers; using Robust.Shared.Exceptions; using Robust.Shared.GameObjects; using Robust.Shared.IoC; @@ -9,7 +10,8 @@ namespace Content.Benchmarks { - public sealed class EntityManagerGetAllComponents + [Virtual] + public class EntityManagerGetAllComponents { private IEntityManager _entityManager; diff --git a/Content.Benchmarks/MapLoadBenchmark.cs b/Content.Benchmarks/MapLoadBenchmark.cs new file mode 100644 index 00000000000..40b03e959ae --- /dev/null +++ b/Content.Benchmarks/MapLoadBenchmark.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using Content.IntegrationTests; +using Content.Server.Maps; +using Robust.Server.Maps; +using Robust.Shared; +using Robust.Shared.Analyzers; +using Robust.Shared.IoC; +using Robust.Shared.Map; +using Robust.Shared.Prototypes; + +namespace Content.Benchmarks; + +[Virtual] +public class MapLoadBenchmark +{ + private PairTracker _pair = default!; + + [GlobalSetup] + public void Setup() + { + ProgramShared.PathOffset = "../../../../"; + _pair = PoolManager.GetServerClient().GetAwaiter().GetResult(); + Paths = _pair.Pair.Server.ResolveDependency() + .EnumeratePrototypes().ToDictionary(x => x.ID, x => x.MapPath.ToString()); + IoCManager.InitThread(_pair.Pair.Server.InstanceDependencyCollection); + } + + public static IEnumerable MapsSource { get; set; } + + [ParamsSource(nameof(MapsSource))] public string Map; + + public static Dictionary Paths; + + [Benchmark] + public void LoadMap() + { + _pair.Pair.Server.ResolveDependency().LoadMap(new MapId(10), Paths[Map]); + } +} diff --git a/Content.Benchmarks/NetSerializerIntBenchmark.cs b/Content.Benchmarks/NetSerializerIntBenchmark.cs index 5480da7f7da..98441852098 100644 --- a/Content.Benchmarks/NetSerializerIntBenchmark.cs +++ b/Content.Benchmarks/NetSerializerIntBenchmark.cs @@ -2,11 +2,13 @@ using System.Buffers.Binary; using System.IO; using BenchmarkDotNet.Attributes; +using Robust.Shared.Analyzers; namespace Content.Benchmarks { [SimpleJob] - public sealed class NetSerializerIntBenchmark + [Virtual] + public class NetSerializerIntBenchmark { private MemoryStream _writeStream; private MemoryStream _readStream; diff --git a/Content.Benchmarks/NetSerializerStringBenchmark.cs b/Content.Benchmarks/NetSerializerStringBenchmark.cs index 6d79d283986..32421fcb8b0 100644 --- a/Content.Benchmarks/NetSerializerStringBenchmark.cs +++ b/Content.Benchmarks/NetSerializerStringBenchmark.cs @@ -7,13 +7,15 @@ using BenchmarkDotNet.Attributes; using Lidgren.Network; using NetSerializer; +using Robust.Shared.Analyzers; namespace Content.Benchmarks { // Code for the *Slow and *Unsafe implementations taken from NetSerializer, licensed under the MIT license. [MemoryDiagnoser] - public sealed class NetSerializerStringBenchmark + [Virtual] + public class NetSerializerStringBenchmark { private const int StringByteBufferLength = 256; private const int StringCharBufferLength = 128; diff --git a/Content.Benchmarks/Program.cs b/Content.Benchmarks/Program.cs index 187c328127c..ae2d7817b93 100644 --- a/Content.Benchmarks/Program.cs +++ b/Content.Benchmarks/Program.cs @@ -1,12 +1,38 @@ -using BenchmarkDotNet.Running; +using System; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; +using Content.IntegrationTests; +using Content.Server.Maps; +using Robust.Benchmarks.Configs; +using Robust.Shared.Prototypes; namespace Content.Benchmarks { internal static class Program { + public static void Main(string[] args) { - BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + MainAsync(args).GetAwaiter().GetResult(); + } + + public static async Task MainAsync(string[] args) + { + var pair = await PoolManager.GetServerClient(); + var gameMaps = pair.Pair.Server.ResolveDependency().EnumeratePrototypes().ToList(); + MapLoadBenchmark.MapsSource = gameMaps.Select(x => x.ID); + +#if DEBUG + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("\nWARNING: YOU ARE RUNNING A DEBUG BUILD, USE A RELEASE BUILD FOR AN ACCURATE BENCHMARK"); + Console.WriteLine("THE DEBUG BUILD IS ONLY GOOD FOR FIXING A CRASHING BENCHMARK\n"); + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new DebugInProcessConfig()); +#else + var config = Environment.GetEnvironmentVariable("ROBUST_BENCHMARKS_ENABLE_SQL") != null ? DefaultSQLConfig.Instance : null; + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); +#endif } } } diff --git a/Content.Benchmarks/StereoToMonoBenchmark.cs b/Content.Benchmarks/StereoToMonoBenchmark.cs index 545c2ff472b..9faadfbc562 100644 --- a/Content.Benchmarks/StereoToMonoBenchmark.cs +++ b/Content.Benchmarks/StereoToMonoBenchmark.cs @@ -1,9 +1,11 @@ using System.Runtime.Intrinsics.X86; using BenchmarkDotNet.Attributes; +using Robust.Shared.Analyzers; namespace Content.Benchmarks { - public sealed class StereoToMonoBenchmark + [Virtual] + public class StereoToMonoBenchmark { [Params(128, 256, 512)] public int N { get; set; } diff --git a/RobustToolbox b/RobustToolbox index 42c6b9976ae..74c86bf08c0 160000 --- a/RobustToolbox +++ b/RobustToolbox @@ -1 +1 @@ -Subproject commit 42c6b9976ae1314553386dac7c3bb3739670f8d3 +Subproject commit 74c86bf08c05e3c4c3db34ddf52ace66b24bf9b8