Support an argument to resume MicroBenchmarks from a previous run (dotnet#2378)

* Support a --resume argument to resume a MicroBenchmarks run from previous results

* Show existing results in the output

Author: jeffhandley

Diff for: scripts/benchmarks_ci.py

@@ -172,13 +172,22 @@ def __is_valid_datetime(dt: str) -> str:
         action='store_true',
         help='Attempts to run the benchmarks without building.',
     )
+    parser.add_argument(
+        '--resume',
+        dest='resume',
+        required=False,
+        default=False,
+        action='store_true',
+        help='Resume a previous run from existing benchmark results',
+    )
     parser.add_argument(
         '--skip-logger-setup',
         dest='skip_logger_setup',
         required=False,
         default=False,
         action='store_true',
-        help='Skips the logger setup, for cases when invoked by another script that already sets logging up')
+        help='Skips the logger setup, for cases when invoked by another script that already sets logging up',
+    )
 
     return parser
 

Diff for: scripts/benchmarks_monthly.py

@@ -75,7 +75,13 @@ def add_arguments(parser: ArgumentParser) -> ArgumentParser:
         '--no-clean',
         dest='no_clean',
         action='store_true',
-        help='Do not clean the SDK and results directories before execution')
+        help='Do not clean the SDK installations before execution')
+
+    parser.add_argument(
+        '--resume',
+        dest='resume',
+        action='store_true',
+        help='Resume a previous run from existing benchmark results')
 
     parser.add_argument(
         '--dry-run',
@@ -131,25 +137,28 @@ def log(text: str):
         resultsPath = os.path.join(rootPath, 'artifacts', 'bin', 'MicroBenchmarks', 'Release', moniker, 'BenchmarkDotNet.Artifacts', 'results')
 
         if not args.no_clean:
-            # Delete any preexisting SDK and results, which allows
+            # Delete any preexisting SDK installations, which allows
             # multiple versions to be run from a single command
             if os.path.isdir(sdkPath):
                 log('rmdir -r ' + sdkPath)
 
                 if not args.dry_run:
                     shutil.rmtree(sdkPath)
 
+        benchmarkArgs = ['--skip-logger-setup', '--filter', args.filter, '--architecture', args.architecture, '-f', version['tfm']]
+
+        if 'build' in version:
+            benchmarkArgs += ['--dotnet-versions', version['build']]
+
+        if args.resume:
+            benchmarkArgs += ['--resume']
+        else:
             if os.path.isdir(resultsPath):
                 log('rmdir -r ' + resultsPath)
 
                 if not args.dry_run:
                     shutil.rmtree(resultsPath)
 
-        benchmarkArgs = ['--skip-logger-setup', '--filter', args.filter, '--architecture', args.architecture, '-f', version['tfm']]
-
-        if 'build' in version:
-            benchmarkArgs += ['--dotnet-versions', version['build']]
-
         if args.bdn_arguments:
             if version['tfm'].startswith('nativeaot'):
                 benchmarkArgs += ['--bdn-arguments', args.bdn_arguments + ' --ilCompilerVersion ' + version['ilc']]

Diff for: scripts/micro_benchmarks.py

@@ -243,6 +243,8 @@ def __get_benchmarkdotnet_arguments(framework: str, args: tuple) -> list:
         ]
     if args.filter:
         run_args += ['--filter'] + args.filter
+    if args.resume:
+        run_args += ['--resume']
 
     # Extra BenchmarkDotNet cli arguments.
     if args.bdn_arguments:

Diff for: src/benchmarks/micro/Program.cs

@@ -23,15 +23,18 @@ static int Main(string[] args)
             List<string> categoryExclusionFilterValue;
             Dictionary<string, string> parameterFilterValue;
             bool getDiffableDisasm;
+            bool resumeRun;
 
             // Parse and remove any additional parameters that we need that aren't part of BDN
-            try {
+            try
+            {
                 argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out partitionCount);
                 argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out partitionIndex);
                 argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out exclusionFilterValue);
                 argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue);
                 argsList = CommandLineOptions.ParseAndRemovePairsParameter(argsList, "--parameter-filter", out parameterFilterValue);
                 CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm);
+                CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--resume", out resumeRun);
 
                 CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex);
             }
@@ -52,7 +55,8 @@ static int Main(string[] args)
                         exclusionFilterValue: exclusionFilterValue,
                         categoryExclusionFilterValue: categoryExclusionFilterValue,
                         parameterFilterValue: parameterFilterValue,
-                        getDiffableDisasm: getDiffableDisasm)
+                        getDiffableDisasm: getDiffableDisasm,
+                        resumeRun: resumeRun)
                     .AddValidator(new NoWasmValidator(Categories.NoWASM)))
                 .ToExitCode();
         }

Diff for: src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs

@@ -1,18 +1,20 @@
-using System.Collections.Immutable;
-using System.IO;
-using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Exporters.Json;
-using Perfolizer.Horology;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
-using System.Collections.Generic;
+using Newtonsoft.Json;
+using Perfolizer.Horology;
 using Reporting;
-using BenchmarkDotNet.Loggers;
-using System.Linq;
-using BenchmarkDotNet.Exporters;
 using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
 
 namespace BenchmarkDotNet.Extensions
 {
@@ -27,7 +29,8 @@ public static IConfig Create(
             List<string> categoryExclusionFilterValue = null,
             Dictionary<string, string> parameterFilterValue = null,
             Job job = null,
-            bool getDiffableDisasm = false)
+            bool getDiffableDisasm = false,
+            bool resumeRun = false)
         {
             if (job is null)
             {
@@ -39,6 +42,12 @@ public static IConfig Create(
                     .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
             }
 
+            if (resumeRun)
+            {
+                exclusionFilterValue ??= new List<string>();
+                exclusionFilterValue.AddRange(GetBenchmarksToResume(artifactsPath));
+            }
+
             var config = ManualConfig.CreateEmpty()
                 .WithBuildTimeout(TimeSpan.FromMinutes(15)) // for slow machines
                 .AddLogger(ConsoleLogger.Default) // log output to console
@@ -83,5 +92,75 @@ private static DisassemblyDiagnoser CreateDisassembler()
                 exportHtml: false,
                 exportCombinedDisassemblyReport: false,
                 exportDiff: false));
+
+        private static IEnumerable<string> GetBenchmarksToResume(DirectoryInfo artifacts)
+        {
+            if (!artifacts.Exists)
+                return new string[0];
+
+            // Get all existing report files, of any export type; order by descending filename length to avoid rename collisions
+            var existingBenchmarks = artifacts.GetFiles($"*-report-*", SearchOption.AllDirectories)
+                .OrderByDescending(resultFile => resultFile.FullName.Length)
+                .SelectMany(resultFile =>
+                {
+                    var reportFileName = resultFile.FullName;
+
+                    // Prepend the report name with -resume, potentially multiple times if multiple reports for the same
+                    // benchmarks exist, so that they don't collide with one another. But don't unnecessarily prepend
+                    // -resume multiple times.
+                    if (!reportFileName.Contains("-resume-report-") || File.Exists(reportFileName.Replace("-resume-report-", "-report-")))
+                    {
+                        var resumeFileName = reportFileName.Replace("-report-", "-resume-report-");
+                        File.Move(reportFileName, resumeFileName);
+
+                        reportFileName = resumeFileName;
+                    }
+
+                    // For JSON reports, load the data to get the benchmarks that have already been reported
+                    if (reportFileName.EndsWith(".json"))
+                    {
+                        try
+                        {
+                            var result = JsonConvert.DeserializeObject<BdnResult>(File.ReadAllText(reportFileName));
+                            var benchmarks = result.Benchmarks.Select(benchmark =>
+                            {
+                                var nameParts = new[] { benchmark.Namespace, benchmark.Type, benchmark.Method };
+                                return string.Join(".", nameParts.Where(part => !string.IsNullOrEmpty(part)));
+                            }).Distinct();
+
+                            return benchmarks;
+                        }
+                        catch (JsonSerializationException)
+                        {
+                        }
+                    }
+
+                    return new string[0];
+                });
+
+                if (existingBenchmarks.Any())
+                {
+                    Console.WriteLine($"// Found {existingBenchmarks.Count()} existing result(s) to be skipped:");
+
+                    foreach (var benchmark in existingBenchmarks.OrderBy(b => b))
+                    {
+                        Console.WriteLine($"// ***** {benchmark}");
+                    }
+                }
+
+                return existingBenchmarks;
+        }
+
+        private class Benchmark
+        {
+            public string Namespace { get; set; }
+            public string Type { get; set; }
+            public string Method { get; set; }
+        }
+
+        private class BdnResult
+        {
+            public List<Benchmark> Benchmarks { get; set; }
+        }
     }
 }