[GR-12629] harness avg no warmup

PullRequest: graalpython/292

Author: cosminbasca

graalpython/benchmarks/src/harness.py

@@ -55,6 +55,109 @@
 ATTR_TEARDOWN = '__teardown__'
 
 
+# ----------------------------------------------------------------------------------------------------------------------
+#
+# the CUSUM method adapted for warmup detection within a given threshold (initial iterations)
+#
+# ----------------------------------------------------------------------------------------------------------------------
+def zeros(n):
+    return [0 for _ in range(n)]
+
+
+def append(arr, val):
+    if isinstance(arr, list):
+        return arr + [val]
+    else:
+        return [val] + arr
+
+
+def cusum(values, threshold=1.0, drift=0.0):
+    csum_pos, csum_neg = zeros(len(values)), zeros(len(values))
+    change_points = []
+    for i in range(1, len(values)):
+        diff = values[i] - values[i - 1]
+        csum_pos[i] = csum_pos[i-1] + diff - drift
+        csum_neg[i] = csum_neg[i-1] - diff - drift
+
+        if csum_pos[i] < 0:
+            csum_pos[i] = 0
+        if csum_neg[i] < 0:
+            csum_neg[i] = 0
+
+        if csum_pos[i] > threshold or csum_neg[i] > threshold:
+            change_points = append(change_points, i)
+            csum_pos[i], csum_neg[i] = 0, 0
+
+    return change_points
+
+
+def avg(values):
+    return float(sum(values)) / len(values)
+
+
+def norm(values):
+    _max, _min  = max(values), min(values)
+    return [float(v - _min) / (_max - _min) * 100.0 for v in values]
+
+
+def pairwise_slopes(values, cp):
+    return [abs(float(values[i+1] - values[i]) / float(cp[i+1] - cp[i])) for i in range(len(values)-1)]
+
+
+def last_n_percent_runs(values, n=0.1):
+    assert 0.0 < n <= 1.0
+    end_runs_idx = len(values) - int(len(values) * n)
+    end_runs_idx = len(values) - 1 if end_runs_idx >= len(values) else end_runs_idx
+    return values[end_runs_idx:], list(range(end_runs_idx, len(values)))
+
+
+def first_n_percent_runs(values, n=0.1):
+    assert 0.0 < n <= 1.0
+    first_run_idx = int(len(values) * n)
+    return first_run_idx -1 if first_run_idx == len(values) else first_run_idx
+
+
+def detect_warmup(values, cp_threshold=0.03, stability_slope_grade=0.01):
+    """
+    detect the point of warmup point (iteration / run)
+
+    :param values: the durations for each run
+    :param cp_threshold:  the percent in value difference for a point to be considered a change point (percentage)
+    :param stability_slope_grade: the slope grade (percentage). A grade of 1% corresponds to a slope of 0.5 degrees
+    :return: the change point or -1 if not detected
+    """
+    # normalize all
+    stability_slope_grade *= 100.0
+    cp_threshold *= 100
+    values = norm(values)
+
+    try:
+        cp = cusum(values, threshold=cp_threshold)
+        rolling_avg = [avg(values[i:]) for i in cp]
+
+        def warmup(cp_index):
+            val_idx = cp[cp_index] + 1
+            return val_idx if val_idx < len(values) else -1
+
+        # find the point where the duration avg is below the cp threshold
+        for i, d in enumerate(rolling_avg):
+            if d <= cp_threshold:
+                return warmup(i)
+
+        # could not find something below the CP threshold (noise in the data), use the stabilisation of slopes
+        last_n_vals, last_n_idx = last_n_percent_runs(values, 0.1)
+        slopes = pairwise_slopes(rolling_avg + last_n_vals, cp + last_n_idx)
+
+        for i, d in enumerate(slopes):
+            if d <= stability_slope_grade:
+                return warmup(i)
+
+        return -1
+    except Exception as e:
+        print("exception occurred while detecting warmup: %s" % e)
+        return -1
+
+
 def ccompile(name, code):
     from importlib import invalidate_caches
     from distutils.core import setup, Extension
@@ -89,7 +192,14 @@ def _as_int(value):
 
 
 class BenchRunner(object):
-    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=0):
+    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0):
+        assert isinstance(iterations, int), \
+            "BenchRunner iterations argument must be an int, got %s instead" % iterations
+        assert isinstance(warmup, int), \
+            "BenchRunner warmup argument must be an int, got %s instead" % warmup
+        assert isinstance(warmup_runs, int), \
+            "BenchRunner warmup_runs argument must be an int, got %s instead" % warmup_runs
+
         if bench_args is None:
             bench_args = []
         self.bench_module = BenchRunner.get_bench_module(bench_file)
@@ -98,10 +208,8 @@ def __init__(self, bench_file, bench_args=None, iterations=1, warmup=0):
         _iterations = _as_int(iterations)
         self._run_once = _iterations <= 1
         self.iterations = 1 if self._run_once else _iterations
-
-        assert isinstance(self.iterations, int)
-        self.warmup = _as_int(warmup)
-        assert isinstance(self.warmup, int)
+        self.warmup_runs = warmup_runs if warmup_runs > 0 else 0
+        self.warmup = warmup if warmup > 0 else -1
 
     @staticmethod
     def get_bench_module(bench_file):
@@ -139,9 +247,10 @@ def _call_attr(self, attr_name, *args):
 
     def run(self):
         if self._run_once:
-            print("### %s, exactly one iteration (no warmup curves)" % (self.bench_module.__name__))
+            print("### %s, exactly one iteration (no warmup curves)" % self.bench_module.__name__)
         else:
-            print("### %s, %s warmup iterations, %s bench iterations " % (self.bench_module.__name__, self.warmup, self.iterations))
+            print("### %s, %s warmup iterations, %s bench iterations " % (self.bench_module.__name__,
+                                                                          self.warmup_runs, self.iterations))
 
         # process the args if the processor function is defined
         args = self._call_attr(ATTR_PROCESS_ARGS, *self.bench_args)
@@ -159,9 +268,9 @@ def run(self):
         bench_func = self._get_attr(ATTR_BENCHMARK)
         durations = []
         if bench_func and hasattr(bench_func, '__call__'):
-            if self.warmup:
-                print("### warming up for %s iterations ... " % self.warmup)
-                for _ in range(self.warmup):
+            if self.warmup_runs:
+                print("### (pre)warming up for %s iterations ... " % self.warmup_runs)
+                for _ in range(self.warmup_runs):
                     bench_func(*args)
 
             for iteration in range(self.iterations):
@@ -173,21 +282,39 @@ def run(self):
                 if self._run_once:
                     print("@@@ name=%s, duration=%s" % (self.bench_module.__name__, duration_str))
                 else:
-                    print("### iteration=%s, name=%s, duration=%s" % (iteration, self.bench_module.__name__, duration_str))
+                    print("### iteration=%s, name=%s, duration=%s" % (iteration, self.bench_module.__name__,
+                                                                      duration_str))
 
         print(_HRULE)
         print("### teardown ... ")
         self._call_attr(ATTR_TEARDOWN)
+        warmup_iter = self.warmup if self.warmup > 0 else detect_warmup(durations)
+        # if we cannot detect a warmup starting point but we performed some pre runs, we take a starting point
+        # after the 10% of the first runs ...
+        if warmup_iter < 0 and self.warmup_runs > 0:
+            print("### warmup could not be detected, but %s pre-runs were executed.\n"
+                  "### we assume the benchmark is warmed up and pick an iteration "
+                  "in the first 10%% of the runs" % self.warmup_runs)
+            warmup_iter = first_n_percent_runs(durations, 0.1)
         print("### benchmark complete")
         print(_HRULE)
-        print("### BEST     duration: %.3f s" % min(durations))
-        print("### WORST    duration: %.3f s" % max(durations))
-        print("### AVG      duration: %.3f" % (sum(durations) / len(durations)))
+        print("### BEST                duration: %.3f s" % min(durations))
+        print("### WORST               duration: %.3f s" % max(durations))
+        print("### AVG (all runs)      duration: %.3f s" % (sum(durations) / len(durations)))
+        if warmup_iter > 0:
+            print("### WARMUP %s at iteration: %d" % ("specified" if self.warmup > 0 else "detected", warmup_iter))
+            no_warmup_durations = durations[warmup_iter:]
+            print("### AVG (no warmup)     duration: %.3f s" % (sum(no_warmup_durations) / len(no_warmup_durations)))
+        else:
+            print("### WARMUP iteration not specified or could not be detected")
+        print(_HRULE)
+        print("### RAW DURATIONS: %s" % str(durations))
         print(_HRULE)
 
 
 def run_benchmark(args):
-    warmup = 0
+    warmup = -1
+    warmup_runs = 0
     iterations = 1
     bench_file = None
     bench_args = []
@@ -208,6 +335,12 @@ def run_benchmark(args):
         elif arg.startswith("--warmup"):
             warmup = _as_int(arg.split("=")[1])
 
+        elif arg == '-r':
+            i += 1
+            warmup_runs = _as_int(args[i])
+        elif arg.startswith("--warmup-runs"):
+            warmup_runs = _as_int(arg.split("=")[1])
+
         elif arg == '-p':
             i += 1
             paths = args[i].split(",")
@@ -229,7 +362,7 @@ def run_benchmark(args):
     else:
         print("### no extra module search paths specified")
 
-    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup).run()
+    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs).run()
 
 
 if __name__ == '__main__':