add benchmarking against pyvsc

Between 12x and 50x faster than pyvsc, depending on testcase

Author: will-keen

README.md

@@ -361,17 +361,17 @@ The user is free to either inherit from `RandObj` (for more complex cases like t
 In local testing, the instruction above can be randomized at a rate of approximately 2000Hz, which is 'fast enough' for most pre-silicon verification simulations.
 
 ## `TODO`
-  - Add meaningful benchmarking against similar libraries (mainly [`pyvsc`](https://github.com/fvutils/pyvsc).)
   - Add equivalent SystemVerilog testcases for benchmarking.
   - Add type hinting.
 
 ## Contributions
 
 Please feel free to contribute to the project, following these guidelines:
 - Please contribute by creating a fork and submitting a pull request.
-- Pull requests should be as small as possible for what issue they are trying to address.
+- Pull requests should be as small as possible to resolve the issue they are trying to address.
 - Pull requests must respect the goals of the library, as stated above.
-- Pull requests should pass all the tests in the `tests/` directory.
+- Pull requests should pass all the tests in the `tests/` directory. Run `python -m tests`.
+- Pull requests should take care not to make performance worse except for cases which require bug fixes. Run `python -m tests` and `python -m benchmarks`.
 
 ## Contact the author(s)
 

benchmarks/__init__.py

@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2023 Imagination Technologies Ltd. All Rights Reserved
+
+'''
+Benchmark against pyvsc library for equivalent testcases.
+'''
+
+import unittest
+import timeit
+import sys
+
+from argparse import ArgumentParser
+
+from constrainedrandom import Random
+from benchmarks.pyvsc.basic import vsc_basic, cr_basic
+from benchmarks.pyvsc.in_keyword import vsc_in, cr_in, cr_in_order
+from benchmarks.pyvsc.ldinstr import vsc_ldinstr
+from examples.ldinstr import ldInstr
+
+
+
+TEST_LENGTH_MULTIPLIER = 1
+
+
+class BenchmarkTests(unittest.TestCase):
+
+    def run_one(self, name, randobj, iterations):
+        '''
+        Benchmark one object that implements the .randomize() function
+        for N iterations.
+        '''
+        start_time = timeit.default_timer()
+        for _ in range(iterations):
+            randobj.randomize()
+        end_time = timeit.default_timer()
+        total_time = end_time - start_time
+        hz = iterations / total_time
+        print(f'{self._testMethodName}: {name} took {total_time:.4g}s for {iterations} iterations ({hz:.1f}Hz)')
+        return total_time, hz
+
+    def run_benchmark(self, randobjs, iterations, check):
+        '''
+        Reusable function to run a fair benchmark between
+        two or more randomizable objects.
+
+        randobjs:   dictionary where key is name, value is an object that implements .randomize()
+        iterations: how many times to call .randomize()
+        check:      function taking a dictionary of results to check.
+        '''
+        iterations *= TEST_LENGTH_MULTIPLIER
+        results = {}
+        winner = None
+        best_hz = 0
+        for name, randobj in randobjs.items():
+            total_time, hz = self.run_one(name, randobj, iterations)
+            if hz > best_hz:
+                winner = name
+                best_hz = hz
+            # Store both total time and Hz in case the test wants to specify
+            # checks on how long should be taken in wall clock time.
+            results[name] = total_time, hz
+        print(f'{self._testMethodName}: The winner is {winner} with {best_hz:.1f}Hz!')
+        # Print summary of hz delta
+        for name, (_total_time, hz) in results.items():
+            if name == winner:
+                continue
+            speedup = best_hz / hz
+            print(f'{self._testMethodName}: {winner} was {speedup:.2f}x faster than {name}')
+        check(results)
+
+    def test_basic(self):
+        '''
+        Test basic randomizable object.
+        '''
+        randobjs = {'vsc': vsc_basic(), 'cr': cr_basic(Random(0))}
+        def check(results):
+            self.assertGreater(results['cr'][1], results['vsc'][1])
+        self.run_benchmark(randobjs, 100, check)
+
+    def test_in(self):
+        '''
+        Test object using 'in' keyword.
+        '''
+        randobjs = {
+            'vsc': vsc_in(),
+            'cr': cr_in(Random(0)),
+            'cr_order': cr_in_order(Random(0)),
+        }
+        def check(results):
+            self.assertGreater(results['cr'][1], results['vsc'][1])
+        self.run_benchmark(randobjs, 100, check)
+
+    def test_ldinstr(self):
+        '''
+        Test LD instruction example.
+        '''
+        randobjs = {
+            'vsc': vsc_ldinstr(),
+            'cr': ldInstr(Random(0)),
+        }
+        def check(results):
+            self.assertGreater(results['cr'][1], results['vsc'][1])
+        self.run_benchmark(randobjs, 100, check)
+
+
+def parse_args():
+    parser = ArgumentParser(description='Run unit tests for constrainedrandom library')
+    parser.add_argument('--length-mul', type=int, default=1, help='Multiplier for test length, when desiring greater certainty on performance.')
+    args, extra = parser.parse_known_args()
+    return args, extra
+
+
+if __name__ == "__main__":
+    args, extra = parse_args()
+    TEST_LENGTH_MULTIPLIER = args.length_mul
+    # Reconstruct argv
+    argv = [sys.argv[0]] + extra
+    unittest.main(argv=argv)

benchmarks/__main__.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2023 Imagination Technologies Ltd. All Rights Reserved
+
+'''
+Basic random object from pyvsc documentation.
+'''
+
+from constrainedrandom import RandObj
+import vsc
+
+
+@vsc.randobj
+class vsc_basic(object):
+
+    def __init__(self):
+        self.a = vsc.rand_bit_t(8)
+        self.b = vsc.rand_bit_t(8)
+        self.c = vsc.rand_bit_t(8)
+        self.d = vsc.rand_bit_t(8)
+
+    @vsc.constraint
+    def ab_c(self):
+        self.a < self.b
+
+
+class cr_basic(RandObj):
+
+    def __init__(self, random):
+        super().__init__(random)
+        self.add_rand_var('a', bits=8)
+        self.add_rand_var('b', bits=8, order=1)
+        self.add_rand_var('c', bits=8)
+        self.add_rand_var('d', bits=8)
+
+        self.add_multi_var_constraint(lambda a, b : a < b, ('a', 'b'))

benchmarks/pyvsc/__init__.py

@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2023 Imagination Technologies Ltd. All Rights Reserved
+
+'''
+Random object using 'in' keyword from pyvsc documentation.
+'''
+
+from constrainedrandom import RandObj
+import vsc
+
+
+@vsc.randobj
+class vsc_in(object):
+
+    def __init__(self):
+        self.a = vsc.rand_bit_t(8)
+        self.b = vsc.rand_bit_t(8)
+        self.c = vsc.rand_bit_t(8)
+        self.d = vsc.rand_bit_t(8)
+
+    @vsc.constraint
+    def ab_c(self):
+
+       self.a in vsc.rangelist(1, 2, vsc.rng(4,8))
+       self.c != 0
+       self.d != 0
+
+       self.c < self.d
+       self.b in vsc.rangelist(vsc.rng(self.c,self.d))
+
+
+class cr_in(RandObj):
+    '''
+    Basic implementation, does thes same thing as vsc_in. No ordering hints.
+    '''
+
+    def __init__(self, random):
+        super().__init__(random)
+
+        self.add_rand_var('a', domain=[1,2] + list(range(4,8)))
+        self.add_rand_var('b', bits=8, constraints=(lambda b : b != 0))
+        self.add_rand_var('c', bits=8)
+        self.add_rand_var('d', bits=8, constraints=(lambda d : d != 0))
+
+        def c_lt_d(c, d):
+            return c < d
+        self.add_multi_var_constraint(c_lt_d, ('c', 'd'))
+
+        def b_in_range(b, c, d):
+            return b in range(c, d)
+        self.add_multi_var_constraint(b_in_range, ('b', 'c', 'd'))
+
+
+class cr_in_order(RandObj):
+    '''
+    cr_in, but with ordering hints.
+    '''
+
+    def __init__(self, random):
+        super().__init__(random)
+
+        self.add_rand_var('a', domain=[1,2] + list(range(4,8)), order=0)
+        self.add_rand_var('b', bits=8, constraints=(lambda b : b != 0), order=2)
+        self.add_rand_var('c', bits=8, order=0)
+        self.add_rand_var('d', bits=8, constraints=(lambda d : d != 0), order=1)
+
+        def c_lt_d(c, d):
+            return c < d
+        self.add_multi_var_constraint(c_lt_d, ('c', 'd'))
+
+        def b_in_range(b, c, d):
+            return b in range(c, d)
+        self.add_multi_var_constraint(b_in_range, ('b', 'c', 'd'))

benchmarks/pyvsc/basic.py

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2023 Imagination Technologies Ltd. All Rights Reserved
+
+'''
+Implement realistic load instruction case from constrainedrandom examples.ldinstr
+'''
+
+import vsc
+
+@vsc.randobj
+class vsc_ldinstr(object):
+
+    def __init__(self):
+        self.imm0 = vsc.rand_bit_t(11)
+        self.src0 = vsc.rand_bit_t(5)
+        self.dst0 = vsc.rand_bit_t(5)
+        self.wb = vsc.rand_bit_t(1)
+        self.enc = 0xfa800000
+        # Make this the same as in examples.ldinstr
+        self.src0_value_getter = lambda : 0xfffffbcd
+
+    @vsc.constraint
+    def wb_src0_dst0(self):
+        with vsc.if_then(self.wb == 1):
+            self.src0 != self.dst0
+
+    @vsc.constraint
+    def sum_src0_imm0(self):
+        self.imm0 + self.src0_value_getter() <= 0xffffffff
+        (self.imm0 + self.src0_value_getter()) & 3 == 0

benchmarks/pyvsc/in_keyword.py

@@ -12,6 +12,7 @@
 from constrainedrandom import Random, RandObj
 from examples.ldinstr import ldInstr
 
+
 TEST_LENGTH_MULTIPLIER = 1