Add Halmos support for formal verification (OpenZeppelin#5034)

Co-authored-by: Hadrien Croubois <[email protected]>

.github/actions/gas-compare/action.yml

@@ -1,4 +1,5 @@
 name: Compare gas costs
+description: Compare gas costs between branches
 inputs:
   token:
     description: github token

.github/actions/setup/action.yml

@@ -1,4 +1,5 @@
 name: Setup
+description: Common environment setup
 
 runs:
   using: composite

.github/actions/storage-layout/action.yml

@@ -1,4 +1,5 @@
 name: Compare storage layouts
+description: Compare storage layouts between branches
 inputs:
   token:
     description: github token

.github/workflows/formal-verification.yml

@@ -48,8 +48,9 @@ jobs:
         with:
           python-version: ${{ env.PIP_VERSION }}
           cache: 'pip'
+          cache-dependency-path: 'fv-requirements.txt'
       - name: Install python packages
-        run: pip install -r requirements.txt
+        run: pip install -r fv-requirements.txt
       - name: Install java
         uses: actions/setup-java@v3
         with:
@@ -66,3 +67,20 @@ jobs:
           node certora/run.js ${{ steps.arguments.outputs.result }} >> "$GITHUB_STEP_SUMMARY"
         env:
           CERTORAKEY: ${{ secrets.CERTORAKEY }}
+
+  halmos:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Install python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PIP_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: 'fv-requirements.txt'
+      - name: Install python packages
+        run: pip install -r fv-requirements.txt
+      - name: Run Halmos
+        run: halmos --match-test '^symbolic|^testSymbolic' -vv

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "lib/erc4626-tests"]
 	path = lib/erc4626-tests
 	url = https://github.com/a16z/erc4626-tests.git
+[submodule "lib/halmos-cheatcodes"]
+	path = lib/halmos-cheatcodes
+	url = https://github.com/a16z/halmos-cheatcodes

fv-requirements.txt

@@ -0,0 +1,4 @@
+certora-cli==4.13.1
+# File uses a custom name (fv-requirements.txt) so that it isn't picked by Netlify's build
+# whose latest Python version is 0.3.8, incompatible with most recent versions of Halmos
+halmos==0.1.12

scripts/generate/templates/SlotDerivation.t.js

@@ -6,17 +6,26 @@ const header = `\
 pragma solidity ^0.8.20;
 
 import {Test} from "forge-std/Test.sol";
-
+import {SymTest} from "halmos-cheatcodes/SymTest.sol";
 import {SlotDerivation} from "@openzeppelin/contracts/utils/SlotDerivation.sol";
 `;
 
 const array = `\
 bytes[] private _array;
 
+function symbolicDeriveArray(uint256 length, uint256 offset) public {
+  vm.assume(length > 0);
+  vm.assume(offset < length);
+  _assertDeriveArray(length, offset);
+}
+
 function testDeriveArray(uint256 length, uint256 offset) public {
   length = bound(length, 1, type(uint256).max);
   offset = bound(offset, 0, length - 1);
+  _assertDeriveArray(length, offset);
+}
 
+function _assertDeriveArray(uint256 length, uint256 offset) public {
   bytes32 baseSlot;
   assembly {
     baseSlot := _array.slot
@@ -33,10 +42,10 @@ function testDeriveArray(uint256 length, uint256 offset) public {
 }
 `;
 
-const mapping = ({ type, name, isValueType }) => `\
+const mapping = ({ type, name }) => `\
 mapping(${type} => bytes) private _${type}Mapping;
 
-function testDeriveMapping${name}(${type} ${isValueType ? '' : 'memory'} key) public {
+function testSymbolicDeriveMapping${name}(${type} key) public {
   bytes32 baseSlot;
   assembly {
     baseSlot := _${type}Mapping.slot
@@ -52,10 +61,37 @@ function testDeriveMapping${name}(${type} ${isValueType ? '' : 'memory'} key) pu
 }
 `;
 
+const boundedMapping = ({ type, name }) => `\
+mapping(${type} => bytes) private _${type}Mapping;
+
+function testDeriveMapping${name}(${type} memory key) public {
+  _assertDeriveMapping${name}(key);
+}
+
+function symbolicDeriveMapping${name}() public {
+  _assertDeriveMapping${name}(svm.create${name}(256, "DeriveMapping${name}Input"));
+}
+
+function _assertDeriveMapping${name}(${type} memory key) internal {
+  bytes32 baseSlot;
+  assembly {
+      baseSlot := _${type}Mapping.slot
+  }
+
+  bytes storage derived = _${type}Mapping[key];
+  bytes32 derivedSlot;
+  assembly {
+      derivedSlot := derived.slot
+  }
+
+  assertEq(baseSlot.deriveMapping(key), derivedSlot);
+}
+`;
+
 // GENERATE
 module.exports = format(
   header.trimEnd(),
-  'contract SlotDerivationTest is Test {',
+  'contract SlotDerivationTest is Test, SymTest {',
   'using SlotDerivation for bytes32;',
   '',
   array,
@@ -68,6 +104,6 @@ module.exports = format(
         isValueType: type.isValueType,
       })),
     ),
-  ).map(type => mapping(type)),
+  ).map(type => (type.isValueType ? mapping(type) : boundedMapping(type))),
   '}',
 );

test/proxy/Clones.t.sol

@@ -6,7 +6,7 @@ import {Test} from "forge-std/Test.sol";
 import {Clones} from "@openzeppelin/contracts/proxy/Clones.sol";
 
 contract ClonesTest is Test {
-    function testPredictDeterministicAddressSpillage(address implementation, bytes32 salt) public {
+    function testSymbolicPredictDeterministicAddressSpillage(address implementation, bytes32 salt) public {
         address predicted = Clones.predictDeterministicAddress(implementation, salt);
         bytes32 spillage;
         /// @solidity memory-safe-assembly

test/utils/Arrays.t.sol

@@ -3,11 +3,27 @@
 pragma solidity ^0.8.20;
 
 import {Test} from "forge-std/Test.sol";
+import {SymTest} from "halmos-cheatcodes/SymTest.sol";
 import {Arrays} from "@openzeppelin/contracts/utils/Arrays.sol";
 
-contract ArraysTest is Test {
+contract ArraysTest is Test, SymTest {
     function testSort(uint256[] memory values) public {
         Arrays.sort(values);
+        _assertSort(values);
+    }
+
+    function symbolicSort() public {
+        uint256[] memory values = new uint256[](3);
+        for (uint256 i = 0; i < 3; i++) {
+            values[i] = svm.createUint256("arrayElement");
+        }
+        Arrays.sort(values);
+        _assertSort(values);
+    }
+
+    /// Asserts
+
+    function _assertSort(uint256[] memory values) internal {
         for (uint256 i = 1; i < values.length; ++i) {
             assertLe(values[i - 1], values[i]);
         }

test/utils/Create2.t.sol

@@ -6,7 +6,7 @@ import {Test} from "forge-std/Test.sol";
 import {Create2} from "@openzeppelin/contracts/utils/Create2.sol";
 
 contract Create2Test is Test {
-    function testComputeAddressSpillage(bytes32 salt, bytes32 bytecodeHash, address deployer) public {
+    function testSymbolicComputeAddressSpillage(bytes32 salt, bytes32 bytecodeHash, address deployer) public {
         address predicted = Create2.computeAddress(salt, bytecodeHash, deployer);
         bytes32 spillage;
         /// @solidity memory-safe-assembly

test/utils/Packing.t.sol

@@ -9,7 +9,7 @@ contract PackingTest is Test {
     using Packing for *;
 
     // Pack a pair of arbitrary uint128, and check that split recovers the correct values
-    function testUint128x2(uint128 first, uint128 second) external {
+    function testSymbolicUint128x2(uint128 first, uint128 second) external {
         Packing.Uint128x2 packed = Packing.pack(first, second);
         assertEq(packed.first(), first);
         assertEq(packed.second(), second);
@@ -20,7 +20,7 @@ contract PackingTest is Test {
     }
 
     // split an arbitrary bytes32 into a pair of uint128, and check that repack matches the input
-    function testUint128x2(bytes32 input) external {
+    function testSymbolicUint128x2(bytes32 input) external {
         (uint128 first, uint128 second) = input.asUint128x2().split();
         assertEq(Packing.pack(first, second).asBytes32(), input);
     }

test/utils/ShortStrings.t.sol

@@ -3,48 +3,102 @@
 pragma solidity ^0.8.20;
 
 import {Test} from "forge-std/Test.sol";
+import {SymTest} from "halmos-cheatcodes/SymTest.sol";
 
 import {ShortStrings, ShortString} from "@openzeppelin/contracts/utils/ShortStrings.sol";
 
-contract ShortStringsTest is Test {
+contract ShortStringsTest is Test, SymTest {
     string _fallback;
 
     function testRoundtripShort(string memory input) external {
         vm.assume(_isShort(input));
+        _assertRoundtripShort(input);
+    }
+
+    function symbolicRoundtripShort() external {
+        string memory input = svm.createString(31, "RoundtripShortInput");
+        _assertRoundtripShort(input);
+    }
+
+    function testRoundtripWithFallback(string memory input, string memory fallbackInitial) external {
+        _assertRoundtripWithFallback(input, fallbackInitial);
+    }
+
+    function symbolicRoundtripWithFallbackLong() external {
+        string memory input = svm.createString(256, "RoundtripWithFallbackInput");
+        string memory fallbackInitial = svm.createString(256, "RoundtripWithFallbackFallbackInitial");
+        _assertRoundtripWithFallback(input, fallbackInitial);
+    }
+
+    function symbolicRoundtripWithFallbackShort() external {
+        string memory input = svm.createString(31, "RoundtripWithFallbackInput");
+        string memory fallbackInitial = svm.createString(31, "RoundtripWithFallbackFallbackInitial");
+        _assertRoundtripWithFallback(input, fallbackInitial);
+    }
+
+    function testRevertLong(string memory input) external {
+        vm.assume(!_isShort(input));
+        _assertRevertLong(input);
+    }
+
+    function testLengthShort(string memory input) external {
+        vm.assume(_isShort(input));
+        _assertLengthShort(input);
+    }
+
+    function symbolicLengthShort() external {
+        string memory input = svm.createString(31, "LengthShortInput");
+        _assertLengthShort(input);
+    }
+
+    function testLengthWithFallback(string memory input, string memory fallbackInitial) external {
+        _fallback = fallbackInitial;
+        _assertLengthWithFallback(input);
+    }
+
+    function symbolicLengthWithFallback() external {
+        uint256 length = 256;
+        string memory input = svm.createString(length, "LengthWithFallbackInput");
+        string memory fallbackInitial = svm.createString(length, "LengthWithFallbackFallbackInitial");
+        _fallback = fallbackInitial;
+        _assertLengthWithFallback(input);
+    }
+
+    /// Assertions
+
+    function _assertRoundtripShort(string memory input) internal {
         ShortString short = ShortStrings.toShortString(input);
         string memory output = ShortStrings.toString(short);
         assertEq(input, output);
     }
 
-    function testRoundtripWithFallback(string memory input, string memory fallbackInitial) external {
+    function _assertRoundtripWithFallback(string memory input, string memory fallbackInitial) internal {
         _fallback = fallbackInitial; // Make sure that the initial value has no effect
         ShortString short = ShortStrings.toShortStringWithFallback(input, _fallback);
         string memory output = ShortStrings.toStringWithFallback(short, _fallback);
         assertEq(input, output);
     }
 
-    function testRevertLong(string memory input) external {
-        vm.assume(!_isShort(input));
+    function _assertRevertLong(string memory input) internal {
         vm.expectRevert(abi.encodeWithSelector(ShortStrings.StringTooLong.selector, input));
         this.toShortString(input);
     }
 
-    function testLengthShort(string memory input) external {
-        vm.assume(_isShort(input));
-        uint256 inputLength = bytes(input).length;
+    function _assertLengthShort(string memory input) internal {
         ShortString short = ShortStrings.toShortString(input);
         uint256 shortLength = ShortStrings.byteLength(short);
+        uint256 inputLength = bytes(input).length;
         assertEq(inputLength, shortLength);
     }
 
-    function testLengthWithFallback(string memory input, string memory fallbackInitial) external {
-        _fallback = fallbackInitial;
+    function _assertLengthWithFallback(string memory input) internal {
         uint256 inputLength = bytes(input).length;
         ShortString short = ShortStrings.toShortStringWithFallback(input, _fallback);
         uint256 shortLength = ShortStrings.byteLengthWithFallback(short, _fallback);
         assertEq(inputLength, shortLength);
     }
 
+    /// Helpers
     function toShortString(string memory input) external pure returns (ShortString) {
         return ShortStrings.toShortString(input);
     }