Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/writing_tests/benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,5 @@ def test_empty_block(
This is a safety check to make sure the benchmark works as expected. For example, if a test uses the `JUMP` instruction but the jump destination is invalid, each transaction will stop early. That means it won't use as much gas as we expected.

This check helps catch such issues. As a result, the post-storage comparison method via `SSTORE` is no longer needed, thereby reducing the additional storage cost.

However, in cases where it is difficult to determine the total gas usage, or if an alternative verification method is used, developers may set `skip_gas_used_validation` to `True` to disable the gas usage check.
2 changes: 2 additions & 0 deletions src/ethereum_test_specs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ class BaseTest(BaseModel):
_gas_optimization_max_gas_limit: int | None = PrivateAttr(None)

expected_benchmark_gas_used: int | None = None
skip_gas_used_validation: bool = False

spec_types: ClassVar[Dict[str, Type["BaseTest"]]] = {}

Expand Down Expand Up @@ -117,6 +118,7 @@ def from_test(
tag=base_test.tag,
t8n_dump_dir=base_test.t8n_dump_dir,
expected_benchmark_gas_used=base_test.expected_benchmark_gas_used,
skip_gas_used_validation=base_test.skip_gas_used_validation,
**kwargs,
)
new_instance._request = base_test._request
Expand Down
12 changes: 7 additions & 5 deletions src/ethereum_test_specs/blockchain.py
Original file line number Diff line number Diff line change
Expand Up @@ -580,11 +580,13 @@ def generate_block_data(
"expected_benchmark_gas_used is not set"
)
gas_used = int(transition_tool_output.result.gas_used)
assert gas_used == expected_benchmark_gas_used, (
f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
f"({expected_benchmark_gas_used})"
f", difference: {gas_used - expected_benchmark_gas_used}"
)

if not self.skip_gas_used_validation:
assert gas_used == expected_benchmark_gas_used, (
f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
f"({expected_benchmark_gas_used})"
f", difference: {gas_used - expected_benchmark_gas_used}"
)

requests_list: List[Bytes] | None = None
if fork.header_requests_required(header.number, header.timestamp):
Expand Down
13 changes: 7 additions & 6 deletions src/ethereum_test_specs/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,15 +366,16 @@ def make_state_test_fixture(

if self._operation_mode == OpMode.BENCHMARKING:
expected_benchmark_gas_used = self.expected_benchmark_gas_used
gas_used = int(transition_tool_output.result.gas_used)
assert expected_benchmark_gas_used is not None, (
"expected_benchmark_gas_used is not set"
)
assert gas_used == expected_benchmark_gas_used, (
f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
f"({expected_benchmark_gas_used})"
f", difference: {gas_used - expected_benchmark_gas_used}"
)
gas_used = int(transition_tool_output.result.gas_used)
if not self.skip_gas_used_validation:
assert gas_used == expected_benchmark_gas_used, (
f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
f"({expected_benchmark_gas_used})"
f", difference: {gas_used - expected_benchmark_gas_used}"
)

return StateFixture(
env=FixtureEnvironment(**env.model_dump(exclude_none=True)),
Expand Down
Loading