Migrate puzzles away from load_clvm to import from chia_puzzles_py (#19055)

* add dependency for chia_puzzles_py

* migrate cat and dao wallets to new repo

* migrate did wallet

* more migration and deduplication

* change singleton launcher hash import in rpc

* stop importing singleton launcher from nft_puzzles in nft_wallet

* remove SINGLETON_LAUNCHER_PUZZLE import from nft_puzzles

* remove unused import

* ruff check and fix

* use graftroot dl offers from chia_puzzles_py

* get rid of more non explicit exports

* remove circular dependency

* ruff check

* migrate py files in puzzles to use chia_puzzles_py

* fix import

* fix more imports

* migrate clawback wallet and fix typo (requires version update of chia_puzzles_py)

* delete puzzles and separate nft puzzles from puzzle_utils

* use 0.19.1 version of chia_puzzles_py

* more nft puzzle detangling

* use latest chia_puzzles_py and migrate pools

* ruff fixes

* delete pool puzzles folder

* remove load_clvm from test_singleton and test_singleton_lifecycle_fast

* mypy fixes

* ruff fixes

* remove manage_clvm check precommit hook

* update version

* update chia_puzzles dependency to 0.19.4

* remove full_node puzzles

* remove test for condition codes

* change generator type to SerializedProgram

* remove remaining load_clvms that can be factored out

* delete consensus puzzles

* fix test_singleton_lifecycle_fast

* cast debug_spend_bundle coin name to bytes32

* ruff debug_spend_bundle

* update chia-puzzles and fix typo

* fix line endings

* fix typo in ROM_BOOTSTRAP_GENERATOR references

* remove bytes32 cast from debug_spend_bundle

* update chia-puzzles-py dependency to version 0.20.1 and refactor ACS_TRANSFER_PROGRAM initialization

* remove unused puzzle files

* singleton_top_layer exports hashs now which are imported from chia_puzzles

* remove .get_tree_hash() in favor of importing the hash

* remaining hash imports

* regenerate poetry.lock and pyproject.toml from main

* import singleton launcher hash from explicit export

* import SINGLETON_LAUNCHER_HASH from chia.wallet.singleton in test_pool_rpc.py

* import SINGLETON_MOD_HASH from chia.wallet.puzzles.singleton_top_layer in test_pool_puzzles_lifecycle.py

* poetry lock --no-update post-rebase

* re-regenerate lock file --no-update

* fix rebased block_tools

Author: matt-o-how

.pre-commit-config.yaml

@@ -57,13 +57,6 @@ repos:
       - id: check-merge-conflict
       - id: check-ast
       - id: debug-statements
-  - repo: local
-    hooks:
-      - id: clvm_hex
-        name: .clsp.hex files
-        entry: ./activated.py python tools/manage_clvm.py check
-        language: system
-        pass_filenames: false
   - repo: local
     hooks:
       - id: chialispp

chia/_tests/clvm/test_chialisp_deserialization.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import pytest
+from chia_puzzles_py.programs import CHIALISP_DESERIALISATION
 
 from chia.types.blockchain_format.program import INFINITE_COST, Program
 from chia.util.byte_types import hexstr_to_bytes
-from chia.wallet.puzzles.load_clvm import load_clvm
 
-DESERIALIZE_MOD = load_clvm("chialisp_deserialisation.clsp", package_or_requirement="chia.consensus.puzzles")
+DESERIALIZE_MOD = Program.from_bytes(CHIALISP_DESERIALISATION)
 
 
 def serialized_atom_overflow(size):

chia/_tests/clvm/test_condition_codes.py

@@ -4,6 +4,14 @@
 import io
 from typing import Any
 
+from chia_puzzles_py.programs import (
+    BLOCK_PROGRAM_ZERO,
+    CHIALISP_DESERIALISATION,
+    DECOMPRESS_COIN_SPEND_ENTRY,
+    DECOMPRESS_COIN_SPEND_ENTRY_WITH_PREFIX,
+    DECOMPRESS_PUZZLE,
+    ROM_BOOTSTRAP_GENERATOR,
+)
 from chia_rs import serialized_length
 from clvm.serialize import sexp_from_stream
 from clvm.SExp import SExp
@@ -15,18 +23,20 @@
 from chia.util.ints import uint32
 from chia.wallet.puzzles.load_clvm import load_clvm
 
-TEST_GEN_DESERIALIZE = load_clvm(
-    "test_generator_deserialize.clsp", package_or_requirement="chia._tests.generator.puzzles"
-)
-DESERIALIZE_MOD = load_clvm("chialisp_deserialisation.clsp", package_or_requirement="chia.consensus.puzzles")
+DESERIALIZE_MOD = Program.from_bytes(CHIALISP_DESERIALISATION)
+
+GENERATOR_MOD: Program = Program.from_bytes(ROM_BOOTSTRAP_GENERATOR)
 
-DECOMPRESS_PUZZLE = load_clvm("decompress_puzzle.clsp", package_or_requirement="chia.full_node.puzzles")
-DECOMPRESS_CSE = load_clvm("decompress_coin_spend_entry.clsp", package_or_requirement="chia.full_node.puzzles")
 
-DECOMPRESS_CSE_WITH_PREFIX = load_clvm(
-    "decompress_coin_spend_entry_with_prefix.clsp", package_or_requirement="chia.full_node.puzzles"
+DECOMPRESS_PUZZLE = Program.from_bytes(DECOMPRESS_PUZZLE)
+DECOMPRESS_CSE = Program.from_bytes(DECOMPRESS_COIN_SPEND_ENTRY)
+
+DECOMPRESS_CSE_WITH_PREFIX = Program.from_bytes(DECOMPRESS_COIN_SPEND_ENTRY_WITH_PREFIX)
+DECOMPRESS_BLOCK = Program.from_bytes(BLOCK_PROGRAM_ZERO)
+
+TEST_GEN_DESERIALIZE = load_clvm(
+    "test_generator_deserialize.clsp", package_or_requirement="chia._tests.generator.puzzles"
 )
-DECOMPRESS_BLOCK = load_clvm("block_program_zero.clsp", package_or_requirement="chia.full_node.puzzles")
 TEST_MULTIPLE = load_clvm(
     "test_multiple_generator_input_arguments.clsp", package_or_requirement="chia._tests.generator.puzzles"
 )

chia/_tests/generator/test_compression.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from chia_puzzles_py.programs import CHIALISP_DESERIALISATION, ROM_BOOTSTRAP_GENERATOR
 from clvm.CLVMObject import CLVMStorage
 from clvm_tools import binutils
 from clvm_tools.clvmc import compile_clvm_text
@@ -13,16 +14,13 @@
 from chia.types.generator_types import BlockGenerator
 from chia.types.spend_bundle_conditions import SpendConditions
 from chia.util.ints import uint32
-from chia.wallet.puzzles.load_clvm import load_clvm, load_serialized_clvm_maybe_recompile
 
-MAX_COST = 10**15
-COST_PER_BYTE = 12000
+DESERIALIZE_MOD = Program.from_bytes(CHIALISP_DESERIALISATION)
 
+GENERATOR_MOD: SerializedProgram = SerializedProgram.from_bytes(ROM_BOOTSTRAP_GENERATOR)
 
-DESERIALIZE_MOD = load_clvm("chialisp_deserialisation.clsp", package_or_requirement="chia.consensus.puzzles")
-GENERATOR_MOD: SerializedProgram = load_serialized_clvm_maybe_recompile(
-    "rom_bootstrap_generator.clsp", package_or_requirement="chia.consensus.puzzles"
-)
+MAX_COST = 10**15
+COST_PER_BYTE = 12000
 
 
 GENERATOR_CODE = """

chia/_tests/generator/test_rom.py

@@ -11,7 +11,6 @@
 from chia._tests.util.key_tool import KeyTool
 from chia.consensus.default_constants import DEFAULT_CONSTANTS
 from chia.pools.pool_puzzles import (
-    SINGLETON_MOD_HASH,
     create_absorb_spend,
     create_p2_singleton_puzzle,
     create_p2_singleton_puzzle_hash,
@@ -41,6 +40,7 @@
     puzzle_for_pk,
     solution_for_conditions,
 )
+from chia.wallet.puzzles.singleton_top_layer import SINGLETON_MOD_HASH
 from chia.wallet.singleton import get_most_recent_singleton_coin_from_coin_spend
 
 """

chia/_tests/pools/test_pool_puzzles_lifecycle.py

@@ -20,7 +20,6 @@
 from chia._tests.util.setup_nodes import setup_simulators_and_wallets_service
 from chia._tests.util.time_out_assert import time_out_assert
 from chia.consensus.constants import ConsensusConstants
-from chia.pools.pool_puzzles import SINGLETON_LAUNCHER_HASH
 from chia.pools.pool_wallet_info import PoolSingletonState, PoolWalletInfo
 from chia.rpc.wallet_rpc_client import WalletRpcClient
 from chia.simulator.add_blocks_in_batches import add_blocks_in_batches
@@ -36,6 +35,7 @@
 from chia.util.config import load_config
 from chia.util.ints import uint32, uint64
 from chia.wallet.derive_keys import find_authentication_sk, find_owner_sk
+from chia.wallet.singleton import SINGLETON_LAUNCHER_PUZZLE_HASH as SINGLETON_LAUNCHER_HASH
 from chia.wallet.transaction_record import TransactionRecord
 from chia.wallet.util.transaction_type import TransactionType
 from chia.wallet.util.tx_config import DEFAULT_TX_CONFIG

chia/_tests/pools/test_pool_rpc.py

@@ -5,22 +5,21 @@
 from pathlib import Path
 from typing import Any
 
+from chia_puzzles_py.programs import CHIALISP_DESERIALISATION
 from chia_rs import Coin
 
 from chia.consensus.constants import ConsensusConstants
+from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.serialized_program import SerializedProgram
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.types.condition_with_args import ConditionWithArgs
 from chia.types.generator_types import BlockGenerator
 from chia.util.ints import uint32, uint64
 from chia.wallet.cat_wallet.cat_utils import match_cat_puzzle
-from chia.wallet.puzzles.load_clvm import load_serialized_clvm_maybe_recompile
 from chia.wallet.uncurried_puzzle import uncurry_puzzle
 
-DESERIALIZE_MOD = load_serialized_clvm_maybe_recompile(
-    "chialisp_deserialisation.clsp", package_or_requirement="chia.consensus.puzzles"
-)
+DESERIALIZE_MOD = Program.from_bytes(CHIALISP_DESERIALISATION)
 
 
 @dataclass

chia/_tests/util/run_block.py

@@ -17,43 +17,35 @@
 from chia.util.errors import Err
 from chia.util.hash import std_hash
 from chia.util.ints import uint32, uint64
-from chia.wallet.cat_wallet.cat_utils import CAT_MOD
+from chia.wallet.cat_wallet.cat_utils import CAT_MOD, CAT_MOD_HASH
 from chia.wallet.dao_wallet.dao_info import DAORules
-from chia.wallet.dao_wallet.dao_utils import curry_singleton, get_p2_singleton_puzhash, get_treasury_puzzle
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.dao_wallet.dao_utils import (
+    DAO_FINISHED_STATE,
+    DAO_FINISHED_STATE_HASH,
+    DAO_LOCKUP_MOD,
+    DAO_PROPOSAL_MOD,
+    DAO_PROPOSAL_TIMER_MOD,
+    DAO_PROPOSAL_TIMER_MOD_HASH,
+    DAO_PROPOSAL_VALIDATOR_MOD,
+    DAO_PROPOSAL_VALIDATOR_MOD_HASH,
+    DAO_TREASURY_MOD,
+    DAO_TREASURY_MOD_HASH,
+    P2_SINGLETON_AGGREGATOR_MOD,
+    P2_SINGLETON_MOD,
+    SPEND_P2_SINGLETON_MOD,
+    curry_singleton,
+    get_p2_singleton_puzhash,
+    get_treasury_puzzle,
+)
+from chia.wallet.dao_wallet.dao_utils import (
+    DAO_UPDATE_PROPOSAL_MOD as DAO_UPDATE_MOD,
+)
+from chia.wallet.singleton import SINGLETON_LAUNCHER_PUZZLE_HASH as SINGLETON_LAUNCHER_HASH
+from chia.wallet.singleton import SINGLETON_TOP_LAYER_MOD as SINGLETON_MOD
+from chia.wallet.singleton import SINGLETON_TOP_LAYER_MOD_HASH as SINGLETON_MOD_HASH
 from chia.wallet.singleton import create_singleton_puzzle_hash
 from chia.wallet.wallet_spend_bundle import WalletSpendBundle
 
-CAT_MOD_HASH: bytes32 = CAT_MOD.get_tree_hash()
-SINGLETON_MOD: Program = load_clvm("singleton_top_layer_v1_1.clsp")
-SINGLETON_MOD_HASH: bytes32 = SINGLETON_MOD.get_tree_hash()
-SINGLETON_LAUNCHER: Program = load_clvm("singleton_launcher.clsp")
-SINGLETON_LAUNCHER_HASH: bytes32 = SINGLETON_LAUNCHER.get_tree_hash()
-DAO_LOCKUP_MOD: Program = load_clvm("dao_lockup.clsp")
-DAO_LOCKUP_MOD_HASH: bytes32 = DAO_LOCKUP_MOD.get_tree_hash()
-DAO_PROPOSAL_TIMER_MOD: Program = load_clvm("dao_proposal_timer.clsp")
-DAO_PROPOSAL_TIMER_MOD_HASH: bytes32 = DAO_PROPOSAL_TIMER_MOD.get_tree_hash()
-DAO_PROPOSAL_MOD: Program = load_clvm("dao_proposal.clsp")
-DAO_PROPOSAL_MOD_HASH: bytes32 = DAO_PROPOSAL_MOD.get_tree_hash()
-DAO_PROPOSAL_VALIDATOR_MOD: Program = load_clvm("dao_proposal_validator.clsp")
-DAO_PROPOSAL_VALIDATOR_MOD_HASH: bytes32 = DAO_PROPOSAL_VALIDATOR_MOD.get_tree_hash()
-DAO_TREASURY_MOD: Program = load_clvm("dao_treasury.clsp")
-DAO_TREASURY_MOD_HASH: bytes32 = DAO_TREASURY_MOD.get_tree_hash()
-SPEND_P2_SINGLETON_MOD: Program = load_clvm("dao_spend_p2_singleton_v2.clsp")
-SPEND_P2_SINGLETON_MOD_HASH: bytes32 = SPEND_P2_SINGLETON_MOD.get_tree_hash()
-DAO_FINISHED_STATE: Program = load_clvm("dao_finished_state.clsp")
-DAO_FINISHED_STATE_HASH: bytes32 = DAO_FINISHED_STATE.get_tree_hash()
-DAO_CAT_TAIL: Program = load_clvm(
-    "genesis_by_coin_id_or_singleton.clsp", package_or_requirement="chia.wallet.cat_wallet.puzzles"
-)
-DAO_CAT_TAIL_HASH: bytes32 = DAO_CAT_TAIL.get_tree_hash()
-P2_SINGLETON_MOD: Program = load_clvm("p2_singleton_via_delegated_puzzle.clsp")
-P2_SINGLETON_MOD_HASH: bytes32 = P2_SINGLETON_MOD.get_tree_hash()
-P2_SINGLETON_AGGREGATOR_MOD: Program = load_clvm("p2_singleton_aggregator.clsp")
-P2_SINGLETON_AGGREGATOR_MOD_HASH: bytes32 = P2_SINGLETON_AGGREGATOR_MOD.get_tree_hash()
-DAO_UPDATE_MOD: Program = load_clvm("dao_update_proposal.clsp")
-DAO_UPDATE_MOD_HASH: bytes32 = DAO_UPDATE_MOD.get_tree_hash()
-
 
 def test_finished_state() -> None:
     """
@@ -63,9 +55,7 @@ def test_finished_state() -> None:
     the lockup puzzle.
     """
     proposal_id = Program.to("proposal_id").get_tree_hash()
-    singleton_struct: Program = Program.to(
-        (SINGLETON_MOD.get_tree_hash(), (proposal_id, SINGLETON_LAUNCHER.get_tree_hash()))
-    )
+    singleton_struct: Program = Program.to((SINGLETON_MOD_HASH, (proposal_id, SINGLETON_LAUNCHER_HASH)))
     finished_inner_puz = DAO_FINISHED_STATE.curry(singleton_struct, DAO_FINISHED_STATE_HASH)
     finished_full_puz = SINGLETON_MOD.curry(singleton_struct, finished_inner_puz)
     inner_sol = Program.to([1])
@@ -93,9 +83,7 @@ def test_proposal() -> None:
     CAT_TAIL_HASH = Program.to("tail").get_tree_hash()
     treasury_id = Program.to("treasury").get_tree_hash()
     singleton_id = Program.to("singleton_id").get_tree_hash()
-    singleton_struct: Program = Program.to(
-        (SINGLETON_MOD.get_tree_hash(), (singleton_id, SINGLETON_LAUNCHER.get_tree_hash()))
-    )
+    singleton_struct: Program = Program.to((SINGLETON_MOD_HASH, (singleton_id, SINGLETON_LAUNCHER_HASH)))
     self_destruct_time = 1000  # number of blocks
     oracle_spend_delay = 10
     active_votes_list = [0xFADEDDAB]  # are the ids of previously voted on proposals?
@@ -387,9 +375,7 @@ def test_proposal_timer() -> None:
     CAT_TAIL_HASH = Program.to("tail").get_tree_hash()
     treasury_id = Program.to("treasury").get_tree_hash()
     singleton_id = Program.to("singleton_id").get_tree_hash()
-    singleton_struct: Program = Program.to(
-        (SINGLETON_MOD.get_tree_hash(), (singleton_id, SINGLETON_LAUNCHER.get_tree_hash()))
-    )
+    singleton_struct: Program = Program.to((SINGLETON_MOD_HASH, (singleton_id, SINGLETON_LAUNCHER_HASH)))
     dao_lockup_self = DAO_LOCKUP_MOD.curry(
         SINGLETON_MOD_HASH,
         SINGLETON_LAUNCHER_HASH,
@@ -482,7 +468,7 @@ def test_validator() -> None:
 
     # Setup the proposal
     proposal_id = Program.to("proposal_id").get_tree_hash()
-    proposal_struct: Program = Program.to((SINGLETON_MOD.get_tree_hash(), (proposal_id, SINGLETON_LAUNCHER_HASH)))
+    proposal_struct: Program = Program.to((SINGLETON_MOD_HASH, (proposal_id, SINGLETON_LAUNCHER_HASH)))
     CAT_TAIL_HASH = Program.to("tail").get_tree_hash()
     acs: Program = Program.to(1)
     acs_ph: bytes32 = acs.get_tree_hash()

chia/_tests/wallet/dao_wallet/test_dao_clvm.py

@@ -10,12 +10,10 @@
 from chia.types.coin_spend import make_spend
 from chia.types.mempool_inclusion_status import MempoolInclusionStatus
 from chia.util.errors import Err
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.db_wallet.db_wallet_puzzles import GRAFTROOT_DL_OFFERS
 from chia.wallet.util.merkle_utils import build_merkle_tree, build_merkle_tree_from_binary_tree, simplify_merkle_proof
 from chia.wallet.wallet_spend_bundle import WalletSpendBundle
 
-GRAFTROOT_MOD = load_clvm("graftroot_dl_offers.clsp", package_or_requirement="chia.data_layer.puzzles")
-
 # Always returns the last value
 # (mod solution
 #
@@ -46,7 +44,7 @@ async def test_graftroot(cost_logger: CostLogger) -> None:
         desired_key_values = ((bytes32.zeros, bytes32([1] * 32)), (bytes32([7] * 32), bytes32([8] * 32)))
         desired_row_hashes: list[bytes32] = [build_merkle_tree_from_binary_tree(kv)[0] for kv in desired_key_values]
         fake_struct: Program = Program.to((ACS_PH, NIL_PH))
-        graftroot_puzzle: Program = GRAFTROOT_MOD.curry(
+        graftroot_puzzle: Program = GRAFTROOT_DL_OFFERS.curry(
             # Do everything twice to test depending on multiple singleton updates
             p2_conditions,
             [fake_struct, fake_struct],

chia/_tests/wallet/db_wallet/test_db_graftroot.py

@@ -12,13 +12,12 @@
 from chia.types.mempool_inclusion_status import MempoolInclusionStatus
 from chia.util.errors import Err
 from chia.wallet.conditions import AssertPuzzleAnnouncement
-from chia.wallet.nft_wallet.nft_puzzles import (
-    NFT_METADATA_UPDATER,
-    NFT_TRANSFER_PROGRAM_DEFAULT,
+from chia.wallet.nft_wallet.nft_puzzle_utils import (
     construct_ownership_layer,
     create_nft_layer_puzzle_with_curry_params,
     metadata_to_program,
 )
+from chia.wallet.nft_wallet.nft_puzzles import NFT_METADATA_UPDATER, NFT_TRANSFER_PROGRAM_DEFAULT
 from chia.wallet.wallet_spend_bundle import WalletSpendBundle
 
 ACS = Program.to(1)