Added JavaRawLevel classes and fixes (#356)

* Added empty Java classes

* Renamed Java raw module

* Moved Java Raw modules

This is now grouped in with the normal level modules

* Added DimensionID alias

* Added some chunk handle methods

* Added first pass of JavaRawDimension

The encoded and decode functions are now methods of the JavaRawDimension class

* Added Java raw chunk alias

* Added the first pass of the Java raw level class

* Changed thread safety doc

This method does not invalidate anything in this class.

* Added java raw dimension implementation

* Moved the implementation back to the cpp file

* Updated nbt dependency

* Added WIP level class

* Updated nbt dependency

* Added java raw wrappers

* Renamed getters

* Added export symbol

* Extended JavaRawLevel wrapper

* Cast to shared_ptr

pybind11 does not seem to cast to the correct holder type automatically.

* Construct unique_ptr in place

The old version had an unnecessary move.

* Fixed incorrect variable

* Added get_level_name implementation

* Changed data_version signatures

* Added lock docstrings

* Renamed mutex getter

* Renamed mutex getter

* Made all_chunk_coords const

* Added JavaRawDimension wrapper

* Improved threading

* Changed to shared lock

* Modified docstrings

* Added docstrings

* Added a public lock to IdRegistry

* Improved docstrings

* Add return type to signal wrapper

* Extended the JavaRawLevel class

Implemented open and close.
Added reload method.
Added signals.
Implemented metadata getters and setters.

* Close JavaRawLevel on destruction

* Write level.dat to temporary file and rename

If the program exits while writing the original will be left intact.

* Release GIL in signal

* Added a nogil holder

The signal destructor acquires a lock so must be called without the GIL otherwise a deadlock can occur.
This just wraps a std::shared_ptr and releases the GIL before resetting the shared_ptr.

* Simplified and improved holder constructor

* Added session.lock locking

* Explicitly unlock file

* Moved get_data_version function

* Moved level.dat file writing to a new function

* Added JavaRawLevel::create

* Fixed region regex

It must be a full match

* Implemented dimension finding

* Fix warning

* Modified VersionNumber constructor

This passes the arguments to the vector constructor so it now supports everything the vector supports.

* Implemented JavaRawLevel::_register_dimension

* Fixed block and version constructors

* Fixed some raw level issues

* Fixed VersionNumber constructor

* Fixed properties

Pybind11 def_property functions do not support call_guard

* Implemented most of _get_dimension_bounds

* Improved LockFile

* Fixed two LockFile issues

* Added LockFile tests

* Fixed open file call

* Explicitly lock in test

Unix file locks are advisory meaning they can be ignored.

* Make the linux lock non-blocking

CMakeLists.txt

@@ -67,6 +67,7 @@ list(REMOVE_ITEM AMULET_HEADERS ${AMULET_EXTENSION_HEADERS})
 
 # Add implementation
 add_library(amulet_core SHARED)
+target_link_libraries( amulet_core PRIVATE pybind11::module ) # TODO: remove this when it doesn't depend on pybind11
 target_link_libraries( amulet_core PRIVATE zlibstatic )
 target_link_libraries( amulet_core PRIVATE lz4_static )
 target_link_libraries( amulet_core PRIVATE amulet_nbt )

pyproject.toml

@@ -5,7 +5,7 @@ requires = [
     "versioneer",
     "pybind11[global] == 2.13.6",
     "Amulet_pybind11_extensions @ git+https://github.com/Amulet-Team/Amulet-pybind11-extensions.git@f781177e52f352ea0c1750e9219a1a84f4ddd92f",
-    "amulet_nbt == 4.0a13",
+    "amulet_nbt == 4.0a14",
     "amulet_leveldb == 2.0a3",
 ]
 build-backend = "setuptools.build_meta"
@@ -30,7 +30,7 @@ dependencies = [
     "pillow ~= 10.0",
     "amulet_runtime_final ~= 1.1",
     "amulet_compiler_target == 1.0",
-    "amulet_nbt == 4.0a13",
+    "amulet_nbt == 4.0a14",
     "amulet_leveldb ~= 2.0a3",
 ]
 

src/amulet/block.hpp

@@ -99,15 +99,20 @@ class BlockStack {
 public:
     const std::vector<Block>& get_blocks() const { return _blocks; }
 
-    template <typename T>
-    BlockStack(const T& blocks)
-        : _blocks(blocks)
+    template <typename... Args>
+    BlockStack(Args&&... args)
+        : _blocks(std::forward<Args>(args)...)
     {
         if (_blocks.empty()) {
             throw std::invalid_argument("A BlockStack must contain at least one block");
         }
     }
 
+    BlockStack(std::initializer_list<Block> blocks)
+        : _blocks(blocks)
+    {
+    }
+
     AMULET_CORE_EXPORT void serialise(BinaryWriter&) const;
     AMULET_CORE_EXPORT static BlockStack deserialise(BinaryReader&);
 

src/amulet/level/abc/chunk_handle.hpp

@@ -5,15 +5,38 @@
 #include <string>
 
 #include <amulet/chunk.hpp>
+#include <amulet/utils/mutex.hpp>
 
 namespace Amulet {
 
+using DimensionID = std::string;
+
 class ChunkHandle {
+private:
+    OrderedMutex _public_mutex;
+    std::string _dimension_id;
+    std::int64_t _cx;
+    std::int64_t _cz;
+
+protected:
+    ChunkHandle(
+        const DimensionID& dimension_id,
+        std::int64_t cx,
+        std::int64_t cz)
+        : _dimension_id(dimension_id)
+        , _cx(cx)
+        , _cz(cz)
+    {
+    }
+
 public:
     virtual ~ChunkHandle() = default;
-    virtual std::string dimension_id() = 0;
-    virtual std::int64_t cx() = 0;
-    virtual std::int64_t cz() = 0;
+
+    AMULET_CORE_EXPORT OrderedMutex& mutex() { return _public_mutex; };
+    AMULET_CORE_EXPORT const std::string& dimension_id() const { return _dimension_id; }
+    AMULET_CORE_EXPORT ::int64_t cx() const { return _cx; }
+    AMULET_CORE_EXPORT std::int64_t cz() const { return _cz; }
+
     virtual bool exists() = 0;
     virtual std::shared_ptr<Chunk> get_chunk() = 0;
     virtual void set_chunk(std::shared_ptr<Chunk>) = 0;

src/amulet/level/abc/dimension.hpp

@@ -10,10 +10,12 @@
 
 namespace Amulet {
 
+using DimensionID = std::string;
+
 class Dimension {
 public:
     virtual ~Dimension() = default;
-    virtual std::string dimension_id() = 0;
+    virtual DimensionID dimension_id() = 0;
     virtual const BlockStack& default_block() = 0;
     virtual const Biome& default_biome() = 0;
     virtual std::shared_ptr<ChunkHandle> get_chunk_handle(std::int64_t, std::int64_t) = 0;

src/amulet/level/abc/registry.cpp

@@ -1,10 +1,16 @@
+#include <shared_mutex>
 #include <stdexcept>
 #include <string>
 
 #include "registry.hpp"
 
 namespace Amulet {
 
+std::shared_mutex& IdRegistry::mutex()
+{
+    return _public_mutex;
+}
+
 NamespacedName IdRegistry::numerical_id_to_namespace_id(std::uint32_t index) const
 {
     return _index_to_name.at(index);

src/amulet/level/abc/registry.hpp

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <map>
+#include <shared_mutex>
 #include <string>
 #include <utility>
 
@@ -14,25 +15,35 @@ using NamespacedName = std::pair<std::string, std::string>;
 // A registry from numerical id to namespaced name.
 class IdRegistry {
 private:
+    std::shared_mutex _public_mutex;
     std::map<std::uint32_t, NamespacedName> _index_to_name;
     std::map<NamespacedName, std::uint32_t> _name_to_index;
 
 public:
     AMULET_CORE_EXPORT IdRegistry() = default;
+
+    // The public mutex.
+    // Thread safe.
+    AMULET_CORE_EXPORT std::shared_mutex& mutex();
+
     // Convert a numerical id to its namespaced id.
-    // Not thread safe. External shared/unique lock must be held while calling this.
+    // External shared lock required.
     AMULET_CORE_EXPORT NamespacedName numerical_id_to_namespace_id(std::uint32_t index) const;
+
     // Convert a namespaced id to its numerical id.
-    // Not thread safe. External shared/unique lock must be held while calling this.
+    // External shared lock required.
     AMULET_CORE_EXPORT std::uint32_t namespace_id_to_numerical_id(const NamespacedName& name) const;
+
     // Register a namespaced id to its numerical id.
-    // Not thread safe. External unique lock must be held while calling this.
+    // External unique lock required.
     AMULET_CORE_EXPORT void register_id(std::uint32_t index, const NamespacedName& name);
+
     // The number of ids registered.
-    // Not thread safe. External shared/unique lock must be held while calling this.
+    // External shared lock required.
     AMULET_CORE_EXPORT size_t size() const;
+
     // A read-only view of ids registered.
-    // Not thread safe. External shared/unique lock must be held while calling and using this.
+    // External shared lock required.
     AMULET_CORE_EXPORT const std::map<std::uint32_t, NamespacedName>& ids() const;
 };
 

src/amulet/level/abc/registry.py.cpp

@@ -19,6 +19,11 @@ py::module init_registry(py::module m_parent)
         "A registry for namespaced ids.\n"
         "External synchronisation is required with this class.");
     IdRegistry.def(py::init<>());
+    IdRegistry.def_property_readonly(
+        "lock",
+        &Amulet::IdRegistry::mutex,
+        py::doc("The public lock.\n"
+                "Thread safe."));
     IdRegistry.def(
         "numerical_id_to_namespace_id",
         [](const Amulet::IdRegistry& self, std::uint32_t index) {
@@ -30,7 +35,7 @@ py::module init_registry(py::module m_parent)
         },
         py::arg("index"),
         py::doc("Convert a numerical id to its namespaced id.\n"
-                "Not thread safe. External shared/unique lock must be held while calling this.\n"));
+                "External shared lock required."));
     IdRegistry.def(
         "namespace_id_to_numerical_id",
         [](const Amulet::IdRegistry& self, const Amulet::NamespacedName& name) {
@@ -42,7 +47,7 @@ py::module init_registry(py::module m_parent)
         },
         py::arg("name"),
         py::doc("Convert a namespaced id to its numerical id.\n"
-                "Not thread safe. External shared/unique lock must be held while calling this.\n"));
+                "External shared lock required."));
     IdRegistry.def(
         "namespace_id_to_numerical_id",
         [](const Amulet::IdRegistry& self, std::string namespace_, std::string base_name) {
@@ -55,20 +60,20 @@ py::module init_registry(py::module m_parent)
         py::arg("namespace"),
         py::arg("base_name"),
         py::doc("Convert a namespaced id to its numerical id.\n"
-                "Not thread safe. External shared/unique lock must be held while calling this.\n"));
+                "External shared lock required."));
     IdRegistry.def(
         "register_id",
         &Amulet::IdRegistry::register_id,
         py::arg("index"),
         py::arg("name"),
         py::doc("Convert a namespaced id to its numerical id.\n"
-                "Not thread safe. External unique lock must be held while calling this.\n"));
+                "External unique lock required."));
     IdRegistry.def(
         "__len__",
         &Amulet::IdRegistry::size,
         py::doc(
             "The number of ids registered.\n"
-            "Not thread safe. External shared/unique lock must be held while calling this.\n"));
+            "External shared lock required."));
     IdRegistry.def(
         "__iter__",
         [](const Amulet::IdRegistry& self) -> pybind11_extensions::collections::abc::Iterator<std::uint32_t> {
@@ -78,7 +83,7 @@ py::module init_registry(py::module m_parent)
         },
         py::keep_alive<0, 1>(),
         py::doc("An iterable of the numerical ids registered.\n"
-                "Not thread safe. External shared/unique lock must be held while calling and using this.\n"));
+                "External shared lock required."));
     IdRegistry.def(
         "__getitem__",
         [](const Amulet::IdRegistry& self, std::uint32_t index) {
@@ -90,7 +95,7 @@ py::module init_registry(py::module m_parent)
         },
         py::arg("index"),
         py::doc("Convert a numerical id to its namespaced id.\n"
-                "Not thread safe. External shared/unique lock must be held while calling this.\n"));
+                "External shared lock required."));
     IdRegistry.def(
         "__getitem__",
         [](const Amulet::IdRegistry& self, const Amulet::NamespacedName& name) {
@@ -102,7 +107,7 @@ py::module init_registry(py::module m_parent)
         },
         py::arg("name"),
         py::doc("Convert a namespaced id to its numerical id.\n"
-                "Not thread safe. External shared/unique lock must be held while calling this.\n"));
+                "External shared lock required."));
 
     Amulet::collections::PyMapping_contains<std::uint32_t>(IdRegistry);
     Amulet::collections::PyMapping_keys<std::uint32_t>(IdRegistry);

src/amulet/level/abc/registry.pyi

@@ -4,6 +4,8 @@ import collections.abc
 import types
 import typing
 
+import amulet.utils.lock
+
 __all__ = ["IdRegistry"]
 
 class IdRegistry:
@@ -18,28 +20,28 @@ class IdRegistry:
     def __getitem__(self, index: int) -> tuple[str, str]:
         """
         Convert a numerical id to its namespaced id.
-        Not thread safe. External shared/unique lock must be held while calling this.
+        External shared lock required.
         """
 
     @typing.overload
     def __getitem__(self, name: tuple[str, str]) -> int:
         """
         Convert a namespaced id to its numerical id.
-        Not thread safe. External shared/unique lock must be held while calling this.
+        External shared lock required.
         """
 
     def __hash__(self) -> int: ...
     def __init__(self) -> None: ...
     def __iter__(self) -> collections.abc.Iterator[int]:
         """
         An iterable of the numerical ids registered.
-        Not thread safe. External shared/unique lock must be held while calling and using this.
+        External shared lock required.
         """
 
     def __len__(self) -> int:
         """
         The number of ids registered.
-        Not thread safe. External shared/unique lock must be held while calling this.
+        External shared lock required.
         """
 
     def get(
@@ -51,26 +53,32 @@ class IdRegistry:
     def namespace_id_to_numerical_id(self, name: tuple[str, str]) -> int:
         """
         Convert a namespaced id to its numerical id.
-        Not thread safe. External shared/unique lock must be held while calling this.
+        External shared lock required.
         """
 
     @typing.overload
     def namespace_id_to_numerical_id(self, namespace: str, base_name: str) -> int:
         """
         Convert a namespaced id to its numerical id.
-        Not thread safe. External shared/unique lock must be held while calling this.
+        External shared lock required.
         """
 
     def numerical_id_to_namespace_id(self, index: int) -> tuple[str, str]:
         """
         Convert a numerical id to its namespaced id.
-        Not thread safe. External shared/unique lock must be held while calling this.
+        External shared lock required.
         """
 
     def register_id(self, index: int, name: tuple[str, str]) -> None:
         """
         Convert a namespaced id to its numerical id.
-        Not thread safe. External unique lock must be held while calling this.
+        External unique lock required.
         """
 
     def values(self) -> collections.abc.ValuesView[tuple[str, str]]: ...
+    @property
+    def lock(self) -> amulet.utils.lock.SharedLock:
+        """
+        The public lock.
+        Thread safe.
+        """

src/amulet/level/java/__init__.pyi

@@ -1,5 +1,18 @@
 from __future__ import annotations
 
-from . import _raw, anvil, chunk, chunk_components, long_array
+from amulet.level.java.raw_dimension import JavaRawDimension
+from amulet.level.java.raw_level import JavaCreateArgsV1, JavaRawLevel
 
-__all__ = ["anvil", "chunk", "chunk_components", "long_array"]
+from . import anvil, chunk, chunk_components, long_array, raw_dimension, raw_level
+
+__all__ = [
+    "JavaCreateArgsV1",
+    "JavaRawDimension",
+    "JavaRawLevel",
+    "anvil",
+    "chunk",
+    "chunk_components",
+    "long_array",
+    "raw_dimension",
+    "raw_level",
+]

src/amulet/level/java/__init_java.py.cpp

@@ -11,7 +11,8 @@ void init_long_array(py::module);
 void init_java_chunk_components(py::module);
 void init_java_chunk(py::module);
 void init_java_anvil(py::module);
-void init_java_raw(py::module);
+py::module init_java_raw_dimension(py::module);
+py::module init_java_raw_level(py::module);
 
 py::module init_java(py::module m_parent)
 {
@@ -21,8 +22,15 @@ py::module init_java(py::module m_parent)
     init_java_chunk_components(m);
     init_java_chunk(m);
     init_java_anvil(m);
-    init_java_raw(m);
 
+    auto raw_dimension = init_java_raw_dimension(m);
+    m.attr("JavaRawDimension") = raw_dimension.attr("JavaRawDimension");
+
+    auto raw_level = init_java_raw_level(m);
+    m.attr("JavaCreateArgsV1") = raw_level.attr("JavaCreateArgsV1");
+    m.attr("JavaRawLevel") = raw_level.attr("JavaRawLevel");
+
+    //    m.attr("JavaInternalDimensionID") = py::module::import("amulet.level.java.raw._typing").attr("JavaInternalDimensionID");
     //    m.attr("JavaLevel") = py::module::import("amulet.level.java._level").attr("JavaLevel");
 
     return m;