[WIP] Interoperability with other Python binding frameworks

CMakeLists.txt

@@ -32,6 +32,7 @@ option(NB_TEST_STABLE_ABI    "Test the stable ABI interface?" OFF)
 option(NB_TEST_SHARED_BUILD  "Build a shared nanobind library for the test suite?" OFF)
 option(NB_TEST_CUDA          "Force the use of the CUDA/NVCC compiler for testing purposes" OFF)
 option(NB_TEST_FREE_THREADED "Build free-threaded extensions for the test suite?" ON)
+option(NB_TEST_NO_INTEROP    "Build without framework interoperability support?" OFF)
 
 if (NOT MSVC)
   option(NB_TEST_SANITIZERS_ASAN  "Build tests with the address sanitizer?" OFF)

cmake/nanobind-config.cmake

@@ -190,6 +190,7 @@ function (nanobind_build_library TARGET_NAME)
     ${NB_DIR}/src/nb_static_property.cpp
     ${NB_DIR}/src/nb_ft.h
     ${NB_DIR}/src/nb_ft.cpp
+    ${NB_DIR}/src/nb_foreign.cpp
     ${NB_DIR}/src/common.cpp
     ${NB_DIR}/src/error.cpp
     ${NB_DIR}/src/trampoline.cpp
@@ -236,6 +237,10 @@ function (nanobind_build_library TARGET_NAME)
     target_compile_definitions(${TARGET_NAME} PUBLIC NB_FREE_THREADED)
   endif()
 
+  if (TARGET_NAME MATCHES "-local")
+    target_compile_definitions(${TARGET_NAME} PRIVATE NB_DISABLE_FOREIGN)
+  endif()
+
   # Nanobind performs many assertion checks -- detailed error messages aren't
   # included in Release/MinSizeRel/RelWithDebInfo modes
   target_compile_definitions(${TARGET_NAME} PRIVATE
@@ -330,7 +335,7 @@ endfunction()
 
 function(nanobind_add_module name)
   cmake_parse_arguments(PARSE_ARGV 1 ARG
-    "STABLE_ABI;FREE_THREADED;NB_STATIC;NB_SHARED;PROTECT_STACK;LTO;NOMINSIZE;NOSTRIP;MUSL_DYNAMIC_LIBCPP;NB_SUPPRESS_WARNINGS"
+    "STABLE_ABI;FREE_THREADED;NB_STATIC;NB_SHARED;PROTECT_STACK;LTO;NOMINSIZE;NOSTRIP;MUSL_DYNAMIC_LIBCPP;NB_SUPPRESS_WARNINGS;NO_INTEROP"
     "NB_DOMAIN" "")
 
   add_library(${name} MODULE ${ARG_UNPARSED_ARGUMENTS})
@@ -375,6 +380,10 @@ function(nanobind_add_module name)
     set(libname "${libname}-ft")
   endif()
 
+  if (ARG_NO_INTEROP)
+    set(libname "${libname}-local")
+  endif()
+
   if (ARG_NB_DOMAIN AND ARG_NB_SHARED)
     set(libname ${libname}-${ARG_NB_DOMAIN})
   endif()

docs/api_cmake.rst

@@ -110,6 +110,10 @@ The high-level interface consists of just one CMake command:
           an optimization that nanobind does by default in this specific case).
           If this explanation sounds confusing, then you can ignore it. See the
           detailed description below for more information on this step.
+      * - ``NO_INTEROP``
+        - Remove support for interoperability with other Python binding
+          frameworks. If you don't need it in your environment, this offers
+          a minor performance and code size benefit.
 
    :cmake:command:`nanobind_add_module` performs the following
    steps to produce bindings.

include/nanobind/nb_class.h

@@ -91,9 +91,6 @@ enum class type_init_flags : uint32_t {
     all_init_flags           = (0x1f << 19)
 };
 
-// See internals.h
-struct nb_alias_chain;
-
 // Implicit conversions for C++ type bindings, used in type_data below
 struct implicit_t {
     const std::type_info **cpp;
@@ -114,7 +111,7 @@ struct type_data {
     const char *name;
     const std::type_info *type;
     PyTypeObject *type_py;
-    nb_alias_chain *alias_chain;
+    void *foreign_bindings;
 #if defined(Py_LIMITED_API)
     PyObject* (*vectorcall)(PyObject *, PyObject * const*, size_t, PyObject *);
 #endif
@@ -332,6 +329,19 @@ inline void *type_get_slot(handle h, int slot_id) {
 #endif
 }
 
+// nanobind interoperability with other binding frameworks
+inline void set_foreign_type_defaults(bool export_all, bool import_all) {
+    detail::nb_type_set_foreign_defaults(export_all, import_all);
+}
+template <class T = void>
+inline void import_foreign_type(handle type) {
+    detail::nb_type_import(type.ptr(),
+                           std::is_void_v<T> ? nullptr : &typeid(T));
+}
+inline void export_type_to_foreign(handle type) {
+    detail::nb_type_export(type.ptr());
+}
+
 template <typename Visitor> struct def_visitor {
   protected:
     // Ensure def_visitor<T> can only be derived from, not constructed

include/nanobind/nb_error.h

@@ -125,7 +125,7 @@ NB_EXCEPTION(next_overload)
 
 inline void register_exception_translator(detail::exception_translator t,
                                           void *payload = nullptr) {
-    detail::register_exception_translator(t, payload);
+    detail::register_exception_translator(t, payload, /*at_end=*/false);
 }
 
 template <typename T>
@@ -142,7 +142,7 @@ class exception : public object {
                 } catch (T &e) {
                     PyErr_SetString((PyObject *) payload, e.what());
                 }
-            }, m_ptr);
+            }, m_ptr, /*at_end=*/false);
     }
 };
 

include/nanobind/nb_lib.h

@@ -341,10 +341,12 @@ NB_CORE const std::type_info *nb_type_info(PyObject *t) noexcept;
 NB_CORE void *nb_inst_ptr(PyObject *o) noexcept;
 
 /// Check if a Python type object wraps an instance of a specific C++ type
-NB_CORE bool nb_type_isinstance(PyObject *obj, const std::type_info *t) noexcept;
+NB_CORE bool nb_type_isinstance(PyObject *obj, const std::type_info *t,
+                                bool foreign_ok) noexcept;
 
-/// Search for the Python type object associated with a C++ type
-NB_CORE PyObject *nb_type_lookup(const std::type_info *t) noexcept;
+/// Search for a Python type object associated with a C++ type
+NB_CORE PyObject *nb_type_lookup(const std::type_info *t,
+                                 bool foreign_ok) noexcept;
 
 /// Allocate an instance of type 't'
 NB_CORE PyObject *nb_inst_alloc(PyTypeObject *t);
@@ -386,6 +388,15 @@ NB_CORE void nb_inst_set_state(PyObject *o, bool ready, bool destruct) noexcept;
 /// Query the 'ready' and 'destruct' flags of an instance
 NB_CORE std::pair<bool, bool> nb_inst_state(PyObject *o) noexcept;
 
+// Set whether types will be shared with other binding frameworks by default
+NB_CORE void nb_type_set_foreign_defaults(bool export_all, bool import_all);
+
+// Teach nanobind about a type bound by another binding framework
+NB_CORE void nb_type_import(PyObject *pytype, const std::type_info *cpptype);
+
+// Teach other binding frameworks about a type bound by nanobind
+NB_CORE void nb_type_export(PyObject *pytype);
+
 // ========================================================================
 
 // Create and install a Python property object
@@ -500,7 +511,8 @@ NB_CORE void print(PyObject *file, PyObject *str, PyObject *end);
 typedef void (*exception_translator)(const std::exception_ptr &, void *);
 
 NB_CORE void register_exception_translator(exception_translator translator,
-                                           void *payload);
+                                           void *payload,
+                                           bool at_end);
 
 NB_CORE PyObject *exception_new(PyObject *mod, const char *name,
                                 PyObject *base);

include/nanobind/nb_types.h

@@ -667,15 +667,19 @@ class iterable : public object {
 
 /// Retrieve the Python type object associated with a C++ class
 template <typename T> handle type() noexcept {
-    return detail::nb_type_lookup(&typeid(detail::intrinsic_t<T>));
+    return detail::nb_type_lookup(&typeid(detail::intrinsic_t<T>), false);
+}
+template <typename T> handle maybe_foreign_type() noexcept {
+    return detail::nb_type_lookup(&typeid(detail::intrinsic_t<T>), true);
 }
 
 template <typename T>
-NB_INLINE bool isinstance(handle h) noexcept {
+NB_INLINE bool isinstance(handle h, bool foreign_ok = false) noexcept {
     if constexpr (std::is_base_of_v<handle, T>)
         return T::check_(h);
     else if constexpr (detail::is_base_caster_v<detail::make_caster<T>>)
-        return detail::nb_type_isinstance(h.ptr(), &typeid(detail::intrinsic_t<T>));
+        return detail::nb_type_isinstance(h.ptr(), &typeid(detail::intrinsic_t<T>),
+                                          foreign_ok);
     else
         return detail::make_caster<T>().from_python(h, 0, nullptr);
 }

include/nanobind/stl/unique_ptr.h

@@ -94,6 +94,11 @@ struct type_caster<std::unique_ptr<T, Deleter>> {
         // Stash source python object
         src = src_;
 
+        // Don't accept foreign types; they can't relinquish ownership
+        if (!src.is_none() && !inst_check(src)) {
+            return false;
+        }
+
         /* Try casting to a pointer of the underlying type. We pass flags=0 and
            cleanup=nullptr to prevent implicit type conversions (they are
            problematic since the instance then wouldn't be owned by 'src') */

src/error.cpp

@@ -217,12 +217,28 @@ builtin_exception::~builtin_exception() { }
 
 NAMESPACE_BEGIN(detail)
 
-void register_exception_translator(exception_translator t, void *payload) {
-    nb_translator_seq *cur  = &internals->translators,
-                      *next = new nb_translator_seq(*cur);
-    cur->next = next;
-    cur->payload = payload;
-    cur->translator = t;
+void register_exception_translator(exception_translator t,
+                                   void *payload,
+                                   bool at_end) {
+    // We will insert the new translator so it is pointed to by `*insert_at`,
+    // i.e., so that it is executed just before the current `*insert_at`
+    nb_maybe_atomic<nb_translator_seq *> *insert_at = &internals->translators;
+    if (at_end) {
+        // Insert before the default exception translator (which is last in
+        // the list)
+        nb_translator_seq *next = insert_at->load_acquire();
+        while (next && next->next.load_relaxed()) {
+            insert_at = &next->next;
+            next = insert_at->load_acquire();
+        }
+    }
+    nb_translator_seq *new_head = new nb_translator_seq{};
+    nb_translator_seq *cur_head = insert_at->load_relaxed();
+    new_head->payload = payload;
+    new_head->translator = t;
+    do {
+        new_head->next.store_release(cur_head);
+    } while (!insert_at->compare_exchange_weak(cur_head, new_head));
 }
 
 NB_CORE PyObject *exception_new(PyObject *scope, const char *name,

src/nb_abi.h

@@ -14,7 +14,7 @@
 
 /// Tracks the version of nanobind's internal data structures
 #ifndef NB_INTERNALS_VERSION
-#  define NB_INTERNALS_VERSION 16
+#  define NB_INTERNALS_VERSION 17
 #endif
 
 #if defined(__MINGW32__)

src/nb_combined.cpp

@@ -79,6 +79,7 @@
 #include "nb_ndarray.cpp"
 #include "nb_static_property.cpp"
 #include "nb_ft.cpp"
+#include "nb_foreign.cpp"
 #include "error.cpp"
 #include "common.cpp"
 #include "implicit.cpp"

src/nb_enum.cpp

@@ -18,23 +18,6 @@ using enum_map = tsl::robin_map<int64_t, int64_t, int64_hash>;
 
 PyObject *enum_create(enum_init_data *ed) noexcept {
     // Update hash table that maps from std::type_info to Python type
-    nb_internals *internals_ = internals;
-    bool success;
-    nb_type_map_slow::iterator it;
-
-    {
-        lock_internals guard(internals_);
-        std::tie(it, success) = internals_->type_c2p_slow.try_emplace(ed->type, nullptr);
-        if (!success) {
-            PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
-                             "nanobind: type '%s' was already registered!\n",
-                             ed->name);
-            PyObject *tp = (PyObject *) it->second->type_py;
-            Py_INCREF(tp);
-            return tp;
-        }
-    }
-
     handle scope(ed->scope);
 
     bool is_arithmetic = ed->flags & (uint32_t) enum_flags::is_arithmetic;
@@ -85,20 +68,7 @@ PyObject *enum_create(enum_init_data *ed) noexcept {
     t->enum_tbl.rev = new enum_map();
     t->scope = ed->scope;
 
-    it.value() = t;
-
-    {
-        lock_internals guard(internals_);
-        internals_->type_c2p_slow[ed->type] = t;
-
-        #if !defined(NB_FREE_THREADED)
-            internals_->type_c2p_fast[ed->type] = t;
-        #endif
-    }
-
-    make_immortal(result.ptr());
-
-    result.attr("__nb_enum__") = capsule(t, [](void *p) noexcept {
+    capsule tie_lifetimes(t, [](void *p) noexcept {
         type_init_data *t = (type_init_data *) p;
         delete (enum_map *) t->enum_tbl.fwd;
         delete (enum_map *) t->enum_tbl.rev;
@@ -107,6 +77,19 @@ PyObject *enum_create(enum_init_data *ed) noexcept {
         delete t;
     });
 
+    if (type_data *conflict; !nb_type_register(t, &conflict)) {
+        PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
+                         "nanobind: type '%s' was already registered!\n",
+                         ed->name);
+        PyObject *tp = (PyObject *) conflict->type_py;
+        Py_INCREF(tp);
+        return tp;
+    }
+
+    result.attr("__nb_enum__") = tie_lifetimes;
+
+    make_immortal(result.ptr());
+
     return result.release().ptr();
 }