Stubs fallback

.gitlab/test_rust.sh

@@ -43,10 +43,27 @@ export PATH=${INSTALL_DIR}/bin:$PATH
 export PKG_CONFIG_PATH=${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}/lib:${UCX_INSTALL_DIR}/lib/pkgconfig:${INSTALL_DIR}/lib/$ARCH-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
 export NIXL_PLUGIN_DIR=${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins
 export NIXL_PREFIX=${INSTALL_DIR}
+export NIXL_NO_STUBS_FALLBACK=1
 
 cargo test -- --test-threads=1
 
+# test that stubs and real wrapper defined APIs / symbols match
+g++ -c ./src/bindings/rust/wrapper.cpp -o wrapper.o -I ./src/api/cpp/
+g++ -c ./src/bindings/rust/stubs.cpp -o stubs.o
+
+diff <(nm -C --defined-only wrapper.o | awk '$2 ~ /^T$/ {print $3}' | sort) \
+     <(nm -C --defined-only stubs.o    | awk '$2 ~ /^T$/ {print $3}' | grep -v nixl_capi_stub_abort | sort)
+
+if [ $? -ne 0 ]; then
+    echo "Stubs API and wrapper API differ"
+    exit 1
+else
+    echo "Stubs API and wrapper API match"
+fi
+
+
 # test stubs build
 cargo build --features stub-api
 
 cargo package
+

README.md

@@ -145,6 +145,11 @@ Common build options:
 - `cudapath_inc`, `cudapath_lib`: Custom CUDA paths
 - `static_plugins`: Comma-separated list of plugins to build statically
 
+#### Environment Variables
+
+There are a few environment variables that can be set to configure the build:
+- `NIXL_NO_STUBS_FALLBACK`: If not set or 0, build NIXL stub library if the library build fails
+
 ### Building Documentation
 
 If you have Doxygen installed, you can build the documentation:

src/bindings/rust/Cargo.toml

@@ -40,6 +40,7 @@ bindgen = "0.71"
 cc = { version = "1.2.23", features = ["parallel"] }
 pkg-config = "0.3"
 os_info = "3.11"
+anyhow = "1"
 
 [dev-dependencies]
 tempfile = "3.20.0"

src/bindings/rust/build.rs

@@ -84,7 +84,7 @@ fn get_nixl_libs() -> Option<Vec<pkg_config::Library>> {
     }
 }
 
-fn build_nixl(cc_builder: &mut cc::Build) {
+fn build_nixl(cc_builder: &mut cc::Build) -> anyhow::Result<()> {
     let nixl_root_path =
         env::var("NIXL_PREFIX").unwrap_or_else(|_| "/opt/nvidia/nvda_nixl".to_string());
 
@@ -129,7 +129,6 @@ fn build_nixl(cc_builder: &mut cc::Build) {
         .file("wrapper.cpp")
         .includes(nixl_include_paths);
 
-
     println!("cargo:rustc-link-search={}", nixl_lib_path);
 
     let etcd_enabled = env::var("HAVE_ETCD").map(|v| v != "0").unwrap_or(false);
@@ -139,7 +138,7 @@ fn build_nixl(cc_builder: &mut cc::Build) {
     }
 
     // Compile the wrapper C++ code
-    cc_builder.compile("nixl_wrapper");
+    cc_builder.try_compile("nixl_wrapper")?;
 
     // Get the output path for bindings
     let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
@@ -183,9 +182,10 @@ fn build_nixl(cc_builder: &mut cc::Build) {
     builder
         .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
         .generate()
-        .expect("Unable to generate bindings")
-        .write_to_file(out_path.join("bindings.rs"))
-        .expect("Couldn't write bindings!");
+        .map_err(|_| anyhow::anyhow!("Unable to generate bindings"))?
+        .write_to_file(out_path.join("bindings.rs"))?;
+
+    Ok(())
 }
 
 fn build_stubs(cc_builder: &mut cc::Build) {
@@ -218,18 +218,38 @@ fn build_stubs(cc_builder: &mut cc::Build) {
         .expect("Couldn't write bindings!");
 }
 
-fn run_build(use_stub_api: bool) {
-    let mut cc_builder = cc::Build::new();
-    cc_builder
+fn create_builder() -> cc::Build {
+    let mut builder = cc::Build::new();
+    builder
         .cpp(true)
         .compiler("g++")
         .flag("-std=c++17")
         .flag("-fPIC")
         .flag("-Wno-unused-parameter")
         .flag("-Wno-unused-variable");
+    builder
+}
+
+fn run_build(use_stub_api: bool) {
+    let mut cc_builder = create_builder();
 
     if !use_stub_api {
-        build_nixl(&mut cc_builder);
+        let no_fallback = env::var("NIXL_NO_STUBS_FALLBACK")
+            .map(|v| v == "1")
+            .unwrap_or(false);
+
+        if let Err(e) = build_nixl(&mut cc_builder) {
+            if !no_fallback {
+                println!(
+                    "cargo:warning=NIXL build failed: {}, falling back to stub API",
+                    e
+                );
+                let mut stub_builder = create_builder();
+                build_stubs(&mut stub_builder);
+            } else {
+                panic!("Failed to build NIXL: {}", e);
+            }
+        }
     } else {
         build_stubs(&mut cc_builder);
     }

src/bindings/rust/src/lib.rs

@@ -67,7 +67,7 @@ use bindings::{
     nixl_capi_query_resp_list_get_params, nixl_capi_prep_xfer_dlist, nixl_capi_release_xfer_dlist_handle,
     nixl_capi_make_xfer_req, nixl_capi_get_local_partial_md,
     nixl_capi_send_local_partial_md, nixl_capi_query_xfer_backend, nixl_capi_opt_args_set_ip_addr,
-    nixl_capi_opt_args_set_port, nixl_capi_get_xfer_telemetry
+    nixl_capi_opt_args_set_port, nixl_capi_get_xfer_telemetry, nixl_capi_is_stub
 };
 
 // Re-export status codes
@@ -559,3 +559,7 @@ impl NixlRegistration for SystemStorage {
         Ok(())
     }
 }
+
+pub fn is_stub() -> bool {
+    unsafe { nixl_capi_is_stub() }
+}

src/bindings/rust/stubs.cpp

@@ -56,11 +56,6 @@ nixl_capi_create_agent(const char* name, nixl_capi_agent_t* agent)
   return nixl_capi_stub_abort();
 }
 
-nixl_capi_status_t
-nixl_capi_default_agent_config(nixl_capi_agent_config_t *out) {
-    return nixl_capi_stub_abort();
-}
-
 nixl_capi_status_t
 nixl_capi_create_configured_agent(const char *name,
                                   const nixl_capi_agent_config_t *cfg,
@@ -94,6 +89,18 @@ nixl_capi_load_remote_md(nixl_capi_agent_t agent, const void* data, size_t len,
   return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_send_local_md(nixl_capi_agent_t agent, nixl_capi_opt_args_t opt_args) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_send_local_partial_md(nixl_capi_agent_t agent,
+                                nixl_capi_reg_dlist_t descs,
+                                nixl_capi_opt_args_t opt_args) {
+    return nixl_capi_stub_abort();
+}
+
 nixl_capi_status_t
 nixl_capi_prep_xfer_dlist(nixl_capi_agent_t agent,
                           const char *agent_name,
@@ -129,6 +136,32 @@ nixl_capi_invalidate_remote_md(nixl_capi_agent_t agent, const char* remote_agent
   return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_invalidate_local_md(nixl_capi_agent_t agent, nixl_capi_opt_args_t opt_args) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_check_remote_md(nixl_capi_agent_t agent,
+                          const char *remote_name,
+                          nixl_capi_xfer_dlist_t descs) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_fetch_remote_md(nixl_capi_agent_t agent,
+                          const char *remote_name,
+                          nixl_capi_opt_args_t opt_args) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_agent_make_connection(nixl_capi_agent_t agent,
+                                const char *remote_agent,
+                                nixl_capi_opt_args_t opt_args) {
+    return nixl_capi_stub_abort();
+}
+
 nixl_capi_status_t
 nixl_capi_get_available_plugins(nixl_capi_agent_t agent, nixl_capi_string_list_t* plugins)
 {
@@ -340,6 +373,36 @@ nixl_capi_xfer_dlist_resize(nixl_capi_xfer_dlist_t dlist, size_t new_size)
   return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_xfer_dlist_get_type(nixl_capi_xfer_dlist_t dlist, nixl_capi_mem_type_t *mem_type) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_xfer_dlist_desc_count(nixl_capi_xfer_dlist_t dlist, size_t *count) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_xfer_dlist_is_empty(nixl_capi_xfer_dlist_t dlist, bool *is_empty) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_xfer_dlist_trim(nixl_capi_xfer_dlist_t dlist) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_xfer_dlist_rem_desc(nixl_capi_xfer_dlist_t dlist, int index) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_xfer_dlist_print(nixl_capi_xfer_dlist_t dlist) {
+    return nixl_capi_stub_abort();
+}
+
 // Registration descriptor list functions
 nixl_capi_status_t
 nixl_capi_create_reg_dlist(nixl_capi_mem_type_t mem_type, nixl_capi_reg_dlist_t *dlist) {
@@ -374,6 +437,36 @@ nixl_capi_reg_dlist_resize(nixl_capi_reg_dlist_t dlist, size_t new_size)
   return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_reg_dlist_get_type(nixl_capi_reg_dlist_t dlist, nixl_capi_mem_type_t *mem_type) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_reg_dlist_desc_count(nixl_capi_reg_dlist_t dlist, size_t *count) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_reg_dlist_is_empty(nixl_capi_reg_dlist_t dlist, bool *is_empty) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_reg_dlist_trim(nixl_capi_reg_dlist_t dlist) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_reg_dlist_rem_desc(nixl_capi_reg_dlist_t dlist, int index) {
+    return nixl_capi_stub_abort();
+}
+
+nixl_capi_status_t
+nixl_capi_reg_dlist_print(nixl_capi_reg_dlist_t dlist) {
+    return nixl_capi_stub_abort();
+}
+
 // Memory registration functions
 nixl_capi_status_t
 nixl_capi_register_mem(nixl_capi_agent_t agent, nixl_capi_reg_dlist_t dlist, nixl_capi_opt_args_t opt_args)
@@ -416,6 +509,16 @@ nixl_capi_query_xfer_backend(nixl_capi_agent_t agent,
     return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_estimate_xfer_cost(nixl_capi_agent_t agent,
+                             nixl_capi_xfer_req_t req_hndl,
+                             nixl_capi_opt_args_t opt_args,
+                             int64_t *duration_us,
+                             int64_t *err_margin_us,
+                             nixl_capi_cost_t *method) {
+    return nixl_capi_stub_abort();
+}
+
 nixl_capi_status_t
 nixl_capi_destroy_xfer_req(nixl_capi_xfer_req_t req)
 {
@@ -434,6 +537,15 @@ nixl_capi_get_notifs(nixl_capi_agent_t agent, nixl_capi_notif_map_t notif_map, n
   return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_gen_notif(nixl_capi_agent_t agent,
+                    const char *remote_agent,
+                    const void *data,
+                    size_t len,
+                    nixl_capi_opt_args_t opt_args) {
+    return nixl_capi_stub_abort();
+}
+
 nixl_capi_status_t
 nixl_capi_create_notif_map(nixl_capi_notif_map_t* notif_map)
 {
@@ -514,4 +626,16 @@ nixl_capi_query_mem(nixl_capi_agent_t agent,
     return nixl_capi_stub_abort();
 }
 
+nixl_capi_status_t
+nixl_capi_get_xfer_telemetry(nixl_capi_agent_t agent,
+                             nixl_capi_xfer_req_t req_hndl,
+                             nixl_capi_xfer_telemetry_t telemetry) {
+    return nixl_capi_stub_abort();
+}
+
+bool
+nixl_capi_is_stub() {
+    return true;
+}
+
 }  // extern "C"

src/bindings/rust/wrapper.cpp

@@ -1903,4 +1903,10 @@ nixl_capi_get_xfer_telemetry(nixl_capi_agent_t agent,
         return NIXL_CAPI_ERROR_BACKEND;
     }
 }
+
+bool
+nixl_capi_is_stub() {
+    return false;
+}
+
 } // extern "C"

src/bindings/rust/wrapper.h

@@ -354,6 +354,8 @@ nixl_capi_get_xfer_telemetry(nixl_capi_agent_t agent,
                              nixl_capi_xfer_req_t req_hndl,
                              nixl_capi_xfer_telemetry_t telemetry);
 
+bool
+nixl_capi_is_stub();
 
 #ifdef __cplusplus
 }