Moving runtime bindings into their own crate

Cargo.toml

@@ -23,7 +23,8 @@ features = ["private-docs-rs", "tensorflow_unstable", "ndarray", "eager"]
 libc = "0.2.132"
 num-complex = { version = "0.4.2", default-features = false }
 tensorflow-internal-macros = { version = "=0.0.3", path = "tensorflow-internal-macros" }
-tensorflow-sys = { version = "0.22.1", path = "tensorflow-sys" }
+tensorflow-sys = { version = "0.22.1", path = "tensorflow-sys", optional = true }
+tensorflow-sys-runtime = { version = "0.1.0", path = "tensorflow-sys-runtime", optional = true }
 byteorder = "1.4.3"
 crc = "3.0.0"
 half = "2.1.0"
@@ -38,9 +39,10 @@ random = "0.12.2"
 serial_test = "0.9.0"
 
 [features]
+default = ["tensorflow-sys"]
 tensorflow_gpu = ["tensorflow-sys/tensorflow_gpu"]
 tensorflow_unstable = []
-tensorflow_runtime_linking = ["tensorflow-sys/runtime_linking"]
+tensorflow_runtime_linking = ["tensorflow-sys-runtime"]
 eager = ["tensorflow-sys/eager"]
 # This is for testing purposes; users should not use this.
 examples_system_alloc = ["tensorflow-sys/examples_system_alloc"]
@@ -69,4 +71,4 @@ name = "xor"
 
 [[example]]
 name = "mobilenetv3"
-required-features = ["eager"]
+required-features = ["eager"]

RELEASING.md

@@ -25,6 +25,11 @@ Note that any crate not mentioned here (e.g. tensorflow-proto-codegen, tensorflo
    1. Bump the version in `tensorflow-sys/Cargo.toml`
    1. Bump the version in `tensorflow-sys/README.md`
    1. Bump the version for `tensorflow-sys` in the root `Cargo.toml`
+1. Bump version number of `tensorflow-sys-runtime` if necessary
+   1. Run `git log v${PREVIOUS_VERSION?}..HEAD tensorflow-sys-runtime` and see if there were any changes. If not, skip.
+   1. Bump the version in `tensorflow-sys-runtime/Cargo.toml`
+   1. Bump the version in `tensorflow-sys-runtime/README.md`
+   1. Bump the version for `tensorflow-sys-runtime` in the root `Cargo.toml`
 1. Bump version number of `tensorflow-internal-macros` if necessary
    1. Run `git log v${PREVIOUS_VERSION?}..HEAD tensorflow-internal-macros` and see if there were any changes. If not, skip.
    1. Bump the version in `tensorflow-internal-macros/Cargo.toml`
@@ -36,6 +41,7 @@ Note that any crate not mentioned here (e.g. tensorflow-proto-codegen, tensorflo
 1. Run `./run-valgrind`
 1. Commit and push the changes. (Push before publishing to ensure that the changes being published are up to date.)
 1. If the version of tensorflow-sys was bumped, run `cargo publish` for tensorflow-sys. (Publish before tagging in case there are problems publishing and we need to add commits to fix them.)
+1. If the version of tensorflow-sys-runtime was bumped, run `cargo publish` for tensorflow-sys-runtime. (Publish before tagging in case there are problems publishing and we need to add commits to fix them.)
 1. If the version of tensorflow-internal-macros was bumped, run `cargo publish` for tensorflow-internal-macros. (Publish before tagging in case there are problems publishing and we need to add commits to fix them.)
 1. Run `cargo publish`. (Publish before tagging in case there are problems publishing and we need to add commits to fix them.)
 1. Add a `v${VERSION?}` tag and push it

src/buffer.rs

@@ -16,7 +16,10 @@ use std::ops::RangeTo;
 use std::os::raw::c_void as std_c_void;
 use std::process;
 use std::slice;
+#[cfg(feature = "default")]
 use tensorflow_sys as tf;
+#[cfg(feature = "tensorflow_runtime_linking")]
+use tensorflow_sys_runtime as tf;
 
 /// Fixed-length heap-allocated vector.
 /// This is basically a `Box<[T]>`, except that that type can't actually be constructed.

src/graph.rs

@@ -28,7 +28,10 @@ use std::slice;
 use std::str::FromStr;
 use std::str::Utf8Error;
 use std::sync::Arc;
+#[cfg(feature = "default")]
 use tensorflow_sys as tf;
+#[cfg(feature = "tensorflow_runtime_linking")]
+use tensorflow_sys_runtime as tf;
 
 #[derive(Debug)]
 struct GraphImpl {

src/lib.rs

@@ -49,7 +49,10 @@ use std::process;
 use std::ptr;
 use std::slice;
 use std::str::Utf8Error;
+#[cfg(feature = "default")]
 use tensorflow_sys as tf;
+#[cfg(feature = "tensorflow_runtime_linking")]
+use tensorflow_sys_runtime as tf;
 
 ////////////////////////
 

src/while_loop.rs

@@ -9,7 +9,10 @@ use std::os::raw::c_int;
 use std::ptr;
 use std::result;
 use std::slice;
+#[cfg(feature = "default")]
 use tensorflow_sys as tf;
+#[cfg(feature = "tensorflow_runtime_linking")]
+use tensorflow_sys_runtime as tf;
 
 // This exists purely to ensure TF_AbortWhile gets called properly, even on panic.
 #[derive(Debug)]

tensorflow-sys-runtime/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "tensorflow-sys-runtime"
+version = "0.1.0"
+authors = ["Brian Jones <[email protected]>"]
+license = "Apache-2.0"
+keywords = ["TensorFlow", "runtime bindings"]
+description = "The package provides runtime bindings to TensorFlow."
+documentation = "https://tensorflow.github.io/rust"
+homepage = "https://github.com/tensorflow/rust"
+repository = "https://github.com/tensorflow/rust"
+edition = "2018"
+
+[dependencies]
+libc = "0.2.132"
+lazy_static = "1.4.0"
+libloading = "0.7.3"
+cfg-if = "1.0.0"
+log = "0.4.17"

tensorflow-sys-runtime/README.md

@@ -0,0 +1,25 @@
+# tensorflow-sys-runtime [![Version][version-icon]][version-page]
+
+The crate provides runtime bindings to [TensorFlow][tensorflow]. Using runtime bindings allows you to avoid
+pulling in additional package dependencies into your project. Users will need to call tensorflow::library::load()
+before any other calls so that the linking is completed before use.
+
+## NOTE
+This crate is meant to be used by [Rust language bindings for Tensorflow][crates-tf]. It is not meant to be used on it's own.
+To use it you will need to disable the default features so you don't also include the tensorflow-sys crate.
+
+```
+[dependencies]
+tensorflow = { version = "0.19.1", default-features = false, features = ["tensorflow_runtime_linking"]}
+```
+
+## Requirements
+
+To use these bindings you must have the Tensorflow C libraries installed. See  [install steps][tensorflow-setup]
+for detailed instructions.
+
+[tensorflow]: https://www.tensorflow.org
+[tensorflow-setup]: https://www.tensorflow.org/install/lang_c
+[crates-tf]: https://crates.io/crates/tensorflow
+[version-icon]: https://img.shields.io/crates/v/tensorflow-sys-runtime.svg
+[version-page]: https://crates.io/crates/tensorflow-sys-runtime

tensorflow-sys-runtime/generate_bindgen_rs.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+if ! which bindgen > /dev/null; then
+    echo "ERROR: Please install 'bindgen' using cargo:"
+    echo "    cargo install bindgen"
+    echo "See https://github.com/servo/rust-bindgen for more information."
+    exit 1
+fi
+
+include_dir="$HOME/git/tensorflow"
+
+bindgen_options_runtime_functions="--allowlist-function TF_.+ --blocklist-type .+ --size_t-is-usize --default-enum-style=rust --generate-inline-functions"
+cmd="bindgen ${bindgen_options_runtime_functions} ${include_dir}/tensorflow/c/c_api.h --output src/runtime_linking/c_api.rs --  -I ${include_dir}"
+echo ${cmd}
+${cmd}
+
+bindgen_options_runtime_types="--allowlist-type TF_.+ --blocklist-function .+ --size_t-is-usize --default-enum-style=rust --generate-inline-functions"
+cmd="bindgen ${bindgen_options_runtime_types} ${include_dir}/tensorflow/c/c_api.h --output src/runtime_linking/types.rs --  -I ${include_dir}"
+echo ${cmd}
+${cmd}
+
+echo "link! {\n$(cat src/runtime_linking/c_api.rs)" > src/runtime_linking/c_api.rs
+echo } >> src/runtime_linking/c_api.rs

tensorflow-sys/src/runtime_linking/c_api.rs → tensorflow-sys-runtime/src/c_api.rs

@@ -1,3 +1,4 @@
+include!("runtime.rs");
 link! {
 /* automatically generated by rust-bindgen 0.59.2 */
 

tensorflow-sys-runtime/src/lib.rs

@@ -0,0 +1,23 @@
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+include!("c_api.rs");
+include!("types.rs");
+include!("finder.rs");
+pub use crate::TF_AttrType::*;
+pub use crate::TF_Code::*;
+pub use crate::TF_DataType::*;
+
+pub mod library {
+    use std::path::PathBuf;
+
+    // Include the definition of `load` here. This allows hiding all of the "extra" linking-related
+    // functions in the same place, without polluting the top-level namespace (which should only
+    // contain foreign functions and types).
+    #[doc(inline)]
+    pub use super::load;
+
+    pub fn find() -> Option<PathBuf> {
+        super::find("tensorflow")
+    }
+}

tensorflow-sys/src/runtime_linking/runtime.rs → tensorflow-sys-runtime/src/runtime.rs

@@ -16,14 +16,12 @@ macro_rules! link {
         // Wrap the loaded functions.
         pub(crate) struct SharedLibrary {
             library: libloading::Library,
-            path: PathBuf,
             pub functions: Functions,
         }
         impl SharedLibrary {
-            fn new(library: libloading::Library, path: PathBuf) -> Self {
+            fn new(library: libloading::Library) -> Self {
                 Self {
                     library,
-                    path,
                     functions: Functions::default(),
                 }
             }
@@ -95,7 +93,7 @@ macro_rules! link {
                         )
                     });
 
-                    let mut library = SharedLibrary::new(library?, path);
+                    let mut library = SharedLibrary::new(library?);
                     $(load::$name(&mut library);)+
                     Ok(library)
                 }

tensorflow-sys/Cargo.toml

@@ -36,7 +36,6 @@ zip = "0.6.2"
 [features]
 tensorflow_gpu = []
 eager = []
-runtime_linking = ["lazy_static", "libloading", "cfg-if", "log"]
 # This is for testing purposes; users should not use this.
 examples_system_alloc = []
 private-docs-rs = [] # DO NOT RELY ON THIS

tensorflow-sys/generate_bindgen_rs.sh

@@ -18,16 +18,3 @@ bindgen_options_eager="--allowlist-function TFE_.+ --allowlist-type TFE_.+ --all
 cmd="bindgen ${bindgen_options_eager} ${include_dir}/tensorflow/c/eager/c_api.h --output src/eager/c_api.rs --  -I ${include_dir}"
 echo ${cmd}
 ${cmd}
-
-bindgen_options_runtime_functions="--allowlist-function TF_.+ --blocklist-type .+ --size_t-is-usize --default-enum-style=rust --generate-inline-functions"
-cmd="bindgen ${bindgen_options_runtime_functions} ${include_dir}/tensorflow/c/c_api.h --output src/runtime_linking/c_api.rs --  -I ${include_dir}"
-echo ${cmd}
-${cmd}
-
-bindgen_options_runtime_types="--allowlist-type TF_.+ --blocklist-function .+ --size_t-is-usize --default-enum-style=rust --generate-inline-functions"
-cmd="bindgen ${bindgen_options_runtime_types} ${include_dir}/tensorflow/c/c_api.h --output src/runtime_linking/types.rs --  -I ${include_dir}"
-echo ${cmd}
-${cmd}
-
-echo "link! {\n$(cat src/runtime_linking/c_api.rs)" > src/runtime_linking/c_api.rs
-echo } >> src/runtime_linking/c_api.rs

tensorflow-sys/src/lib.rs

@@ -6,31 +6,8 @@
 mod eager;
 #[cfg(feature = "eager")]
 pub use eager::*;
-
-#[cfg(feature = "runtime_linking")]
-mod runtime_linking;
-#[cfg(feature = "runtime_linking")]
-pub use runtime_linking::*;
-
-#[cfg(not(feature = "runtime_linking"))]
 include!("c_api.rs");
 
-#[cfg(not(feature = "runtime_linking"))]
 pub use crate::TF_AttrType::*;
 pub use crate::TF_Code::*;
 pub use crate::TF_DataType::*;
-
-#[cfg(feature = "runtime_linking")]
-pub mod library {
-    use std::path::PathBuf;
-
-    // Include the definition of `load` here. This allows hiding all of the "extra" linking-related
-    // functions in the same place, without polluting the top-level namespace (which should only
-    // contain foreign functions and types).
-    #[doc(inline)]
-    pub use super::runtime_linking::load;
-
-    pub fn find() -> Option<PathBuf> {
-        super::runtime_linking::find("tensorflow")
-    }
-}