*: fix builds (#351)

This PR breaks the circle dependencies between grpcio and grpcio-proto
by adding a new crate named tests and examples. We can bring things back
once rust-lang/cargo#6915 is fixed.

The new structure also fixes a problem that crates.io denies to accept
circle dependencies in dev-dependencies. More see rust-lang/cargo#4242.

This PR also updates protobuf-build to the latest version.

Author: BusyJay

Cargo.toml

@@ -26,7 +26,7 @@ bytes = { version = "0.4.11", optional = true }
 log = "0.4"
 
 [workspace]
-members = ["proto", "benchmark", "compiler", "interop"]
+members = ["proto", "benchmark", "compiler", "interop", "tests-and-examples"]
 
 [features]
 default = ["protobuf-codec", "secure"]
@@ -37,34 +37,6 @@ openssl = ["secure", "grpcio-sys/openssl"]
 openssl-vendored = ["secure", "grpcio-sys/openssl-vendored"]
 no-omit-frame-pointer = ["grpcio-sys/no-omit-frame-pointer"]
 
-[[example]]
-name = "route_guide_client"
-path = "examples/route_guide/client.rs"
-
-[[example]]
-name = "route_guide_server"
-path = "examples/route_guide/server.rs"
-
-[[example]]
-name = "greeter_client"
-path = "examples/hello_world/client.rs"
-
-[[example]]
-name = "greeter_server"
-path = "examples/hello_world/server.rs"
-
-[dev-dependencies]
-serde_json = "1.0"
-serde = "1.0"
-serde_derive = "1.0"
-grpcio-proto = { path = "proto", version = "0.5.0-alpha.2", default-features = false }
-rand = "0.4"
-slog = "2.0"
-slog-async = "2.1"
-slog-stdlog = "3.0"
-slog-scope = "4.0"
-slog-term = "2.2"
-
 [profile.release]
 debug = true
 

benchmark/src/util.rs

@@ -17,7 +17,7 @@ use std::time::{Duration, Instant};
 use grpc_proto::testing::stats::HistogramData;
 use grpc_sys;
 
-#[path = "../../examples/log_util.rs"]
+#[path = "../../tests-and-examples/examples/log_util.rs"]
 pub mod log_util;
 
 pub struct Sample {

proto/Cargo.toml

@@ -14,8 +14,8 @@ build = "build.rs"
 
 [features]
 default = ["protobuf-codec"]
-protobuf-codec = ["grpcio/protobuf-codec", "protobuf-codegen", "protobuf", "grpcio-compiler/protobuf-codec"]
-prost-codec = ["prost-derive", "bytes", "lazy_static", "grpcio/prost-codec", "prost-types", "prost", "grpcio-compiler/prost-codec"]
+protobuf-codec = ["grpcio/protobuf-codec", "grpcio-compiler/protobuf-codec", "protobuf-build/grpcio-protobuf-codec"]
+prost-codec = ["prost-derive", "bytes", "lazy_static", "grpcio/prost-codec", "prost", "grpcio-compiler/prost-codec", "protobuf-build/grpcio-prost-codec"]
 
 [dependencies]
 futures = "0.1"
@@ -27,9 +27,6 @@ protobuf = "2"
 lazy_static = { version = "1.3", optional = true }
 
 [build-dependencies]
-protobuf-build = "0.7"
+protobuf-build = { version = "0.8", default-features = false }
 grpcio-compiler = { path = "../compiler", version = "0.5.0-alpha.2", default-features = false }
-prost = { version = "0.5", optional = true }
-prost-types = { version = "0.5", optional = true }
-protobuf = { version = "2", optional = true }
-protobuf-codegen = { version = "2", optional = true }
+walkdir = "2.2"

proto/build.rs

@@ -11,144 +11,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
+use std::env;
 
 fn main() {
-    for package in &["testing", "example", "health"] {
-        compile(format!("{}.desc", package), package);
-    }
-}
-
-#[cfg(feature = "prost-codec")]
-fn compile<P: AsRef<Path>>(desc_path: P, _package: &str) {
-    prost::desc_to_module(desc_path.as_ref());
-}
-
-#[cfg(feature = "prost-codec")]
-mod prost {
-    use std::env;
-    use std::fs;
-    use std::path::{Path, PathBuf};
-
-    use prost::Message;
-    use prost_types::FileDescriptorSet as ProstFileDescriptorSet;
-
-    /// Descriptor file to module file using Prost.
-    pub fn desc_to_module(descriptor: &Path) {
-        let buf = fs::read(descriptor).unwrap();
-        let proto_set: ProstFileDescriptorSet = Message::decode(buf).unwrap();
-        let files: Vec<_> = proto_set
-            .file
+    let out_dir = env::var("OUT_DIR").unwrap();
+    let modules = &[
+        ("grpc/testing", "testing"),
+        ("grpc/health/v1/", "health"),
+        ("grpc/example", "example"),
+    ];
+    for (dir, package) in modules {
+        let out_dir = format!("{}/{}", out_dir, package);
+        let files: Vec<_> = walkdir::WalkDir::new(format!("proto/{}", dir))
             .into_iter()
-            .map(|f| {
-                let mut path = PathBuf::new();
-                path.push("proto");
-                path.push(f.name.unwrap());
-                path.to_str().unwrap().to_owned()
+            .filter_map(|p| {
+                let dent = p.expect("Error happened when search protos");
+                if !dent.file_type().is_file() {
+                    return None;
+                }
+                Some(format!("{}", dent.path().display()))
             })
             .collect();
-
-        let packages = protobuf_build::generate_prost_files(&files, &env::var("OUT_DIR").unwrap());
-        assert!(!packages.is_empty());
-        protobuf_build::generate_wrappers(
-            &packages
-                .iter()
-                .map(|m| format!("{}/{}.rs", env::var("OUT_DIR").unwrap(), m))
-                .collect::<Vec<_>>(),
-            &env::var("OUT_DIR").unwrap(),
-            protobuf_build::GenOpt::all(),
-        );
-    }
-}
-
-#[cfg(feature = "protobuf-codec")]
-fn compile<P: AsRef<Path>>(desc_path: P, package: &str) {
-    protobuf::compile(desc_path, package);
-}
-
-#[cfg(feature = "protobuf-codec")]
-mod protobuf {
-    use std::env;
-    use std::fs::{self, File};
-    use std::io::Write;
-    use std::path::Path;
-
-    use grpcio_compiler::codegen as grpc_gen;
-    use protobuf::compiler_plugin::GenResult;
-    use protobuf::descriptor::{FileDescriptorProto, FileDescriptorSet};
-    use protobuf_codegen::{self as pb_gen, Customize};
-
-    /// Compile all related proto file to `FileDescriptorSet` and use it to generate
-    /// rust source using rust-protobuf.
-    ///
-    /// Using `FileDescriptorSet` here so we don't need to compile the binaries like
-    /// protoc-gen-rust and grpc_rust_plugin.
-    pub fn compile<P: AsRef<Path>>(desc_path: P, package: &str) {
-        let out_dir = env::var("OUT_DIR").unwrap();
-        let module_path = Path::new(&out_dir).join(package);
-        if !module_path.exists() {
-            fs::create_dir_all(&module_path).unwrap();
-        }
-
-        let mod_rs = module_path.join("mod.rs");
-        let mut module_file = File::create(mod_rs).unwrap();
-
-        desc_to_module(
-            desc_path.as_ref(),
-            &module_path,
-            |a, b| {
-                let c = Customize::default();
-                pb_gen::gen(a, b, &c)
-            },
-            &mut module_file,
-        );
-        desc_to_module(
-            desc_path.as_ref(),
-            &module_path,
-            grpc_gen::gen,
-            &mut module_file,
-        );
-    }
-
-    /// Descriptor file to module file using rust-protobuf.
-    fn desc_to_module<G, W>(descriptor: &Path, output_dir: &Path, mut generate_files: G, out: W)
-    where
-        G: FnMut(&[FileDescriptorProto], &[String]) -> Vec<GenResult>,
-        W: Write,
-    {
-        let mut f = File::open(descriptor).unwrap();
-        let proto_set: FileDescriptorSet = protobuf::parse_from_reader(&mut f).unwrap();
-
-        let files: Vec<_> = proto_set
-            .get_file()
-            .into_iter()
-            .map(|f| f.get_name().to_owned())
-            .collect();
-
-        // All files need to be generated in our case.
-        let results = generate_files(proto_set.get_file(), &files);
-        write_files(
-            results.into_iter().map(|res| (res.name, res.content)),
-            output_dir,
-            out,
-        );
-    }
-
-    fn write_files<W: Write>(
-        results: impl Iterator<Item = (String, Vec<u8>)>,
-        output_dir: &Path,
-        mut out: W,
-    ) {
-        if !output_dir.exists() {
-            fs::create_dir_all(output_dir).unwrap();
-        }
-
-        for (name, content) in results {
-            let out_file = output_dir.join(&name);
-            let mut f = File::create(&out_file).unwrap();
-            f.write_all(&content).unwrap();
-            let (module_name, _) = name.split_at(name.len() - 3); // ".rs".len() == 3
-            writeln!(out, "pub mod {};", module_name).unwrap();
-        }
+        protobuf_build::generate_files(&["proto".to_owned()], &files, &out_dir);
     }
 }

proto/build.sh

@@ -11,11 +11,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod testing {
+    include!(concat!(env!("OUT_DIR"), "/testing/mod.rs"));
+
+    #[cfg(feature = "prost-codec")]
+    pub use self::grpc::testing::*;
+}
+
+pub mod example {
+    include!(concat!(env!("OUT_DIR"), "/example/mod.rs"));
+}
+
+pub mod health {
+    pub mod v1 {
+        include!(concat!(env!("OUT_DIR"), "/health/mod.rs"));
+
+        #[cfg(feature = "prost-codec")]
+        pub use self::grpc::health::v1::*;
+    }
+}
+
 #[cfg(feature = "prost-codec")]
 #[allow(clippy::large_enum_variant)]
-pub mod testing {
-    include!(concat!(env!("OUT_DIR"), "/grpc.testing.rs"));
-    include!(concat!(env!("OUT_DIR"), "/wrapper_grpc.testing.rs"));
+pub mod help {
+
+    use super::testing::*;
 
     // Wrapper functions for oneof fields.
     impl ClientArgs {
@@ -114,49 +134,4 @@ pub mod testing {
     }
 }
 
-#[cfg(feature = "prost-codec")]
-pub mod example {
-    pub mod helloworld {
-        include!(concat!(env!("OUT_DIR"), "/helloworld.rs"));
-        include!(concat!(env!("OUT_DIR"), "/wrapper_helloworld.rs"));
-    }
-    pub mod route_guide {
-        include!(concat!(env!("OUT_DIR"), "/routeguide.rs"));
-        include!(concat!(env!("OUT_DIR"), "/wrapper_routeguide.rs"));
-    }
-}
-
-#[cfg(feature = "prost-codec")]
-pub mod health {
-    pub mod v1 {
-        pub mod health {
-            include!(concat!(env!("OUT_DIR"), "/grpc.health.v1.rs"));
-            include!(concat!(env!("OUT_DIR"), "/wrapper_grpc.health.v1.rs"));
-        }
-    }
-}
-
-#[cfg(feature = "protobuf-codec")]
-#[allow(bare_trait_objects)]
-#[allow(renamed_and_removed_lints)]
-pub mod testing {
-    include!(concat!(env!("OUT_DIR"), "/testing/mod.rs"));
-}
-
-#[cfg(feature = "protobuf-codec")]
-#[allow(bare_trait_objects)]
-#[allow(renamed_and_removed_lints)]
-pub mod example {
-    include!(concat!(env!("OUT_DIR"), "/example/mod.rs"));
-}
-
-#[cfg(feature = "protobuf-codec")]
-pub mod health {
-    #[allow(bare_trait_objects)]
-    #[allow(renamed_and_removed_lints)]
-    pub mod v1 {
-        include!(concat!(env!("OUT_DIR"), "/health/mod.rs"));
-    }
-}
-
 pub mod util;