Merge pull request #262 from helsing-ai/mara/fs-deps

Local Dependencies

Author: mara-schulke

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "buffrs"
-version = "0.8.1"
+version = "0.9.0"
 edition = "2021"
 description = "Modern protobuf package management"
 authors = [

Cargo.toml

@@ -9,6 +9,7 @@ Buffrs to develop and consume protocol buffer packages.
 * [Dependencies](./dependencies.md)
 * [Creating a Package](./creating-a-package.md)
 * [Consuming Packages](./consuming-packages.md)
+* [Local Dependencies](./local-dependencies.md)
 * [Import System](./import-system.md)
 * [Project Layout](./project-layout.md)
 * [Manifest vs Lockfile](./manifest-vs-lockfile.md)

docs/src/guide/index.md

@@ -0,0 +1,96 @@
+## Local Dependencies
+
+When working on larger projects or in monorepo projects setups you may find
+yourself in the situation to consume a locally defined buffrs project.
+
+Imagine the following project setup:
+
+```
+mono
+├── build.rs
+├── Cargo.toml
+├── Proto.toml
+├── Proto.toml
+├── proto
+|   └── mono.proto
+└── src
+    └── main.rs
+```
+
+In this scenario the buffrs project for `mono-api` and the cargo project for
+the `mono-server` are setup in the very same directory, which is totally fine
+as long as this server does not require other buffrs api packages to be
+compiled!
+
+#### Problem
+
+Adding a dependency on other (unrelated api packages to `mono-api`) is
+complicated in the above scenario because its not clear to buffrs whether you
+are trying to reuse third party api definitions for your api or just wanting to
+install protos for compilation.
+
+Hence buffrs will throw you an error if you try to publish an api package with
+dependencies on other apis!
+
+```
+Error:   × failed to publish `mono-api` to `http://<registry-uri>/<repository>`
+  ╰─▶ depending on API packages is not allowed
+```
+
+#### Solution
+
+Gladly buffrs offers a builtin solution for this! You can separate the
+`mono-api` buffrs package (used to publish your api) from the `mono-server`
+buffrs projects (used to install protos for compiling the server).
+
+A monorepo setup here could look like this:
+
+```
+mono
+├── mono-api
+|   ├── Proto.toml
+|   └── proto
+|       └── mono.proto
+└── mono-server
+    ├── build.rs
+    ├── Cargo.toml
+    ├── Proto.toml
+    ├── proto
+    |   └── vendor
+    └── src
+        └── main.rs
+```
+
+Where `mono/mono-api/Proto.toml` has this content:
+
+```
+edition = "0.9"
+
+[package]
+type = "api"
+name = "mono-api"
+version = "0.1.0"
+```
+
+And `mono/mono-server/Proto.toml` has this content:
+
+```
+edition = "0.9"
+
+[dependencies]
+mono-api = { path = "../api" }
+third-party-api = { version = "=1.0.0", repository = "some-repo", registry = "http://..." }
+```
+
+This enables you to:
+
+- Independently publish `mono-api` using `buffrs publish` / `buffrs package`
+- Independently declare dependencies for `mono-server`
+
+#### Caveats
+
+Please note that projects containing any local dependencies can not be
+published anymore. The ability to declare local filesystem dependencies is
+mainly useful for the above scenario where you want to install buffrs packages
+for your server from different locations on the filesystem (monorepo, git
+submodules etc).

docs/src/guide/local-dependencies.md

@@ -19,7 +19,7 @@ use crate::{
     manifest::{Dependency, Manifest, PackageManifest, MANIFEST_FILE},
     package::{PackageName, PackageStore, PackageType},
     registry::{Artifactory, RegistryUri},
-    resolver::DependencyGraph,
+    resolver::{DependencyGraph, ResolvedDependency},
 };
 
 use async_recursion::async_recursion;
@@ -390,26 +390,31 @@ pub async fn install() -> miette::Result<()> {
             "unexpected error: missing dependency in dependency graph"
         ))?;
 
-        store.unpack(&resolved.package).await.wrap_err(miette!(
+        store.unpack(resolved.package()).await.wrap_err(miette!(
             "failed to unpack package {}",
-            &resolved.package.name()
+            &resolved.package().name()
         ))?;
 
         tracing::info!(
             "{} installed {}@{}",
             if prefix.is_empty() { "::" } else { &prefix },
             name,
-            resolved.package.version()
+            resolved.package().version()
         );
 
-        locked.push(resolved.package.lock(
-            resolved.registry.clone(),
-            resolved.repository.clone(),
-            resolved.dependants.len(),
-        ));
+        if let ResolvedDependency::Remote {
+            package,
+            registry,
+            repository,
+            dependants,
+            ..
+        } = &resolved
+        {
+            locked.push(package.lock(registry.clone(), repository.clone(), dependants.len()));
+        }
 
-        for (index, dependency) in resolved.depends_on.iter().enumerate() {
-            let tree_char = if index + 1 == resolved.depends_on.len() {
+        for (index, dependency) in resolved.depends_on().iter().enumerate() {
+            let tree_char = if index + 1 == resolved.depends_on().len() {
                 '┗'
             } else {
                 '┣'

src/command.rs

@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
     fmt::{self, Display},
-    path::Path,
+    path::{Path, PathBuf},
     str::FromStr,
 };
 use tokio::fs;
@@ -46,6 +46,8 @@ pub enum Edition {
     /// at any time. Users are responsible for consulting documentation and
     /// help channels if errors occur.
     Canary,
+    /// The canary edition used by buffrs 0.8.x
+    Canary08,
     /// The canary edition used by buffrs 0.7.x
     Canary07,
     /// Unknown edition of manifests
@@ -66,6 +68,7 @@ impl From<&str> for Edition {
     fn from(value: &str) -> Self {
         match value {
             self::CANARY_EDITION => Self::Canary,
+            "0.8" => Self::Canary08,
             "0.7" => Self::Canary07,
             _ => Self::Unknown,
         }
@@ -76,6 +79,7 @@ impl From<Edition> for &'static str {
     fn from(value: Edition) -> Self {
         match value {
             Edition::Canary => CANARY_EDITION,
+            Edition::Canary08 => "0.8",
             Edition::Canary07 => "0.7",
             Edition::Unknown => "unknown",
         }
@@ -206,7 +210,7 @@ mod deserializer {
                     };
 
                     match Edition::from(edition.as_str()) {
-                        Edition::Canary | Edition::Canary07 => Ok(RawManifest::Canary {
+                        Edition::Canary | Edition::Canary08 | Edition::Canary07 => Ok(RawManifest::Canary {
                             package,
                             dependencies,
                         }),
@@ -231,7 +235,7 @@ impl From<Manifest> for RawManifest {
             .collect();
 
         match manifest.edition {
-            Edition::Canary | Edition::Canary07 => RawManifest::Canary {
+            Edition::Canary | Edition::Canary08 | Edition::Canary07 => RawManifest::Canary {
                 package: manifest.package,
                 dependencies,
             },
@@ -408,47 +412,92 @@ impl Dependency {
     ) -> Self {
         Self {
             package,
-            manifest: DependencyManifest {
+            manifest: RemoteDependencyManifest {
                 repository,
                 version,
                 registry,
-            },
+            }
+            .into(),
         }
     }
 
     /// Creates a copy of this dependency with a pinned version
     pub fn with_version(&self, version: &Version) -> Dependency {
         let mut dependency = self.clone();
-        dependency.manifest.version = VersionReq {
-            comparators: vec![semver::Comparator {
-                op: semver::Op::Exact,
-                major: version.major,
-                minor: Some(version.minor),
-                patch: Some(version.patch),
-                pre: version.pre.clone(),
-            }],
-        };
+
+        if let DependencyManifest::Remote(ref mut manifest) = dependency.manifest {
+            manifest.version = VersionReq {
+                comparators: vec![semver::Comparator {
+                    op: semver::Op::Exact,
+                    major: version.major,
+                    minor: Some(version.minor),
+                    patch: Some(version.patch),
+                    pre: version.pre.clone(),
+                }],
+            };
+        }
+
         dependency
     }
 }
 
 impl Display for Dependency {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "{}/{}@{}",
-            self.manifest.repository, self.package, self.manifest.version
-        )
+        match &self.manifest {
+            DependencyManifest::Remote(manifest) => write!(
+                f,
+                "{}/{}@{}",
+                manifest.repository, self.package, manifest.version
+            ),
+            DependencyManifest::Local(manifest) => {
+                write!(f, "{}@{}", self.package, manifest.path.display())
+            }
+        }
     }
 }
 
 /// Manifest format for dependencies
 #[derive(Debug, Clone, Hash, Serialize, Deserialize, PartialEq, Eq)]
-pub struct DependencyManifest {
+#[serde(untagged)]
+pub enum DependencyManifest {
+    /// A remote dependency from artifactory
+    Remote(RemoteDependencyManifest),
+    /// A local dependency located on the filesystem
+    Local(LocalDependencyManifest),
+}
+
+impl DependencyManifest {
+    pub(crate) fn is_local(&self) -> bool {
+        matches!(self, DependencyManifest::Local(_))
+    }
+}
+
+/// Manifest format for dependencies
+#[derive(Debug, Clone, Hash, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RemoteDependencyManifest {
     /// Version requirement in the buffrs format, currently only supports pinning
     pub version: VersionReq,
     /// Artifactory repository to pull dependency from
     pub repository: String,
     /// Artifactory registry to pull from
     pub registry: RegistryUri,
 }
+
+impl From<RemoteDependencyManifest> for DependencyManifest {
+    fn from(value: RemoteDependencyManifest) -> Self {
+        Self::Remote(value)
+    }
+}
+
+/// Manifest format for local filesystem dependencies
+#[derive(Debug, Clone, Hash, Serialize, Deserialize, PartialEq, Eq)]
+pub struct LocalDependencyManifest {
+    /// Path to local buffrs package
+    pub path: PathBuf,
+}
+
+impl From<LocalDependencyManifest> for DependencyManifest {
+    fn from(value: LocalDependencyManifest) -> Self {
+        Self::Local(value)
+    }
+}

src/manifest.rs

@@ -124,9 +124,10 @@ impl PackageStore {
     pub async fn resolve(&self, package: &PackageName) -> miette::Result<Manifest> {
         let manifest = self.locate(package).join(MANIFEST_FILE);
 
-        let manifest = Manifest::try_read_from(manifest)
-            .await?
-            .ok_or(miette!("the package store is corrupted"))?;
+        let manifest = Manifest::try_read_from(&manifest).await?.ok_or(miette!(
+            "the package store is corrupted: `{}` is not present",
+            manifest.display()
+        ))?;
 
         Ok(manifest)
     }