Remove blobs as a physical file

crates/spfs-cli/main/Cargo.toml

@@ -69,4 +69,5 @@ version = "0.51"
 
 [dev-dependencies]
 rstest = { workspace = true }
+spfs = { workspace = true, features = ["test-fixtures"] }
 tempfile = { workspace = true }

crates/spfs-cli/main/src/cmd_check.rs

@@ -90,29 +90,29 @@ impl CmdCheck {
         drop(checker); // clean up progress bars
         let spfs::check::CheckSummary {
             missing_tags,
-            checked_tags,
+            valid_tags,
             missing_objects,
             repaired_objects,
-            checked_objects,
+            valid_objects,
             missing_payloads,
             repaired_payloads,
-            checked_payloads,
-            checked_payload_bytes,
+            valid_payloads,
+            valid_payload_bytes,
         } = summary;
         let missing_objects = missing_objects.len();
         let missing_payloads = missing_payloads.len();
 
         println!("{} after {duration:.0?}:", "Finished".bold());
         let missing = "missing".red().italic();
         let repaired = "repaired".cyan().italic();
-        println!("{checked_tags:>12} tags visited     ({missing_tags} {missing})");
+        println!("{valid_tags:>12} tags visited     ({missing_tags} {missing})");
         println!(
-            "{checked_objects:>12} objects visited  ({missing_objects} {missing}, {repaired_objects} {repaired})",
+            "{valid_objects:>12} objects visited  ({missing_objects} {missing}, {repaired_objects} {repaired})",
         );
         println!(
-            "{checked_payloads:>12} payloads visited ({missing_payloads} {missing}, {repaired_payloads} {repaired})",
+            "{valid_payloads:>12} payloads visited ({missing_payloads} {missing}, {repaired_payloads} {repaired})",
         );
-        let human_bytes = match NumberPrefix::binary(checked_payload_bytes as f64) {
+        let human_bytes = match NumberPrefix::binary(valid_payload_bytes as f64) {
             NumberPrefix::Standalone(amt) => format!("{amt} bytes"),
             NumberPrefix::Prefixed(p, amt) => format!("{amt:.2} {}B", p.symbol()),
         };

crates/spfs-cli/main/src/cmd_info.rs

@@ -13,9 +13,16 @@ use spfs::find_path::ObjectPathEntry;
 use spfs::graph::Annotation;
 use spfs::io::{self, DigestFormat, Pluralize};
 use spfs::prelude::*;
-use spfs::{self};
+use spfs::{
+    Error,
+    {self},
+};
 use spfs_cli_common as cli;
 
+#[cfg(test)]
+#[path = "./cmd_info_test.rs"]
+mod cmd_info_test;
+
 /// Display information about the current environment, or specific items
 #[derive(Debug, Args)]
 pub struct CmdInfo {
@@ -69,9 +76,21 @@ impl CmdInfo {
                     self.pretty_print_file(&reference, &repo, self.logging.verbose as usize)
                         .await?;
                 } else {
-                    let item = repo.read_ref(reference.as_str()).await?;
-                    self.pretty_print_ref(item, &repo, self.logging.verbose as usize)
-                        .await?;
+                    match repo.read_ref(reference.as_str()).await {
+                        Ok(item) => {
+                            self.pretty_print_ref(item, &repo, self.logging.verbose as usize)
+                                .await?;
+                        }
+                        Err(Error::UnknownObject(_)) => {
+                            let digest = repo.resolve_ref(reference.as_str()).await?;
+                            let payload_size = repo.payload_size(digest).await?;
+                            self.pretty_print_payload(digest, payload_size, &repo)
+                                .await?;
+                        }
+                        Err(err) => {
+                            return Err(err.into());
+                        }
+                    }
                 }
                 if !self.to_process.is_empty() {
                     println!();
@@ -216,6 +235,31 @@ impl CmdInfo {
         Ok(())
     }
 
+    /// Display the spfs payload information
+    async fn pretty_print_payload(
+        &mut self,
+        digest: spfs::encoding::Digest,
+        payload_size: u64,
+        repo: &spfs::storage::RepositoryHandle,
+    ) -> Result<()> {
+        println!(
+            "{}:\n {}:",
+            self.format_digest(digest, repo).await?,
+            "blob".green()
+        );
+        println!(
+            " {} {}",
+            "digest:".bright_blue(),
+            self.format_digest(digest, repo).await?
+        );
+        println!(
+            " {} {}",
+            "size:".bright_blue(),
+            spfs::io::format_size(payload_size)
+        );
+        Ok(())
+    }
+
     /// Display the status of the current runtime.
     async fn print_global_info(&self, repo: &spfs::storage::RepositoryHandle) -> Result<()> {
         let runtime = spfs::active_runtime().await?;

crates/spfs-cli/main/src/cmd_info_test.rs

@@ -0,0 +1,55 @@
+// Copyright (c) Contributors to the SPK project.
+// SPDX-License-Identifier: Apache-2.0
+// https://github.com/spkenv/spk
+
+use clap::Parser;
+use rstest::rstest;
+use spfs::Config;
+use spfs::fixtures::*;
+use spfs::prelude::*;
+
+use super::CmdInfo;
+
+#[derive(Parser)]
+struct Opt {
+    #[command(flatten)]
+    info: CmdInfo,
+}
+
+#[rstest]
+#[case::fs(tmprepo("fs"))]
+#[tokio::test]
+async fn info_on_payload(
+    #[case]
+    #[future]
+    repo: TempRepo,
+    #[values(true, false)] use_partial_digest: bool,
+) {
+    let repo = repo.await;
+
+    let manifest = generate_tree(&repo).await.to_graph_manifest();
+    let file = manifest
+        .iter_entries()
+        .find(|entry| entry.is_regular_file())
+        .expect("at least one regular file");
+
+    let digest_arg = if use_partial_digest {
+        file.object()
+            .to_string()
+            .chars()
+            .take(12)
+            .collect::<String>()
+    } else {
+        file.object().to_string()
+    };
+
+    let mut opt =
+        Opt::try_parse_from(["info", "-r", &repo.address().to_string(), &digest_arg]).unwrap();
+    let config = Config::default();
+    let code = opt
+        .info
+        .run(&config)
+        .await
+        .expect("`spfs info` on a file digest is successful");
+    assert_eq!(code, 0, "`spfs info` on a file digest returns exit code 0");
+}

crates/spfs-cli/main/src/cmd_pull.rs

@@ -4,6 +4,7 @@
 
 use clap::Args;
 use miette::Result;
+use spfs::sync::reporter::Summary;
 use spfs_cli_common as cli;
 
 /// Pull one or more objects to the local repository

crates/spfs-cli/main/src/cmd_push.rs

@@ -4,6 +4,7 @@
 
 use clap::Args;
 use miette::Result;
+use spfs::sync::reporter::Summary;
 use spfs_cli_common as cli;
 
 /// Push one or more objects to a remote repository

crates/spfs-cli/main/src/cmd_read.rs

@@ -8,6 +8,10 @@ use spfs::Error;
 use spfs::prelude::*;
 use spfs_cli_common as cli;
 
+#[cfg(test)]
+#[path = "./cmd_read_test.rs"]
+mod cmd_read_test;
+
 /// Output the contents of a blob to stdout
 #[derive(Debug, Args)]
 #[clap(visible_aliases = &["read-file", "cat", "cat-file"])]
@@ -26,24 +30,27 @@ pub struct CmdRead {
 
 impl CmdRead {
     pub async fn run(&mut self, config: &spfs::Config) -> Result<i32> {
+        use spfs::graph::object::Enum;
+
         let repo =
             spfs::config::open_repository_from_string(config, self.repos.remote.as_ref()).await?;
 
         #[cfg(feature = "sentry")]
         tracing::info!(target: "sentry", "using repo: {}", repo.address());
 
-        let item = repo.read_ref(&self.reference.to_string()).await?;
-        use spfs::graph::object::Enum;
-        let blob = match item.to_enum() {
-            Enum::Blob(blob) => blob,
-            _ => {
+        let digest = match repo.read_ref(&self.reference.to_string()).await.map(|fb| {
+            let fb_enum = fb.to_enum();
+            (fb, fb_enum)
+        }) {
+            Ok((_, Enum::Blob(blob))) => *blob.digest(),
+            Ok((obj, _)) => {
                 let path = match &self.path {
                     None => {
-                        miette::bail!("PATH must be given to read from {:?}", item.kind());
+                        miette::bail!("PATH must be given to read from {:?}", obj.kind());
                     }
                     Some(p) => p.strip_prefix("/spfs").unwrap_or(p).to_string(),
                 };
-                let manifest = spfs::compute_object_manifest(item, &repo).await?;
+                let manifest = spfs::compute_object_manifest(obj, &repo).await?;
                 let entry = match manifest.get_path(&path) {
                     Some(e) => e,
                     None => {
@@ -55,11 +62,15 @@ impl CmdRead {
                     tracing::error!("path is a directory or masked file: {path}");
                     return Ok(1);
                 }
-                repo.read_blob(entry.object).await?
+                entry.object
+            }
+            Err(Error::UnknownObject(digest)) => digest,
+            Err(err) => {
+                return Err(err.into());
             }
         };
 
-        let (mut payload, filename) = repo.open_payload(*blob.digest()).await?;
+        let (mut payload, filename) = repo.open_payload(digest).await?;
         tokio::io::copy(&mut payload, &mut tokio::io::stdout())
             .await
             .map_err(|err| Error::StorageReadError("copy of payload to stdout", filename, err))?;

crates/spfs-cli/main/src/cmd_read_test.rs

@@ -0,0 +1,55 @@
+// Copyright (c) Contributors to the SPK project.
+// SPDX-License-Identifier: Apache-2.0
+// https://github.com/spkenv/spk
+
+use clap::Parser;
+use rstest::rstest;
+use spfs::Config;
+use spfs::fixtures::*;
+use spfs::prelude::*;
+
+use super::CmdRead;
+
+#[derive(Parser)]
+struct Opt {
+    #[command(flatten)]
+    read: CmdRead,
+}
+
+#[rstest]
+#[case::fs(tmprepo("fs"))]
+#[tokio::test]
+async fn read_on_payload(
+    #[case]
+    #[future]
+    repo: TempRepo,
+    #[values(true, false)] use_partial_digest: bool,
+) {
+    let repo = repo.await;
+
+    let manifest = generate_tree(&repo).await.to_graph_manifest();
+    let file = manifest
+        .iter_entries()
+        .find(|entry| entry.is_regular_file())
+        .expect("at least one regular file");
+
+    let digest_arg = if use_partial_digest {
+        file.object()
+            .to_string()
+            .chars()
+            .take(12)
+            .collect::<String>()
+    } else {
+        file.object().to_string()
+    };
+
+    let mut opt =
+        Opt::try_parse_from(["read", "-r", &repo.address().to_string(), &digest_arg]).unwrap();
+    let config = Config::default();
+    let code = opt
+        .read
+        .run(&config)
+        .await
+        .expect("`spfs read` on a file digest is successful");
+    assert_eq!(code, 0, "`spfs read` on a file digest returns exit code 0");
+}

crates/spfs-cli/main/src/cmd_write.rs

@@ -55,7 +55,7 @@ impl CmdWrite {
             None => Box::pin(tokio::io::BufReader::new(tokio::io::stdin())),
         };
 
-        let digest = repo.commit_blob(reader).await?;
+        let digest = repo.commit_payload(reader).await?;
 
         tracing::info!(%digest, "created");
         for tag in self.tags.iter() {

crates/spfs-encoding/src/hash.rs

@@ -259,7 +259,10 @@ impl Decodable for String {
     }
 }
 
-/// The first N bytes of a digest that may still be unambiguous as a reference
+/// The first N bytes of a digest that may still be unambiguous as a reference.
+///
+/// This type can represent a full digest, but it does not require that an item
+/// exists with that digest or what type of item it is.
 #[derive(Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Clone)]
 pub struct PartialDigest(Vec<u8>);
 

crates/spfs/Cargo.toml

@@ -14,6 +14,7 @@ workspace = true
 
 [features]
 default = []
+test-fixtures = ["dep:rstest", "dep:tracing-subscriber"]
 
 # If enabled, will create the "local" repository in a subdirectory
 # of the standard storage root, named "ci/pipeline_${CI_PIPELINE_ID}".
@@ -68,6 +69,8 @@ prost = { workspace = true }
 rand = { workspace = true }
 relative-path = { workspace = true, features = ["serde"] }
 ring = { workspace = true }
+criterion = { version = "0.3", features = ["async_tokio", "html_reports"] }
+rstest = { workspace = true, optional = true }
 semver = "1.0"
 sentry = { workspace = true, optional = true }
 serde = { workspace = true, features = ["derive"] }
@@ -97,6 +100,9 @@ tokio-stream = { version = "0.1", features = ["fs", "net"] }
 tokio-util = { version = "0.7.3", features = ["compat", "io"] }
 tonic = { workspace = true }
 tracing = { workspace = true }
+tracing-subscriber = { workspace = true, features = [
+    "env-filter",
+], optional = true }
 ulid = { workspace = true }
 unix_mode = "0.1.3"
 url = { version = "2.2", features = ["serde"] }