diff --git a/Makefile b/Makefile index 0765ae2b6e43..2d06018d0d71 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ COMPONENTS += agent COMPONENTS += dragonball COMPONENTS += runtime COMPONENTS += runtime-rs +COMPONENTS += tarfs +COMPONENTS += tardev-snapshotter +COMPONENTS += overlay +COMPONENTS += utarfs # List of available tools TOOLS = diff --git a/src/overlay/Cargo.lock b/src/overlay/Cargo.lock new file mode 100644 index 000000000000..b987e65bc3ba --- /dev/null +++ b/src/overlay/Cargo.lock @@ -0,0 +1,381 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401a4694d2bf92537b6867d94de48c4842089645fdcdf6c71865b175d836e9c2" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72394f3339a76daf211e57d4bcb374410f3965dcc606dd0e03738c7888766980" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "kata-overlay" +version = "0.1.0" +dependencies = [ + "base64", + "clap", + "tempfile", +] + +[[package]] +name = "libc" +version = "0.2.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10b47e4921acc65b7d824cd92bc7b67a26382d8f4eadf3da65cb50aee6e6f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/src/overlay/Cargo.toml b/src/overlay/Cargo.toml new file mode 100644 index 000000000000..f610c1cfed3d --- /dev/null +++ b/src/overlay/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "kata-overlay" +version = "0.1.0" +edition = "2021" + +[dependencies] +clap = { version = "4.3.2", features = ["derive"] } +base64 = "0.21.2" +tempfile = "3.3.0" diff --git a/src/overlay/src/main.rs b/src/overlay/src/main.rs new file mode 100644 index 000000000000..c4605c0e84f0 --- /dev/null +++ b/src/overlay/src/main.rs @@ -0,0 +1,189 @@ +use base64::{engine::general_purpose, Engine as _}; +use clap::Parser; +use std::io::{self, Error, ErrorKind}; +use std::path::{Path, PathBuf}; +use std::{env::set_current_dir, process::Command}; + +#[derive(Parser, Debug)] +struct Args { + /// The source tarfs file. + source: String, + + /// The directory on which to mount. + directory: String, + + /// The filesystem type. + #[arg(short)] + r#type: Option, + + /// The filesystem options. + #[arg(short, long)] + options: Vec, +} + +const LAYER: &str = "io.katacontainers.fs-opt.layer="; +const LAYER_SRC_PREFIX: &str = "io.katacontainers.fs-opt.layer-src-prefix="; + +struct Layer { + src: PathBuf, + fs: String, + opts: String, +} + +fn parse_layers(args: &Args) -> io::Result> { + let mut layers = Vec::new(); + let mut prefix = Path::new(""); + + for group in &args.options { + for opt in group.split(',') { + if let Some(p) = opt.strip_prefix(LAYER_SRC_PREFIX) { + prefix = Path::new(p); + continue; + } + + let encoded = if let Some(e) = opt.strip_prefix(LAYER) { + e + } else { + continue; + }; + + let decoded = general_purpose::STANDARD + .decode(encoded) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let info = std::str::from_utf8(&decoded) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + + let mut fields = info.split(','); + let src = if let Some(p) = fields.next() { + if !p.is_empty() && p.as_bytes()[0] != b'/' { + prefix.join(Path::new(p)) + } else { + Path::new(p).to_path_buf() + } + } else { + return Err(Error::new( + ErrorKind::InvalidInput, + format!("Missing path from {info}"), + )); + }; + + let fs = if let Some(f) = fields.next() { + f.into() + } else { + return Err(Error::new( + ErrorKind::InvalidInput, + format!("Missing filesystem type from {info}"), + )); + }; + + let fs_opts = fields + .filter(|o| !o.starts_with("io.katacontainers.")) + .fold(String::new(), |a, b| { + if a.is_empty() { + b.into() + } else { + format!("{a},{b}") + } + }); + layers.push(Layer { + src, + fs, + opts: fs_opts, + }); + } + } + + Ok(layers) +} + +struct Unmounter(Vec, tempfile::TempDir); +impl Drop for Unmounter { + fn drop(&mut self) { + for n in &self.0 { + let p = self.1.path().join(n); + match Command::new("umount").arg(&p).status() { + Err(e) => eprintln!("Unable to run umount command: {e}"), + Ok(s) => { + if !s.success() { + eprintln!("Unable to unmount {:?}: {s}", p); + } + } + } + } + } +} + +fn main() -> io::Result<()> { + let args = &Args::parse(); + let layers = parse_layers(args)?; + let mut unmounter = Unmounter(Vec::new(), tempfile::tempdir()?); + + // Mount all layers. + // + // We use the `mount` command instead of a syscall because we want leverage the additional work + // that `mount` does, for example, using helper binaries to mount. + for (i, layer) in layers.iter().enumerate() { + let n = i.to_string(); + let p = unmounter.1.path().join(&n); + std::fs::create_dir_all(&p)?; + println!("Mounting {:?} to {:?}", layer.src, p); + + let status = Command::new("mount") + .arg(&layer.src) + .arg(&p) + .arg("-t") + .arg(&layer.fs) + .arg("-o") + .arg(&layer.opts) + .status()?; + if !status.success() { + return Err(Error::new( + ErrorKind::Other, + format!("failed to mount {:?}: {status}", &layer.src), + )); + } + + unmounter.0.push(n); + } + + // Mont the overlay if we have multiple layers, otherwise do a bind-mount. + let mp = std::fs::canonicalize(&args.directory)?; + if unmounter.0.len() == 1 { + let p = unmounter.1.path().join(unmounter.0.first().unwrap()); + let status = Command::new("mount") + .arg(&p) + .arg(&mp) + .args(&["-t", "bind", "-o", "bind"]) + .status()?; + if !status.success() { + return Err(Error::new( + ErrorKind::Other, + format!("failed to bind mount: {status}"), + )); + } + } else { + let saved = std::env::current_dir()?; + set_current_dir(unmounter.1.path())?; + + let status = Command::new("mount") + .arg("none") + .arg(&mp) + .args(&[ + "-t", + "overlay", + "-o", + &format!("lowerdir={}", unmounter.0.join(":")), + ]) + .status()?; + if !status.success() { + return Err(Error::new( + ErrorKind::Other, + format!("failed to mount overlay: {status}"), + )); + } + + set_current_dir(saved)?; + } + + Ok(()) +} diff --git a/src/tardev-snapshotter/Cargo.lock b/src/tardev-snapshotter/Cargo.lock new file mode 100644 index 000000000000..20b9005a454e --- /dev/null +++ b/src/tardev-snapshotter/Cargo.lock @@ -0,0 +1,1589 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + +[[package]] +name = "containerd-client" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbd55a5b186b60273ed7361d18d566ede8d66db962bafd702dd4db7fd30f23f" +dependencies = [ + "prost", + "prost-types", + "tokio", + "tonic", + "tonic-build", + "tower", +] + +[[package]] +name = "containerd-snapshots" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d52d5b3981fc915b0ffb2a8a6dcf11ca2b75819c6a5b7532334e74c96f3bd8f" +dependencies = [ + "async-stream", + "futures", + "pin-utils", + "prost", + "prost-types", + "serde", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", +] + +[[package]] +name = "cpufeatures" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "filetime" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.16", + "windows-sys 0.48.0", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "libz-ng-sys", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "libz-ng-sys" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468756f34903b582fe7154dc1ffdebd89d0562c4a43b53c621bb0f1b1043ccb" +dependencies = [ + "cmake", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "petgraph" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "proc-macro2" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tardev-snapshotter" +version = "0.1.0" +dependencies = [ + "async-stream", + "base64", + "containerd-client", + "containerd-snapshots", + "env_logger", + "flate2", + "futures", + "log", + "serde_json", + "sha2", + "tarindex", + "tempfile", + "tokio", + "tokio-stream", + "tonic", + "verity", +] + +[[package]] +name = "tarfs-defs" +version = "0.1.0" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "tarindex" +version = "0.1.0" +dependencies = [ + "tar", + "tarfs-defs", + "zerocopy", +] + +[[package]] +name = "tempfile" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tokio" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "verity" +version = "0.1.0" +dependencies = [ + "generic-array", + "sha2", + "zerocopy", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + +[[package]] +name = "zerocopy" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] diff --git a/src/tardev-snapshotter/Cargo.toml b/src/tardev-snapshotter/Cargo.toml new file mode 100644 index 000000000000..3319e41ceb9a --- /dev/null +++ b/src/tardev-snapshotter/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "tardev-snapshotter" +version = "0.1.0" +edition = "2021" + +[dependencies] +async-stream = "0.3.2" +futures = "0.3.17" +serde_json = "1.0" +tonic = "0.9.2" +tokio = {version = "1.26", features = ["full"]} +tokio-stream = "0.1.8" +containerd-snapshots = "0.3.0" +sha2 = "0.10.6" +tarindex = { path = "./tarindex" } +verity = { path = "./verity" } +log = "0.4.17" +env_logger = "0.10.0" +tempfile = "3.3.0" +flate2 = { version = "1.0.26", features = ["zlib-ng"], default-features = false } +base64 = "0.21.2" +containerd-client = "0.4.0" diff --git a/src/tardev-snapshotter/Makefile b/src/tardev-snapshotter/Makefile new file mode 100644 index 000000000000..c169b1cac79b --- /dev/null +++ b/src/tardev-snapshotter/Makefile @@ -0,0 +1,5 @@ +all: + RUSTC_BOOTSTRAP=1 cargo build --release + +clean: + cargo clean diff --git a/src/tardev-snapshotter/src/main.rs b/src/tardev-snapshotter/src/main.rs new file mode 100644 index 000000000000..8480fdd78b3c --- /dev/null +++ b/src/tardev-snapshotter/src/main.rs @@ -0,0 +1,83 @@ +#![feature(impl_trait_in_assoc_type)] +#![feature(type_alias_impl_trait)] + +use containerd_snapshots::server; +use log::{error, info, warn}; +use snapshotter::TarDevSnapshotter; +use std::{env, io, path::Path, process, sync::Arc}; +use tokio::net::UnixListener; +use tonic::transport::Server; + +mod snapshotter; + +#[tokio::main] +pub async fn main() { + env_logger::init(); + + let argv: Vec = env::args().collect(); + if argv.len() != 3 && argv.len() != 4 { + error!( + "Usage: {} [containerd-socket]", + argv[0] + ); + process::exit(1); + } + + let containerd_socket = if argv.len() >= 4 { + &argv[3] + } else { + "/var/run/containerd/containerd.sock" + }; + + // TODO: Check that the directory is accessible. + + let incoming = { + let uds = match bind(&argv[2]) { + Ok(l) => l, + Err(e) => { + error!("UnixListener::bind failed: {e:?}"); + process::exit(1); + } + }; + + async_stream::stream! { + loop { + let item = uds.accept().await.map(|p| p.0); + yield item; + } + } + }; + + info!("Snapshotter started"); + if let Err(e) = Server::builder() + .add_service(server(Arc::new(TarDevSnapshotter::new( + Path::new(&argv[1]), + containerd_socket.to_string(), + )))) + .serve_with_incoming(incoming) + .await + { + error!("serve_with_incoming failed: {:?}", e); + process::exit(1); + } +} + +fn bind(addr: &str) -> io::Result { + // Try to bind. Return on success or failure other than "address in use". + match UnixListener::bind(addr) { + Ok(l) => return Ok(l), + Err(e) => { + if e.kind() != io::ErrorKind::AddrInUse { + return Err(e); + } + } + } + + // Try to remove the existing socket and bind again. + warn!( + "Listen address ({}) already exists, trying to remove it", + addr + ); + let _ = std::fs::remove_file(addr); + UnixListener::bind(addr) +} diff --git a/src/tardev-snapshotter/src/snapshotter.rs b/src/tardev-snapshotter/src/snapshotter.rs new file mode 100644 index 000000000000..40f0feb9b049 --- /dev/null +++ b/src/tardev-snapshotter/src/snapshotter.rs @@ -0,0 +1,498 @@ +use base64::prelude::{Engine, BASE64_STANDARD}; +use containerd_client::{services::v1::ReadContentRequest, tonic::Request, with_namespace, Client}; +use containerd_snapshots::{api, Info, Kind, Snapshotter, Usage}; +use log::{debug, info, trace}; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; +use std::{collections::HashMap, fs, fs::OpenOptions, io, io::Seek, os::unix::ffi::OsStrExt}; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; +use tokio::sync::RwLock; +use tonic::Status; + +const ROOT_HASH_LABEL: &str = "io.katacontainers.dm-verity.root-hash"; +const TARGET_LAYER_DIGEST_LABEL: &str = "containerd.io/snapshot/cri.layer-digest"; + +struct Store { + root: PathBuf, +} + +impl Store { + fn new(root: &Path) -> Self { + Self { root: root.into() } + } + + /// Creates the name of the directory that containerd can use to extract a layer into. + fn extract_dir(&self, name: &str) -> PathBuf { + self.root.join("staging").join(name_to_hash(name)) + } + + /// Creates a directory that containerd can use to extract a layer into. + /// + /// It's a temporary directory that will be thrown away by the snapshotter. + fn extract_dir_to_write(&self, name: &str) -> io::Result { + let path = self.extract_dir(name); + fs::create_dir_all(&path)?; + Ok(path) + } + + /// Creates a temporary staging directory for layers. + fn staging_dir(&self) -> io::Result { + let path = self.root.join("staging"); + fs::create_dir_all(&path)?; + tempfile::tempdir_in(path) + } + + /// Creates the snapshot file path from its name. + /// + /// If `write` is `true`, it also ensures that the directory exists. + fn snapshot_path(&self, name: &str, write: bool) -> Result { + let path = self.root.join("snapshots").join(name_to_hash(name)); + if write { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + } + + Ok(path) + } + + /// Creates the layer file path from its name. + fn layer_path(&self, name: &str) -> PathBuf { + self.root.join("layers").join(name_to_hash(name)) + } + + /// Creates the layer file path from its name and ensures that the directory exists. + fn layer_path_to_write(&self, name: &str) -> Result { + let path = self.layer_path(name); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + Ok(path) + } + + /// Reads the information from storage for the given snapshot name. + fn read_snapshot(&self, name: &str) -> Result { + let path = self.snapshot_path(name, false)?; + let file = fs::File::open(path)?; + serde_json::from_reader(file).map_err(|_| Status::unknown("unable to read snapshot")) + } + + /// Writes to storage the given snapshot information. + /// + /// It fails if a snapshot with the given name already exists. + fn write_snapshot( + &mut self, + kind: Kind, + key: String, + parent: String, + labels: HashMap, + ) -> Result<(), Status> { + let info = Info { + kind, + name: key, + parent, + labels, + ..Info::default() + }; + let name = self.snapshot_path(&info.name, true)?; + // TODO: How to specify the file mode (e.g., 0600)? + let file = OpenOptions::new().write(true).create_new(true).open(name)?; + serde_json::to_writer_pretty(file, &info) + .map_err(|_| Status::internal("unable to write snapshot")) + } + + /// Creates a new snapshot for use. + /// + /// It checks that the parent chain exists and that all ancestors are committed and consist of + /// layers before writing the new snapshot. + fn prepare_snapshot_for_use( + &mut self, + kind: Kind, + key: String, + parent: String, + labels: HashMap, + ) -> Result, Status> { + let mounts = self.mounts_from_snapshot(&parent)?; + self.write_snapshot(kind, key, parent, labels)?; + Ok(mounts) + } + + fn mounts_from_snapshot(&self, parent: &str) -> Result, Status> { + const PREFIX: &str = "io.katacontainers.fs-opt"; + + // Get chain of layers. + let mut next_parent = Some(parent.to_string()); + let mut layers = Vec::new(); + let mut opts = vec![format!( + "{PREFIX}.layer-src-prefix={}", + self.root.join("layers").to_string_lossy() + )]; + while let Some(p) = next_parent { + let info = self.read_snapshot(&p)?; + if info.kind != Kind::Committed { + return Err(Status::failed_precondition( + "parent snapshot is not committed", + )); + } + + let root_hash = if let Some(rh) = info.labels.get(ROOT_HASH_LABEL) { + rh + } else { + return Err(Status::failed_precondition( + "parent snapshot has no root hash stored", + )); + }; + + let name = name_to_hash(&p); + let layer_info = format!( + "{name},tar,ro,{PREFIX}.block_device=file,{PREFIX}.is-layer,{PREFIX}.root-hash={root_hash}"); + layers.push(name); + + opts.push(format!( + "{PREFIX}.layer={}", + BASE64_STANDARD.encode(layer_info.as_bytes()) + )); + + next_parent = (!info.parent.is_empty()).then_some(info.parent); + } + + opts.push(format!("{PREFIX}.overlay-rw")); + opts.push(format!("lowerdir={}", layers.join(":"))); + + Ok(vec![api::types::Mount { + r#type: "fuse3.kata-overlay".into(), + source: "/".into(), + target: String::new(), + options: opts, + }]) + } +} + +/// The snapshotter that creates tar devices. +pub(crate) struct TarDevSnapshotter { + store: RwLock, + containerd_path: String, + containerd_client: RwLock>, +} + +impl TarDevSnapshotter { + /// Creates a new instance of the snapshotter. + /// + /// `root` is the root directory where the snapshotter state is to be stored. + pub(crate) fn new(root: &Path, containerd_path: String) -> Self { + Self { + containerd_path, + store: RwLock::new(Store::new(root)), + containerd_client: RwLock::new(None), + } + } + + async fn prepare_unpack_dir( + &self, + key: String, + parent: String, + labels: HashMap, + ) -> Result, Status> { + let extract_dir; + { + let mut store = self.store.write().await; + extract_dir = store.extract_dir_to_write(&key)?; + store.write_snapshot(Kind::Active, key, parent, labels)?; + } + Ok(vec![api::types::Mount { + r#type: "bind".into(), + source: extract_dir.to_string_lossy().into(), + target: String::new(), + options: vec!["bind".into()], + }]) + } + + async fn get_layer_image(&self, fname: &PathBuf, digest: &str) -> Result<(), Status> { + let mut file = tokio::fs::File::create(fname).await?; + let req = ReadContentRequest { + digest: digest.to_string(), + offset: 0, + size: 0, + }; + let req = with_namespace!(req, "k8s.io"); + + loop { + let guard = self.containerd_client.read().await; + let Some(client) = &*guard else { + drop(guard); + info!("Connecting to containerd at {}", self.containerd_path); + let c = Client::from_path(&self.containerd_path) + .await + .map_err(|_| Status::unknown("unable to connect to containerd"))?; + *self.containerd_client.write().await = Some(c); + continue; + }; + let mut c = client.content(); + let resp = c.read(req).await?; + let mut stream = resp.into_inner(); + while let Some(chunk) = stream.message().await? { + if chunk.offset < 0 { + debug!("Containerd reported a negative offset: {}", chunk.offset); + return Err(Status::invalid_argument("negative offset")); + } + file.seek(io::SeekFrom::Start(chunk.offset as u64)).await?; + file.write_all(&chunk.data).await?; + } + + return Ok(()); + } + } + + /// Creates a new snapshot for an image layer. + /// + /// It downloads, decompresses, and creates the index for the layer before writing the new + /// snapshot. + async fn prepare_image_layer( + &self, + key: String, + parent: String, + mut labels: HashMap, + ) -> Result<(), Status> { + let dir = self.store.read().await.staging_dir()?; + + { + let Some(digest_str) = labels.get(TARGET_LAYER_DIGEST_LABEL) else { + return Err(Status::invalid_argument( + "missing target layer digest label", + )); + }; + + let name = dir.path().join(name_to_hash(&key)); + let mut gzname = name.clone(); + gzname.set_extension("gz"); + trace!("Fetching layer image to {:?}", &gzname); + self.get_layer_image(&gzname, digest_str).await?; + + // TODO: Decompress in stream instead of reopening. + // Decompress data. + trace!("Decompressing {:?} to {:?}", &gzname, &name); + let root_hash = tokio::task::spawn_blocking(move || -> io::Result<_> { + let compressed = fs::File::open(&gzname)?; + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(&name)?; + let mut gz_decoder = flate2::read::GzDecoder::new(compressed); + std::io::copy(&mut gz_decoder, &mut file)?; + + trace!("Appending index to {:?}", &name); + file.rewind()?; + tarindex::append_index(&mut file)?; + + trace!("Appending dm-verity tree to {:?}", &name); + let root_hash = verity::append_tree::(&mut file)?; + + trace!("Root hash for {:?} is {:x}", &name, root_hash); + Ok(root_hash) + }) + .await + .map_err(|_| Status::unknown("error in worker task"))??; + + // Store a label with the root hash so that we can recall it later when mounting. + labels.insert(ROOT_HASH_LABEL.into(), format!("{:x}", root_hash)); + } + + // Move file to its final location and write the snapshot. + { + let from = dir.path().join(name_to_hash(&key)); + let mut store = self.store.write().await; + let to = store.layer_path_to_write(&key)?; + trace!("Renaming from {:?} to {:?}", &from, &to); + tokio::fs::rename(from, to).await?; + store.write_snapshot(Kind::Committed, key, parent, labels)?; + } + + trace!("Layer prepared"); + Ok(()) + } +} + +#[tonic::async_trait] +impl Snapshotter for TarDevSnapshotter { + type Error = Status; + + async fn stat(&self, key: String) -> Result { + trace!("stat({})", key); + self.store.read().await.read_snapshot(&key) + } + + async fn update( + &self, + info: Info, + fieldpaths: Option>, + ) -> Result { + trace!("update({:?}, {:?})", info, fieldpaths); + Err(Status::unimplemented("no support for updating snapshots")) + } + + async fn usage(&self, key: String) -> Result { + trace!("usage({})", key); + let store = self.store.read().await; + + let info = store.read_snapshot(&key)?; + if info.kind != Kind::Committed { + // Only committed snapshots consume storage. + return Ok(Usage { inodes: 0, size: 0 }); + } + + let mut file = tokio::fs::File::open(store.layer_path(&key)).await?; + let len = file.seek(io::SeekFrom::End(0)).await?; + Ok(Usage { + // TODO: Read the index "header" to determine the inode count. + inodes: 1, + size: len as _, + }) + } + + async fn mounts(&self, key: String) -> Result, Self::Error> { + trace!("mounts({})", key); + let store = self.store.read().await; + let info = store.read_snapshot(&key)?; + + if info.kind != Kind::View && info.kind != Kind::Active { + return Err(Status::failed_precondition( + "snapshot is not active nor a view", + )); + } + + if info.labels.get(TARGET_LAYER_DIGEST_LABEL).is_some() { + let extract_dir = store.extract_dir(&key); + Ok(vec![api::types::Mount { + r#type: "bind".into(), + source: extract_dir.to_string_lossy().into(), + target: String::new(), + options: Vec::new(), + }]) + } else { + store.mounts_from_snapshot(&info.parent) + } + } + + async fn prepare( + &self, + key: String, + parent: String, + labels: HashMap, + ) -> Result, Status> { + trace!("prepare({}, {}, {:?})", key, parent, labels); + + // There are two reasons for preparing a snapshot: to build an image and to actually use it + // as a container image. We determine the reason by the presence of the snapshot-ref label. + if labels.get(TARGET_LAYER_DIGEST_LABEL).is_some() { + self.prepare_unpack_dir(key, parent, labels).await + } else { + self.store + .write() + .await + .prepare_snapshot_for_use(Kind::Active, key, parent, labels) + } + } + + async fn view( + &self, + key: String, + parent: String, + labels: HashMap, + ) -> Result, Self::Error> { + trace!("view({}, {}, {:?})", key, parent, labels); + self.store + .write() + .await + .prepare_snapshot_for_use(Kind::View, key, parent, labels) + } + + async fn commit( + &self, + name: String, + key: String, + labels: HashMap, + ) -> Result<(), Self::Error> { + trace!("commit({}, {}, {:?})", name, key, labels); + + let info; + { + let store = self.store.write().await; + info = store.read_snapshot(&key)?; + if info.kind != Kind::Active { + return Err(Status::failed_precondition("snapshot is not active")); + } + } + + if info.labels.get(TARGET_LAYER_DIGEST_LABEL).is_some() { + self.prepare_image_layer(name, info.parent, labels).await + } else { + Err(Status::unimplemented( + "no support for commiting arbitrary snapshots", + )) + } + } + + async fn remove(&self, key: String) -> Result<(), Self::Error> { + trace!("remove({})", key); + let store = self.store.write().await; + + // TODO: Move this to store. + if let Ok(info) = store.read_snapshot(&key) { + match info.kind { + Kind::Committed => { + if info.labels.get(TARGET_LAYER_DIGEST_LABEL).is_some() { + // Try to delete a layer. It's ok if it's not found. + if let Err(e) = fs::remove_file(store.layer_path(&key)) { + if e.kind() != io::ErrorKind::NotFound { + return Err(e.into()); + } + } + } + } + Kind::Active => { + if let Err(e) = tokio::fs::remove_dir_all(store.extract_dir(&key)).await { + if e.kind() != io::ErrorKind::NotFound { + return Err(e.into()); + } + } + } + _ => {} + } + } + + let name = store.snapshot_path(&key, false)?; + fs::remove_file(name)?; + + Ok(()) + } + + type InfoStream = impl tokio_stream::Stream> + Send + 'static; + async fn list(&self, _: String, _: Vec) -> Result { + trace!("walk()"); + let store = self.store.read().await; + let snapshots_dir = store.root.join("snapshots"); + Ok(async_stream::try_stream! { + let mut files = tokio::fs::read_dir(snapshots_dir).await?; + while let Some(p) = files.next_entry().await? { + if let Ok(f) = fs::File::open(p.path()) { + if let Ok(i) = serde_json::from_reader(f) { + yield i; + } + } + } + }) + } +} + +/// Converts the given name to a string representation of its sha256 hash. +fn name_to_hash(name: &str) -> String { + let path = Path::new(name); + let mut hasher = Sha256::new(); + match path.file_name() { + Some(n) => hasher.update(n.as_bytes()), + None => hasher.update(name), + } + format!("{:x}", hasher.finalize()) +} diff --git a/src/tardev-snapshotter/tardev-snapshotter.service b/src/tardev-snapshotter/tardev-snapshotter.service new file mode 100644 index 000000000000..915f3781b6f3 --- /dev/null +++ b/src/tardev-snapshotter/tardev-snapshotter.service @@ -0,0 +1,11 @@ +[Unit] +Description=tardev containerd snapshotter daemon +After=network.target + +[Service] +ExecStart=/usr/bin/tardev-snapshotter /var/lib/containerd/io.containerd.snapshotter.v1.tardev /run/containerd/tardev-snapshotter.sock +Environment="RUST_LOG=tardev_snapshotter=trace" +Restart=on-failure + +[Install] +WantedBy=containerd.service diff --git a/src/tardev-snapshotter/tarfs-defs/Cargo.toml b/src/tardev-snapshotter/tarfs-defs/Cargo.toml new file mode 100644 index 000000000000..4b925dc033ce --- /dev/null +++ b/src/tardev-snapshotter/tarfs-defs/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "tarfs-defs" +version = "0.1.0" +edition = "2021" + +[dependencies] +zerocopy = "0.6.1" diff --git a/src/tardev-snapshotter/tarfs-defs/src/lib.rs b/src/tardev-snapshotter/tarfs-defs/src/lib.rs new file mode 100644 index 000000000000..3d4ea28739b8 --- /dev/null +++ b/src/tardev-snapshotter/tarfs-defs/src/lib.rs @@ -0,0 +1,125 @@ +use zerocopy::byteorder::{LE, U16, U32, U64}; + +/// Flags used in [`Inode::flags`]. +pub mod inode_flags { + /// Indicates that the inode is opaque. + /// + /// When set, inode will have the "trusted.overlay.opaque" set to "y" at runtime. + pub const OPAQUE: u8 = 0x1; +} + +/// An inode in the tarfs inode table. +#[derive(zerocopy::AsBytes, zerocopy::FromBytes, zerocopy::Unaligned)] +#[repr(C)] +pub struct Inode { + /// The mode of the inode. + /// + /// The bottom 9 bits are the rwx bits for owner, group, all. + /// + /// The bits in the [`S_IFMT`] mask represent the file mode. + pub mode: U16, + + /// Tarfs flags for the inode. + /// + /// Values are drawn from the [`inode_flags`] module. + pub flags: u8, + + /// The bottom 4 bits represent the top 4 bits of mtime. + pub hmtime: u8, + + /// The owner of the inode. + pub owner: U32, + + /// The group of the inode. + pub group: U32, + + /// The bottom 32 bits of mtime. + pub lmtime: U32, + + /// Size of the contents of the inode. + pub size: U64, + + /// Either the offset to the data, or the major and minor numbers of a device. + /// + /// For the latter, the 32 LSB are the minor, and the 32 MSB are the major numbers. + pub offset: U64, +} + +/// An entry in a tarfs directory entry table. +#[derive(zerocopy::AsBytes, zerocopy::FromBytes, zerocopy::Unaligned)] +#[repr(C)] +pub struct DirEntry { + /// The inode number this entry refers to. + pub ino: U64, + + /// The offset to the name of the entry. + pub name_offset: U64, + + /// The length of the name of the entry. + pub name_len: U64, + + /// The type of entry. + pub etype: u8, + + /// Unused padding. + pub _padding: [u8; 7], +} + +/// The super-block of a tarfs instance. +#[derive(zerocopy::AsBytes, zerocopy::FromBytes, zerocopy::Unaligned)] +#[repr(C)] +pub struct SuperBlock { + /// The offset to the beginning of the inode-table. + pub inode_table_offset: U64, + + /// The number of inodes in the file system. + pub inode_count: U64, +} + +/// A mask to be applied to [`Inode::mode`] to extract the inode's type. +pub const S_IFMT: u16 = 0o0170000; + +/// A socket. +pub const S_IFSOCK: u16 = 0o0140000; + +/// A symbolic link. +pub const S_IFLNK: u16 = 0o0120000; + +/// A regular file. +pub const S_IFREG: u16 = 0o0100000; + +/// A block device. +pub const S_IFBLK: u16 = 0o0060000; + +/// A directory. +pub const S_IFDIR: u16 = 0o0040000; + +/// A character device. +pub const S_IFCHR: u16 = 0o0020000; + +/// A (fifo) pipe. +pub const S_IFIFO: u16 = 0o0010000; + +/// Unknown directory entry type. +pub const DT_UNKNOWN: u8 = 0; + +/// A (fifo) pipe. +pub const DT_FIFO: u8 = 1; + +/// A character device. +pub const DT_CHR: u8 = 2; + +/// A directory. +pub const DT_DIR: u8 = 4; + +/// A block device. +pub const DT_BLK: u8 = 6; + +/// A regular file. +pub const DT_REG: u8 = 8; + +/// A symbolic link. +pub const DT_LNK: u8 = 10; + +/// A socket. +pub const DT_SOCK: u8 = 12; diff --git a/src/tardev-snapshotter/tarindex/Cargo.toml b/src/tardev-snapshotter/tarindex/Cargo.toml new file mode 100644 index 000000000000..550291f4dbc1 --- /dev/null +++ b/src/tardev-snapshotter/tarindex/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "tarindex" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "tarindex" +path = "src/bin/tarindex.rs" + +[dependencies] +tar = "0.4" +zerocopy = "0.6.1" +tarfs-defs = { path = "../tarfs-defs" } diff --git a/src/tardev-snapshotter/tarindex/src/bin/tarindex.rs b/src/tardev-snapshotter/tarindex/src/bin/tarindex.rs new file mode 100644 index 000000000000..4a773373726a --- /dev/null +++ b/src/tardev-snapshotter/tarindex/src/bin/tarindex.rs @@ -0,0 +1,13 @@ +use std::{env, fs::OpenOptions, io, process}; +use tarindex::append_index; + +fn main() -> io::Result<()> { + let argv: Vec = env::args().collect(); + if argv.len() != 2 { + eprintln!("Usage: {} ", argv[0]); + process::exit(1); + } + + let mut file = OpenOptions::new().read(true).write(true).open(&argv[1])?; + append_index(&mut file) +} diff --git a/src/tardev-snapshotter/tarindex/src/lib.rs b/src/tardev-snapshotter/tarindex/src/lib.rs new file mode 100644 index 000000000000..a46d35a47cdf --- /dev/null +++ b/src/tardev-snapshotter/tarindex/src/lib.rs @@ -0,0 +1,491 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::{cell::RefCell, io, mem, rc::Rc}; +use tar::Archive; +use tarfs_defs::*; +use zerocopy::AsBytes; + +#[derive(Default)] +struct Entry { + offset: u64, + size: u64, + children: BTreeMap, Rc>>, + mode: u16, + ino: u64, + emitted: bool, + is_opaque: bool, + + mtime: u64, + owner: u32, + group: u32, +} + +impl Entry { + fn find_or_create_child(&mut self, name: &[u8]) -> Rc> { + self.children + .entry(name.to_vec()) + .or_insert_with(|| Rc::new(RefCell::new(Entry::default()))) + .clone() + } +} + +fn visit_breadth_first_mut( + root: Rc>, + mut visitor: impl FnMut(&mut Entry) -> io::Result<()>, +) -> io::Result<()> { + let mut q = VecDeque::new(); + q.push_back(root); + + while let Some(e) = q.pop_front() { + visitor(&mut e.borrow_mut())?; + + for child in e.borrow().children.values() { + q.push_back(child.clone()); + } + } + + Ok(()) +} + +fn read_all_entries( + reader: &mut (impl io::Read + io::Seek), + root: &mut Rc>, + mut cb: impl FnMut(&mut Rc>, &[u8], &Entry), + mut hardlink: impl FnMut(&mut Rc>, &[u8], &[u8]), +) -> io::Result { + let mut ar = Archive::new(reader); + + for file in ar.entries()? { + let f = file?; + let h = f.header(); + + let mut mode = if let Ok(m) = h.mode() { + m as u16 & 0x1ff + } else { + continue; + }; + + let entry_size; + let entry_offset; + match h.entry_type() { + tar::EntryType::Regular => { + mode |= S_IFREG; + entry_size = f.size(); + entry_offset = f.raw_file_position(); + } + tar::EntryType::Directory => { + mode |= S_IFDIR; + entry_size = 0; + entry_offset = 0; + } + tar::EntryType::Fifo => { + mode |= S_IFIFO; + entry_size = 0; + entry_offset = 0; + } + tar::EntryType::Char => { + mode |= S_IFCHR; + let major = if let Ok(Some(v)) = h.device_major() { + v as u64 + } else { + eprintln!( + "Skipping chr device without a major device number: {}", + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + }; + let minor = if let Ok(Some(v)) = h.device_minor() { + v as u64 + } else { + eprintln!( + "Skipping chr device without a minor device number: {}", + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + }; + entry_offset = minor | (major << 32); + entry_size = 0; + } + tar::EntryType::Block => { + mode |= S_IFBLK; + let major = if let Ok(Some(v)) = h.device_major() { + v as u64 + } else { + eprintln!( + "Skipping blk device without a major device number: {}", + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + }; + let minor = if let Ok(Some(v)) = h.device_minor() { + v as u64 + } else { + eprintln!( + "Skipping blk device without a minor device number: {}", + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + }; + entry_offset = minor | (major << 32); + entry_size = 0; + } + tar::EntryType::Symlink => { + mode |= S_IFLNK; + match f.link_name_bytes() { + Some(name) => { + let hname = h + .link_name_bytes() + .unwrap_or(std::borrow::Cow::Borrowed(b"")); + if *hname != *name { + // TODO: Handle this case by duplicating the full name. + eprintln!( + "Skipping symlink with long link name ({}, {} bytes, {}, {} bytes): {}", + String::from_utf8_lossy(&name), name.len(), + String::from_utf8_lossy(&hname), hname.len(), + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + } + + entry_size = name.len() as u64; + entry_offset = f.raw_header_position() + 157; + } + None => { + eprintln!( + "Skipping symlink without a link name: {}", + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + } + } + } + tar::EntryType::Link => { + match f.link_name_bytes() { + Some(name) => hardlink(root, &f.path_bytes(), &name), + None => { + eprintln!( + "Skipping hardlink without a link name: {}", + String::from_utf8_lossy(&f.path_bytes()) + ); + } + } + continue; + } + _ => { + eprintln!( + "Skipping unhandled file due to its type ({:?}): {}", + h.entry_type(), + String::from_utf8_lossy(&f.path_bytes()) + ); + continue; + } + } + + cb( + root, + &f.path_bytes(), + &Entry { + size: entry_size, + offset: entry_offset, + children: BTreeMap::new(), + is_opaque: false, + mode, + ino: 0, + emitted: false, + mtime: h.mtime().unwrap_or(0), + owner: h.uid().unwrap_or(0) as u32, // TODO: This can be a u64 in `tar`. + group: h.gid().unwrap_or(0) as u32, // TODO: This can be a u64 in `tar`. + }, + ); + } + + ar.into_inner().seek(io::SeekFrom::End(0)) +} + +fn clean_path(str: &[u8]) -> Option> { + let mut ret = Vec::new(); + + for component in str.split(|&c| c == b'/') { + match component { + // Empty entries or "." are just ignored. + b"" | b"." => {} + + // Pop an element when we see "..". + b".." => { + if ret.is_empty() { + return None; + } + ret.pop(); + } + + // Add anything else. + _ => { + ret.push(component); + } + } + } + + Some(ret) +} + +/// Initilises the `offset` of all `Entry` instances that represent directories. +/// +/// Returns the next available offset. +/// +/// `first_offset` is the offset of the first directory entry. +fn init_direntry_offset(root: Rc>, first_offset: u64) -> io::Result { + let mut offset = first_offset; + visit_breadth_first_mut(root, |e| { + if e.mode & S_IFMT != S_IFDIR { + return Ok(()); + } + + e.offset = offset; + e.size = mem::size_of::() as u64 * e.children.len() as u64; + + offset += e.size; + Ok(()) + })?; + Ok(offset) +} + +/// Writes all directory entries to the given file. +/// +/// Returns the next available offset for the strings. +/// +/// `first_string_offset` is the offset of the first string. +fn write_direntry_bodies( + root: Rc>, + first_string_offset: u64, + file: &mut impl io::Write, +) -> io::Result { + let mut offset = first_string_offset; + visit_breadth_first_mut(root, |e| { + if e.mode & S_IFMT != S_IFDIR { + return Ok(()); + } + + for (name, child) in &e.children { + let child = child.borrow(); + let dirent = DirEntry { + ino: child.ino.into(), + name_offset: offset.into(), + name_len: (name.len() as u64).into(), + etype: match child.mode & S_IFMT { + S_IFSOCK => DT_SOCK, + S_IFLNK => DT_LNK, + S_IFREG => DT_REG, + S_IFBLK => DT_BLK, + S_IFDIR => DT_DIR, + S_IFCHR => DT_CHR, + S_IFIFO => DT_FIFO, + _ => DT_UNKNOWN, + }, + _padding: [0; 7], + }; + file.write_all(dirent.as_bytes())?; + offset += u64::from(dirent.name_len); + } + + Ok(()) + })?; + Ok(offset) +} + +fn traverse_path(root: &Rc>, path: &[&[u8]]) -> Rc> { + let mut ptr = root.clone(); + for component in path { + let new = ptr.borrow_mut().find_or_create_child(component); + ptr = new; + } + + ptr +} + +pub fn append_index(data: &mut (impl io::Read + io::Write + io::Seek)) -> io::Result<()> { + let mut root = Rc::new(RefCell::new(Entry { + mode: S_IFDIR | 0o555, + ..Entry::default() + })); + + let contents_size = read_all_entries( + data, + &mut root, + |root, name, e| { + // Break the name into path components. + let mut path = if let Some(p) = clean_path(name) { + p + } else { + // Skip files that don't point into the root. + eprintln!("Skipping malformed name: {}", String::from_utf8_lossy(name)); + return; + }; + + if let Some(n) = path.last_mut() { + if n == b".wh..wh..opq" { + // Set the opaque flag on the parent directory. + let ptr = traverse_path(&root, &path[..path.len() - 1]); + ptr.borrow_mut().is_opaque = true; + return; + } + + if n.starts_with(b".wh.") { + // Find the file and make it a char device with (0, 0) as major and minor. This + // indicates to overlayfs that it shouldn't look at lower layers. + *n = &n[4..]; + let ptr = traverse_path(&root, &path); + let mut cur = ptr.borrow_mut(); + cur.children = BTreeMap::new(); + cur.mode = (cur.mode & !S_IFMT) | S_IFCHR; + cur.size = 0; + cur.offset = 0; + return; + } + } + + // Find the right entry in the tree. + let ptr = traverse_path(&root, &path); + let mut cur = ptr.borrow_mut(); + + // Update the entry. We remove any previous existing entry. + *cur = Entry { + children: BTreeMap::new(), + mode: e.mode, + size: e.size, + offset: e.offset, + mtime: e.mtime, + owner: e.owner, + group: e.group, + ino: e.ino, + emitted: e.emitted, + is_opaque: e.is_opaque, + }; + }, + |root, name, linkname| { + // Find the destination. + let path = if let Some(p) = clean_path(linkname) { + p + } else { + // Skip files that don't point into the root. + eprintln!( + "Skipping malformed linkname name: {}", + String::from_utf8_lossy(linkname) + ); + return; + }; + + // Find existing file. + let mut existing = root.clone(); + for component in path { + let new = existing.borrow_mut().find_or_create_child(component); + existing = new; + } + + if existing.borrow().mode & S_IFMT != S_IFREG { + eprintln!( + "Skipping link to non-file: {}", + String::from_utf8_lossy(linkname) + ); + return; + } + + // Find the file to create. + let path = if let Some(p) = clean_path(name) { + p + } else { + // Skip files that don't point into the root. + eprintln!("Skipping malformed name: {}", String::from_utf8_lossy(name)); + return; + }; + + if path.is_empty() { + *root = existing; + } else { + let mut ptr = root.clone(); + for component in path.iter().take(path.len() - 1) { + let new = ptr.borrow_mut().find_or_create_child(component); + ptr = new; + } + ptr.borrow_mut() + .children + .insert(path.last().unwrap().to_vec(), existing); + } + }, + )?; + + data.seek(io::SeekFrom::End(0))?; + + // Assign i-node numbers only for the entries that survided conversion to tree. + let mut ino_count = 0u64; + visit_breadth_first_mut(root.clone(), |e| { + if e.ino == 0 { + ino_count += 1; + e.ino = ino_count; + } + Ok(()) + })?; + + // Calculate the offsets for directory entries. + let inode_table_size: u64 = mem::size_of::() as u64 * ino_count; + let string_table_offset = init_direntry_offset(root.clone(), contents_size + inode_table_size)?; + + // Write the i-node table. + visit_breadth_first_mut(root.clone(), |e| { + if e.emitted { + return Ok(()); + } + + e.emitted = true; + let inode = Inode { + mode: e.mode.into(), + flags: if e.is_opaque { + tarfs_defs::inode_flags::OPAQUE + } else { + 0 + }, + hmtime: (e.mtime >> 32 & 0xf) as u8, + owner: e.owner.into(), + group: e.group.into(), + lmtime: (e.mtime as u32).into(), + size: e.size.into(), + offset: e.offset.into(), + }; + data.write_all(inode.as_bytes())?; + Ok(()) + })?; + + // Write the directory bodies. + let mut end_offset = write_direntry_bodies(root.clone(), string_table_offset, data)?; + + // Write the strings. + visit_breadth_first_mut(root, |e| { + if e.mode & S_IFMT != S_IFDIR { + return Ok(()); + } + + for name in e.children.keys() { + data.write_all(name)?; + end_offset += name.len() as u64; + } + + Ok(()) + })?; + + // Write the "super-block". + const ALIGNMENT: u64 = 4096; + const fn align(v: u64) -> u64 { + (v + (ALIGNMENT - 1)) / ALIGNMENT * ALIGNMENT + } + end_offset = align(end_offset); + data.seek(io::SeekFrom::Start(end_offset + ALIGNMENT - 512))?; + let sb = SuperBlock { + inode_table_offset: contents_size.into(), + inode_count: ino_count.into(), + }; + data.write_all(sb.as_bytes())?; + + // Write padding to align to a 4096-byte boundary. + data.seek(io::SeekFrom::Start(end_offset + (ALIGNMENT - 1)))?; + data.write_all(&[0])?; + + Ok(()) +} diff --git a/src/tardev-snapshotter/verity/Cargo.toml b/src/tardev-snapshotter/verity/Cargo.toml new file mode 100644 index 000000000000..ff27d893d6c3 --- /dev/null +++ b/src/tardev-snapshotter/verity/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "verity" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "verity" +path = "src/bin/verity.rs" + +[[bin]] +name = "verity-info" +path = "src/bin/verity-info.rs" + +[dependencies] +sha2 = "0.10.6" +generic-array = "0.14.6" +zerocopy = "0.6.1" diff --git a/src/tardev-snapshotter/verity/src/bin/verity-info.rs b/src/tardev-snapshotter/verity/src/bin/verity-info.rs new file mode 100644 index 000000000000..8689b8a8580c --- /dev/null +++ b/src/tardev-snapshotter/verity/src/bin/verity-info.rs @@ -0,0 +1,46 @@ +use std::{env, fs::File, io, io::Read, io::Seek, process}; +use zerocopy::AsBytes; + +fn main() -> io::Result<()> { + let argv: Vec = env::args().collect(); + if argv.len() != 2 { + eprintln!("Usage: {} ", argv[0]); + process::exit(1); + } + + let mut file = File::open(&argv[1])?; + let size = file.seek(io::SeekFrom::End(0))?; + if size < 4096 { + eprintln!("File is too small: {size}"); + process::exit(1); + } + + file.seek(std::io::SeekFrom::End(-4096))?; + let mut buf = [0u8; 4096]; + file.read_exact(&mut buf)?; + + let mut sb = verity::SuperBlock::default(); + sb.as_bytes_mut() + .copy_from_slice(&buf[4096 - 512..][..std::mem::size_of::()]); + let data_block_size = u64::from(sb.data_block_size.get()); + let hash_block_size = u64::from(sb.hash_block_size.get()); + let data_size = if let Some(v) = sb.data_block_count.get().checked_mul(data_block_size) { + v + } else { + eprintln!("Overflow when calculating the data size"); + process::exit(1); + }; + + if data_size > size { + eprintln!("Data size ({data_size}) is greater than device size ({size})"); + process::exit(1); + } + + println!("Data block size: {data_block_size}"); + println!("Data block clount: {}", sb.data_block_count.get()); + println!("Hash block size: {hash_block_size}"); + println!("Hash offset: {data_size}"); + println!("veritysetup verify --data-block-size={data_block_size} --data-blocks={} --hash-block-size={hash_block_size} --hash-offset={data_size} --no-superblock -s 0000000000000000000000000000000000000000000000000000000000000000 {} {}", sb.data_block_count.get(), argv[1], argv[1]); + + Ok(()) +} diff --git a/src/tardev-snapshotter/verity/src/bin/verity.rs b/src/tardev-snapshotter/verity/src/bin/verity.rs new file mode 100644 index 000000000000..a5368b48dfc8 --- /dev/null +++ b/src/tardev-snapshotter/verity/src/bin/verity.rs @@ -0,0 +1,68 @@ +use generic_array::typenum::Unsigned; +use sha2::{digest::OutputSizeUser, Sha256}; +use std::{env, fs::File, fs::OpenOptions, io, io::Seek, process}; +use verity::{append_tree, traverse_file, Verity}; + +fn main() -> io::Result<()> { + let argv: Vec = env::args().collect(); + if argv.len() != 3 && argv.len() != 4 { + eprintln!("Usage: {} [tree.bin]", argv[0]); + process::exit(1); + } + + let mut reader = File::open(&argv[2])?; + let file_size = reader.seek(io::SeekFrom::End(0))?; + reader.rewind()?; + + if file_size == 0 { + eprintln!("Empty input file."); + process::exit(1); + } + + let salt = [0u8; ::OutputSize::USIZE]; + + match argv[1].as_ref() { + // Append the tree to the file. + "a" => { + let mut file = OpenOptions::new().read(true).write(true).open(&argv[2])?; + println!("Root hash: {:x}", append_tree::(&mut file)?); + } + + // Create the tree in a separate file. + "t" => { + if argv.len() != 4 { + eprintln!("Must specify the name of the output file"); + process::exit(1); + } + + let mut writer = File::create(&argv[3])?; + let verity = Verity::::new(file_size, 4096, 4096, &salt, 0)?; + println!( + "Root hash: {:x}", + traverse_file( + &mut reader, + 0, + false, + verity, + &mut verity::write_to(&mut writer) + )? + ); + } + + // Calculate the root hash without writing the tree. + "r" => { + let verity = Verity::::new(file_size, 4096, 4096, &salt, 0)?; + println!( + "Root hash: {:x}", + traverse_file(&mut reader, 0, false, verity, &mut verity::no_write)? + ); + } + + _ => { + eprintln!("Unknown command: {}", argv[1]); + process::exit(1); + } + } + + Ok(()) +} diff --git a/src/tardev-snapshotter/verity/src/lib.rs b/src/tardev-snapshotter/verity/src/lib.rs new file mode 100644 index 000000000000..30c59e87cfda --- /dev/null +++ b/src/tardev-snapshotter/verity/src/lib.rs @@ -0,0 +1,241 @@ +use generic_array::{typenum::Unsigned, GenericArray}; +use sha2::{digest::OutputSizeUser, Digest}; +use std::fs::File; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use zerocopy::byteorder::{LE, U32, U64}; +use zerocopy::AsBytes; + +#[derive(Default, zerocopy::AsBytes, zerocopy::FromBytes, zerocopy::Unaligned)] +#[repr(C)] +pub struct SuperBlock { + pub data_block_size: U32, + pub hash_block_size: U32, + pub data_block_count: U64, +} + +#[derive(Clone)] +struct Level { + next_index: usize, + file_offset: u64, + data: Vec, +} + +pub struct Verity { + levels: Vec, + seeded: T, + data_block_size: usize, + hash_block_size: usize, + block_remaining_count: u64, + super_block: SuperBlock, +} + +impl Verity { + const HASH_SIZE: usize = T::OutputSize::USIZE; + + /// Creates a new `Verity` instance. + pub fn new( + data_size: u64, + data_block_size: usize, + hash_block_size: usize, + salt: &[u8], + mut write_file_offset: u64, + ) -> io::Result { + let level_count = { + let mut max_size = data_block_size as u64; + let mut count = 0usize; + + while max_size < data_size { + count += 1; + max_size *= (hash_block_size / Self::HASH_SIZE) as u64; + } + count + }; + + let mut data = Vec::new(); + data.resize(hash_block_size, 0); + + let mut levels = Vec::new(); + levels.resize( + level_count, + Level { + next_index: 0, + file_offset: 0, + data, + }, + ); + + for (i, l) in levels.iter_mut().enumerate() { + let entry_size = (data_block_size as u64) + * ((hash_block_size / Self::HASH_SIZE) as u64).pow(level_count as u32 - i as u32); + let count = (data_size + entry_size - 1) / entry_size; + l.file_offset = write_file_offset; + write_file_offset += hash_block_size as u64 * count; + } + + let block_count = data_size / (data_block_size as u64); + Ok(Self { + levels, + seeded: T::new_with_prefix(salt), + data_block_size, + block_remaining_count: block_count, + hash_block_size, + super_block: SuperBlock { + data_block_size: (data_block_size as u32).into(), + hash_block_size: (hash_block_size as u32).into(), + data_block_count: block_count.into(), + }, + }) + } + + /// Determines if more blocks are expected. + /// + /// This is based on file size specified when this instance was created. + fn more_blocks(&self) -> bool { + self.block_remaining_count > 0 + } + + /// Adds the given hash to the level. + /// + /// Returns `true` is the level is now full; `false` is there is still room for more hashes. + fn add_hash(&mut self, l: usize, hash: &[u8]) -> bool { + let level = &mut self.levels[l]; + level.data[level.next_index * Self::HASH_SIZE..][..Self::HASH_SIZE].copy_from_slice(hash); + level.next_index += 1; + level.next_index >= self.hash_block_size / Self::HASH_SIZE + } + + /// Finalises the level despite potentially not having filled it. + /// + /// It zeroes out the remaining bytes of the level so that its hash can be calculated + /// consistently. + fn finalize_level(&mut self, l: usize) { + let level = &mut self.levels[l]; + for b in &mut level.data[level.next_index * Self::HASH_SIZE..] { + *b = 0; + } + level.next_index = 0; + } + + fn uplevel(&mut self, l: usize, reader: &mut File, writer: &mut F) -> io::Result + where + F: FnMut(&mut File, &[u8], u64) -> io::Result<()>, + { + self.finalize_level(l); + writer(reader, &self.levels[l].data, self.levels[l].file_offset)?; + self.levels[l].file_offset += self.hash_block_size as u64; + let h = self.digest(&self.levels[l].data); + Ok(self.add_hash(l - 1, h.as_slice())) + } + + fn digest(&self, block: &[u8]) -> GenericArray { + let mut hasher = self.seeded.clone(); + hasher.update(block); + hasher.finalize() + } + + fn add_block(&mut self, b: &[u8], reader: &mut File, writer: &mut F) -> io::Result<()> + where + F: FnMut(&mut File, &[u8], u64) -> io::Result<()>, + { + if self.block_remaining_count == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected block", + )); + } + + self.block_remaining_count -= 1; + + let count = self.levels.len(); + let hash = self.digest(b); + if self.add_hash(count - 1, hash.as_slice()) { + // Go up the levels as far as it can. + for l in (1..count).rev() { + if !self.uplevel(l, reader, writer)? { + break; + } + } + } + Ok(()) + } + + fn finalize( + mut self, + write_superblock: bool, + reader: &mut File, + writer: &mut impl FnMut(&mut File, &[u8], u64) -> io::Result<()>, + ) -> io::Result> { + let len = self.levels.len(); + for mut l in (1..len).rev() { + if self.levels[l].next_index != 0 { + while l > 0 { + self.uplevel(l, reader, writer)?; + l -= 1; + } + break; + } + } + + self.finalize_level(0); + + writer(reader, &self.levels[0].data, self.levels[0].file_offset)?; + self.levels[0].file_offset += self.hash_block_size as u64; + + if write_superblock { + writer( + reader, + self.super_block.as_bytes(), + self.levels[len - 1].file_offset + 4096 - 512, + )?; + + // TODO: Align to the hash_block_size... + // Align to 4096 bytes. + writer(reader, &[0u8], self.levels[len - 1].file_offset + 4095)?; + } + + Ok(self.digest(&self.levels[0].data)) + } +} + +pub fn traverse_file( + file: &mut File, + mut read_offset: u64, + write_superblock: bool, + mut verity: Verity, + writer: &mut impl FnMut(&mut File, &[u8], u64) -> io::Result<()>, +) -> io::Result> { + let mut buf = Vec::new(); + buf.resize(verity.data_block_size, 0); + while verity.more_blocks() { + file.seek(SeekFrom::Start(read_offset))?; + file.read_exact(&mut buf)?; + verity.add_block(&buf, file, writer)?; + read_offset += verity.data_block_size as u64; + } + verity.finalize(write_superblock, file, writer) +} + +pub fn no_write(_: &mut File, _: &[u8], _: u64) -> io::Result<()> { + Ok(()) +} + +pub fn write_to(f: &mut File) -> impl FnMut(&mut File, &[u8], u64) -> io::Result<()> + '_ { + |_, data, offset| { + f.seek(SeekFrom::Start(offset))?; + f.write_all(data) + } +} + +pub fn append_tree( + file: &mut File, +) -> io::Result> { + let file_size = file.seek(io::SeekFrom::End(0))?; + file.rewind()?; + let mut salt = Vec::new(); + salt.resize(::OutputSize::USIZE, 0); + let verity = Verity::::new(file_size, 4096, 4096, &salt, file_size)?; + traverse_file(file, 0, true, verity, &mut |f, data, offset| { + f.seek(SeekFrom::Start(offset))?; + f.write_all(data) + }) +} diff --git a/src/tarfs/Makefile b/src/tarfs/Makefile new file mode 100644 index 000000000000..207b50adea06 --- /dev/null +++ b/src/tarfs/Makefile @@ -0,0 +1,21 @@ +ifneq ($(KERNELRELEASE),) + +obj-m := tarfs.o + +else + +KDIR ?= /lib/modules/`uname -r`/build +INSTALL_MOD_PATH ?= $$PWD/_install + +default: + $(MAKE) -C $(KDIR) M=$$PWD + +install: + $(MAKE) -C $(KDIR) M=$$PWD INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) modules_install + +clean: + rm -rf _install + $(MAKE) -C $(KDIR) M=$$PWD clean + + +endif diff --git a/src/tarfs/tarfs.c b/src/tarfs/tarfs.c new file mode 100644 index 000000000000..bd13021e45b8 --- /dev/null +++ b/src/tarfs/tarfs.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TARFS_MAGIC (0x54415246535f) + +struct tarfs_super { + u64 inode_table_offset; + u64 inode_count; +} __packed; + +struct tarfs_state { + struct tarfs_super super; + u64 data_size; +}; + +#define TARFS_INODE_FLAG_OPAQUE 0x1 + +struct tarfs_inode { + u16 mode; + u8 flags; + u8 hmtime; /* High 4 bits of mtime. */ + u32 owner; + u32 group; + u32 lmtime; /* Lower 32 bits of mtime. */ + u64 size; + u64 offset; /* 64 bits of offset, or 32 LSB are minor dev and 32 MSB are major dev. */ +} __packed; + +struct tarfs_direntry { + u64 ino; + u64 nameoffset; + u64 namelen; + u8 type; + u8 padding[7]; +} __packed; + +struct tarfs_inode_info { + struct inode inode; + u64 data_offset; + u8 flags; +}; + +#define TARFS_I(ptr) (container_of(ptr, struct tarfs_inode_info, inode)) + +#define TARFS_BSIZE 4096 + +static struct kmem_cache *tarfs_inode_cachep; + +static struct dentry *tarfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags); + +static int tarfs_dev_read(struct super_block *sb, u64 pos, void *buf, size_t buflen) +{ + struct buffer_head *bh; + unsigned long offset; + size_t segment; + const struct tarfs_state *state = sb->s_fs_info; + + /* Check for overflows. */ + if (pos + buflen < pos) + return -ERANGE; + + /* Check that the read range is within the data part of the device. */ + if (pos + buflen > state->data_size) + return -EIO; + + while (buflen > 0) { + offset = pos & (TARFS_BSIZE - 1); + segment = min_t(size_t, buflen, TARFS_BSIZE - offset); + bh = sb_bread(sb, pos / TARFS_BSIZE); + if (!bh) + return -EIO; + memcpy(buf, bh->b_data + offset, segment); + brelse(bh); + buf += segment; + buflen -= segment; + pos += segment; + } + + return 0; +} + +static int tarfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct tarfs_direntry disk_dentry; + u64 offset = TARFS_I(inode)->data_offset; + int ret = 0; + char *name_buffer = NULL; + u64 name_len = 0; + u64 cur = ctx->pos; + u64 size = i_size_read(inode) / sizeof(disk_dentry) * sizeof(disk_dentry); + + /* cur must be aligned to a directory entry. */ + if (ctx->pos % sizeof(struct tarfs_direntry)) + return -ENOENT; + + /* Make sure we can't overflow the read offset. */ + if (offset + size < offset) + return -ERANGE; + + /* Make sure the increment of cur won't overflow by limiting size. */ + if (size >= U64_MAX - sizeof(disk_dentry)) + return -ERANGE; + + for (cur = ctx->pos; cur < size; cur += sizeof(disk_dentry)) { + u64 disk_len; + u8 type; + + ret = tarfs_dev_read(inode->i_sb, offset + cur, &disk_dentry, sizeof(disk_dentry)); + if (ret) + break; + + disk_len = le64_to_cpu(disk_dentry.namelen); + if (disk_len > name_len) { + kfree(name_buffer); + + if (disk_len > SIZE_MAX) + return -ENOMEM; + + name_buffer = kmalloc(disk_len, GFP_KERNEL); + if (!name_buffer) + return -ENOMEM; + name_len = disk_len; + } + + ret = tarfs_dev_read(inode->i_sb, + le64_to_cpu(disk_dentry.nameoffset), + name_buffer, disk_len); + if (ret) + break; + + /* Filter out bad types. */ + type = disk_dentry.type; + switch (type) { + case DT_FIFO: + case DT_CHR: + case DT_DIR: + case DT_BLK: + case DT_REG: + case DT_LNK: + case DT_SOCK: + break; + default: + type = DT_UNKNOWN; + } + + if (!dir_emit(ctx, name_buffer, disk_len, le64_to_cpu(disk_dentry.ino), type)) { + kfree(name_buffer); + return 0; + } + } + + kfree(name_buffer); + + if (!ret) + ctx->pos = cur; + + return ret; +} + +static int tarfs_readpage(struct file *file, struct page *page) +{ + struct inode *inode = page->mapping->host; + loff_t offset, size; + unsigned long fillsize, pos; + void *buf; + int ret; + + buf = kmap_local_page(page); + if (!buf) + return -ENOMEM; + + offset = page_offset(page); + size = i_size_read(inode); + fillsize = 0; + ret = 0; + if (offset < size) { + size -= offset; + fillsize = size > PAGE_SIZE ? PAGE_SIZE : size; + + pos = TARFS_I(inode)->data_offset + offset; + + ret = tarfs_dev_read(inode->i_sb, pos, buf, fillsize); + if (ret < 0) { + SetPageError(page); + fillsize = 0; + ret = -EIO; + } + } + + if (fillsize < PAGE_SIZE) + memset(buf + fillsize, 0, PAGE_SIZE - fillsize); + if (ret == 0) + SetPageUptodate(page); + + flush_dcache_page(page); + kunmap(page); + unlock_page(page); + return ret; +} + +#if KERNEL_VERSION(5, 19, 0) <= LINUX_VERSION_CODE +static int tarfs_read_folio(struct file *file, struct folio *folio) +{ + return tarfs_readpage(file, &folio->page); +} +#else +static inline void * +alloc_inode_sb(struct super_block *sb, struct kmem_cache *cache, gfp_t gfp) +{ + return kmem_cache_alloc(cache, gfp); +} +#endif + +static struct inode *tarfs_iget(struct super_block *sb, u64 ino) +{ + static const struct inode_operations tarfs_symlink_inode_operations = { + .get_link = page_get_link, + }; + static const struct inode_operations tarfs_dir_inode_operations = { + .lookup = tarfs_lookup, + }; + static const struct file_operations tarfs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = tarfs_readdir, + .llseek = generic_file_llseek, + }; + static const struct address_space_operations tarfs_aops = { +#if KERNEL_VERSION(5, 19, 0) <= LINUX_VERSION_CODE + .read_folio = tarfs_read_folio, +#else + .readpage = tarfs_readpage, +#endif + }; + struct tarfs_inode_info *info; + struct tarfs_inode disk_inode; + struct inode *inode; + const struct tarfs_state *state = sb->s_fs_info; + int ret; + u16 mode; + u64 offset; + + if (!ino || ino > state->super.inode_count) + return ERR_PTR(-ENOENT); + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode->i_state & I_NEW)) + return inode; + + /* + * The checks in tarfs_fill_super ensure that we don't overflow while trying to calculate + * offset of the inode table entry as long as the inode number is less than inode_count. + */ + ret = tarfs_dev_read(sb, + state->super.inode_table_offset + sizeof(struct tarfs_inode) * (ino - 1), + &disk_inode, sizeof(disk_inode)); + if (ret < 0) + goto discard; + + i_uid_write(inode, le32_to_cpu(disk_inode.owner)); + i_gid_write(inode, le32_to_cpu(disk_inode.group)); + + offset = le64_to_cpu(disk_inode.offset); + mode = le16_to_cpu(disk_inode.mode); + + /* Ignore inodes that have unknown mode bits. */ + if (mode & ~(S_IFMT | 0777)) { + ret = -ENOENT; + goto discard; + } + + switch (mode & S_IFMT) { + case S_IFREG: + inode->i_fop = &generic_ro_fops; + inode->i_data.a_ops = &tarfs_aops; + break; + + case S_IFDIR: + inode->i_op = &tarfs_dir_inode_operations; + inode->i_fop = &tarfs_dir_operations; + break; + + case S_IFLNK: + inode->i_data.a_ops = &tarfs_aops; + inode->i_op = &tarfs_symlink_inode_operations; + inode_nohighmem(inode); + break; + + case S_IFSOCK: + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + init_special_inode(inode, mode, MKDEV(offset >> 32, offset & MINORMASK)); + offset = 0; + break; + + default: + ret = -ENOENT; + goto discard; + } + + set_nlink(inode, 1); + + inode->i_mtime.tv_sec = inode->i_atime.tv_sec = inode->i_ctime.tv_sec = + (((u64)disk_inode.hmtime & 0xf) << 32) | le32_to_cpu(disk_inode.lmtime); + inode->i_mtime.tv_nsec = inode->i_atime.tv_nsec = inode->i_ctime.tv_nsec = 0; + + inode->i_mode = mode; + inode->i_size = le64_to_cpu(disk_inode.size); + inode->i_blocks = (inode->i_size + TARFS_BSIZE - 1) / TARFS_BSIZE; + + info = TARFS_I(inode); + info->data_offset = offset; + info->flags = disk_inode.flags; + + unlock_new_inode(inode); + return inode; + +discard: + discard_new_inode(inode); + return ERR_PTR(ret); +} + +static int tarfs_strcmp(struct super_block *sb, unsigned long pos, + const char *str, size_t size) +{ + struct buffer_head *bh; + unsigned long offset; + size_t segment; + bool matched; + + /* compare string up to a block at a time. */ + while (size) { + offset = pos & (TARFS_BSIZE - 1); + segment = min_t(size_t, size, TARFS_BSIZE - offset); + bh = sb_bread(sb, pos / TARFS_BSIZE); + if (!bh) + return -EIO; + matched = memcmp(bh->b_data + offset, str, segment) == 0; + brelse(bh); + if (!matched) + return 0; + + size -= segment; + pos += segment; + str += segment; + } + + return 1; +} + +static struct dentry *tarfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode; + struct tarfs_direntry disk_dentry; + u64 offset = TARFS_I(dir)->data_offset; + int ret; + const char *name = dentry->d_name.name; + size_t len = dentry->d_name.len; + u64 size = i_size_read(dir) / sizeof(disk_dentry) * sizeof(disk_dentry); + u64 cur; + + /* Make sure we can't overflow the read offset. */ + if (offset + size < offset) + return ERR_PTR(-ERANGE); + + /* Make sure the increment of cur won't overflow by limiting size. */ + if (size >= U64_MAX - sizeof(disk_dentry)) + return ERR_PTR(-ERANGE); + + for (cur = 0; cur < size; cur += sizeof(disk_dentry)) { + u64 disk_len; + + ret = tarfs_dev_read(dir->i_sb, offset + cur, &disk_dentry, sizeof(disk_dentry)); + if (ret) + return ERR_PTR(ret); + + disk_len = le64_to_cpu(disk_dentry.namelen); + if (len != disk_len || disk_len > SIZE_MAX) + continue; + + ret = tarfs_strcmp(dir->i_sb, le64_to_cpu(disk_dentry.nameoffset), name, len); + if (ret < 0) + return ERR_PTR(ret); + + if (ret == 1) { + inode = tarfs_iget(dir->i_sb, le64_to_cpu(disk_dentry.ino)); + return d_splice_alias(inode, dentry); + } + } + + /* We reached the end of the directory. */ + return ERR_PTR(-ENOENT); +} + +static int tarfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + const struct tarfs_state *state = sb->s_fs_info; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = TARFS_MAGIC; + buf->f_namelen = LONG_MAX; + buf->f_bsize = TARFS_BSIZE; + buf->f_bfree = buf->f_bavail = buf->f_ffree = 0; + buf->f_blocks = state->super.inode_table_offset / TARFS_BSIZE; + buf->f_files = state->super.inode_count; + buf->f_fsid = u64_to_fsid(id); + return 0; +} + +static struct inode *tarfs_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + return tarfs_iget(sb, ino); +} + +static struct dentry *tarfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + tarfs_nfs_get_inode); +} + +static struct dentry *tarfs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + tarfs_nfs_get_inode); +} + +static struct inode *tarfs_alloc_inode(struct super_block *sb) +{ + struct tarfs_inode_info *info; + + info = alloc_inode_sb(sb, tarfs_inode_cachep, GFP_KERNEL); + if (!info) + return NULL; + + return &info->inode; +} + +static void tarfs_free_inode(struct inode *inode) +{ + kmem_cache_free(tarfs_inode_cachep, TARFS_I(inode)); +} + +int tarfs_xattr_trusted_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct tarfs_inode_info *info = TARFS_I(inode); + bool opaque = (info->flags & TARFS_INODE_FLAG_OPAQUE) != 0; + + if (opaque && strcmp(name, "overlay.opaque") == 0) { + if (size == 0) + return 1; + *(char *)buffer = 'y'; + return 1; + } + + return -ENODATA; +} + +static int tarfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + static const struct export_operations tarfs_export_ops = { + .fh_to_dentry = tarfs_fh_to_dentry, + .fh_to_parent = tarfs_fh_to_parent, + }; + static const struct super_operations super_ops = { + .alloc_inode = tarfs_alloc_inode, + .free_inode = tarfs_free_inode, + .statfs = tarfs_statfs, + }; + static const struct xattr_handler xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = tarfs_xattr_trusted_get, + }; + static const struct xattr_handler *xattr_handlers[] = { + &xattr_trusted_handler, + NULL, + }; + struct inode *root; + sector_t scount; + struct tarfs_state *state; + struct buffer_head *bh; + const struct tarfs_super *super; + u64 inode_table_end; + + sb_set_blocksize(sb, TARFS_BSIZE); + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_magic = TARFS_MAGIC; + sb->s_flags |= SB_RDONLY | SB_NOATIME; + sb->s_time_min = 0; + sb->s_time_max = 0; + sb->s_op = &super_ops; + sb->s_xattr = xattr_handlers; + + scount = bdev_nr_sectors(sb->s_bdev); + if (!scount) + return -ENXIO; + + state = kmalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + /* + * state will be freed by kill_sb even if we fail in one of the + * functions below. + */ + sb->s_fs_info = state; + + /* Read super block then init state. */ + bh = sb_bread(sb, scount * SECTOR_SIZE / TARFS_BSIZE - 1); + if (!bh) + return -EIO; + + super = (const struct tarfs_super *)&bh->b_data[TARFS_BSIZE - 512]; + state->super.inode_count = le64_to_cpu(super->inode_count); + state->super.inode_table_offset = + le64_to_cpu(super->inode_table_offset); + state->data_size = scount * SECTOR_SIZE; + + brelse(bh); + + /* This is used to indicate to overlayfs when this superblock limits inodes to 32 bits. */ + if (state->super.inode_count <= U32_MAX) + sb->s_export_op = &tarfs_export_ops; + + /* Check that the inode table starts within the device data. */ + if (state->super.inode_table_offset >= state->data_size) + return -E2BIG; + + /* Check that we don't overflow while calculating the offset of the last inode. */ + if (state->super.inode_count > U64_MAX / sizeof(struct tarfs_inode)) + return -ERANGE; + + /* Check that we don't overflow calculating the end of the inode table. */ + inode_table_end = state->super.inode_count * sizeof(struct tarfs_inode) + + state->super.inode_table_offset; + + if (inode_table_end < state->super.inode_table_offset) + return -ERANGE; + + /* Check that the inode tanble ends within the device data. */ + if (inode_table_end > state->data_size) + return -E2BIG; + + root = tarfs_iget(sb, 1); + if (IS_ERR(root)) + return PTR_ERR(root); + + sb->s_root = d_make_root(root); + if (!sb->s_root) + return -ENOMEM; + + return 0; +} + +static int tarfs_get_tree(struct fs_context *fc) +{ + int ret; + + ret = get_tree_bdev(fc, tarfs_fill_super); + if (ret) { + pr_err("get_tree_bdev failed: %d\n", ret); + return ret; + } + + return 0; +} + +static int tarfs_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static int tarfs_init_fs_context(struct fs_context *fc) +{ + static const struct fs_context_operations ops = { + .get_tree = tarfs_get_tree, + .reconfigure = tarfs_reconfigure, + }; + fc->ops = &ops; + return 0; +} + +static void tarfs_kill_sb(struct super_block *sb) +{ + if (sb->s_bdev) + kill_block_super(sb); + kfree(sb->s_fs_info); +} + +static void tarfs_inode_init_once(void *ptr) +{ + struct tarfs_inode_info *info = ptr; + inode_init_once(&info->inode); +} + +static struct file_system_type tarfs_fs_type = { + .owner = THIS_MODULE, + .name = "tar", + .init_fs_context = tarfs_init_fs_context, + .kill_sb = tarfs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("tar"); + +static int __init tarfs_init(void) +{ + int ret; + + tarfs_inode_cachep = kmem_cache_create("tarfs_inode_cache", + sizeof(struct tarfs_inode_info), 0, + (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| + SLAB_ACCOUNT), + tarfs_inode_init_once); + if (!tarfs_inode_cachep) { + pr_err("kmem_cache_create failed\n"); + return -ENOMEM; + } + + ret = register_filesystem(&tarfs_fs_type); + if (ret) { + pr_err("register_filesystem failed: %d\n", ret); + kmem_cache_destroy(tarfs_inode_cachep); + return ret; + } + + return 0; +} + +static void __exit tarfs_exit(void) +{ + unregister_filesystem(&tarfs_fs_type); + kmem_cache_destroy(tarfs_inode_cachep); +} + +module_init(tarfs_init); +module_exit(tarfs_exit); + +MODULE_DESCRIPTION("tarfs"); +MODULE_AUTHOR("Wedson Almeida Filho "); +MODULE_LICENSE("GPL"); diff --git a/src/utarfs/.gitignore b/src/utarfs/.gitignore new file mode 100644 index 000000000000..ea8c4bf7f35f --- /dev/null +++ b/src/utarfs/.gitignore @@ -0,0 +1 @@ +/target diff --git a/src/utarfs/Cargo.lock b/src/utarfs/Cargo.lock new file mode 100644 index 000000000000..81757f14cb26 --- /dev/null +++ b/src/utarfs/Cargo.lock @@ -0,0 +1,575 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401a4694d2bf92537b6867d94de48c4842089645fdcdf6c71865b175d836e9c2" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72394f3339a76daf211e57d4bcb374410f3965dcc606dd0e03738c7888766980" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.19", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "daemonize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" +dependencies = [ + "libc", +] + +[[package]] +name = "env_logger" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fuser" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5910691a0ececcc6eba8bb14029025c2d123e96a53db1533f6a4602861a5aaf7" +dependencies = [ + "libc", + "log", + "memchr", + "page_size", + "pkg-config", + "smallvec", + "users", + "zerocopy", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "libc" +version = "0.2.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10b47e4921acc65b7d824cd92bc7b67a26382d8f4eadf3da65cb50aee6e6f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tarfs-defs" +version = "0.1.0" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "users" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" +dependencies = [ + "libc", + "log", +] + +[[package]] +name = "utarfs" +version = "0.1.0" +dependencies = [ + "clap", + "daemonize", + "env_logger", + "fuser", + "libc", + "log", + "memmap", + "tarfs-defs", + "zerocopy", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "zerocopy" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] diff --git a/src/utarfs/Cargo.toml b/src/utarfs/Cargo.toml new file mode 100644 index 000000000000..b5be7b8366dc --- /dev/null +++ b/src/utarfs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "utarfs" +version = "0.1.0" +edition = "2021" + +[dependencies] +fuser = "0.12.0" +libc = "0.2.51" +log = "0.4.17" +env_logger = "0.6.0" +memmap = "0.7.0" +zerocopy = "0.6.1" +tarfs-defs = { path = "../tardev-snapshotter/tarfs-defs" } +clap = { version = "4.3.2", features = ["derive"] } +daemonize = "0.5.0" diff --git a/src/utarfs/Makefile b/src/utarfs/Makefile new file mode 100644 index 000000000000..78b2b65500a7 --- /dev/null +++ b/src/utarfs/Makefile @@ -0,0 +1,5 @@ +all: + cargo build --release + +clean: + cargo clean diff --git a/src/utarfs/src/fs.rs b/src/utarfs/src/fs.rs new file mode 100644 index 000000000000..ead837c7a7f6 --- /dev/null +++ b/src/utarfs/src/fs.rs @@ -0,0 +1,266 @@ +use fuser::{FileType, Request}; +use libc::{makedev, EINVAL, ENODATA, ENOENT}; +use std::io::{self, Error, ErrorKind}; +use std::time::{Duration, UNIX_EPOCH}; +use std::{ffi::OsStr, mem::size_of, os::unix::ffi::OsStrExt}; +use tarfs_defs::*; +use zerocopy::FromBytes; + +pub struct Tar { + inode_table_offset: u64, + inode_count: u64, + data: memmap::Mmap, +} + +const TARFS_BSIZE: u64 = 4096; + +impl Tar { + pub fn new(data: memmap::Mmap, last_offset: u64) -> io::Result { + if last_offset > data.len() as u64 { + return Err(Error::new( + ErrorKind::UnexpectedEof, + "last_offset beyond end of file", + )); + } + + if last_offset < 512 { + return Err(Error::new( + ErrorKind::UnexpectedEof, + "last_offset too small", + )); + } + + // TODO: Validate that offsets are and that inode count is also ok. + let sb = SuperBlock::read_from_prefix(&data[(last_offset - 512) as usize..]).unwrap(); + Ok(Self { + inode_table_offset: sb.inode_table_offset.into(), + inode_count: sb.inode_count.into(), + data, + }) + } + + fn inode(&self, ino: u64) -> Result { + if ino < 1 || ino > self.inode_count { + return Err(ENOENT); + } + + // TODO: Remove this unwrap and check we're within range. + Ok(Inode::read_from_prefix( + &self.data + [(self.inode_table_offset + (ino - 1) * size_of::() as u64) as usize..], + ) + .unwrap()) + } + + fn attr(&self, ino: u64) -> Result { + let inode = self.inode(ino)?; + let kind = match u16::from(inode.mode) & S_IFMT { + S_IFIFO => FileType::NamedPipe, + S_IFCHR => FileType::CharDevice, + S_IFDIR => FileType::Directory, + S_IFBLK => FileType::BlockDevice, + S_IFREG => FileType::RegularFile, + S_IFLNK => FileType::Symlink, + S_IFSOCK => FileType::Socket, + _ => return Err(ENOENT), + }; + + let d = Duration::from_secs(u64::from(inode.lmtime) | (u64::from(inode.hmtime) << 32)); + let ts = UNIX_EPOCH.checked_add(d).unwrap_or(UNIX_EPOCH); + + let rdev = match kind { + FileType::BlockDevice | FileType::CharDevice => { + let offset: u64 = inode.offset.into(); + makedev((offset >> 32) as _, offset as _) as u32 + } + _ => 0, + }; + + Ok(fuser::FileAttr { + ino, + size: inode.size.into(), + blocks: (u64::from(inode.size) + TARFS_BSIZE - 1) / TARFS_BSIZE, + atime: ts, + mtime: ts, + ctime: ts, + crtime: ts, + kind, + perm: u16::from(inode.mode) & 0o777, + nlink: 1, + uid: inode.owner.into(), + gid: inode.group.into(), + rdev, + flags: 0, + blksize: TARFS_BSIZE as _, + }) + } + + fn for_each( + &self, + parent: u64, + first: i64, + mut cb: impl FnMut(&DirEntry, u64) -> Result, i32>, + ) -> Result, i32> { + let inode = self.inode(parent)?; + + if u16::from(inode.mode) & S_IFMT != S_IFDIR { + return Err(ENOENT); + } + + if first < 0 || first % size_of::() as i64 != 0 { + return Err(ENOENT); + } + + for offset in (first as u64..inode.size.into()).step_by(size_of::()) { + let dentry = DirEntry::read_from_prefix( + &self.data[(u64::from(inode.offset) + offset) as usize..], + ) + .unwrap(); + if let Some(v) = cb(&dentry, offset)? { + return Ok(Some(v)); + } + } + + Ok(None) + } +} + +impl fuser::Filesystem for Tar { + fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: fuser::ReplyEntry) { + let ret = self.for_each(parent, 0, |dentry, _| { + let ename = OsStr::from_bytes( + &self.data[u64::from(dentry.name_offset) as usize..] + [..u64::from(dentry.name_len) as usize], + ); + + Ok(if ename != name { + None + } else { + Some(self.attr(dentry.ino.into())?) + }) + }); + + match ret { + Ok(Some(a)) => reply.entry(&Duration::MAX, &a, 0), + Ok(None) => reply.error(ENOENT), + Err(e) => reply.error(e), + } + } + + fn getattr(&mut self, _req: &Request, ino: u64, reply: fuser::ReplyAttr) { + match self.attr(ino) { + Ok(a) => reply.attr(&Duration::MAX, &a), + Err(e) => reply.error(e), + } + } + + fn read( + &mut self, + _req: &Request, + ino: u64, + _fh: u64, + offset: i64, + size: u32, + _flags: i32, + _lock: Option, + reply: fuser::ReplyData, + ) { + let inode = match self.inode(ino) { + Ok(i) => i, + Err(e) => return reply.error(e), + }; + if u16::from(inode.mode) & S_IFMT != S_IFREG { + return reply.error(ENOENT); + } + let fsize = u64::from(inode.size); + if offset < 0 { + return reply.error(EINVAL); + } + + if offset as u64 >= fsize { + return reply.data(&[]); + } + + let available = fsize - offset as u64; + reply.data( + &self.data[(u64::from(inode.offset) + offset as u64) as usize..] + [..std::cmp::min(available, size.into()) as usize], + ); + } + + fn readlink(&mut self, _req: &Request, ino: u64, reply: fuser::ReplyData) { + let inode = match self.inode(ino) { + Ok(i) => i, + Err(e) => return reply.error(e), + }; + if u16::from(inode.mode) & S_IFMT != S_IFLNK { + return reply.error(ENOENT); + } + reply + .data(&self.data[u64::from(inode.offset) as usize..][..u64::from(inode.size) as usize]); + } + + fn readdir( + &mut self, + _req: &Request, + ino: u64, + _fh: u64, + first: i64, + mut reply: fuser::ReplyDirectory, + ) { + let ret = self.for_each(ino, first, |dentry, offset| { + let etype = match dentry.etype { + DT_FIFO => FileType::NamedPipe, + DT_CHR => FileType::CharDevice, + DT_DIR => FileType::Directory, + DT_BLK => FileType::BlockDevice, + DT_REG => FileType::RegularFile, + DT_LNK => FileType::Symlink, + DT_SOCK => FileType::Socket, + _ => return Ok(None), + }; + + if reply.add( + dentry.ino.into(), + (offset + size_of::() as u64) as i64, + etype, + OsStr::from_bytes( + &self.data[u64::from(dentry.name_offset) as usize..] + [..u64::from(dentry.name_len) as usize], + ), + ) { + Ok(Some(())) + } else { + Ok(None) + } + }); + + match ret { + Err(e) => reply.error(e), + Ok(_) => reply.ok(), + } + } + + fn getxattr( + &mut self, + _req: &Request, + ino: u64, + name: &OsStr, + size: u32, + reply: fuser::ReplyXattr, + ) { + let inode = match self.inode(ino) { + Ok(i) => i, + Err(e) => return reply.error(e), + }; + if inode.flags & inode_flags::OPAQUE == 0 || name != "trusted.overlay.opaque" { + return reply.error(ENODATA); + } + + if size == 0 { + reply.size(1); + } else { + reply.data(b"y"); + } + } +} diff --git a/src/utarfs/src/main.rs b/src/utarfs/src/main.rs new file mode 100644 index 000000000000..06bad1c977fd --- /dev/null +++ b/src/utarfs/src/main.rs @@ -0,0 +1,104 @@ +use clap::Parser; +use fuser::MountOption; +use log::debug; +use std::io::{self, Error, ErrorKind}; +use zerocopy::byteorder::{LE, U32, U64}; +use zerocopy::FromBytes; + +mod fs; + +// TODO: Remove this and import from dm-verity crate. +#[derive(Default, zerocopy::AsBytes, zerocopy::FromBytes, zerocopy::Unaligned)] +#[repr(C)] +pub struct VeritySuperBlock { + pub data_block_size: U32, + pub hash_block_size: U32, + pub data_block_count: U64, +} + +#[derive(Parser, Debug)] +struct Args { + /// The source tarfs file. + source: String, + + /// The directory on which to mount. + directory: String, + + /// The filesystem type. + #[arg(short)] + r#type: Option, + + /// The filesystem options. + #[arg(short, long)] + options: Vec, +} + +fn main() -> io::Result<()> { + env_logger::init(); + let args = Args::parse(); + let mountpoint = std::fs::canonicalize(&args.directory)?; + let file = std::fs::File::open(&args.source)?; + + // Check that the filesystem is tar. + if let Some(t) = &args.r#type { + if t != "tar" { + debug!("Bad file system: {t}"); + return Err(Error::new( + ErrorKind::InvalidInput, + "File system (-t) must be \"tar\"", + )); + } + } + + // Parse all options. + let mut options = Vec::new(); + for opts in &args.options { + for opt in opts.split(',') { + debug!("Parsing option {opt}"); + let fsopt = match opt { + "dev" => MountOption::Dev, + "nodev" => MountOption::NoDev, + "suid" => MountOption::Suid, + "nosuid" => MountOption::NoSuid, + "ro" => MountOption::RO, + "exec" => MountOption::Exec, + "noexec" => MountOption::NoExec, + "atime" => MountOption::Atime, + "noatime" => MountOption::NoAtime, + "dirsync" => MountOption::DirSync, + "sync" => MountOption::Sync, + "async" => MountOption::Async, + "rw" => { + return Err(Error::new( + ErrorKind::InvalidInput, + "Tar file system are always read-only", + )); + } + _ => { + return Err(Error::new( + ErrorKind::InvalidInput, + format!("Unknown option ({opt})"), + )); + } + }; + options.push(fsopt); + } + } + + let contents = unsafe { memmap::Mmap::map(&file)? }; + let vsb = VeritySuperBlock::read_from_prefix(&contents[contents.len() - 512..]).unwrap(); + + debug!("Size: {}", contents.len()); + debug!("Data block size: {}", vsb.data_block_size); + debug!("Hash block size: {}", vsb.hash_block_size); + debug!("Data block count: {}", vsb.data_block_count); + + let sb_offset = u64::from(vsb.data_block_size) * u64::from(vsb.data_block_count); + let tar = fs::Tar::new(contents, sb_offset)?; + + daemonize::Daemonize::new() + .start() + .map_err(|e| Error::new(ErrorKind::Other, e))?; + + fuser::mount2(tar, mountpoint, &options) +}