feat: patchable init mirroring

Cargo.toml

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.dependencies]
 clap = { version = "4.5.27", features = ["derive"] }
-git2 = "0.20.0"
+git2 = "0.20.1"
 serde = { version = "1.0.217", features = ["derive"] }
 snafu = "0.8.5"
 tempfile = "3.16.0"

druid/stackable/patches/30.0.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/druid.git"
 base = "09d36ee324747f1407705c27618b6d415c3fa8a9"
+mirror = "https://github.com/stackabletech/druid.git"

druid/stackable/patches/30.0.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/druid.git"
 base = "a30af7a91d528e5c3a90356a5592abc7119191c6"
+mirror = "https://github.com/stackabletech/druid.git"

druid/stackable/patches/31.0.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/druid.git"
 base = "520482cb9638e452b0553595b4f29bb397a63758"
+mirror = "https://github.com/stackabletech/druid.git"

hadoop/stackable/patches/3.3.4/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hadoop.git"
 base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/3.3.6/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hadoop.git"
 base = "1be78238728da9266a4f88195058f08fd012bf9c"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/3.4.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hadoop.git"
 base = "bd8b77f398f626bb7791783192ee7a5dfaeec760"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/3.4.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hadoop.git"
 base = "4d7825309348956336b8f06a08322b78422849b1"
+mirror = "https://github.com/stackabletech/hadoop.git"

hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hbase-operator-tools.git"
 base = "478af00af79f82624264fd2bb447b97fecc8e790"
+mirror = "https://github.com/stackabletech/hbase-operator-tools.git"

hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hbase-operator-tools.git"
 base = "fd5a5fb90755949a90c502c76de8313130403fa3"
+mirror = "https://github.com/stackabletech/hbase-operator-tools.git"

hbase/phoenix/stackable/patches/5.2.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/phoenix.git"
 base = "b738d66cb5863b759bb98eaa417b3b5731d41f95"
+mirror = "https://github.com/stackabletech/phoenix.git"

hbase/stackable/patches/2.4.18/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hbase.git"
 base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e"
+mirror = "https://github.com/stackabletech/hbase.git"

hbase/stackable/patches/2.6.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hbase.git"
 base = "de99f8754135ea69adc39da48d2bc2b2710a5366"
+mirror = "https://github.com/stackabletech/hbase.git"

hbase/stackable/patches/2.6.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hbase.git"
 base = "7ed50b4dd742269a78875fb32112215f831284ff"
+mirror = "https://github.com/stackabletech/hbase.git"

hive/stackable/patches/3.1.3/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hive.git"
 base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975"
+mirror = "https://github.com/stackabletech/hive.git"

hive/stackable/patches/4.0.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hive.git"
 base = "183f8cb41d3dbed961ffd27999876468ff06690c"
+mirror = "https://github.com/stackabletech/hive.git"

hive/stackable/patches/4.0.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hive.git"
 base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264"
+mirror = "https://github.com/stackabletech/hive.git"

kafka/stackable/patches/3.7.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/kafka.git"
 base = "e2494e6ffb89f8288ed2aeb9b5596c755210bffd"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/3.7.2/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/kafka.git"
 base = "79a8f2b5f44f9d5a6867190d1dfc463d08d60b82"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/3.8.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/kafka.git"
 base = "771b9576b00ecf5b64ab6e8bedf04156fbdb5cd6"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/3.9.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/kafka.git"
 base = "84caaa6e9da06435411510a81fa321d4f99c351f"
+mirror = "https://github.com/stackabletech/kafka.git"

nifi/stackable/patches/1.27.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/nifi.git"
 base = "e0c4461d90bd4f6e5f2b81765bcff5cd97ed3e18"
+mirror = "https://github.com/stackabletech/nifi.git"

nifi/stackable/patches/1.28.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/nifi.git"
 base = "883338fe28883733417d10f6ffa9319e75f5ea06"
+mirror = "https://github.com/stackabletech/nifi.git"

nifi/stackable/patches/2.2.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/nifi.git"
 base = "b33ffac8aa10992482f7fa54e6cfccc46a5e8e27"
+mirror = "https://github.com/stackabletech/nifi.git"

omid/stackable/patches/1.1.0/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/phoenix-omid.git"
 base = "3b9e16b7537adbc90a7403507fb8aabd8d1fab0c"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/1.1.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/phoenix-omid.git"
 base = "cd546d58d93f380fec9bf65dbfa618f53493f662"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/1.1.2/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/phoenix-omid.git"
 base = "88812c9e127063f3b3016262f81ea3e8b48ec157"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/phoenix-omid.git"
 base = "c3e4da626fdb27060fd139a809e057965e52d163"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

rust/patchable/src/main.rs

@@ -9,7 +9,7 @@ use std::{fs::File, io::Write, path::PathBuf};
 
 use git2::{Oid, Repository};
 use serde::{Deserialize, Serialize};
-use snafu::{OptionExt, ResultExt as _, Snafu};
+use snafu::{ensure, OptionExt, ResultExt as _, Snafu};
 use tracing_indicatif::IndicatifLayer;
 use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _};
 
@@ -29,6 +29,7 @@ struct ProductVersionConfig {
     upstream: String,
     #[serde(with = "utils::oid_serde")]
     base: Oid,
+    mirror: Option<String>,
 }
 
 struct ProductVersionContext {
@@ -149,6 +150,11 @@ enum Cmd {
         /// Refs (such as tags and branches) will be resolved to commit IDs.
         #[clap(long)]
         base: String,
+
+        /// Assume a mirror exists at stackabletech/<repo_name> and push the base ref to it.
+        /// The mirror URL will be stored in patchable.toml instead of the original upstream.
+        #[clap(long)]
+        mirrored: bool,
     },
 
     /// Shows the patch directory for a given product version
@@ -197,6 +203,18 @@ pub enum Error {
         path: PathBuf,
     },
 
+    #[snafu(display("failed to parse upstream URL {url:?} to extract repository name"))]
+    ParseUpstreamUrl { url: String },
+    #[snafu(display("failed to add temporary mirror remote for {url:?}"))]
+    AddMirrorRemote { source: git2::Error, url: String },
+    #[snafu(display("failed to push commit {commit} (as {refspec}) to mirror {url:?}"))]
+    PushToMirror {
+        source: git2::Error,
+        url: String,
+        refspec: String,
+        commit: Oid,
+    },
+
     #[snafu(display("failed to find images repository"))]
     FindImagesRepo { source: repo::Error },
     #[snafu(display("images repository has no work directory"))]
@@ -287,7 +305,7 @@ fn main() -> Result<()> {
             let base_commit = repo::resolve_and_fetch_commitish(
                 &product_repo,
                 &config.base.to_string(),
-                &config.upstream,
+                config.mirror.as_deref().unwrap_or(&config.upstream),
             )
             .context(FetchBaseCommitSnafu)?;
             let base_branch = ctx.base_branch();
@@ -397,7 +415,12 @@ fn main() -> Result<()> {
             );
         }
 
-        Cmd::Init { pv, upstream, base } => {
+        Cmd::Init {
+            pv,
+            upstream,
+            base,
+            mirrored,
+        } => {
             let ctx = ProductVersionContext {
                 pv,
                 images_repo_root,
@@ -414,13 +437,85 @@ fn main() -> Result<()> {
             // --base can be a reference, but patchable.toml should always have a resolved commit id,
             // so that it cannot be changed under our feet (without us knowing so, anyway...).
             tracing::info!(?base, "resolving base commit-ish");
-            let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream)
-                .context(FetchBaseCommitSnafu)?;
+            let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?;
+            let mut upstream_mirror = None;
+
+            if mirrored {
+                // Parse e.g. "https://github.com/apache/druid.git" into "druid"
+                let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?;
+
+                ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream });
+
+                let mirror_url = format!("https://github.com/stackabletech/{}.git", repo_name);
+                tracing::info!(%mirror_url, "using mirror repository");
+
+                // Add mirror remote
+                let mut mirror_remote = product_repo
+                    .remote_anonymous(&mirror_url)
+                    .context(AddMirrorRemoteSnafu { url: mirror_url.clone() })?;
+
+                // Push the base commit to the mirror
+                tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror");
+                let mut callbacks = git2::RemoteCallbacks::new();
+                callbacks.credentials(|_url, username_from_url, _allowed_types| {
+                    git2::Cred::credential_helper(
+                        &git2::Config::open_default().unwrap(), // Use default git config
+                        _url,
+                        username_from_url,
+                    )
+                });
+
+                // Add progress tracking for push operation
+                let (span_push, mut quant_push) = utils::setup_progress_tracking(tracing::info_span!("pushing"));
+                callbacks.push_transfer_progress(move |current, total, _| {
+                    if total > 0 {
+                        quant_push.update_span_progress(
+                            current,
+                            total,
+                            &span_push,
+                        );
+                    }
+                });
+
+                let mut push_options = git2::PushOptions::new();
+                push_options.remote_callbacks(callbacks);
+
+                // Check if the reference is a tag or branch by inspecting the git repository
+                let refspec = {
+                    let tag_ref = format!("refs/tags/{}", base);
+                    let is_tag = product_repo
+                        .find_reference(&tag_ref)
+                        .is_ok();
+
+                    if is_tag {
+                        format!("{}:refs/tags/{}", base_commit, base)
+                    } else {
+                        // Assume it's a branch as default behavior
+                        format!("{}:refs/heads/{}", base_commit, base)
+                    }
+                };
+
+                tracing::info!(refspec = refspec, "constructed push refspec");
+
+                mirror_remote
+                    .push(&[&refspec], Some(&mut push_options))
+                    .context(PushToMirrorSnafu {
+                        url: mirror_url.clone(),
+                        refspec: &refspec,
+                        commit: base_commit,
+                    })?;
+
+                tracing::info!("successfully pushed base ref to mirror");
+
+                upstream_mirror = Some(mirror_url);
+            };
+
             tracing::info!(?base, base.commit = ?base_commit, "resolved base commit");
 
             tracing::info!("saving configuration");
             let config = ProductVersionConfig {
                 upstream,
+                mirror: upstream_mirror,
                 base: base_commit,
             };
             let config_path = ctx.config_path();

rust/patchable/src/repo.rs

@@ -5,11 +5,10 @@ use git2::{
     WorktreeAddOptions,
 };
 use snafu::{ResultExt, Snafu};
-use tracing_indicatif::span_ext::IndicatifSpanExt;
 
 use crate::{
     error::{self, CommitRef},
-    utils::{progress_bar_style, Quantizer},
+    utils::setup_progress_tracking,
 };
 
 #[derive(Debug, Snafu)]
@@ -149,15 +148,11 @@ pub fn resolve_and_fetch_commitish(
                 error = &err as &dyn std::error::Error,
                 "base commit not found locally, fetching from upstream"
             );
-            let span_recv = tracing::info_span!("receiving");
-            let span_index = tracing::info_span!("indexing");
-            span_recv.pb_set_style(&progress_bar_style());
-            span_index.pb_set_style(&progress_bar_style());
-            let _ = span_recv.enter();
-            let _ = span_index.enter();
+
+            let (span_recv, mut quant_recv) = setup_progress_tracking(tracing::info_span!("receiving"));
+            let (span_index, mut quant_index) = setup_progress_tracking(tracing::info_span!("indexing"));
+
             let mut callbacks = RemoteCallbacks::new();
-            let mut quant_recv = Quantizer::percent();
-            let mut quant_index = Quantizer::percent();
             callbacks.transfer_progress(move |progress| {
                 quant_recv.update_span_progress(
                     progress.received_objects(),
@@ -171,6 +166,7 @@ pub fn resolve_and_fetch_commitish(
                 );
                 true
             });
+
             repo.remote_anonymous(upstream_url)
                 .context(CreateRemoteSnafu {
                     repo,
@@ -180,7 +176,9 @@ pub fn resolve_and_fetch_commitish(
                     &[commitish],
                     Some(
                         FetchOptions::new()
-                            .update_fetchhead(true)
+                            // Tags need to be present to later determine whether `commitish` is a tag or not
+                            // Patchable needs to know this when initializing a repository with the `--mirrored` option (to construct the refspec)
+                            .download_tags(git2::AutotagOption::Auto)
                             .remote_callbacks(callbacks)
                             // TODO: could be 1, CLI option maybe?
                             .depth(0),
@@ -193,8 +191,7 @@ pub fn resolve_and_fetch_commitish(
                     refs: vec![commitish.to_string()],
                 })?;
             tracing::info!("fetched base commit");
-            // FETCH_HEAD is written by Remote::fetch to be the last reference fetched
-            repo.revparse_single("FETCH_HEAD")
+            repo.revparse_single(commitish)
                 .and_then(|obj| obj.peel_to_commit())
         }
         Err(err) => Err(err),

rust/patchable/src/utils.rs

@@ -4,13 +4,6 @@ use git2::Repository;
 use tracing::Span;
 use tracing_indicatif::{span_ext::IndicatifSpanExt, style::ProgressStyle};
 
-pub fn progress_bar_style() -> ProgressStyle {
-    ProgressStyle::with_template(
-        "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%",
-    )
-    .expect("hard-coded template should be valid")
-}
-
 /// Runs a function whenever a `value` changes "enough".
 ///
 /// See [`Self::update`], and especially [`Self::update_span_progress`].
@@ -94,3 +87,14 @@ pub mod oid_serde {
             .and_then(|oid| Oid::from_str(&oid).map_err(<D::Error as serde::de::Error>::custom))
     }
 }
+
+/// Sets up progress tracking for Git operations with a progress bar.
+pub fn setup_progress_tracking(span: tracing::Span) -> (tracing::Span, Quantizer) {
+    span.pb_set_style(&ProgressStyle::with_template(
+        "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%",
+    )
+    .expect("hard-coded template should be valid"));
+    let _ = span.enter();
+    let quantizer = Quantizer::percent();
+    (span, quantizer)
+}

spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/hbase-connectors.git"
 base = "e5217d13ed729703580ff2d1b02378ada2d94f4d"
+mirror = "https://github.com/stackabletech/hbase-connectors.git"

spark-k8s/stackable/patches/3.5.2/patchable.toml

@@ -1,2 +1,3 @@
 upstream = "https://github.com/apache/spark.git"
 base = "bb7846dd487f259994fdc69e18e03382e3f64f42"
+mirror = "https://github.com/stackabletech/spark.git"