feat: use gix-negotiate in fetch machinery.

Thanks to it we are finally able to do pack negotiations just like git can,
as many rounds as it takes and with all available algorithms.

Author: Byron

Cargo.lock

@@ -670,7 +670,7 @@ See its [README.md](https://github.com/Byron/gitoxide/blob/main/gix-lock/README.
         * [x] fetch
            * [x] shallow (remains shallow, options to adjust shallow boundary)
            * [ ] a way to auto-explode small packs to avoid them to pile up
-           * [ ] 'ref-in-want'
+           * [x] 'ref-in-want'
            * [ ] standard negotiation algorithms (right now we only have a 'naive' one)
         * [ ] push
         * [x] ls-refs

crate-status.md

@@ -131,6 +131,7 @@ gix-object = { version = "^0.29.2", path = "../gix-object" }
 gix-actor = { version = "^0.20.0", path = "../gix-actor" }
 gix-pack = { version = "^0.35.0", path = "../gix-pack", features = ["object-cache-dynamic"] }
 gix-revision = { version = "^0.14.0", path = "../gix-revision" }
+gix-negotiate = { version = "0.1.0", path = "../gix-negotiate" }
 
 gix-path = { version = "^0.8.0", path = "../gix-path" }
 gix-url = { version = "^0.18.0", path = "../gix-url" }

gix/Cargo.toml

@@ -1,20 +1,261 @@
 use crate::remote::fetch;
-use crate::remote::fetch::negotiate::Algorithm;
+use crate::remote::fetch::Shallow;
+use gix_negotiate::Flags;
+use gix_pack::Find;
+use std::borrow::Cow;
+type Queue = gix_revision::PriorityQueue<gix_revision::graph::CommitterTimestamp, gix_hash::ObjectId>;
 
 /// The error returned during negotiation.
 #[derive(Debug, thiserror::Error)]
 #[allow(missing_docs)]
 pub enum Error {
     #[error("We were unable to figure out what objects the server should send after {rounds} round(s)")]
     NegotiationFailed { rounds: usize },
+    #[error(transparent)]
+    LookupCommitInGraph(#[from] gix_revision::graph::lookup::commit::Error),
+    #[error(transparent)]
+    InitRefsIterator(#[from] crate::reference::iter::init::Error),
+    #[error(transparent)]
+    InitRefsIteratorPlatform(#[from] crate::reference::iter::Error),
+    #[error(transparent)]
+    ObtainRefDuringIteration(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error(transparent)]
+    LoadIndex(#[from] gix_odb::store::load_index::Error),
+}
+
+#[must_use]
+pub(crate) enum Action {
+    /// None of the remote refs moved compared to our last recorded state (via tracking refs), so there is nothing to do at all,
+    /// not even a ref update.
+    NoChange,
+    /// Don't negotiate, don't fetch the pack, skip right to updating the references.
+    ///
+    /// This happens if we already have all local objects even though the server seems to have changed.
+    SkipToRefUpdate,
+    /// We can't know for sure if fetching *is not* needed, so we go ahead and negotiate.
+    MustNegotiate {
+        /// Each `ref_map.mapping` has a slot here which is `true` if we have the object the remote ref points to locally.
+        remote_ref_target_known: Vec<bool>,
+    },
+}
+
+/// This function is modeled after the similarly named one in the git codebase to do the following:
+///
+/// * figure out all advertised refs on the remote *that we already have* and keep track of the oldest one as cutoff date.
+/// * mark all of our own refs as tips for a traversal.
+/// * mark all their parents, recursively, up to (and including) the cutoff date up to which we have seen the servers commit that we have.
+/// * pass all known-to-be-common-with-remote commits to the negotiator as common commits.
+///
+/// This is done so that we already find the most recent common commits, even if we are ahead, which is *potentially* better than
+/// what we would get if we would rely on tracking refs alone, particularly if one wouldn't trust the tracking refs for some reason.
+///
+/// Note that git doesn't trust its own tracking refs as the server *might* have changed completely, for instance by force-pushing, so
+/// marking our local tracking refs as known is something that's actually not proven to be correct so it's not done.
+///
+/// Additionally, it does what's done in `transport.c` and we check if a fetch is actually needed as at least one advertised ref changed.
+///
+/// Finally, we also mark tips in the `negotiator` in one go to avoid traversing all refs twice, since we naturally encounter all tips during
+/// our own walk.
+///
+/// Return whether or not we should negotiate, along with a queue for later use.
+pub(crate) fn mark_complete_and_common_ref(
+    repo: &crate::Repository,
+    negotiator: &mut dyn gix_negotiate::Negotiator,
+    graph: &mut gix_negotiate::Graph<'_>,
+    ref_map: &fetch::RefMap,
+    shallow: &fetch::Shallow,
+) -> Result<Action, Error> {
+    if let fetch::Shallow::Deepen(0) = shallow {
+        // Avoid deepening (relative) with zero as it seems to upset the server. Git also doesn't actually
+        // perform the negotiation for some reason (couldn't find it in code).
+        return Ok(Action::NoChange);
+    }
+
+    // Compute the cut-off date by checking which of the refs advertised (and matched in refspecs) by the remote we have,
+    // and keep the oldest one.
+    let mut cutoff_date = None::<gix_revision::graph::CommitterTimestamp>;
+    let mut num_mappings_with_change = 0;
+    let mut remote_ref_target_known: Vec<bool> = std::iter::repeat(false).take(ref_map.mappings.len()).collect();
+
+    for (mapping_idx, mapping) in ref_map.mappings.iter().enumerate() {
+        let want_id = mapping.remote.as_id();
+        let have_id = mapping.local.as_ref().and_then(|name| {
+            // this is the only time git uses the peer-id.
+            let r = repo.find_reference(name).ok()?;
+            r.target().try_id().map(ToOwned::to_owned)
+        });
+
+        // Like git, we don't let known unchanged mappings participate in the tree traversal
+        if want_id.zip(have_id).map_or(true, |(want, have)| want != have) {
+            num_mappings_with_change += 1;
+        }
+
+        if let Some(commit) = want_id
+            .and_then(|id| graph.try_lookup_or_insert_commit(id.into(), |_| {}).transpose())
+            .transpose()?
+        {
+            remote_ref_target_known[mapping_idx] = true;
+            cutoff_date = cutoff_date.unwrap_or_default().max(commit.commit_time).into();
+        } else if want_id.map_or(false, |maybe_annotated_tag| repo.objects.contains(maybe_annotated_tag)) {
+            remote_ref_target_known[mapping_idx] = true;
+        }
+    }
+
+    // If any kind of shallowing operation is desired, the server may still create a pack for us.
+    if matches!(shallow, Shallow::NoChange) {
+        if num_mappings_with_change == ref_map.mappings.len() {
+            return Ok(Action::NoChange);
+        } else if remote_ref_target_known.iter().all(|known| *known) {
+            return Ok(Action::SkipToRefUpdate);
+        }
+    }
+
+    // color our commits as complete as identified by references, unconditionally
+    // (`git` is conditional here based on `deepen`, but it doesn't make sense and it's hard to extract from history when that happened).
+    let mut queue = Queue::new();
+    mark_all_refs_in_repo(repo, graph, &mut queue, Flags::COMPLETE)?;
+    mark_alternate_complete(repo, graph, &mut queue)?;
+    // Keep track of the tips, which happen to be on our queue right, before we traverse the graph with cutoff.
+    let tips = if let Some(cutoff) = cutoff_date {
+        let tips = Cow::Owned(queue.clone());
+        // color all their parents up to the cutoff date, the oldest commit we know the server has.
+        mark_recent_complete_commits(&mut queue, graph, cutoff)?;
+        tips
+    } else {
+        Cow::Borrowed(&queue)
+    };
+
+    // mark all complete advertised refs as common refs.
+    for mapping in ref_map
+        .mappings
+        .iter()
+        .zip(remote_ref_target_known.iter().copied())
+        // We need this filter as the graph wouldn't contain annotated tags.
+        .filter_map(|(mapping, known)| (!known).then_some(mapping))
+    {
+        let want_id = mapping.remote.as_id();
+        if let Some(common_id) = want_id
+            .and_then(|id| graph.get(id).map(|c| (c, id)))
+            .filter(|(c, _)| c.data.flags.contains(Flags::COMPLETE))
+            .map(|(_, id)| id)
+        {
+            negotiator.known_common(common_id.into(), graph)?;
+        }
+    }
+
+    // As negotiators currently may rely on getting `known_common` calls first and tips after, we adhere to that which is the only
+    // reason we cached the set of tips.
+    for tip in tips.iter_unordered() {
+        negotiator.add_tip(*tip, graph)?;
+    }
+
+    Ok(Action::MustNegotiate {
+        remote_ref_target_known,
+    })
+}
+
+/// Add all `wants` to `arguments`, which is the unpeeled direct target that the advertised remote ref points to.
+pub(crate) fn add_wants(
+    arguments: &mut gix_protocol::fetch::Arguments,
+    mappings: &[fetch::Mapping],
+    mapping_known: &[bool],
+) {
+    let wants = mappings
+        .iter()
+        .zip(mapping_known)
+        .filter_map(|(m, known)| (!*known).then_some(m));
+    for want in wants {
+        if !arguments.can_use_ref_in_want() || matches!(want.remote, fetch::Source::ObjectId(_)) {
+            if let Some(id) = want.remote.as_id() {
+                arguments.want(id);
+            }
+        } else {
+            arguments.want_ref(
+                want.remote
+                    .as_name()
+                    .expect("name available if this isn't an object id"),
+            )
+        }
+    }
+}
+
+/// Remove all commits that are more recent than the cut-off, which is the commit time of the oldest common commit we have with the server.
+fn mark_recent_complete_commits(
+    queue: &mut Queue,
+    graph: &mut gix_negotiate::Graph<'_>,
+    cutoff: gix_revision::graph::CommitterTimestamp,
+) -> Result<(), Error> {
+    while let Some(id) = queue
+        .peek()
+        .and_then(|(commit_time, id)| (commit_time >= &cutoff).then_some(*id))
+    {
+        queue.pop();
+        let commit = graph.get(&id).expect("definitely set when adding tips or parents");
+        for parent_id in commit.parents.clone() {
+            let mut was_complete = false;
+            if let Some(parent) = graph
+                .try_lookup_or_insert_commit(parent_id, |md| {
+                    was_complete = md.flags.contains(Flags::COMPLETE);
+                    md.flags |= Flags::COMPLETE
+                })?
+                .filter(|_| !was_complete)
+            {
+                queue.insert(parent.commit_time, parent_id)
+            }
+        }
+    }
+    Ok(())
+}
+
+fn mark_all_refs_in_repo(
+    repo: &crate::Repository,
+    graph: &mut gix_negotiate::Graph<'_>,
+    queue: &mut Queue,
+    mark: Flags,
+) -> Result<(), Error> {
+    for local_ref in repo.references()?.all()?.peeled() {
+        let local_ref = local_ref?;
+        let id = local_ref.id().detach();
+        let mut is_complete = false;
+        if let Some(commit) = graph
+            .try_lookup_or_insert_commit(id, |md| {
+                is_complete = md.flags.contains(Flags::COMPLETE);
+                md.flags |= mark
+            })?
+            .filter(|_| !is_complete)
+        {
+            queue.insert(commit.commit_time, id);
+        };
+    }
+    Ok(())
+}
+
+fn mark_alternate_complete(
+    repo: &crate::Repository,
+    graph: &mut gix_negotiate::Graph<'_>,
+    queue: &mut Queue,
+) -> Result<(), Error> {
+    for alternate_repo in repo
+        .objects
+        .store_ref()
+        .alternate_db_paths()?
+        .into_iter()
+        .filter_map(|path| {
+            path.ancestors()
+                .nth(1)
+                .and_then(|git_dir| crate::open_opts(git_dir, repo.options.clone()).ok())
+        })
+    {
+        mark_all_refs_in_repo(&alternate_repo, graph, queue, Flags::ALTERNATE | Flags::COMPLETE)?;
+    }
+    Ok(())
 }
 
 /// Negotiate one round with `algo` by looking at `ref_map` and adjust `arguments` to contain the haves and wants.
 /// If this is not the first round, the `previous_response` is set with the last recorded server response.
 /// Returns `true` if the negotiation is done from our side so the server won't keep asking.
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn one_round(
-    algo: Algorithm,
     round: usize,
     repo: &crate::Repository,
     ref_map: &fetch::RefMap,
@@ -23,64 +264,54 @@ pub(crate) fn one_round(
     _previous_response: Option<&gix_protocol::fetch::Response>,
     shallow: Option<&fetch::Shallow>,
 ) -> Result<bool, Error> {
-    let tag_refspec_to_ignore = fetch_tags
-        .to_refspec()
-        .filter(|_| matches!(fetch_tags, crate::remote::fetch::Tags::Included));
     if let Some(fetch::Shallow::Deepen(0)) = shallow {
         // Avoid deepening (relative) with zero as it seems to upset the server. Git also doesn't actually
         // perform the negotiation for some reason (couldn't find it in code).
         return Ok(true);
     }
 
-    match algo {
-        Algorithm::Noop | Algorithm::Skipping | Algorithm::Consecutive => {
-            todo!()
+    let tag_refspec_to_ignore = fetch_tags
+        .to_refspec()
+        .filter(|_| matches!(fetch_tags, crate::remote::fetch::Tags::Included));
+
+    // Use actual negotiation code here, this is the NAIVE implementation/hack
+    assert_eq!(round, 1, "Naive always finishes after the first round, it claims.");
+    let mut has_missing_tracking_branch = false;
+    for mapping in &ref_map.mappings {
+        if tag_refspec_to_ignore.map_or(false, |tag_spec| {
+            mapping
+                .spec_index
+                .implicit_index()
+                .and_then(|idx| ref_map.extra_refspecs.get(idx))
+                .map_or(false, |spec| spec.to_ref() == tag_spec)
+        }) {
+            continue;
         }
-        Algorithm::Naive => {
-            assert_eq!(round, 1, "Naive always finishes after the first round, it claims.");
-            let mut has_missing_tracking_branch = false;
-            for mapping in &ref_map.mappings {
-                if tag_refspec_to_ignore.map_or(false, |tag_spec| {
-                    mapping
-                        .spec_index
-                        .implicit_index()
-                        .and_then(|idx| ref_map.extra_refspecs.get(idx))
-                        .map_or(false, |spec| spec.to_ref() == tag_spec)
-                }) {
-                    continue;
-                }
-                let have_id = mapping.local.as_ref().and_then(|name| {
-                    repo.find_reference(name)
-                        .ok()
-                        .and_then(|r| r.target().try_id().map(ToOwned::to_owned))
-                });
-                match have_id {
-                    Some(have_id) => {
-                        if let Some(want_id) = mapping.remote.as_id() {
-                            if want_id != have_id {
-                                arguments.want(want_id);
-                                arguments.have(have_id);
-                            }
-                        }
-                    }
-                    None => {
-                        if let Some(want_id) = mapping.remote.as_id() {
-                            arguments.want(want_id);
-                            has_missing_tracking_branch = true;
-                        }
-                    }
-                }
+        let have_id = mapping.local.as_ref().and_then(|name| {
+            repo.find_reference(name)
+                .ok()
+                .and_then(|r| r.target().try_id().map(ToOwned::to_owned))
+        });
+        match (have_id, mapping.remote.as_id()) {
+            (Some(have_id), Some(want_id)) if want_id != have_id => {
+                arguments.want(want_id);
+                arguments.have(have_id);
             }
+            (None, Some(want_id)) => {
+                arguments.want(want_id);
+                has_missing_tracking_branch = true;
+            }
+            _ => {}
+        }
+    }
 
-            if has_missing_tracking_branch || (shallow.is_some() && arguments.is_empty()) {
-                if let Ok(Some(r)) = repo.head_ref() {
-                    if let Some(id) = r.target().try_id() {
-                        arguments.have(id);
-                        arguments.want(id);
-                    }
-                }
+    if has_missing_tracking_branch || (shallow.is_some() && arguments.is_empty()) {
+        if let Ok(Some(r)) = repo.head_ref() {
+            if let Some(id) = r.target().try_id() {
+                arguments.have(id);
+                arguments.want(id);
             }
-            Ok(true)
         }
     }
+    Ok(true)
 }