Skip to content

Commit 42cc748

Browse files
jonathantanmygitster
authored andcommitted
negotiator/skipping: skip commits during fetch
Introduce a new negotiation algorithm used during fetch that skips commits in an effort to find common ancestors faster. The skips grow similarly to the Fibonacci sequence as the commit walk proceeds further away from the tips. The skips may cause unnecessary commits to be included in the packfile, but the negotiation step typically ends more quickly. Usage of this algorithm is guarded behind the configuration flag fetch.negotiationAlgorithm. Signed-off-by: Jonathan Tan <[email protected]> Signed-off-by: Junio C Hamano <[email protected]>
1 parent ec06283 commit 42cc748

File tree

8 files changed

+461
-4
lines changed

8 files changed

+461
-4
lines changed

Documentation/config.txt

+9
Original file line numberDiff line numberDiff line change
@@ -1491,6 +1491,15 @@ fetch.output::
14911491
`full` and `compact`. Default value is `full`. See section
14921492
OUTPUT in linkgit:git-fetch[1] for detail.
14931493

1494+
fetch.negotiationAlgorithm::
1495+
Control how information about the commits in the local repository is
1496+
sent when negotiating the contents of the packfile to be sent by the
1497+
server. Set to "skipping" to use an algorithm that skips commits in an
1498+
effort to converge faster, but may result in a larger-than-necessary
1499+
packfile; any other value instructs Git to use the default algorithm
1500+
that never skips commits (unless the server has acknowledged it or one
1501+
of its descendants).
1502+
14941503
format.attach::
14951504
Enable multipart/mixed attachments as the default for
14961505
'format-patch'. The value can also be a double quoted string

Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -893,6 +893,7 @@ LIB_OBJS += merge-recursive.o
893893
LIB_OBJS += mergesort.o
894894
LIB_OBJS += name-hash.o
895895
LIB_OBJS += negotiator/default.o
896+
LIB_OBJS += negotiator/skipping.o
896897
LIB_OBJS += notes.o
897898
LIB_OBJS += notes-cache.o
898899
LIB_OBJS += notes-merge.o

fetch-negotiator.c

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
#include "git-compat-util.h"
22
#include "fetch-negotiator.h"
33
#include "negotiator/default.h"
4+
#include "negotiator/skipping.h"
45

5-
void fetch_negotiator_init(struct fetch_negotiator *negotiator)
6+
void fetch_negotiator_init(struct fetch_negotiator *negotiator,
7+
const char *algorithm)
68
{
9+
if (algorithm && !strcmp(algorithm, "skipping")) {
10+
skipping_negotiator_init(negotiator);
11+
return;
12+
}
713
default_negotiator_init(negotiator);
814
}

fetch-negotiator.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ struct fetch_negotiator {
5252
void *data;
5353
};
5454

55-
void fetch_negotiator_init(struct fetch_negotiator *negotiator);
55+
void fetch_negotiator_init(struct fetch_negotiator *negotiator,
56+
const char *algorithm);
5657

5758
#endif

fetch-pack.c

+5-2
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ static int agent_supported;
3333
static int server_supports_filtering;
3434
static struct lock_file shallow_lock;
3535
static const char *alternate_shallow_file;
36+
static char *negotiation_algorithm;
3637

3738
/* Remember to update object flag allocation in object.h */
3839
#define COMPLETE (1U << 0)
@@ -889,7 +890,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
889890
const char *agent_feature;
890891
int agent_len;
891892
struct fetch_negotiator negotiator;
892-
fetch_negotiator_init(&negotiator);
893+
fetch_negotiator_init(&negotiator, negotiation_algorithm);
893894

894895
sort_ref_list(&ref, ref_compare_name);
895896
QSORT(sought, nr_sought, cmp_ref_by_name);
@@ -1271,7 +1272,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
12711272
int in_vain = 0;
12721273
int haves_to_send = INITIAL_FLUSH;
12731274
struct fetch_negotiator negotiator;
1274-
fetch_negotiator_init(&negotiator);
1275+
fetch_negotiator_init(&negotiator, negotiation_algorithm);
12751276
packet_reader_init(&reader, fd[0], NULL, 0,
12761277
PACKET_READ_CHOMP_NEWLINE);
12771278

@@ -1350,6 +1351,8 @@ static void fetch_pack_config(void)
13501351
git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
13511352
git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
13521353
git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1354+
git_config_get_string("fetch.negotiationalgorithm",
1355+
&negotiation_algorithm);
13531356

13541357
git_config(git_default_config, NULL);
13551358
}

negotiator/skipping.c

+250
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
#include "cache.h"
2+
#include "skipping.h"
3+
#include "../commit.h"
4+
#include "../fetch-negotiator.h"
5+
#include "../prio-queue.h"
6+
#include "../refs.h"
7+
#include "../tag.h"
8+
9+
/* Remember to update object flag allocation in object.h */
10+
/*
11+
* Both us and the server know that both parties have this object.
12+
*/
13+
#define COMMON (1U << 2)
14+
/*
15+
* The server has told us that it has this object. We still need to tell the
16+
* server that we have this object (or one of its descendants), but since we are
17+
* going to do that, we do not need to tell the server about its ancestors.
18+
*/
19+
#define ADVERTISED (1U << 3)
20+
/*
21+
* This commit has entered the priority queue.
22+
*/
23+
#define SEEN (1U << 4)
24+
/*
25+
* This commit has left the priority queue.
26+
*/
27+
#define POPPED (1U << 5)
28+
29+
static int marked;
30+
31+
/*
32+
* An entry in the priority queue.
33+
*/
34+
struct entry {
35+
struct commit *commit;
36+
37+
/*
38+
* Used only if commit is not COMMON.
39+
*/
40+
uint16_t original_ttl;
41+
uint16_t ttl;
42+
};
43+
44+
struct data {
45+
struct prio_queue rev_list;
46+
47+
/*
48+
* The number of non-COMMON commits in rev_list.
49+
*/
50+
int non_common_revs;
51+
};
52+
53+
static int compare(const void *a_, const void *b_, void *unused)
54+
{
55+
const struct entry *a = a_;
56+
const struct entry *b = b_;
57+
return compare_commits_by_commit_date(a->commit, b->commit, NULL);
58+
}
59+
60+
static struct entry *rev_list_push(struct data *data, struct commit *commit, int mark)
61+
{
62+
struct entry *entry;
63+
commit->object.flags |= mark | SEEN;
64+
65+
entry = xcalloc(1, sizeof(*entry));
66+
entry->commit = commit;
67+
prio_queue_put(&data->rev_list, entry);
68+
69+
if (!(mark & COMMON))
70+
data->non_common_revs++;
71+
return entry;
72+
}
73+
74+
static int clear_marks(const char *refname, const struct object_id *oid,
75+
int flag, void *cb_data)
76+
{
77+
struct object *o = deref_tag(parse_object(oid), refname, 0);
78+
79+
if (o && o->type == OBJ_COMMIT)
80+
clear_commit_marks((struct commit *)o,
81+
COMMON | ADVERTISED | SEEN | POPPED);
82+
return 0;
83+
}
84+
85+
/*
86+
* Mark this SEEN commit and all its SEEN ancestors as COMMON.
87+
*/
88+
static void mark_common(struct data *data, struct commit *c)
89+
{
90+
struct commit_list *p;
91+
92+
if (c->object.flags & COMMON)
93+
return;
94+
c->object.flags |= COMMON;
95+
if (!(c->object.flags & POPPED))
96+
data->non_common_revs--;
97+
98+
if (!c->object.parsed)
99+
return;
100+
for (p = c->parents; p; p = p->next) {
101+
if (p->item->object.flags & SEEN)
102+
mark_common(data, p->item);
103+
}
104+
}
105+
106+
/*
107+
* Ensure that the priority queue has an entry for to_push, and ensure that the
108+
* entry has the correct flags and ttl.
109+
*
110+
* This function returns 1 if an entry was found or created, and 0 otherwise
111+
* (because the entry for this commit had already been popped).
112+
*/
113+
static int push_parent(struct data *data, struct entry *entry,
114+
struct commit *to_push)
115+
{
116+
struct entry *parent_entry;
117+
118+
if (to_push->object.flags & SEEN) {
119+
int i;
120+
if (to_push->object.flags & POPPED)
121+
/*
122+
* The entry for this commit has already been popped,
123+
* due to clock skew. Pretend that this parent does not
124+
* exist.
125+
*/
126+
return 0;
127+
/*
128+
* Find the existing entry and use it.
129+
*/
130+
for (i = 0; i < data->rev_list.nr; i++) {
131+
parent_entry = data->rev_list.array[i].data;
132+
if (parent_entry->commit == to_push)
133+
goto parent_found;
134+
}
135+
BUG("missing parent in priority queue");
136+
parent_found:
137+
;
138+
} else {
139+
parent_entry = rev_list_push(data, to_push, 0);
140+
}
141+
142+
if (entry->commit->object.flags & (COMMON | ADVERTISED)) {
143+
mark_common(data, to_push);
144+
} else {
145+
uint16_t new_original_ttl = entry->ttl
146+
? entry->original_ttl : entry->original_ttl * 3 / 2 + 1;
147+
uint16_t new_ttl = entry->ttl
148+
? entry->ttl - 1 : new_original_ttl;
149+
if (parent_entry->original_ttl < new_original_ttl) {
150+
parent_entry->original_ttl = new_original_ttl;
151+
parent_entry->ttl = new_ttl;
152+
}
153+
}
154+
155+
return 1;
156+
}
157+
158+
static const struct object_id *get_rev(struct data *data)
159+
{
160+
struct commit *to_send = NULL;
161+
162+
while (to_send == NULL) {
163+
struct entry *entry;
164+
struct commit *commit;
165+
struct commit_list *p;
166+
int parent_pushed = 0;
167+
168+
if (data->rev_list.nr == 0 || data->non_common_revs == 0)
169+
return NULL;
170+
171+
entry = prio_queue_get(&data->rev_list);
172+
commit = entry->commit;
173+
commit->object.flags |= POPPED;
174+
if (!(commit->object.flags & COMMON))
175+
data->non_common_revs--;
176+
177+
if (!(commit->object.flags & COMMON) && !entry->ttl)
178+
to_send = commit;
179+
180+
parse_commit(commit);
181+
for (p = commit->parents; p; p = p->next)
182+
parent_pushed |= push_parent(data, entry, p->item);
183+
184+
if (!(commit->object.flags & COMMON) && !parent_pushed)
185+
/*
186+
* This commit has no parents, or all of its parents
187+
* have already been popped (due to clock skew), so send
188+
* it anyway.
189+
*/
190+
to_send = commit;
191+
192+
free(entry);
193+
}
194+
195+
return &to_send->object.oid;
196+
}
197+
198+
static void known_common(struct fetch_negotiator *n, struct commit *c)
199+
{
200+
if (c->object.flags & SEEN)
201+
return;
202+
rev_list_push(n->data, c, ADVERTISED);
203+
}
204+
205+
static void add_tip(struct fetch_negotiator *n, struct commit *c)
206+
{
207+
n->known_common = NULL;
208+
if (c->object.flags & SEEN)
209+
return;
210+
rev_list_push(n->data, c, 0);
211+
}
212+
213+
static const struct object_id *next(struct fetch_negotiator *n)
214+
{
215+
n->known_common = NULL;
216+
n->add_tip = NULL;
217+
return get_rev(n->data);
218+
}
219+
220+
static int ack(struct fetch_negotiator *n, struct commit *c)
221+
{
222+
int known_to_be_common = !!(c->object.flags & COMMON);
223+
if (!(c->object.flags & SEEN))
224+
die("received ack for commit %s not sent as 'have'\n",
225+
oid_to_hex(&c->object.oid));
226+
mark_common(n->data, c);
227+
return known_to_be_common;
228+
}
229+
230+
static void release(struct fetch_negotiator *n)
231+
{
232+
clear_prio_queue(&((struct data *)n->data)->rev_list);
233+
FREE_AND_NULL(n->data);
234+
}
235+
236+
void skipping_negotiator_init(struct fetch_negotiator *negotiator)
237+
{
238+
struct data *data;
239+
negotiator->known_common = known_common;
240+
negotiator->add_tip = add_tip;
241+
negotiator->next = next;
242+
negotiator->ack = ack;
243+
negotiator->release = release;
244+
negotiator->data = data = xcalloc(1, sizeof(*data));
245+
data->rev_list.compare = compare;
246+
247+
if (marked)
248+
for_each_ref(clear_marks, NULL);
249+
marked = 1;
250+
}

negotiator/skipping.h

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#ifndef NEGOTIATOR_SKIPPING_H
2+
#define NEGOTIATOR_SKIPPING_H
3+
4+
struct fetch_negotiator;
5+
6+
void skipping_negotiator_init(struct fetch_negotiator *negotiator);
7+
8+
#endif

0 commit comments

Comments
 (0)