negotiator/skipping: skip commits during fetch
Introduce a new negotiation algorithm used during fetch that skips commits in an effort to find common ancestors faster. The skips grow similarly to the Fibonacci sequence as the commit walk proceeds further away from the tips. The skips may cause unnecessary commits to be included in the packfile, but the negotiation step typically ends more quickly. Usage of this algorithm is guarded behind the configuration flag fetch.negotiationAlgorithm. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jul 16, 2018 at 11:44 UTC
42cc7485a2ec49ecc440c921d2eb0cae4da80549
8 files changed
+461
-4
Documentation/config.txt
+9
@@ -1491,6 +1491,15 @@ fetch.output::
1491
`full` and `compact`. Default value is `full`. See section
1492
OUTPUT in linkgit:git-fetch[1] for detail.
1493
1494
+fetch.negotiationAlgorithm::
1495
+ Control how information about the commits in the local repository is
1496
+ sent when negotiating the contents of the packfile to be sent by the
1497
+ server. Set to "skipping" to use an algorithm that skips commits in an
1498
+ effort to converge faster, but may result in a larger-than-necessary
1499
+ packfile; any other value instructs Git to use the default algorithm
1500
+ that never skips commits (unless the server has acknowledged it or one
1501
+ of its descendants).
1502
+
1503
format.attach::
1504
Enable multipart/mixed attachments as the default for
1505
'format-patch'. The value can also be a double quoted string
Makefile
+1
@@ -893,6 +893,7 @@ LIB_OBJS += merge-recursive.o
893
LIB_OBJS += mergesort.o
894
LIB_OBJS += name-hash.o
895
LIB_OBJS += negotiator/default.o
896
+LIB_OBJS += negotiator/skipping.o
897
LIB_OBJS += notes.o
898
LIB_OBJS += notes-cache.o
899
LIB_OBJS += notes-merge.o
fetch-negotiator.c
+7
-1
@@ -1,8 +1,14 @@
1
#include "git-compat-util.h"
2
#include "fetch-negotiator.h"
3
#include "negotiator/default.h"
4
+#include "negotiator/skipping.h"
5
5
-void fetch_negotiator_init(struct fetch_negotiator *negotiator)
6
+void fetch_negotiator_init(struct fetch_negotiator *negotiator,
7
+ const char *algorithm)
8
{
9
+ if (algorithm && !strcmp(algorithm, "skipping")) {
10
+ skipping_negotiator_init(negotiator);
11
+ return;
12
+ }
13
default_negotiator_init(negotiator);
14
}
fetch-negotiator.h
+2
-1
@@ -52,6 +52,7 @@ struct fetch_negotiator {
52
void *data;
53
};
54
55
-void fetch_negotiator_init(struct fetch_negotiator *negotiator);
55
+void fetch_negotiator_init(struct fetch_negotiator *negotiator,
56
+ const char *algorithm);
57
58
#endif
fetch-pack.c
+5
-2
@@ -33,6 +33,7 @@ static int agent_supported;
33
static int server_supports_filtering;
34
static struct lock_file shallow_lock;
35
static const char *alternate_shallow_file;
36
+static char *negotiation_algorithm;
37
38
/* Remember to update object flag allocation in object.h */
39
#define COMPLETE (1U << 0)
@@ -889,7 +890,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
890
const char *agent_feature;
891
int agent_len;
892
struct fetch_negotiator negotiator;
892
- fetch_negotiator_init(&negotiator);
893
+ fetch_negotiator_init(&negotiator, negotiation_algorithm);
894
895
sort_ref_list(&ref, ref_compare_name);
896
QSORT(sought, nr_sought, cmp_ref_by_name);
@@ -1271,7 +1272,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1272
int in_vain = 0;
1273
int haves_to_send = INITIAL_FLUSH;
1274
struct fetch_negotiator negotiator;
1274
- fetch_negotiator_init(&negotiator);
1275
+ fetch_negotiator_init(&negotiator, negotiation_algorithm);
1276
packet_reader_init(&reader, fd[0], NULL, 0,
1277
PACKET_READ_CHOMP_NEWLINE);
1278
@@ -1350,6 +1351,8 @@ static void fetch_pack_config(void)
1351
git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1352
git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1353
git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1354
+ git_config_get_string("fetch.negotiationalgorithm",
1355
+ &negotiation_algorithm);
1356
1357
git_config(git_default_config, NULL);
1358
}
negotiator/skipping.c
new
+250
@@ -0,0 +1,250 @@
1
+#include "cache.h"
2
+#include "skipping.h"
3
+#include "../commit.h"
4
+#include "../fetch-negotiator.h"
5
+#include "../prio-queue.h"
6
+#include "../refs.h"
7
+#include "../tag.h"
8
+
9
+/* Remember to update object flag allocation in object.h */
10
+/*
11
+ * Both us and the server know that both parties have this object.
12
+ */
13
+#define COMMON (1U << 2)
14
+/*
15
+ * The server has told us that it has this object. We still need to tell the
16
+ * server that we have this object (or one of its descendants), but since we are
17
+ * going to do that, we do not need to tell the server about its ancestors.
18
+ */
19
+#define ADVERTISED (1U << 3)
20
+/*
21
+ * This commit has entered the priority queue.
22
+ */
23
+#define SEEN (1U << 4)
24
+/*
25
+ * This commit has left the priority queue.
26
+ */
27
+#define POPPED (1U << 5)
28
+
29
+static int marked;
30
+
31
+/*
32
+ * An entry in the priority queue.
33
+ */
34
+struct entry {
35
+ struct commit *commit;
36
+
37
+ /*
38
+ * Used only if commit is not COMMON.
39
+ */
40
+ uint16_t original_ttl;
41
+ uint16_t ttl;
42
+};
43
+
44
+struct data {
45
+ struct prio_queue rev_list;
46
+
47
+ /*
48
+ * The number of non-COMMON commits in rev_list.
49
+ */
50
+ int non_common_revs;
51
+};
52
+
53
+static int compare(const void *a_, const void *b_, void *unused)
54
+{
55
+ const struct entry *a = a_;
56
+ const struct entry *b = b_;
57
+ return compare_commits_by_commit_date(a->commit, b->commit, NULL);
58
+}
59
+
60
+static struct entry *rev_list_push(struct data *data, struct commit *commit, int mark)
61
+{
62
+ struct entry *entry;
63
+ commit->object.flags |= mark | SEEN;
64
+
65
+ entry = xcalloc(1, sizeof(*entry));
66
+ entry->commit = commit;
67
+ prio_queue_put(&data->rev_list, entry);
68
+
69
+ if (!(mark & COMMON))
70
+ data->non_common_revs++;
71
+ return entry;
72
+}
73
+
74
+static int clear_marks(const char *refname, const struct object_id *oid,
75
+ int flag, void *cb_data)
76
+{
77
+ struct object *o = deref_tag(parse_object(oid), refname, 0);
78
+
79
+ if (o && o->type == OBJ_COMMIT)
80
+ clear_commit_marks((struct commit *)o,
81
+ COMMON | ADVERTISED | SEEN | POPPED);
82
+ return 0;
83
+}
84
+
85
+/*
86
+ * Mark this SEEN commit and all its SEEN ancestors as COMMON.
87
+ */
88
+static void mark_common(struct data *data, struct commit *c)
89
+{
90
+ struct commit_list *p;
91
+
92
+ if (c->object.flags & COMMON)
93
+ return;
94
+ c->object.flags |= COMMON;
95
+ if (!(c->object.flags & POPPED))
96
+ data->non_common_revs--;
97
+
98
+ if (!c->object.parsed)
99
+ return;
100
+ for (p = c->parents; p; p = p->next) {
101
+ if (p->item->object.flags & SEEN)
102
+ mark_common(data, p->item);
103
+ }
104
+}
105
+
106
+/*
107
+ * Ensure that the priority queue has an entry for to_push, and ensure that the
108
+ * entry has the correct flags and ttl.
109
+ *
110
+ * This function returns 1 if an entry was found or created, and 0 otherwise
111
+ * (because the entry for this commit had already been popped).
112
+ */
113
+static int push_parent(struct data *data, struct entry *entry,
114
+ struct commit *to_push)
115
+{
116
+ struct entry *parent_entry;
117
+
118
+ if (to_push->object.flags & SEEN) {
119
+ int i;
120
+ if (to_push->object.flags & POPPED)
121
+ /*
122
+ * The entry for this commit has already been popped,
123
+ * due to clock skew. Pretend that this parent does not
124
+ * exist.
125
+ */
126
+ return 0;
127
+ /*
128
+ * Find the existing entry and use it.
129
+ */
130
+ for (i = 0; i < data->rev_list.nr; i++) {
131
+ parent_entry = data->rev_list.array[i].data;
132
+ if (parent_entry->commit == to_push)
133
+ goto parent_found;
134
+ }
135
+ BUG("missing parent in priority queue");
136
+parent_found:
137
+ ;
138
+ } else {
139
+ parent_entry = rev_list_push(data, to_push, 0);
140
+ }
141
+
142
+ if (entry->commit->object.flags & (COMMON | ADVERTISED)) {
143
+ mark_common(data, to_push);
144
+ } else {
145
+ uint16_t new_original_ttl = entry->ttl
146
+ ? entry->original_ttl : entry->original_ttl * 3 / 2 + 1;
147
+ uint16_t new_ttl = entry->ttl
148
+ ? entry->ttl - 1 : new_original_ttl;
149
+ if (parent_entry->original_ttl < new_original_ttl) {
150
+ parent_entry->original_ttl = new_original_ttl;
151
+ parent_entry->ttl = new_ttl;
152
+ }
153
+ }
154
+
155
+ return 1;
156
+}
157
+
158
+static const struct object_id *get_rev(struct data *data)
159
+{
160
+ struct commit *to_send = NULL;
161
+
162
+ while (to_send == NULL) {
163
+ struct entry *entry;
164
+ struct commit *commit;
165
+ struct commit_list *p;
166
+ int parent_pushed = 0;
167
+
168
+ if (data->rev_list.nr == 0 || data->non_common_revs == 0)
169
+ return NULL;
170
+
171
+ entry = prio_queue_get(&data->rev_list);
172
+ commit = entry->commit;
173
+ commit->object.flags |= POPPED;
174
+ if (!(commit->object.flags & COMMON))
175
+ data->non_common_revs--;
176
+
177
+ if (!(commit->object.flags & COMMON) && !entry->ttl)
178
+ to_send = commit;
179
+
180
+ parse_commit(commit);
181
+ for (p = commit->parents; p; p = p->next)
182
+ parent_pushed |= push_parent(data, entry, p->item);
183
+
184
+ if (!(commit->object.flags & COMMON) && !parent_pushed)
185
+ /*
186
+ * This commit has no parents, or all of its parents
187
+ * have already been popped (due to clock skew), so send
188
+ * it anyway.
189
+ */
190
+ to_send = commit;
191
+
192
+ free(entry);
193
+ }
194
+
195
+ return &to_send->object.oid;
196
+}
197
+
198
+static void known_common(struct fetch_negotiator *n, struct commit *c)
199
+{
200
+ if (c->object.flags & SEEN)
201
+ return;
202
+ rev_list_push(n->data, c, ADVERTISED);
203
+}
204
+
205
+static void add_tip(struct fetch_negotiator *n, struct commit *c)
206
+{
207
+ n->known_common = NULL;
208
+ if (c->object.flags & SEEN)
209
+ return;
210
+ rev_list_push(n->data, c, 0);
211
+}
212
+
213
+static const struct object_id *next(struct fetch_negotiator *n)
214
+{
215
+ n->known_common = NULL;
216
+ n->add_tip = NULL;
217
+ return get_rev(n->data);
218
+}
219
+
220
+static int ack(struct fetch_negotiator *n, struct commit *c)
221
+{
222
+ int known_to_be_common = !!(c->object.flags & COMMON);
223
+ if (!(c->object.flags & SEEN))
224
+ die("received ack for commit %s not sent as 'have'\n",
225
+ oid_to_hex(&c->object.oid));
226
+ mark_common(n->data, c);
227
+ return known_to_be_common;
228
+}
229
+
230
+static void release(struct fetch_negotiator *n)
231
+{
232
+ clear_prio_queue(&((struct data *)n->data)->rev_list);
233
+ FREE_AND_NULL(n->data);
234
+}
235
+
236
+void skipping_negotiator_init(struct fetch_negotiator *negotiator)
237
+{
238
+ struct data *data;
239
+ negotiator->known_common = known_common;
240
+ negotiator->add_tip = add_tip;
241
+ negotiator->next = next;
242
+ negotiator->ack = ack;
243
+ negotiator->release = release;
244
+ negotiator->data = data = xcalloc(1, sizeof(*data));
245
+ data->rev_list.compare = compare;
246
+
247
+ if (marked)
248
+ for_each_ref(clear_marks, NULL);
249
+ marked = 1;
250
+}
negotiator/skipping.h
new
+8
@@ -0,0 +1,8 @@
1
+#ifndef NEGOTIATOR_SKIPPING_H
2
+#define NEGOTIATOR_SKIPPING_H
3
+
4
+struct fetch_negotiator;
5
+
6
+void skipping_negotiator_init(struct fetch_negotiator *negotiator);
7
+
8
+#endif
t/t5552-skipping-fetch-negotiator.sh
new
+179
@@ -0,0 +1,179 @@
1
+#!/bin/sh
2
+
3
+test_description='test skipping fetch negotiator'
4
+. ./test-lib.sh
5
+
6
+have_sent () {
7
+ while test "$#" -ne 0
8
+ do
9
+ grep "fetch> have $(git -C client rev-parse $1)" trace
10
+ if test $? -ne 0
11
+ then
12
+ echo "No have $(git -C client rev-parse $1) ($1)"
13
+ return 1
14
+ fi
15
+ shift
16
+ done
17
+}
18
+
19
+have_not_sent () {
20
+ while test "$#" -ne 0
21
+ do
22
+ grep "fetch> have $(git -C client rev-parse $1)" trace
23
+ if test $? -eq 0
24
+ then
25
+ return 1
26
+ fi
27
+ shift
28
+ done
29
+}
30
+
31
+test_expect_success 'commits with no parents are sent regardless of skip distance' '
32
+ git init server &&
33
+ test_commit -C server to_fetch &&
34
+
35
+ git init client &&
36
+ for i in $(seq 7)
37
+ do
38
+ test_commit -C client c$i
39
+ done &&
40
+
41
+ # We send: "c7" (skip 1) "c5" (skip 2) "c2" (skip 4). After that, since
42
+ # "c1" has no parent, it is still sent as "have" even though it would
43
+ # normally be skipped.
44
+ test_config -C client fetch.negotiationalgorithm skipping &&
45
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" &&
46
+ have_sent c7 c5 c2 c1 &&
47
+ have_not_sent c6 c4 c3
48
+'
49
+
50
+test_expect_success 'when two skips collide, favor the larger one' '
51
+ rm -rf server client trace &&
52
+ git init server &&
53
+ test_commit -C server to_fetch &&
54
+
55
+ git init client &&
56
+ for i in $(seq 11)
57
+ do
58
+ test_commit -C client c$i
59
+ done &&
60
+ git -C client checkout c5 &&
61
+ test_commit -C client c5side &&
62
+
63
+ # Before reaching c5, we send "c5side" (skip 1) and "c11" (skip 1) "c9"
64
+ # (skip 2) "c6" (skip 4). The larger skip (skip 4) takes precedence, so
65
+ # the next "have" sent will be "c1" (from "c6" skip 4) and not "c4"
66
+ # (from "c5side" skip 1).
67
+ test_config -C client fetch.negotiationalgorithm skipping &&
68
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" &&
69
+ have_sent c5side c11 c9 c6 c1 &&
70
+ have_not_sent c10 c8 c7 c5 c4 c3 c2
71
+'
72
+
73
+test_expect_success 'use ref advertisement to filter out commits' '
74
+ rm -rf server client trace &&
75
+ git init server &&
76
+ test_commit -C server c1 &&
77
+ test_commit -C server c2 &&
78
+ test_commit -C server c3 &&
79
+ git -C server tag -d c1 c2 c3 &&
80
+
81
+ git clone server client &&
82
+ test_commit -C client c4 &&
83
+ test_commit -C client c5 &&
84
+ git -C client checkout c4^^ &&
85
+ test_commit -C client c2side &&
86
+
87
+ git -C server checkout --orphan anotherbranch &&
88
+ test_commit -C server to_fetch &&
89
+
90
+ # The server advertising "c3" (as "refs/heads/master") means that we do
91
+ # not need to send any ancestors of "c3", but we still need to send "c3"
92
+ # itself.
93
+ test_config -C client fetch.negotiationalgorithm skipping &&
94
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch origin to_fetch &&
95
+ have_sent c5 c4^ c2side &&
96
+ have_not_sent c4 c4^^ c4^^^
97
+'
98
+
99
+test_expect_success 'handle clock skew' '
100
+ rm -rf server client trace &&
101
+ git init server &&
102
+ test_commit -C server to_fetch &&
103
+
104
+ git init client &&
105
+
106
+ # 2 regular commits
107
+ test_tick=2000000000 &&
108
+ test_commit -C client c1 &&
109
+ test_commit -C client c2 &&
110
+
111
+ # 4 old commits
112
+ test_tick=1000000000 &&
113
+ git -C client checkout c1 &&
114
+ test_commit -C client old1 &&
115
+ test_commit -C client old2 &&
116
+ test_commit -C client old3 &&
117
+ test_commit -C client old4 &&
118
+
119
+ # "c2" and "c1" are popped first, then "old4" to "old1". "old1" would
120
+ # normally be skipped, but is treated as a commit without a parent here
121
+ # and sent, because (due to clock skew) its only parent has already been
122
+ # popped off the priority queue.
123
+ test_config -C client fetch.negotiationalgorithm skipping &&
124
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" &&
125
+ have_sent c2 c1 old4 old2 old1 &&
126
+ have_not_sent old3
127
+'
128
+
129
+test_expect_success 'do not send "have" with ancestors of commits that server ACKed' '
130
+ rm -rf server client trace &&
131
+ git init server &&
132
+ test_commit -C server to_fetch &&
133
+
134
+ git init client &&
135
+ for i in $(seq 8)
136
+ do
137
+ git -C client checkout --orphan b$i &&
138
+ test_commit -C client b$i.c0
139
+ done &&
140
+ for j in $(seq 19)
141
+ do
142
+ for i in $(seq 8)
143
+ do
144
+ git -C client checkout b$i &&
145
+ test_commit -C client b$i.c$j
146
+ done
147
+ done &&
148
+
149
+ # Copy this branch over to the server and add a commit on it so that it
150
+ # is reachable but not advertised.
151
+ git -C server fetch --no-tags "$(pwd)/client" b1:refs/heads/b1 &&
152
+ git -C server checkout b1 &&
153
+ test_commit -C server commit-on-b1 &&
154
+
155
+ test_config -C client fetch.negotiationalgorithm skipping &&
156
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" to_fetch &&
157
+ grep " fetch" trace &&
158
+
159
+ # fetch-pack sends 2 requests each containing 16 "have" lines before
160
+ # processing the first response. In these 2 requests, 4 commits from
161
+ # each branch are sent. Just check the first branch.
162
+ have_sent b1.c19 b1.c17 b1.c14 b1.c9 &&
163
+ have_not_sent b1.c18 b1.c16 b1.c15 b1.c13 b1.c12 b1.c11 b1.c10 &&
164
+
165
+ # While fetch-pack is processing the first response, it should read that
166
+ # the server ACKs b1.c19 and b1.c17.
167
+ grep "fetch< ACK $(git -C client rev-parse b1.c19) common" trace &&
168
+ grep "fetch< ACK $(git -C client rev-parse b1.c17) common" trace &&
169
+
170
+ # fetch-pack should thus not send any more commits in the b1 branch, but
171
+ # should still send the others (in this test, just check b2).
172
+ for i in $(seq 0 8)
173
+ do
174
+ have_not_sent b1.c$i
175
+ done &&
176
+ have_sent b2.c1 b2.c0
177
+'
178
+
179
+test_done