fetch-pack: support negotiation tip whitelist
During negotiation, fetch-pack eventually reports as "have" lines all commits reachable from all refs. Allow the user to restrict the commits sent in this way by providing a whitelist of tips; only the tips themselves and their ancestors will be sent. Both globs and single objects are supported. This feature is only supported for protocols that support connect or stateless-connect (such as HTTP with protocol v2). This will speed up negotiation when the repository has multiple relatively independent branches (for example, when a repository interacts with multiple repositories, such as with linux-next [1] and torvalds/linux [2]), and the user knows which local branch is likely to have commits in common with the upstream branch they are fetching. [1] https://kernel.googlesource.com/pub/scm/linux/kernel/git/next/linux-next/ [2] https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/ Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jul 2, 2018 at 15:39 UTC
3390e42adb3b84a9d61b3d46f4105f4cb6ba5edd
8 files changed
+176
-2
Documentation/fetch-options.txt
+16
@@ -42,6 +42,22 @@ the current repository has the same history as the source repository.
42
.git/shallow. This option updates .git/shallow and accept such
43
refs.
44
45
+--negotiation-tip=<commit|glob>::
46
+ By default, Git will report, to the server, commits reachable
47
+ from all local refs to find common commits in an attempt to
48
+ reduce the size of the to-be-received packfile. If specified,
49
+ Git will only report commits reachable from the given tips.
50
+ This is useful to speed up fetches when the user knows which
51
+ local ref is likely to have commits in common with the
52
+ upstream ref being fetched.
53
++
54
+This option may be specified more than once; if so, Git will report
55
+commits reachable from any of the given commits.
56
++
57
+The argument to this option may be a glob on ref names, a ref, or the (possibly
58
+abbreviated) SHA-1 of a commit. Specifying a glob is equivalent to specifying
59
+this option multiple times, one for each matching ref name.
60
+
61
ifndef::git-pull[]
62
--dry-run::
63
Show what would be done, without making any changes.
builtin/fetch.c
+43
@@ -63,6 +63,7 @@ static int shown_url = 0;
63
static struct refspec refmap = REFSPEC_INIT_FETCH;
64
static struct list_objects_filter_options filter_options;
65
static struct string_list server_options = STRING_LIST_INIT_DUP;
66
+static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP;
67
68
static int git_fetch_config(const char *k, const char *v, void *cb)
69
{
@@ -174,6 +175,8 @@ static struct option builtin_fetch_options[] = {
175
TRANSPORT_FAMILY_IPV4),
176
OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"),
177
TRANSPORT_FAMILY_IPV6),
178
+ OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"),
179
+ N_("report that we have only objects reachable from this object")),
180
OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
181
OPT_END()
182
};
@@ -1049,6 +1052,40 @@ static void set_option(struct transport *transport, const char *name, const char
1052
name, transport->url);
1053
}
1054
1055
+
1056
+static int add_oid(const char *refname, const struct object_id *oid, int flags,
1057
+ void *cb_data)
1058
+{
1059
+ struct oid_array *oids = cb_data;
1060
+
1061
+ oid_array_append(oids, oid);
1062
+ return 0;
1063
+}
1064
+
1065
+static void add_negotiation_tips(struct git_transport_options *smart_options)
1066
+{
1067
+ struct oid_array *oids = xcalloc(1, sizeof(*oids));
1068
+ int i;
1069
+
1070
+ for (i = 0; i < negotiation_tip.nr; i++) {
1071
+ const char *s = negotiation_tip.items[i].string;
1072
+ int old_nr;
1073
+ if (!has_glob_specials(s)) {
1074
+ struct object_id oid;
1075
+ if (get_oid(s, &oid))
1076
+ die("%s is not a valid object", s);
1077
+ oid_array_append(oids, &oid);
1078
+ continue;
1079
+ }
1080
+ old_nr = oids->nr;
1081
+ for_each_glob_ref(add_oid, s, oids);
1082
+ if (old_nr == oids->nr)
1083
+ warning("Ignoring --negotiation-tip=%s because it does not match any refs",
1084
+ s);
1085
+ }
1086
+ smart_options->negotiation_tips = oids;
1087
+}
1088
+
1089
static struct transport *prepare_transport(struct remote *remote, int deepen)
1090
{
1091
struct transport *transport;
@@ -1075,6 +1112,12 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
1112
filter_options.filter_spec);
1113
set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
1114
}
1115
+ if (negotiation_tip.nr) {
1116
+ if (transport->smart_options)
1117
+ add_negotiation_tips(transport->smart_options);
1118
+ else
1119
+ warning("Ignoring --negotiation-tip because the protocol does not support it.");
1120
+ }
1121
return transport;
1122
}
1123
fetch-pack.c
+18
-2
@@ -213,6 +213,22 @@ static int next_flush(int stateless_rpc, int count)
213
return count;
214
}
215
216
+static void mark_tips(struct fetch_negotiator *negotiator,
217
+ const struct oid_array *negotiation_tips)
218
+{
219
+ int i;
220
+
221
+ if (!negotiation_tips) {
222
+ for_each_ref(rev_list_insert_ref_oid, negotiator);
223
+ return;
224
+ }
225
+
226
+ for (i = 0; i < negotiation_tips->nr; i++)
227
+ rev_list_insert_ref(negotiator, NULL,
228
+ &negotiation_tips->oid[i]);
229
+ return;
230
+}
231
+
232
static int find_common(struct fetch_negotiator *negotiator,
233
struct fetch_pack_args *args,
234
int fd[2], struct object_id *result_oid,
@@ -230,7 +246,7 @@ static int find_common(struct fetch_negotiator *negotiator,
246
if (args->stateless_rpc && multi_ack == 1)
247
die(_("--stateless-rpc requires multi_ack_detailed"));
248
233
- for_each_ref(rev_list_insert_ref_oid, negotiator);
249
+ mark_tips(negotiator, args->negotiation_tips);
250
for_each_cached_alternate(negotiator, insert_one_alternate_object);
251
252
fetching = 0;
@@ -1295,7 +1311,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1311
else
1312
state = FETCH_SEND_REQUEST;
1313
1298
- for_each_ref(rev_list_insert_ref_oid, &negotiator);
1314
+ mark_tips(&negotiator, args->negotiation_tips);
1315
for_each_cached_alternate(&negotiator,
1316
insert_one_alternate_object);
1317
break;
fetch-pack.h
+7
@@ -16,6 +16,13 @@ struct fetch_pack_args {
16
const struct string_list *deepen_not;
17
struct list_objects_filter_options filter_options;
18
const struct string_list *server_options;
19
+
20
+ /*
21
+ * If not NULL, during packfile negotiation, fetch-pack will send "have"
22
+ * lines only with these tips and their ancestors.
23
+ */
24
+ const struct oid_array *negotiation_tips;
25
+
26
unsigned deepen_relative:1;
27
unsigned quiet:1;
28
unsigned keep_pack:1;
t/t5510-fetch.sh
+78
@@ -865,4 +865,82 @@ test_expect_success C_LOCALE_OUTPUT 'fetch compact output' '
865
test_cmp expect actual
866
'
867
868
+setup_negotiation_tip () {
869
+ SERVER="$1"
870
+ URL="$2"
871
+ USE_PROTOCOL_V2="$3"
872
+
873
+ rm -rf "$SERVER" client trace &&
874
+ git init "$SERVER" &&
875
+ test_commit -C "$SERVER" alpha_1 &&
876
+ test_commit -C "$SERVER" alpha_2 &&
877
+ git -C "$SERVER" checkout --orphan beta &&
878
+ test_commit -C "$SERVER" beta_1 &&
879
+ test_commit -C "$SERVER" beta_2 &&
880
+
881
+ git clone "$URL" client &&
882
+
883
+ if test "$USE_PROTOCOL_V2" -eq 1
884
+ then
885
+ git -C "$SERVER" config protocol.version 2 &&
886
+ git -C client config protocol.version 2
887
+ fi &&
888
+
889
+ test_commit -C "$SERVER" beta_s &&
890
+ git -C "$SERVER" checkout master &&
891
+ test_commit -C "$SERVER" alpha_s &&
892
+ git -C "$SERVER" tag -d alpha_1 alpha_2 beta_1 beta_2
893
+}
894
+
895
+check_negotiation_tip () {
896
+ # Ensure that {alpha,beta}_1 are sent as "have", but not {alpha_beta}_2
897
+ ALPHA_1=$(git -C client rev-parse alpha_1) &&
898
+ grep "fetch> have $ALPHA_1" trace &&
899
+ BETA_1=$(git -C client rev-parse beta_1) &&
900
+ grep "fetch> have $BETA_1" trace &&
901
+ ALPHA_2=$(git -C client rev-parse alpha_2) &&
902
+ ! grep "fetch> have $ALPHA_2" trace &&
903
+ BETA_2=$(git -C client rev-parse beta_2) &&
904
+ ! grep "fetch> have $BETA_2" trace
905
+}
906
+
907
+test_expect_success '--negotiation-tip limits "have" lines sent' '
908
+ setup_negotiation_tip server server 0 &&
909
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
910
+ --negotiation-tip=alpha_1 --negotiation-tip=beta_1 \
911
+ origin alpha_s beta_s &&
912
+ check_negotiation_tip
913
+'
914
+
915
+test_expect_success '--negotiation-tip understands globs' '
916
+ setup_negotiation_tip server server 0 &&
917
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
918
+ --negotiation-tip=*_1 \
919
+ origin alpha_s beta_s &&
920
+ check_negotiation_tip
921
+'
922
+
923
+test_expect_success '--negotiation-tip understands abbreviated SHA-1' '
924
+ setup_negotiation_tip server server 0 &&
925
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
926
+ --negotiation-tip=$(git -C client rev-parse --short alpha_1) \
927
+ --negotiation-tip=$(git -C client rev-parse --short beta_1) \
928
+ origin alpha_s beta_s &&
929
+ check_negotiation_tip
930
+'
931
+
932
+. "$TEST_DIRECTORY"/lib-httpd.sh
933
+start_httpd
934
+
935
+test_expect_success '--negotiation-tip limits "have" lines sent with HTTP protocol v2' '
936
+ setup_negotiation_tip "$HTTPD_DOCUMENT_ROOT_PATH/server" \
937
+ "$HTTPD_URL/smart/server" 1 &&
938
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
939
+ --negotiation-tip=alpha_1 --negotiation-tip=beta_1 \
940
+ origin alpha_s beta_s &&
941
+ check_negotiation_tip
942
+'
943
+
944
+stop_httpd
945
+
946
test_done
transport-helper.c
+3
@@ -684,6 +684,9 @@ static int fetch(struct transport *transport,
684
transport, "filter",
685
data->transport_options.filter_options.filter_spec);
686
687
+ if (data->transport_options.negotiation_tips)
688
+ warning("Ignoring --negotiation-tip because the protocol does not support it.");
689
+
690
if (data->fetch)
691
return fetch_with_fetch(transport, nr_heads, to_fetch);
692
transport.c
+1
@@ -318,6 +318,7 @@ static int fetch_refs_via_pack(struct transport *transport,
318
args.filter_options = data->options.filter_options;
319
args.stateless_rpc = transport->stateless_rpc;
320
args.server_options = transport->server_options;
321
+ args.negotiation_tips = data->options.negotiation_tips;
322
323
if (!data->got_remote_heads)
324
refs_tmp = get_refs_via_connect(transport, 0, NULL);
transport.h
+10
@@ -25,6 +25,16 @@ struct git_transport_options {
25
const char *receivepack;
26
struct push_cas_option *cas;
27
struct list_objects_filter_options filter_options;
28
+
29
+ /*
30
+ * This is only used during fetch. See the documentation of
31
+ * negotiation_tips in struct fetch_pack_args.
32
+ *
33
+ * This field is only supported by transports that support connect or
34
+ * stateless_connect. Set this field directly instead of using
35
+ * transport_set_option().
36
+ */
37
+ struct oid_array *negotiation_tips;
38
};
39
40
enum transport_family {