@matheusbsbs / matheus-bagliano / commits / 9c1e657a8f

fetch: teach independent negotiation (no packfile)

Currently, the packfile negotiation step within a Git fetch cannot be done independent of sending the packfile, even though there is at least one application wherein this is useful. Therefore, make it possible for this negotiation step to be done independently. A subsequent commit will use this for one such application - push negotiation. This feature is for protocol v2 only. (An implementation for protocol v0 would require a separate implementation in the fetch, transport, and transport helper code.) In the protocol, the main hindrance towards independent negotiation is that the server can unilaterally decide to send the packfile. This is solved by a "wait-for-done" argument: the server will then wait for the client to say "done". In practice, the client will never say it; instead it will cease requests once it is satisfied. In the client, the main change lies in the transport and transport helper code. fetch_refs_via_pack() performs everything needed - protocol version and capability checks, and the negotiation itself. There are 2 code paths that do not go through fetch_refs_via_pack() that needed to be individually excluded: the bundle transport (excluded through requiring smart_options, which the bundle transport doesn't support) and transport helpers that do not support takeover. If or when we support independent negotiation for protocol v0, we will need to modify these 2 code paths to support it. But for now, report failure if independent negotiation is requested in these cases. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jonathan Tan committed May 4, 2021 at 14:16 UTC 9c1e657a8fd26fa3ed8d13fb8c796cef8db8b124
11 files changed +300 -17
Documentation/technical/protocol-v2.txt
+8
@@ -346,6 +346,14 @@ explained below.
346 client should download from all given URIs. Currently, the
347 protocols supported are "http" and "https".
348
349 +If the 'wait-for-done' feature is advertised, the following argument
350 +can be included in the client's request.
351 +
352 + wait-for-done
353 + Indicates to the server that it should never send "ready", but
354 + should wait for the client to say "done" before sending the
355 + packfile.
356 +
357 The response of `fetch` is broken into a number of sections separated by
358 delimiter packets (0001), with each section beginning with its section
359 header. Most sections are sent only when the packfile is sent.
builtin/fetch.c
+26 -1
@@ -82,6 +82,7 @@ static struct string_list server_options = STRING_LIST_INIT_DUP;
82 static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP;
83 static int fetch_write_commit_graph = -1;
84 static int stdin_refspecs = 0;
85 +static int negotiate_only;
86
87 static int git_fetch_config(const char *k, const char *v, void *cb)
88 {
@@ -202,6 +203,8 @@ static struct option builtin_fetch_options[] = {
203 TRANSPORT_FAMILY_IPV6),
204 OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"),
205 N_("report that we have only objects reachable from this object")),
206 + OPT_BOOL(0, "negotiate-only", &negotiate_only,
207 + N_("do not fetch a packfile; instead, print ancestors of negotiation tips")),
208 OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
209 OPT_BOOL(0, "auto-maintenance", &enable_auto_gc,
210 N_("run 'maintenance --auto' after fetching")),
@@ -1986,7 +1989,29 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
1989 }
1990 }
1991
1989 - if (remote) {
1992 + if (negotiate_only) {
1993 + struct oidset acked_commits = OIDSET_INIT;
1994 + struct oidset_iter iter;
1995 + const struct object_id *oid;
1996 +
1997 + if (!remote)
1998 + die(_("must supply remote when using --negotiate-only"));
1999 + gtransport = prepare_transport(remote, 1);
2000 + if (gtransport->smart_options) {
2001 + gtransport->smart_options->acked_commits = &acked_commits;
2002 + } else {
2003 + warning(_("Protocol does not support --negotiate-only, exiting."));
2004 + return 1;
2005 + }
2006 + if (server_options.nr)
2007 + gtransport->server_options = &server_options;
2008 + result = transport_fetch_refs(gtransport, NULL);
2009 +
2010 + oidset_iter_init(&acked_commits, &iter);
2011 + while ((oid = oidset_iter_next(&iter)))
2012 + printf("%s\n", oid_to_hex(oid));
2013 + oidset_clear(&acked_commits);
2014 + } else if (remote) {
2015 if (filter_options.choice || has_promisor_remote())
2016 fetch_one_setup_partial(remote);
2017 result = fetch_one(remote, argc, argv, prune_tags_ok, stdin_refspecs);
fetch-pack.c
+107 -4
@@ -23,6 +23,8 @@
23 #include "fetch-negotiator.h"
24 #include "fsck.h"
25 #include "shallow.h"
26 +#include "commit-reach.h"
27 +#include "commit-graph.h"
28
29 static int transfer_unpack_limit = -1;
30 static int fetch_unpack_limit = -1;
@@ -45,6 +47,8 @@ static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
47 /* Remember to update object flag allocation in object.h */
48 #define COMPLETE (1U << 0)
49 #define ALTERNATE (1U << 1)
50 +#define COMMON (1U << 6)
51 +#define REACH_SCRATCH (1U << 7)
52
53 /*
54 * After sending this many "have"s if we do not get any new ACK , we
@@ -1523,10 +1527,10 @@ enum fetch_state {
1527 FETCH_DONE,
1528 };
1529
1526 -static void do_check_stateless_delimiter(const struct fetch_pack_args *args,
1530 +static void do_check_stateless_delimiter(int stateless_rpc,
1531 struct packet_reader *reader)
1532 {
1529 - check_stateless_delimiter(args->stateless_rpc, reader,
1533 + check_stateless_delimiter(stateless_rpc, reader,
1534 _("git fetch-pack: expected response end packet"));
1535 }
1536
@@ -1622,7 +1626,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1626 */
1627 state = FETCH_GET_PACK;
1628 } else {
1625 - do_check_stateless_delimiter(args, &reader);
1629 + do_check_stateless_delimiter(args->stateless_rpc, &reader);
1630 state = FETCH_SEND_REQUEST;
1631 }
1632 break;
@@ -1645,7 +1649,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1649 packfile_uris.nr ? &index_pack_args : NULL,
1650 sought, nr_sought, &fsck_options.gitmodules_found))
1651 die(_("git fetch-pack: fetch failed."));
1648 - do_check_stateless_delimiter(args, &reader);
1652 + do_check_stateless_delimiter(args->stateless_rpc, &reader);
1653
1654 state = FETCH_DONE;
1655 break;
@@ -1962,6 +1966,105 @@ cleanup:
1966 return ref_cpy;
1967 }
1968
1969 +static int add_to_object_array(const struct object_id *oid, void *data)
1970 +{
1971 + struct object_array *a = data;
1972 +
1973 + add_object_array(lookup_object(the_repository, oid), "", a);
1974 + return 0;
1975 +}
1976 +
1977 +static void clear_common_flag(struct oidset *s)
1978 +{
1979 + struct oidset_iter iter;
1980 + const struct object_id *oid;
1981 + oidset_iter_init(s, &iter);
1982 +
1983 + while ((oid = oidset_iter_next(&iter))) {
1984 + struct object *obj = lookup_object(the_repository, oid);
1985 + obj->flags &= ~COMMON;
1986 + }
1987 +}
1988 +
1989 +void negotiate_using_fetch(const struct oid_array *negotiation_tips,
1990 + const struct string_list *server_options,
1991 + int stateless_rpc,
1992 + int fd[],
1993 + struct oidset *acked_commits)
1994 +{
1995 + struct fetch_negotiator negotiator;
1996 + struct packet_reader reader;
1997 + struct object_array nt_object_array = OBJECT_ARRAY_INIT;
1998 + struct strbuf req_buf = STRBUF_INIT;
1999 + int haves_to_send = INITIAL_FLUSH;
2000 + int in_vain = 0;
2001 + int seen_ack = 0;
2002 + int last_iteration = 0;
2003 + timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
2004 +
2005 + fetch_negotiator_init(the_repository, &negotiator);
2006 + mark_tips(&negotiator, negotiation_tips);
2007 +
2008 + packet_reader_init(&reader, fd[0], NULL, 0,
2009 + PACKET_READ_CHOMP_NEWLINE |
2010 + PACKET_READ_DIE_ON_ERR_PACKET);
2011 +
2012 + oid_array_for_each((struct oid_array *) negotiation_tips,
2013 + add_to_object_array,
2014 + &nt_object_array);
2015 +
2016 + while (!last_iteration) {
2017 + int haves_added;
2018 + struct object_id common_oid;
2019 + int received_ready = 0;
2020 +
2021 + strbuf_reset(&req_buf);
2022 + write_fetch_command_and_capabilities(&req_buf, server_options);
2023 +
2024 + packet_buf_write(&req_buf, "wait-for-done");
2025 +
2026 + haves_added = add_haves(&negotiator, &req_buf, &haves_to_send);
2027 + in_vain += haves_added;
2028 + if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN))
2029 + last_iteration = 1;
2030 +
2031 + /* Send request */
2032 + packet_buf_flush(&req_buf);
2033 + if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0)
2034 + die_errno(_("unable to write request to remote"));
2035 +
2036 + /* Process ACKs/NAKs */
2037 + process_section_header(&reader, "acknowledgments", 0);
2038 + while (process_ack(&negotiator, &reader, &common_oid,
2039 + &received_ready)) {
2040 + struct commit *commit = lookup_commit(the_repository,
2041 + &common_oid);
2042 + if (commit) {
2043 + timestamp_t generation;
2044 +
2045 + parse_commit_or_die(commit);
2046 + commit->object.flags |= COMMON;
2047 + generation = commit_graph_generation(commit);
2048 + if (generation < min_generation)
2049 + min_generation = generation;
2050 + }
2051 + in_vain = 0;
2052 + seen_ack = 1;
2053 + oidset_insert(acked_commits, &common_oid);
2054 + }
2055 + if (received_ready)
2056 + die(_("unexpected 'ready' from remote"));
2057 + else
2058 + do_check_stateless_delimiter(stateless_rpc, &reader);
2059 + if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
2060 + REACH_SCRATCH, 0,
2061 + min_generation))
2062 + last_iteration = 1;
2063 + }
2064 + clear_common_flag(acked_commits);
2065 + strbuf_release(&req_buf);
2066 +}
2067 +
2068 int report_unmatched_refs(struct ref **sought, int nr_sought)
2069 {
2070 int i, ret = 0;
fetch-pack.h
+14
@@ -5,6 +5,7 @@
5 #include "run-command.h"
6 #include "protocol.h"
7 #include "list-objects-filter-options.h"
8 +#include "oidset.h"
9
10 struct oid_array;
11
@@ -81,6 +82,19 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
82 struct string_list *pack_lockfiles,
83 enum protocol_version version);
84
85 +/*
86 + * Execute the --negotiate-only mode of "git fetch", adding all known common
87 + * commits to acked_commits.
88 + *
89 + * In the capability advertisement that has happened prior to invoking this
90 + * function, the "wait-for-done" capability must be present.
91 + */
92 +void negotiate_using_fetch(const struct oid_array *negotiation_tips,
93 + const struct string_list *server_options,
94 + int stateless_rpc,
95 + int fd[],
96 + struct oidset *acked_commits);
97 +
98 /*
99 * Print an appropriate error message for each sought ref that wasn't
100 * matched. Return 0 if all sought refs were matched, otherwise 1.
object.h
+1 -1
@@ -60,7 +60,7 @@ struct object_array {
60 /*
61 * object flag allocation:
62 * revision.h: 0---------10 15 23------26
63 - * fetch-pack.c: 01
63 + * fetch-pack.c: 01 67
64 * negotiator/default.c: 2--5
65 * walker.c: 0-2
66 * upload-pack.c: 4 11-----14 16-----19
t/t5701-git-serve.sh
+1 -1
@@ -16,7 +16,7 @@ test_expect_success 'test capability advertisement' '
16 version 2
17 agent=git/$(git version | cut -d" " -f3)
18 ls-refs=unborn
19 - fetch=shallow
19 + fetch=shallow wait-for-done
20 server-option
21 object-format=$(test_oid algo)
22 0000
t/t5702-protocol-v2.sh
+89
@@ -585,6 +585,49 @@ test_expect_success 'deepen-relative' '
585 test_cmp expected actual
586 '
587
588 +setup_negotiate_only () {
589 + SERVER="$1"
590 + URI="$2"
591 +
592 + rm -rf "$SERVER" client
593 +
594 + git init "$SERVER"
595 + test_commit -C "$SERVER" one
596 + test_commit -C "$SERVER" two
597 +
598 + git clone "$URI" client
599 + test_commit -C client three
600 +}
601 +
602 +test_expect_success 'file:// --negotiate-only' '
603 + SERVER="server" &&
604 + URI="file://$(pwd)/server" &&
605 +
606 + setup_negotiate_only "$SERVER" "$URI" &&
607 +
608 + git -c protocol.version=2 -C client fetch \
609 + --no-tags \
610 + --negotiate-only \
611 + --negotiation-tip=$(git -C client rev-parse HEAD) \
612 + origin >out &&
613 + COMMON=$(git -C "$SERVER" rev-parse two) &&
614 + grep "$COMMON" out
615 +'
616 +
617 +test_expect_success 'file:// --negotiate-only with protocol v0' '
618 + SERVER="server" &&
619 + URI="file://$(pwd)/server" &&
620 +
621 + setup_negotiate_only "$SERVER" "$URI" &&
622 +
623 + test_must_fail git -c protocol.version=0 -C client fetch \
624 + --no-tags \
625 + --negotiate-only \
626 + --negotiation-tip=$(git -C client rev-parse HEAD) \
627 + origin 2>err &&
628 + test_i18ngrep "negotiate-only requires protocol v2" err
629 +'
630 +
631 # Test protocol v2 with 'http://' transport
632 #
633 . "$TEST_DIRECTORY"/lib-httpd.sh
@@ -1035,6 +1078,52 @@ test_expect_success 'packfile-uri with transfer.fsckobjects fails when .gitmodul
1078 test_i18ngrep "disallowed submodule name" err
1079 '
1080
1081 +test_expect_success 'http:// --negotiate-only' '
1082 + SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" &&
1083 + URI="$HTTPD_URL/smart/server" &&
1084 +
1085 + setup_negotiate_only "$SERVER" "$URI" &&
1086 +
1087 + git -c protocol.version=2 -C client fetch \
1088 + --no-tags \
1089 + --negotiate-only \
1090 + --negotiation-tip=$(git -C client rev-parse HEAD) \
1091 + origin >out &&
1092 + COMMON=$(git -C "$SERVER" rev-parse two) &&
1093 + grep "$COMMON" out
1094 +'
1095 +
1096 +test_expect_success 'http:// --negotiate-only without wait-for-done support' '
1097 + SERVER="server" &&
1098 + URI="$HTTPD_URL/one_time_perl/server" &&
1099 +
1100 + setup_negotiate_only "$SERVER" "$URI" &&
1101 +
1102 + echo "s/ wait-for-done/ xxxx-xxx-xxxx/" \
1103 + >"$HTTPD_ROOT_PATH/one-time-perl" &&
1104 +
1105 + test_must_fail git -c protocol.version=2 -C client fetch \
1106 + --no-tags \
1107 + --negotiate-only \
1108 + --negotiation-tip=$(git -C client rev-parse HEAD) \
1109 + origin 2>err &&
1110 + test_i18ngrep "server does not support wait-for-done" err
1111 +'
1112 +
1113 +test_expect_success 'http:// --negotiate-only with protocol v0' '
1114 + SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" &&
1115 + URI="$HTTPD_URL/smart/server" &&
1116 +
1117 + setup_negotiate_only "$SERVER" "$URI" &&
1118 +
1119 + test_must_fail git -c protocol.version=0 -C client fetch \
1120 + --no-tags \
1121 + --negotiate-only \
1122 + --negotiation-tip=$(git -C client rev-parse HEAD) \
1123 + origin 2>err &&
1124 + test_i18ngrep "negotiate-only requires protocol v2" err
1125 +'
1126 +
1127 # DO NOT add non-httpd-specific tests here, because the last part of this
1128 # test script is only executed when httpd is available and enabled.
1129
transport-helper.c
+10
@@ -684,6 +684,16 @@ static int fetch(struct transport *transport,
684 return transport->vtable->fetch(transport, nr_heads, to_fetch);
685 }
686
687 + /*
688 + * If we reach here, then the server, the client, and/or the transport
689 + * helper does not support protocol v2. --negotiate-only requires
690 + * protocol v2.
691 + */
692 + if (data->transport_options.acked_commits) {
693 + warning(_("--negotiate-only requires protocol v2"));
694 + return -1;
695 + }
696 +
697 if (!data->get_refs_list_called)
698 get_refs_list_using_list(transport, 0);
699
transport.c
+25 -5
@@ -392,16 +392,29 @@ static int fetch_refs_via_pack(struct transport *transport,
392 else if (data->version <= protocol_v1)
393 die_if_server_options(transport);
394
395 + if (data->options.acked_commits) {
396 + if (data->version < protocol_v2) {
397 + warning(_("--negotiate-only requires protocol v2"));
398 + ret = -1;
399 + } else if (!server_supports_feature("fetch", "wait-for-done", 0)) {
400 + warning(_("server does not support wait-for-done"));
401 + ret = -1;
402 + } else {
403 + negotiate_using_fetch(data->options.negotiation_tips,
404 + transport->server_options,
405 + transport->stateless_rpc,
406 + data->fd,
407 + data->options.acked_commits);
408 + ret = 0;
409 + }
410 + goto cleanup;
411 + }
412 +
413 refs = fetch_pack(&args, data->fd,
414 refs_tmp ? refs_tmp : transport->remote_refs,
415 to_fetch, nr_heads, &data->shallow,
416 &transport->pack_lockfiles, data->version);
417
400 - close(data->fd[0]);
401 - close(data->fd[1]);
402 - if (finish_connect(data->conn))
403 - ret = -1;
404 - data->conn = NULL;
418 data->got_remote_heads = 0;
419 data->options.self_contained_and_connected =
420 args.self_contained_and_connected;
@@ -412,6 +425,13 @@ static int fetch_refs_via_pack(struct transport *transport,
425 if (report_unmatched_refs(to_fetch, nr_heads))
426 ret = -1;
427
428 +cleanup:
429 + close(data->fd[0]);
430 + close(data->fd[1]);
431 + if (finish_connect(data->conn))
432 + ret = -1;
433 + data->conn = NULL;
434 +
435 free_refs(refs_tmp);
436 free_refs(refs);
437 return ret;
transport.h
+6
@@ -47,6 +47,12 @@ struct git_transport_options {
47 * transport_set_option().
48 */
49 struct oid_array *negotiation_tips;
50 +
51 + /*
52 + * If allocated, whenever transport_fetch_refs() is called, add known
53 + * common commits to this oidset instead of fetching any packfiles.
54 + */
55 + struct oidset *acked_commits;
56 };
57
58 enum transport_family {
upload-pack.c
+13 -5
@@ -103,6 +103,7 @@ struct upload_pack_data {
103 unsigned use_ofs_delta : 1;
104 unsigned no_progress : 1;
105 unsigned use_include_tag : 1;
106 + unsigned wait_for_done : 1;
107 unsigned allow_filter : 1;
108 unsigned allow_filter_fallback : 1;
109 unsigned long tree_filter_max_depth;
@@ -1496,6 +1497,10 @@ static void process_args(struct packet_reader *request,
1497 data->done = 1;
1498 continue;
1499 }
1500 + if (!strcmp(arg, "wait-for-done")) {
1501 + data->wait_for_done = 1;
1502 + continue;
1503 + }
1504
1505 /* Shallow related arguments */
1506 if (process_shallow(arg, &data->shallows))
@@ -1578,7 +1583,7 @@ static int send_acks(struct upload_pack_data *data, struct oid_array *acks)
1583 oid_to_hex(&acks->oid[i]));
1584 }
1585
1581 - if (ok_to_give_up(data)) {
1586 + if (!data->wait_for_done && ok_to_give_up(data)) {
1587 /* Send Ready */
1588 packet_writer_write(&data->writer, "ready\n");
1589 return 1;
@@ -1668,10 +1673,13 @@ int upload_pack_v2(struct repository *r, struct strvec *keys,
1673 case FETCH_PROCESS_ARGS:
1674 process_args(request, &data);
1675
1671 - if (!data.want_obj.nr) {
1676 + if (!data.want_obj.nr && !data.wait_for_done) {
1677 /*
1673 - * Request didn't contain any 'want' lines,
1674 - * guess they didn't want anything.
1678 + * Request didn't contain any 'want' lines (and
1679 + * the request does not contain
1680 + * "wait-for-done", in which it is reasonable
1681 + * to just send 'have's without 'want's); guess
1682 + * they didn't want anything.
1683 */
1684 state = FETCH_DONE;
1685 } else if (data.haves.nr) {
@@ -1723,7 +1731,7 @@ int upload_pack_advertise(struct repository *r,
1731 int allow_sideband_all_value;
1732 char *str = NULL;
1733
1726 - strbuf_addstr(value, "shallow");
1734 + strbuf_addstr(value, "shallow wait-for-done");
1735
1736 if (!repo_config_get_bool(the_repository,
1737 "uploadpack.allowfilter",