clone: request the 'bundle-uri' command when available
Set up all the needed client parts of the 'bundle-uri' protocol v2 command, without actually doing anything with the bundle URIs. If the server says it supports 'bundle-uri' teach Git to issue the 'bundle-uri' command after the 'ls-refs' during 'git clone'. The returned key=value pairs are passed to the bundle list code which is tested using a different ingest mechanism in t5750-bundle-uri-parse.sh. At this point, Git does nothing with that bundle list. It will not download any of the bundles. That will come in a later change after these protocol bits are finalized. The no-op client is initially used only by 'git clone' to test the basic functionality, and eventually will bootstrap the initial download of Git objects during a fresh clone. The bundle URI client will not be integrated into other fetches until a mechanism is created to select a subset of bundles for download. Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ævar Arnfjörð Bjarmason committed
Dec 22, 2022 at 15:14 UTC
0cfde740f0b2c9474aae3a381d1d6e97c7468e7a
8 files changed
+164
builtin/clone.c
+6
@@ -1271,6 +1271,12 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
1271
if (refs)
1272
mapped_refs = wanted_peer_refs(refs, &remote->fetch);
1273
1274
+ /*
1275
+ * Populate transport->got_remote_bundle_uri and
1276
+ * transport->bundle_uri. We might get nothing.
1277
+ */
1278
+ transport_get_remote_bundle_uri(transport);
1279
+
1280
if (mapped_refs) {
1281
int hash_algo = hash_algo_by_ptr(transport_get_hash_algo(transport));
1282
connect.c
+44
@@ -15,6 +15,7 @@
15
#include "version.h"
16
#include "protocol.h"
17
#include "alias.h"
18
+#include "bundle-uri.h"
19
20
static char *server_capabilities_v1;
21
static struct strvec server_capabilities_v2 = STRVEC_INIT;
@@ -491,6 +492,49 @@ static void send_capabilities(int fd_out, struct packet_reader *reader)
492
}
493
}
494
495
+int get_remote_bundle_uri(int fd_out, struct packet_reader *reader,
496
+ struct bundle_list *bundles, int stateless_rpc)
497
+{
498
+ int line_nr = 1;
499
+
500
+ /* Assert bundle-uri support */
501
+ server_supports_v2("bundle-uri", 1);
502
+
503
+ /* (Re-)send capabilities */
504
+ send_capabilities(fd_out, reader);
505
+
506
+ /* Send command */
507
+ packet_write_fmt(fd_out, "command=bundle-uri\n");
508
+ packet_delim(fd_out);
509
+
510
+ packet_flush(fd_out);
511
+
512
+ /* Process response from server */
513
+ while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
514
+ const char *line = reader->line;
515
+ line_nr++;
516
+
517
+ if (!bundle_uri_parse_line(bundles, line))
518
+ continue;
519
+
520
+ return error(_("error on bundle-uri response line %d: %s"),
521
+ line_nr, line);
522
+ }
523
+
524
+ if (reader->status != PACKET_READ_FLUSH)
525
+ return error(_("expected flush after bundle-uri listing"));
526
+
527
+ /*
528
+ * Might die(), but obscure enough that that's OK, e.g. in
529
+ * serve.c we'll call BUG() on its equivalent (the
530
+ * PACKET_READ_RESPONSE_END check).
531
+ */
532
+ check_stateless_delimiter(stateless_rpc, reader,
533
+ _("expected response end packet after ref listing"));
534
+
535
+ return 0;
536
+}
537
+
538
struct ref **get_remote_refs(int fd_out, struct packet_reader *reader,
539
struct ref **list, int for_push,
540
struct transport_ls_refs_options *transport_options,
remote.h
+5
@@ -234,6 +234,11 @@ struct ref **get_remote_refs(int fd_out, struct packet_reader *reader,
234
const struct string_list *server_options,
235
int stateless_rpc);
236
237
+/* Used for protocol v2 in order to retrieve refs from a remote */
238
+struct bundle_list;
239
+int get_remote_bundle_uri(int fd_out, struct packet_reader *reader,
240
+ struct bundle_list *bundles, int stateless_rpc);
241
+
242
int resolve_remote_symref(struct ref *ref, struct ref *list);
243
244
/*
t/lib-bundle-uri-protocol.sh
+19
@@ -83,3 +83,22 @@ test_expect_success "connect with $BUNDLE_URI_PROTOCOL:// using protocol v2: hav
83
# Server advertised bundle-uri capability
84
grep "< bundle-uri" log
85
'
86
+
87
+test_expect_success "clone with $BUNDLE_URI_PROTOCOL:// using protocol v2: request bundle-uris" '
88
+ test_when_finished "rm -rf log cloned" &&
89
+
90
+ GIT_TRACE_PACKET="$PWD/log" \
91
+ git \
92
+ -c protocol.version=2 \
93
+ clone "$BUNDLE_URI_REPO_URI" cloned \
94
+ >actual 2>err &&
95
+
96
+ # Server responded using protocol v2
97
+ grep "< version 2" log &&
98
+
99
+ # Server advertised bundle-uri capability
100
+ grep "< bundle-uri" log &&
101
+
102
+ # Client issued bundle-uri command
103
+ grep "> command=bundle-uri" log
104
+'
transport-helper.c
+13
@@ -1267,9 +1267,22 @@ static struct ref *get_refs_list_using_list(struct transport *transport,
1267
return ret;
1268
}
1269
1270
+static int get_bundle_uri(struct transport *transport)
1271
+{
1272
+ get_helper(transport);
1273
+
1274
+ if (process_connect(transport, 0)) {
1275
+ do_take_over(transport);
1276
+ return transport->vtable->get_bundle_uri(transport);
1277
+ }
1278
+
1279
+ return -1;
1280
+}
1281
+
1282
static struct transport_vtable vtable = {
1283
.set_option = set_helper_option,
1284
.get_refs_list = get_refs_list,
1285
+ .get_bundle_uri = get_bundle_uri,
1286
.fetch_refs = fetch_refs,
1287
.push_refs = push_refs,
1288
.connect = connect_helper,
transport-internal.h
+7
@@ -26,6 +26,13 @@ struct transport_vtable {
26
struct ref *(*get_refs_list)(struct transport *transport, int for_push,
27
struct transport_ls_refs_options *transport_options);
28
29
+ /**
30
+ * Populates the remote side's bundle-uri under protocol v2,
31
+ * if the "bundle-uri" capability was advertised. Returns 0 if
32
+ * OK, negative values on error.
33
+ */
34
+ int (*get_bundle_uri)(struct transport *transport);
35
+
36
/**
37
* Fetch the objects for the given refs. Note that this gets
38
* an array, and should ignore the list structure.
transport.c
+51
@@ -22,6 +22,7 @@
22
#include "protocol.h"
23
#include "object-store.h"
24
#include "color.h"
25
+#include "bundle-uri.h"
26
27
static int transport_use_color = -1;
28
static char transport_colors[][COLOR_MAXLEN] = {
@@ -359,6 +360,32 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus
360
return handshake(transport, for_push, options, 1);
361
}
362
363
+static int get_bundle_uri(struct transport *transport)
364
+{
365
+ struct git_transport_data *data = transport->data;
366
+ struct packet_reader reader;
367
+ int stateless_rpc = transport->stateless_rpc;
368
+
369
+ if (!transport->bundles) {
370
+ CALLOC_ARRAY(transport->bundles, 1);
371
+ init_bundle_list(transport->bundles);
372
+ }
373
+
374
+ /*
375
+ * "Support" protocol v0 and v2 without bundle-uri support by
376
+ * silently degrading to a NOOP.
377
+ */
378
+ if (!server_supports_v2("bundle-uri", 0))
379
+ return 0;
380
+
381
+ packet_reader_init(&reader, data->fd[0], NULL, 0,
382
+ PACKET_READ_CHOMP_NEWLINE |
383
+ PACKET_READ_GENTLE_ON_EOF);
384
+
385
+ return get_remote_bundle_uri(data->fd[1], &reader,
386
+ transport->bundles, stateless_rpc);
387
+}
388
+
389
static int fetch_refs_via_pack(struct transport *transport,
390
int nr_heads, struct ref **to_fetch)
391
{
@@ -902,6 +929,7 @@ static int disconnect_git(struct transport *transport)
929
930
static struct transport_vtable taken_over_vtable = {
931
.get_refs_list = get_refs_via_connect,
932
+ .get_bundle_uri = get_bundle_uri,
933
.fetch_refs = fetch_refs_via_pack,
934
.push_refs = git_transport_push,
935
.disconnect = disconnect_git
@@ -1054,6 +1082,7 @@ static struct transport_vtable bundle_vtable = {
1082
1083
static struct transport_vtable builtin_smart_vtable = {
1084
.get_refs_list = get_refs_via_connect,
1085
+ .get_bundle_uri = get_bundle_uri,
1086
.fetch_refs = fetch_refs_via_pack,
1087
.push_refs = git_transport_push,
1088
.connect = connect_git,
@@ -1068,6 +1097,9 @@ struct transport *transport_get(struct remote *remote, const char *url)
1097
ret->progress = isatty(2);
1098
string_list_init_dup(&ret->pack_lockfiles);
1099
1100
+ CALLOC_ARRAY(ret->bundles, 1);
1101
+ init_bundle_list(ret->bundles);
1102
+
1103
if (!remote)
1104
BUG("No remote provided to transport_get()");
1105
@@ -1482,6 +1514,23 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs)
1514
return rc;
1515
}
1516
1517
+int transport_get_remote_bundle_uri(struct transport *transport)
1518
+{
1519
+ const struct transport_vtable *vtable = transport->vtable;
1520
+
1521
+ /* Check config only once. */
1522
+ if (transport->got_remote_bundle_uri)
1523
+ return 0;
1524
+ transport->got_remote_bundle_uri = 1;
1525
+
1526
+ if (!vtable->get_bundle_uri)
1527
+ return error(_("bundle-uri operation not supported by protocol"));
1528
+
1529
+ if (vtable->get_bundle_uri(transport) < 0)
1530
+ return error(_("could not retrieve server-advertised bundle-uri list"));
1531
+ return 0;
1532
+}
1533
+
1534
void transport_unlock_pack(struct transport *transport, unsigned int flags)
1535
{
1536
int in_signal_handler = !!(flags & TRANSPORT_UNLOCK_PACK_IN_SIGNAL_HANDLER);
@@ -1512,6 +1561,8 @@ int transport_disconnect(struct transport *transport)
1561
ret = transport->vtable->disconnect(transport);
1562
if (transport->got_remote_refs)
1563
free_refs((void *)transport->remote_refs);
1564
+ clear_bundle_list(transport->bundles);
1565
+ free(transport->bundles);
1566
free(transport);
1567
return ret;
1568
}
transport.h
+19
@@ -62,6 +62,7 @@ enum transport_family {
62
TRANSPORT_FAMILY_IPV6
63
};
64
65
+struct bundle_list;
66
struct transport {
67
const struct transport_vtable *vtable;
68
@@ -76,6 +77,18 @@ struct transport {
77
*/
78
unsigned got_remote_refs : 1;
79
80
+ /**
81
+ * Indicates whether we already called get_bundle_uri_list(); set by
82
+ * transport.c::transport_get_remote_bundle_uri().
83
+ */
84
+ unsigned got_remote_bundle_uri : 1;
85
+
86
+ /*
87
+ * The results of "command=bundle-uri", if both sides support
88
+ * the "bundle-uri" capability.
89
+ */
90
+ struct bundle_list *bundles;
91
+
92
/*
93
* Transports that call take-over destroys the data specific to
94
* the transport type while doing so, and cannot be reused.
@@ -281,6 +294,12 @@ void transport_ls_refs_options_release(struct transport_ls_refs_options *opts);
294
const struct ref *transport_get_remote_refs(struct transport *transport,
295
struct transport_ls_refs_options *transport_options);
296
297
+/**
298
+ * Retrieve bundle URI(s) from a remote. Populates "struct
299
+ * transport"'s "bundle_uri" and "got_remote_bundle_uri".
300
+ */
301
+int transport_get_remote_bundle_uri(struct transport *transport);
302
+
303
/*
304
* Fetch the hash algorithm used by a remote.
305
*