clone: unbundle the advertised bundles
A previous change introduced the transport methods to acquire a bundle list from the 'bundle-uri' protocol v2 command, when advertised _and_ when the client has chosen to enable the feature. Teach Git to download and unbundle the data advertised by those bundles during 'git clone'. This takes place between the ref advertisement and the object data download, and stateful connections will linger while the client downloads bundles. In the future, we should consider closing the remote connection during this process. Also, since the --bundle-uri option exists, we do not want to mix the advertised bundles with the user-specified bundles. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Dec 22, 2022 at 15:14 UTC
876094ac16870727207e6afc0b26060101a52359
3 files changed
+98
-7
builtin/clone.c
+20
-5
@@ -1271,11 +1271,26 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
1271
if (refs)
1272
mapped_refs = wanted_peer_refs(refs, &remote->fetch);
1273
1274
- /*
1275
- * Populate transport->got_remote_bundle_uri and
1276
- * transport->bundle_uri. We might get nothing.
1277
- */
1278
- transport_get_remote_bundle_uri(transport);
1274
+ if (!bundle_uri) {
1275
+ /*
1276
+ * Populate transport->got_remote_bundle_uri and
1277
+ * transport->bundle_uri. We might get nothing.
1278
+ */
1279
+ transport_get_remote_bundle_uri(transport);
1280
+
1281
+ if (transport->bundles &&
1282
+ hashmap_get_size(&transport->bundles->bundles)) {
1283
+ /* At this point, we need the_repository to match the cloned repo. */
1284
+ if (repo_init(the_repository, git_dir, work_tree))
1285
+ warning(_("failed to initialize the repo, skipping bundle URI"));
1286
+ else if (fetch_bundle_list(the_repository,
1287
+ transport->bundles))
1288
+ warning(_("failed to fetch advertised bundles"));
1289
+ } else {
1290
+ clear_bundle_list(transport->bundles);
1291
+ FREE_AND_NULL(transport->bundles);
1292
+ }
1293
+ }
1294
1295
if (mapped_refs) {
1296
int hash_algo = hash_algo_by_ptr(transport_get_hash_algo(transport));
t/lib-bundle-uri-protocol.sh
+19
-2
@@ -85,7 +85,7 @@ test_expect_success "connect with $BUNDLE_URI_PROTOCOL:// using protocol v2: hav
85
'
86
87
test_expect_success "clone with $BUNDLE_URI_PROTOCOL:// using protocol v2: request bundle-uris" '
88
- test_when_finished "rm -rf log cloned cloned2" &&
88
+ test_when_finished "rm -rf log* cloned*" &&
89
90
GIT_TRACE_PACKET="$PWD/log" \
91
git \
@@ -117,7 +117,24 @@ test_expect_success "clone with $BUNDLE_URI_PROTOCOL:// using protocol v2: reque
117
grep "< bundle-uri" log &&
118
119
# Client issued bundle-uri command
120
- grep "> command=bundle-uri" log
120
+ grep "> command=bundle-uri" log &&
121
+
122
+ GIT_TRACE_PACKET="$PWD/log3" \
123
+ git \
124
+ -c transfer.bundleURI=true \
125
+ -c protocol.version=2 \
126
+ clone --bundle-uri="$BUNDLE_URI_BUNDLE_URI" \
127
+ "$BUNDLE_URI_REPO_URI" cloned3 \
128
+ >actual 2>err &&
129
+
130
+ # Server responded using protocol v2
131
+ grep "< version 2" log3 &&
132
+
133
+ # Server advertised bundle-uri capability
134
+ grep "< bundle-uri" log3 &&
135
+
136
+ # Client did not issue bundle-uri command (--bundle-uri override)
137
+ ! grep "> command=bundle-uri" log3
138
'
139
140
# The remaining tests will all assume transfer.bundleURI=true
t/t5601-clone.sh
+59
@@ -772,6 +772,65 @@ test_expect_success 'reject cloning shallow repository using HTTP' '
772
git clone --no-reject-shallow $HTTPD_URL/smart/repo.git repo
773
'
774
775
+test_expect_success 'auto-discover bundle URI from HTTP clone' '
776
+ test_when_finished rm -rf trace.txt repo2 "$HTTPD_DOCUMENT_ROOT_PATH/repo2.git" &&
777
+ git -C src bundle create "$HTTPD_DOCUMENT_ROOT_PATH/everything.bundle" --all &&
778
+ git clone --bare --no-local src "$HTTPD_DOCUMENT_ROOT_PATH/repo2.git" &&
779
+
780
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo2.git" config \
781
+ uploadpack.advertiseBundleURIs true &&
782
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo2.git" config \
783
+ bundle.version 1 &&
784
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo2.git" config \
785
+ bundle.mode all &&
786
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo2.git" config \
787
+ bundle.everything.uri "$HTTPD_URL/everything.bundle" &&
788
+
789
+ GIT_TRACE2_EVENT="$(pwd)/trace.txt" \
790
+ git -c protocol.version=2 \
791
+ -c transfer.bundleURI=true clone \
792
+ $HTTPD_URL/smart/repo2.git repo2 &&
793
+ cat >pattern <<-EOF &&
794
+ "event":"child_start".*"argv":\["git-remote-https","$HTTPD_URL/everything.bundle"\]
795
+ EOF
796
+ grep -f pattern trace.txt
797
+'
798
+
799
+test_expect_success 'auto-discover multiple bundles from HTTP clone' '
800
+ test_when_finished rm -rf trace.txt repo3 "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" &&
801
+
802
+ test_commit -C src new &&
803
+ git -C src bundle create "$HTTPD_DOCUMENT_ROOT_PATH/new.bundle" HEAD~1..HEAD &&
804
+ git clone --bare --no-local src "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" &&
805
+
806
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" config \
807
+ uploadpack.advertiseBundleURIs true &&
808
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" config \
809
+ bundle.version 1 &&
810
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" config \
811
+ bundle.mode all &&
812
+
813
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" config \
814
+ bundle.everything.uri "$HTTPD_URL/everything.bundle" &&
815
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo3.git" config \
816
+ bundle.new.uri "$HTTPD_URL/new.bundle" &&
817
+
818
+ GIT_TRACE2_EVENT="$(pwd)/trace.txt" \
819
+ git -c protocol.version=2 \
820
+ -c transfer.bundleURI=true clone \
821
+ $HTTPD_URL/smart/repo3.git repo3 &&
822
+
823
+ # We should fetch _both_ bundles
824
+ cat >pattern <<-EOF &&
825
+ "event":"child_start".*"argv":\["git-remote-https","$HTTPD_URL/everything.bundle"\]
826
+ EOF
827
+ grep -f pattern trace.txt &&
828
+ cat >pattern <<-EOF &&
829
+ "event":"child_start".*"argv":\["git-remote-https","$HTTPD_URL/new.bundle"\]
830
+ EOF
831
+ grep -f pattern trace.txt
832
+'
833
+
834
# DO NOT add non-httpd-specific tests here, because the last part of this
835
# test script is only executed when httpd is available and enabled.
836