@matheusbsbs / matheus-bagliano / commits / 7f0cc04f2c

fetch: fetch from an external bundle URI

When a user specifies a URI via 'git clone --bundle-uri', that URI may be a bundle list that advertises a 'bundle.heuristic' value. In that case, the Git client stores a 'fetch.bundleURI' config value storing that URI. Teach 'git fetch' to check for this config value and download bundles from that URI before fetching from the Git remote(s). Likely, the bundle provider has configured a heuristic (such as "creationToken") that will allow the Git client to download only a portion of the bundles before continuing the fetch. Since this URI is completely independent of the remote server, we want to be sure that we connect to the bundle URI before creating a connection to the Git remote. We do not want to hold a stateful connection for too long if we can avoid it. To test that this works correctly, extend the previous tests that set 'fetch.bundleURI' to do follow-up fetches. The bundle list is updated incrementally at each phase to demonstrate that the heuristic avoids downloading older bundles. This includes the middle fetch downloading the objects in bundle-3.bundle from the Git remote, and therefore not needing that bundle in the third fetch. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Jan 31, 2023 at 13:29 UTC 7f0cc04f2ca1653d573dbe981090895dc959ce65
2 files changed +118 -1
builtin/fetch.c
+6
@@ -29,6 +29,7 @@
29 #include "commit-graph.h"
30 #include "shallow.h"
31 #include "worktree.h"
32 +#include "bundle-uri.h"
33
34 #define FORCED_UPDATES_DELAY_WARNING_IN_MS (10 * 1000)
35
@@ -2109,6 +2110,7 @@ static int fetch_one(struct remote *remote, int argc, const char **argv,
2110 int cmd_fetch(int argc, const char **argv, const char *prefix)
2111 {
2112 int i;
2113 + const char *bundle_uri;
2114 struct string_list list = STRING_LIST_INIT_DUP;
2115 struct remote *remote = NULL;
2116 int result = 0;
@@ -2194,6 +2196,10 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
2196 if (dry_run)
2197 write_fetch_head = 0;
2198
2199 + if (!git_config_get_string_tmp("fetch.bundleuri", &bundle_uri) &&
2200 + fetch_bundle_uri(the_repository, bundle_uri, NULL))
2201 + warning(_("failed to fetch bundles from '%s'"), bundle_uri);
2202 +
2203 if (all) {
2204 if (argc == 1)
2205 die(_("fetch --all does not take a repository argument"));
t/t5558-clone-bundle-uri.sh
+112 -1
@@ -440,7 +440,55 @@ test_expect_success 'clone incomplete bundle list (http, creationToken)' '
440 EOF
441
442 test_remote_https_urls <trace-clone.txt >actual &&
443 - test_cmp expect actual
443 + test_cmp expect actual &&
444 +
445 + # We now have only one bundle ref.
446 + git -C clone-token-http for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&
447 + cat >expect <<-\EOF &&
448 + refs/bundles/base
449 + EOF
450 + test_cmp expect refs &&
451 +
452 + # Add remaining bundles, exercising the "deepening" strategy
453 + # for downloading via the creationToken heurisitc.
454 + cat >>"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
455 + [bundle "bundle-2"]
456 + uri = bundle-2.bundle
457 + creationToken = 2
458 +
459 + [bundle "bundle-3"]
460 + uri = bundle-3.bundle
461 + creationToken = 3
462 +
463 + [bundle "bundle-4"]
464 + uri = bundle-4.bundle
465 + creationToken = 4
466 + EOF
467 +
468 + GIT_TRACE2_EVENT="$(pwd)/trace1.txt" \
469 + git -C clone-token-http fetch origin --no-tags \
470 + refs/heads/merge:refs/heads/merge &&
471 +
472 + cat >expect <<-EOF &&
473 + $HTTPD_URL/bundle-list
474 + $HTTPD_URL/bundle-4.bundle
475 + $HTTPD_URL/bundle-3.bundle
476 + $HTTPD_URL/bundle-2.bundle
477 + EOF
478 +
479 + test_remote_https_urls <trace1.txt >actual &&
480 + test_cmp expect actual &&
481 +
482 + # We now have all bundle refs.
483 + git -C clone-token-http for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&
484 +
485 + cat >expect <<-\EOF &&
486 + refs/bundles/base
487 + refs/bundles/left
488 + refs/bundles/merge
489 + refs/bundles/right
490 + EOF
491 + test_cmp expect refs
492 '
493
494 test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
@@ -477,6 +525,69 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
525 cat >expect <<-\EOF &&
526 refs/bundles/base
527 EOF
528 + test_cmp expect refs &&
529 +
530 + cat >>"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
531 + [bundle "bundle-2"]
532 + uri = bundle-2.bundle
533 + creationToken = 2
534 + EOF
535 +
536 + # Fetch the objects for bundle-2 _and_ bundle-3.
537 + GIT_TRACE2_EVENT="$(pwd)/trace1.txt" \
538 + git -C fetch-http-4 fetch origin --no-tags \
539 + refs/heads/left:refs/heads/left \
540 + refs/heads/right:refs/heads/right &&
541 +
542 + cat >expect <<-EOF &&
543 + $HTTPD_URL/bundle-list
544 + $HTTPD_URL/bundle-2.bundle
545 + EOF
546 +
547 + test_remote_https_urls <trace1.txt >actual &&
548 + test_cmp expect actual &&
549 +
550 + # received left from bundle-2
551 + git -C fetch-http-4 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&
552 + cat >expect <<-\EOF &&
553 + refs/bundles/base
554 + refs/bundles/left
555 + EOF
556 + test_cmp expect refs &&
557 +
558 + cat >>"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
559 + [bundle "bundle-3"]
560 + uri = bundle-3.bundle
561 + creationToken = 3
562 +
563 + [bundle "bundle-4"]
564 + uri = bundle-4.bundle
565 + creationToken = 4
566 + EOF
567 +
568 + # This fetch should skip bundle-3.bundle, since its objects are
569 + # already local (we have the requisite commits for bundle-4.bundle).
570 + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
571 + git -C fetch-http-4 fetch origin --no-tags \
572 + refs/heads/merge:refs/heads/merge &&
573 +
574 + cat >expect <<-EOF &&
575 + $HTTPD_URL/bundle-list
576 + $HTTPD_URL/bundle-4.bundle
577 + EOF
578 +
579 + test_remote_https_urls <trace2.txt >actual &&
580 + test_cmp expect actual &&
581 +
582 + # received merge ref from bundle-4, but right is missing
583 + # because we did not download bundle-3.
584 + git -C fetch-http-4 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&
585 +
586 + cat >expect <<-\EOF &&
587 + refs/bundles/base
588 + refs/bundles/left
589 + refs/bundles/merge
590 + EOF
591 test_cmp expect refs
592 '
593