@matheusbsbs / matheus-bagliano / commits / 026df9e047

bundle-uri: test missing bundles with heuristic

The creationToken heuristic uses a different mechanism for downloading bundles from the "standard" approach. Specifically: it uses a concrete order based on the creationToken values and attempts to download as few bundles as possible. It also modifies local config to store a value for future fetches to avoid downloading bundles, if possible. However, if any of the individual bundles has a failed download, then the logic for the ordering comes into question. It is important to avoid infinite loops, assigning invalid creation token values in config, but also to be opportunistic as possible when downloading as many bundles as seem appropriate. These tests were used to inform the implementation of fetch_bundles_by_token() in bundle-uri.c, but are being added independently here to allow focusing on faulty downloads. There may be more cases that could be added that result in modifications to fetch_bundles_by_token() as interesting data shapes reveal themselves in real scenarios. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Jan 31, 2023 at 13:29 UTC 026df9e04789e2af7271566f312aae323740309f

1 file changed +400

t/t5558-clone-bundle-uri.sh

+400

@@ -618,6 +618,406 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '

618 test_cmp expect actual

619 '

620

621 +test_expect_success 'creationToken heuristic with failed downloads (clone)' '

622 + test_when_finished rm -rf download-* trace*.txt &&

623 +

624 + # Case 1: base bundle does not exist, nothing can unbundle

625 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

626 + [bundle]

627 + version = 1

628 + mode = all

629 + heuristic = creationToken

630 +

631 + [bundle "bundle-1"]

632 + uri = fake.bundle

633 + creationToken = 1

634 +

635 + [bundle "bundle-2"]

636 + uri = bundle-2.bundle

637 + creationToken = 2

638 +

639 + [bundle "bundle-3"]

640 + uri = bundle-3.bundle

641 + creationToken = 3

642 +

643 + [bundle "bundle-4"]

644 + uri = bundle-4.bundle

645 + creationToken = 4

646 + EOF

647 +

648 + GIT_TRACE2_EVENT="$(pwd)/trace-clone-1.txt" \

649 + git clone --single-branch --branch=base \

650 + --bundle-uri="$HTTPD_URL/bundle-list" \

651 + "$HTTPD_URL/smart/fetch.git" download-1 &&

652 +

653 + # Bundle failure does not set these configs.

654 + test_must_fail git -C download-1 config fetch.bundleuri &&

655 + test_must_fail git -C download-1 config fetch.bundlecreationtoken &&

656 +

657 + cat >expect <<-EOF &&

658 + $HTTPD_URL/bundle-list

659 + $HTTPD_URL/bundle-4.bundle

660 + $HTTPD_URL/bundle-3.bundle

661 + $HTTPD_URL/bundle-2.bundle

662 + $HTTPD_URL/fake.bundle

663 + EOF

664 + test_remote_https_urls <trace-clone-1.txt >actual &&

665 + test_cmp expect actual &&

666 +

667 + # All bundles failed to unbundle

668 + git -C download-1 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&

669 + test_must_be_empty refs &&

670 +

671 + # Case 2: middle bundle does not exist, only two bundles can unbundle

672 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

673 + [bundle]

674 + version = 1

675 + mode = all

676 + heuristic = creationToken

677 +

678 + [bundle "bundle-1"]

679 + uri = bundle-1.bundle

680 + creationToken = 1

681 +

682 + [bundle "bundle-2"]

683 + uri = fake.bundle

684 + creationToken = 2

685 +

686 + [bundle "bundle-3"]

687 + uri = bundle-3.bundle

688 + creationToken = 3

689 +

690 + [bundle "bundle-4"]

691 + uri = bundle-4.bundle

692 + creationToken = 4

693 + EOF

694 +

695 + GIT_TRACE2_EVENT="$(pwd)/trace-clone-2.txt" \

696 + git clone --single-branch --branch=base \

697 + --bundle-uri="$HTTPD_URL/bundle-list" \

698 + "$HTTPD_URL/smart/fetch.git" download-2 &&

699 +

700 + # Bundle failure does not set these configs.

701 + test_must_fail git -C download-2 config fetch.bundleuri &&

702 + test_must_fail git -C download-2 config fetch.bundlecreationtoken &&

703 +

704 + cat >expect <<-EOF &&

705 + $HTTPD_URL/bundle-list

706 + $HTTPD_URL/bundle-4.bundle

707 + $HTTPD_URL/bundle-3.bundle

708 + $HTTPD_URL/fake.bundle

709 + $HTTPD_URL/bundle-1.bundle

710 + EOF

711 + test_remote_https_urls <trace-clone-2.txt >actual &&

712 + test_cmp expect actual &&

713 +

714 + # bundle-1 and bundle-3 could unbundle, but bundle-4 could not

715 + git -C download-2 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&

716 + cat >expect <<-EOF &&

717 + refs/bundles/base

718 + refs/bundles/right

719 + EOF

720 + test_cmp expect refs &&

721 +

722 + # Case 3: top bundle does not exist, rest unbundle fine.

723 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

724 + [bundle]

725 + version = 1

726 + mode = all

727 + heuristic = creationToken

728 +

729 + [bundle "bundle-1"]

730 + uri = bundle-1.bundle

731 + creationToken = 1

732 +

733 + [bundle "bundle-2"]

734 + uri = bundle-2.bundle

735 + creationToken = 2

736 +

737 + [bundle "bundle-3"]

738 + uri = bundle-3.bundle

739 + creationToken = 3

740 +

741 + [bundle "bundle-4"]

742 + uri = fake.bundle

743 + creationToken = 4

744 + EOF

745 +

746 + GIT_TRACE2_EVENT="$(pwd)/trace-clone-3.txt" \

747 + git clone --single-branch --branch=base \

748 + --bundle-uri="$HTTPD_URL/bundle-list" \

749 + "$HTTPD_URL/smart/fetch.git" download-3 &&

750 +

751 + # As long as we have continguous successful downloads,

752 + # we _do_ set these configs.

753 + test_cmp_config -C download-3 "$HTTPD_URL/bundle-list" fetch.bundleuri &&

754 + test_cmp_config -C download-3 3 fetch.bundlecreationtoken &&

755 +

756 + cat >expect <<-EOF &&

757 + $HTTPD_URL/bundle-list

758 + $HTTPD_URL/fake.bundle

759 + $HTTPD_URL/bundle-3.bundle

760 + $HTTPD_URL/bundle-2.bundle

761 + $HTTPD_URL/bundle-1.bundle

762 + EOF

763 + test_remote_https_urls <trace-clone-3.txt >actual &&

764 + test_cmp expect actual &&

765 +

766 + # fake.bundle did not unbundle, but the others did.

767 + git -C download-3 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&

768 + cat >expect <<-EOF &&

769 + refs/bundles/base

770 + refs/bundles/left

771 + refs/bundles/right

772 + EOF

773 + test_cmp expect refs

774 +'

775 +

776 +# Expand the bundle list to include other interesting shapes, specifically

777 +# interesting for use when fetching from a previous state.

778 +#

779 +# ---------------- bundle-7

780 +# 7

781 +# _/|\_

782 +# ---/--|--\------ bundle-6

783 +# 5 | 6

784 +# --|---|---|----- bundle-4

785 +# | 4 |

786 +# | / \ /

787 +# --|-|---|/------ bundle-3 (the client will be caught up to this point.)

788 +# \ | 3

789 +# ---\|---|------- bundle-2

790 +# 2 |

791 +# ----|---|------- bundle-1

792 +# \ /

793 +# 1

794 +# |

795 +# (previous commits)

796 +test_expect_success 'expand incremental bundle list' '

797 + (

798 + cd clone-from &&

799 + git checkout -b lefter left &&

800 + test_commit 5 &&

801 + git checkout -b righter right &&

802 + test_commit 6 &&

803 + git checkout -b top lefter &&

804 + git merge -m "7" merge righter &&

805 +

806 + git bundle create bundle-6.bundle lefter righter --not left right &&

807 + git bundle create bundle-7.bundle top --not lefter merge righter &&

808 +

809 + cp bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/"

810 + ) &&

811 + git -C "$HTTPD_DOCUMENT_ROOT_PATH/fetch.git" fetch origin +refs/heads/*:refs/heads/*

812 +'

813 +

814 +test_expect_success 'creationToken heuristic with failed downloads (fetch)' '

815 + test_when_finished rm -rf download-* trace*.txt &&

816 +

817 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

818 + [bundle]

819 + version = 1

820 + mode = all

821 + heuristic = creationToken

822 +

823 + [bundle "bundle-1"]

824 + uri = bundle-1.bundle

825 + creationToken = 1

826 +

827 + [bundle "bundle-2"]

828 + uri = bundle-2.bundle

829 + creationToken = 2

830 +

831 + [bundle "bundle-3"]

832 + uri = bundle-3.bundle

833 + creationToken = 3

834 + EOF

835 +

836 + git clone --single-branch --branch=left \

837 + --bundle-uri="$HTTPD_URL/bundle-list" \

838 + "$HTTPD_URL/smart/fetch.git" fetch-base &&

839 + test_cmp_config -C fetch-base "$HTTPD_URL/bundle-list" fetch.bundleURI &&

840 + test_cmp_config -C fetch-base 3 fetch.bundleCreationToken &&

841 +

842 + # Case 1: all bundles exist: successful unbundling of all bundles

843 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

844 + [bundle]

845 + version = 1

846 + mode = all

847 + heuristic = creationToken

848 +

849 + [bundle "bundle-1"]

850 + uri = bundle-1.bundle

851 + creationToken = 1

852 +

853 + [bundle "bundle-2"]

854 + uri = bundle-2.bundle

855 + creationToken = 2

856 +

857 + [bundle "bundle-3"]

858 + uri = bundle-3.bundle

859 + creationToken = 3

860 +

861 + [bundle "bundle-4"]

862 + uri = bundle-4.bundle

863 + creationToken = 4

864 +

865 + [bundle "bundle-6"]

866 + uri = bundle-6.bundle

867 + creationToken = 6

868 +

869 + [bundle "bundle-7"]

870 + uri = bundle-7.bundle

871 + creationToken = 7

872 + EOF

873 +

874 + cp -r fetch-base fetch-1 &&

875 + GIT_TRACE2_EVENT="$(pwd)/trace-fetch-1.txt" \

876 + git -C fetch-1 fetch origin &&

877 + test_cmp_config -C fetch-1 7 fetch.bundlecreationtoken &&

878 +

879 + cat >expect <<-EOF &&

880 + $HTTPD_URL/bundle-list

881 + $HTTPD_URL/bundle-7.bundle

882 + $HTTPD_URL/bundle-6.bundle

883 + $HTTPD_URL/bundle-4.bundle

884 + EOF

885 + test_remote_https_urls <trace-fetch-1.txt >actual &&

886 + test_cmp expect actual &&

887 +

888 + # Check which bundles have unbundled by refs

889 + git -C fetch-1 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&

890 + cat >expect <<-EOF &&

891 + refs/bundles/base

892 + refs/bundles/left

893 + refs/bundles/lefter

894 + refs/bundles/merge

895 + refs/bundles/right

896 + refs/bundles/righter

897 + refs/bundles/top

898 + EOF

899 + test_cmp expect refs &&

900 +

901 + # Case 2: middle bundle does not exist, only bundle-4 can unbundle

902 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

903 + [bundle]

904 + version = 1

905 + mode = all

906 + heuristic = creationToken

907 +

908 + [bundle "bundle-1"]

909 + uri = bundle-1.bundle

910 + creationToken = 1

911 +

912 + [bundle "bundle-2"]

913 + uri = bundle-2.bundle

914 + creationToken = 2

915 +

916 + [bundle "bundle-3"]

917 + uri = bundle-3.bundle

918 + creationToken = 3

919 +

920 + [bundle "bundle-4"]

921 + uri = bundle-4.bundle

922 + creationToken = 4

923 +

924 + [bundle "bundle-6"]

925 + uri = fake.bundle

926 + creationToken = 6

927 +

928 + [bundle "bundle-7"]

929 + uri = bundle-7.bundle

930 + creationToken = 7

931 + EOF

932 +

933 + cp -r fetch-base fetch-2 &&

934 + GIT_TRACE2_EVENT="$(pwd)/trace-fetch-2.txt" \

935 + git -C fetch-2 fetch origin &&

936 +

937 + # Since bundle-7 fails to unbundle, do not update creation token.

938 + test_cmp_config -C fetch-2 3 fetch.bundlecreationtoken &&

939 +

940 + cat >expect <<-EOF &&

941 + $HTTPD_URL/bundle-list

942 + $HTTPD_URL/bundle-7.bundle

943 + $HTTPD_URL/fake.bundle

944 + $HTTPD_URL/bundle-4.bundle

945 + EOF

946 + test_remote_https_urls <trace-fetch-2.txt >actual &&

947 + test_cmp expect actual &&

948 +

949 + # Check which bundles have unbundled by refs

950 + git -C fetch-2 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&

951 + cat >expect <<-EOF &&

952 + refs/bundles/base

953 + refs/bundles/left

954 + refs/bundles/merge

955 + refs/bundles/right

956 + EOF

957 + test_cmp expect refs &&

958 +

959 + # Case 3: top bundle does not exist, rest unbundle fine.

960 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&

961 + [bundle]

962 + version = 1

963 + mode = all

964 + heuristic = creationToken

965 +

966 + [bundle "bundle-1"]

967 + uri = bundle-1.bundle

968 + creationToken = 1

969 +

970 + [bundle "bundle-2"]

971 + uri = bundle-2.bundle

972 + creationToken = 2

973 +

974 + [bundle "bundle-3"]

975 + uri = bundle-3.bundle

976 + creationToken = 3

977 +

978 + [bundle "bundle-4"]

979 + uri = bundle-4.bundle

980 + creationToken = 4

981 +

982 + [bundle "bundle-6"]

983 + uri = bundle-6.bundle

984 + creationToken = 6

985 +

986 + [bundle "bundle-7"]

987 + uri = fake.bundle

988 + creationToken = 7

989 + EOF

990 +

991 + cp -r fetch-base fetch-3 &&

992 + GIT_TRACE2_EVENT="$(pwd)/trace-fetch-3.txt" \

993 + git -C fetch-3 fetch origin &&

994 +

995 + # As long as we have continguous successful downloads,

996 + # we _do_ set the maximum creation token.

997 + test_cmp_config -C fetch-3 6 fetch.bundlecreationtoken &&

998 +

999 + # NOTE: the fetch skips bundle-4 since bundle-6 successfully

1000 + # unbundles itself and bundle-7 failed to download.

1001 + cat >expect <<-EOF &&

1002 + $HTTPD_URL/bundle-list

1003 + $HTTPD_URL/fake.bundle

1004 + $HTTPD_URL/bundle-6.bundle

1005 + EOF

1006 + test_remote_https_urls <trace-fetch-3.txt >actual &&

1007 + test_cmp expect actual &&

1008 +

1009 + # Check which bundles have unbundled by refs

1010 + git -C fetch-3 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&

1011 + cat >expect <<-EOF &&

1012 + refs/bundles/base

1013 + refs/bundles/left

1014 + refs/bundles/lefter

1015 + refs/bundles/right

1016 + refs/bundles/righter

1017 + EOF

1018 + test_cmp expect refs

1019 +'

1020 +

1021 # Do not add tests here unless they use the HTTP server, as they will

1022 # not run unless the HTTP dependencies exist.

1023