| 1 | #!/bin/sh |
| 2 | |
| 3 | test_description='git partial clone' |
| 4 | |
| 5 | GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main |
| 6 | export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME |
| 7 | |
| 8 | . ./test-lib.sh |
| 9 | |
| 10 | # create a normal "src" repo where we can later create new commits. |
| 11 | # expect_1.oids will contain a list of the OIDs of all blobs. |
| 12 | test_expect_success 'setup normal src repo' ' |
| 13 | echo "{print \$1}" >print_1.awk && |
| 14 | echo "{print \$2}" >print_2.awk && |
| 15 | |
| 16 | git init src && |
| 17 | for n in 1 2 3 4 |
| 18 | do |
| 19 | echo "This is file: $n" > src/file.$n.txt && |
| 20 | git -C src add file.$n.txt && |
| 21 | git -C src commit -m "file $n" && |
| 22 | git -C src ls-files -s file.$n.txt >>temp || return 1 |
| 23 | done && |
| 24 | awk -f print_2.awk <temp | sort >expect_1.oids && |
| 25 | test_line_count = 4 expect_1.oids |
| 26 | ' |
| 27 | |
| 28 | # bare clone "src" giving "srv.bare" for use as our server. |
| 29 | test_expect_success 'setup bare clone for server' ' |
| 30 | git clone --bare "file://$(pwd)/src" srv.bare && |
| 31 | git -C srv.bare config --local uploadpack.allowfilter 1 && |
| 32 | git -C srv.bare config --local uploadpack.allowanysha1inwant 1 |
| 33 | ' |
| 34 | |
| 35 | # do basic partial clone from "srv.bare" |
| 36 | # confirm we are missing all of the known blobs. |
| 37 | # confirm partial clone was registered in the local config. |
| 38 | test_expect_success 'do partial clone 1' ' |
| 39 | git clone --no-checkout --filter=blob:none "file://$(pwd)/srv.bare" pc1 && |
| 40 | |
| 41 | git -C pc1 rev-list --quiet --objects --missing=print HEAD >revs && |
| 42 | awk -f print_1.awk revs | |
| 43 | sed "s/?//" | |
| 44 | sort >observed.oids && |
| 45 | |
| 46 | test_cmp expect_1.oids observed.oids && |
| 47 | test "$(git -C pc1 config --local core.repositoryformatversion)" = "1" && |
| 48 | test "$(git -C pc1 config --local remote.origin.promisor)" = "true" && |
| 49 | test "$(git -C pc1 config --local remote.origin.partialclonefilter)" = "blob:none" |
| 50 | ' |
| 51 | |
| 52 | test_expect_success 'rev-list --missing=allow-promisor on partial clone' ' |
| 53 | git -C pc1 rev-list --objects --missing=allow-promisor HEAD >actual && |
| 54 | git -C pc1 rev-list --objects --missing=print HEAD >expect.raw && |
| 55 | grep -v "^?" expect.raw >expect && |
| 56 | test_cmp expect actual |
| 57 | ' |
| 58 | |
| 59 | test_expect_success 'verify that .promisor file contains refs fetched' ' |
| 60 | ls pc1/.git/objects/pack/pack-*.promisor >promisorlist && |
| 61 | test_line_count = 1 promisorlist && |
| 62 | git -C srv.bare rev-parse --verify HEAD >headhash && |
| 63 | grep "$(cat headhash) HEAD" $(cat promisorlist) && |
| 64 | grep "$(cat headhash) refs/heads/main" $(cat promisorlist) |
| 65 | ' |
| 66 | |
| 67 | # checkout main to force dynamic object fetch of blobs at HEAD. |
| 68 | test_expect_success 'verify checkout with dynamic object fetch' ' |
| 69 | git -C pc1 rev-list --quiet --objects --missing=print HEAD >observed && |
| 70 | test_line_count = 4 observed && |
| 71 | git -C pc1 checkout main && |
| 72 | git -C pc1 rev-list --quiet --objects --missing=print HEAD >observed && |
| 73 | test_line_count = 0 observed |
| 74 | ' |
| 75 | |
| 76 | # create new commits in "src" repo to establish a blame history on file.1.txt |
| 77 | # and push to "srv.bare". |
| 78 | test_expect_success 'push new commits to server' ' |
| 79 | git -C src remote add srv "file://$(pwd)/srv.bare" && |
| 80 | for x in a b c d e |
| 81 | do |
| 82 | echo "Mod file.1.txt $x" >>src/file.1.txt && |
| 83 | git -C src add file.1.txt && |
| 84 | git -C src commit -m "mod $x" || return 1 |
| 85 | done && |
| 86 | git -C src blame main -- file.1.txt >expect.blame && |
| 87 | git -C src push -u srv main |
| 88 | ' |
| 89 | |
| 90 | # (partial) fetch in the partial clone repo from the promisor remote. |
| 91 | # verify that fetch inherited the filter-spec from the config and DOES NOT |
| 92 | # have the new blobs. |
| 93 | test_expect_success 'partial fetch inherits filter settings' ' |
| 94 | git -C pc1 fetch origin && |
| 95 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 96 | main..origin/main >observed && |
| 97 | test_line_count = 5 observed |
| 98 | ' |
| 99 | |
| 100 | # force dynamic object fetch using diff. |
| 101 | # we should only get 1 new blob (for the file in origin/main). |
| 102 | test_expect_success 'verify diff causes dynamic object fetch' ' |
| 103 | git -C pc1 diff main..origin/main -- file.1.txt && |
| 104 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 105 | main..origin/main >observed && |
| 106 | test_line_count = 4 observed |
| 107 | ' |
| 108 | |
| 109 | # force full dynamic object fetch of the file's history using blame. |
| 110 | # we should get the intermediate blobs for the file. |
| 111 | test_expect_success 'verify blame causes dynamic object fetch' ' |
| 112 | git -C pc1 blame origin/main -- file.1.txt >observed.blame && |
| 113 | test_cmp expect.blame observed.blame && |
| 114 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 115 | main..origin/main >observed && |
| 116 | test_line_count = 0 observed |
| 117 | ' |
| 118 | |
| 119 | # create new commits in "src" repo to establish a history on file.2.txt |
| 120 | # and push to "srv.bare". |
| 121 | test_expect_success 'push new commits to server for file.2.txt' ' |
| 122 | for x in a b c d e f |
| 123 | do |
| 124 | echo "Mod file.2.txt $x" >>src/file.2.txt && |
| 125 | git -C src add file.2.txt && |
| 126 | git -C src commit -m "mod $x" || return 1 |
| 127 | done && |
| 128 | git -C src push -u srv main |
| 129 | ' |
| 130 | |
| 131 | # Do FULL fetch by disabling inherited filter-spec using --no-filter. |
| 132 | # Verify we have all the new blobs. |
| 133 | test_expect_success 'override inherited filter-spec using --no-filter' ' |
| 134 | git -C pc1 fetch --no-filter origin && |
| 135 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 136 | main..origin/main >observed && |
| 137 | test_line_count = 0 observed |
| 138 | ' |
| 139 | |
| 140 | # create new commits in "src" repo to establish a history on file.3.txt |
| 141 | # and push to "srv.bare". |
| 142 | test_expect_success 'push new commits to server for file.3.txt' ' |
| 143 | for x in a b c d e f |
| 144 | do |
| 145 | echo "Mod file.3.txt $x" >>src/file.3.txt && |
| 146 | git -C src add file.3.txt && |
| 147 | git -C src commit -m "mod $x" || return 1 |
| 148 | done && |
| 149 | git -C src push -u srv main |
| 150 | ' |
| 151 | |
| 152 | # Do a partial fetch and then try to manually fetch the missing objects. |
| 153 | # This can be used as the basis of a pre-command hook to bulk fetch objects |
| 154 | # perhaps combined with a command in dry-run mode. |
| 155 | test_expect_success 'manual prefetch of missing objects' ' |
| 156 | git -C pc1 fetch --filter=blob:none origin && |
| 157 | |
| 158 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 159 | main..origin/main >revs && |
| 160 | awk -f print_1.awk revs | |
| 161 | sed "s/?//" | |
| 162 | sort >observed.oids && |
| 163 | |
| 164 | test_line_count = 6 observed.oids && |
| 165 | git -C pc1 fetch-pack --stdin "file://$(pwd)/srv.bare" <observed.oids && |
| 166 | |
| 167 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 168 | main..origin/main >revs && |
| 169 | awk -f print_1.awk revs | |
| 170 | sed "s/?//" | |
| 171 | sort >observed.oids && |
| 172 | |
| 173 | test_line_count = 0 observed.oids |
| 174 | ' |
| 175 | |
| 176 | # create new commits in "src" repo to establish a history on file.4.txt |
| 177 | # and push to "srv.bare". |
| 178 | test_expect_success 'push new commits to server for file.4.txt' ' |
| 179 | for x in a b c d e f |
| 180 | do |
| 181 | echo "Mod file.4.txt $x" >src/file.4.txt && |
| 182 | if list_contains "a,b" "$x"; then |
| 183 | printf "%10000s" X >>src/file.4.txt |
| 184 | fi && |
| 185 | if list_contains "c,d" "$x"; then |
| 186 | printf "%20000s" X >>src/file.4.txt |
| 187 | fi && |
| 188 | git -C src add file.4.txt && |
| 189 | git -C src commit -m "mod $x" || return 1 |
| 190 | done && |
| 191 | git -C src push -u srv main |
| 192 | ' |
| 193 | |
| 194 | # Do partial fetch to fetch smaller files; then verify that without --refetch |
| 195 | # applying a new filter does not refetch missing large objects. Then use |
| 196 | # --refetch to apply the new filter on existing commits. Test it under both |
| 197 | # protocol v2 & v0. |
| 198 | test_expect_success 'apply a different filter using --refetch' ' |
| 199 | git -C pc1 fetch --filter=blob:limit=999 origin && |
| 200 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 201 | main..origin/main >observed && |
| 202 | test_line_count = 4 observed && |
| 203 | |
| 204 | git -C pc1 fetch --filter=blob:limit=19999 --refetch origin && |
| 205 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 206 | main..origin/main >observed && |
| 207 | test_line_count = 2 observed && |
| 208 | |
| 209 | git -c protocol.version=0 -C pc1 fetch --filter=blob:limit=29999 \ |
| 210 | --refetch origin && |
| 211 | git -C pc1 rev-list --quiet --objects --missing=print \ |
| 212 | main..origin/main >observed && |
| 213 | test_line_count = 0 observed |
| 214 | ' |
| 215 | |
| 216 | test_expect_success 'fetch --refetch works with a shallow clone' ' |
| 217 | git clone --no-checkout --depth=1 --filter=blob:none "file://$(pwd)/srv.bare" pc1s && |
| 218 | git -C pc1s rev-list --objects --missing=print HEAD >observed && |
| 219 | test_line_count = 6 observed && |
| 220 | |
| 221 | GIT_TRACE=1 git -C pc1s fetch --filter=blob:limit=999 --refetch origin && |
| 222 | git -C pc1s rev-list --objects --missing=print HEAD >observed && |
| 223 | test_line_count = 6 observed |
| 224 | ' |
| 225 | |
| 226 | test_expect_success 'fetch --refetch triggers repacking' ' |
| 227 | GIT_TRACE2_CONFIG_PARAMS=gc.autoPackLimit,maintenance.incremental-repack.auto && |
| 228 | export GIT_TRACE2_CONFIG_PARAMS && |
| 229 | |
| 230 | GIT_TRACE2_EVENT="$PWD/trace1.event" \ |
| 231 | git -C pc1 fetch --refetch origin && |
| 232 | test_subcommand git maintenance run --auto --no-quiet --no-detach <trace1.event && |
| 233 | grep \"param\":\"gc.autopacklimit\",\"value\":\"1\" trace1.event && |
| 234 | grep \"param\":\"maintenance.incremental-repack.auto\",\"value\":\"-1\" trace1.event && |
| 235 | |
| 236 | GIT_TRACE2_EVENT="$PWD/trace2.event" \ |
| 237 | git -c protocol.version=0 \ |
| 238 | -c gc.autoPackLimit=0 \ |
| 239 | -c maintenance.incremental-repack.auto=1234 \ |
| 240 | -C pc1 fetch --refetch origin && |
| 241 | test_subcommand git maintenance run --auto --no-quiet --no-detach <trace2.event && |
| 242 | grep \"param\":\"gc.autopacklimit\",\"value\":\"0\" trace2.event && |
| 243 | grep \"param\":\"maintenance.incremental-repack.auto\",\"value\":\"-1\" trace2.event && |
| 244 | |
| 245 | GIT_TRACE2_EVENT="$PWD/trace3.event" \ |
| 246 | git -c protocol.version=0 \ |
| 247 | -c gc.autoPackLimit=1234 \ |
| 248 | -c maintenance.incremental-repack.auto=0 \ |
| 249 | -C pc1 fetch --refetch origin && |
| 250 | test_subcommand git maintenance run --auto --no-quiet --no-detach <trace3.event && |
| 251 | grep \"param\":\"gc.autopacklimit\",\"value\":\"1\" trace3.event && |
| 252 | grep \"param\":\"maintenance.incremental-repack.auto\",\"value\":\"0\" trace3.event |
| 253 | ' |
| 254 | |
| 255 | test_expect_success 'partial clone with transfer.fsckobjects=1 works with submodules' ' |
| 256 | test_create_repo submodule && |
| 257 | test_commit -C submodule mycommit && |
| 258 | |
| 259 | test_create_repo src_with_sub && |
| 260 | git -C src_with_sub config uploadpack.allowfilter 1 && |
| 261 | git -C src_with_sub config uploadpack.allowanysha1inwant 1 && |
| 262 | |
| 263 | test_config_global protocol.file.allow always && |
| 264 | |
| 265 | git -C src_with_sub submodule add "file://$(pwd)/submodule" mysub && |
| 266 | git -C src_with_sub commit -m "commit with submodule" && |
| 267 | |
| 268 | git -c transfer.fsckobjects=1 \ |
| 269 | clone --filter="blob:none" "file://$(pwd)/src_with_sub" dst && |
| 270 | test_when_finished rm -rf dst |
| 271 | ' |
| 272 | |
| 273 | test_expect_success 'lazily fetched .gitmodules works' ' |
| 274 | git clone --filter="blob:none" --no-checkout "file://$(pwd)/src_with_sub" dst && |
| 275 | git -C dst fetch && |
| 276 | test_when_finished rm -rf dst |
| 277 | ' |
| 278 | |
| 279 | test_expect_success 'partial clone with transfer.fsckobjects=1 uses index-pack --fsck-objects' ' |
| 280 | git init src && |
| 281 | test_commit -C src x && |
| 282 | test_config -C src uploadpack.allowfilter 1 && |
| 283 | test_config -C src uploadpack.allowanysha1inwant 1 && |
| 284 | |
| 285 | GIT_TRACE="$(pwd)/trace" git -c transfer.fsckobjects=1 \ |
| 286 | clone --filter="blob:none" "file://$(pwd)/src" dst && |
| 287 | grep "git index-pack.*--fsck-objects" trace |
| 288 | ' |
| 289 | |
| 290 | test_expect_success 'use fsck before and after manually fetching a missing subtree' ' |
| 291 | # push new commit so server has a subtree |
| 292 | mkdir src/dir && |
| 293 | echo "in dir" >src/dir/file.txt && |
| 294 | git -C src add dir/file.txt && |
| 295 | git -C src commit -m "file in dir" && |
| 296 | git -C src push -u srv main && |
| 297 | SUBTREE=$(git -C src rev-parse HEAD:dir) && |
| 298 | |
| 299 | rm -rf dst && |
| 300 | git clone --no-checkout --filter=tree:0 "file://$(pwd)/srv.bare" dst && |
| 301 | git -C dst fsck && |
| 302 | |
| 303 | # Make sure we only have commits, and all trees and blobs are missing. |
| 304 | git -C dst rev-list --missing=allow-any --objects main \ |
| 305 | >fetched_objects && |
| 306 | awk -f print_1.awk fetched_objects | |
| 307 | xargs -n1 git -C dst cat-file -t >fetched_types && |
| 308 | |
| 309 | sort -u fetched_types >unique_types.observed && |
| 310 | echo commit >unique_types.expected && |
| 311 | test_cmp unique_types.expected unique_types.observed && |
| 312 | |
| 313 | # Auto-fetch a tree with cat-file. |
| 314 | git -C dst cat-file -p $SUBTREE >tree_contents && |
| 315 | grep file.txt tree_contents && |
| 316 | |
| 317 | # fsck still works after an auto-fetch of a tree. |
| 318 | git -C dst fsck && |
| 319 | |
| 320 | # Auto-fetch all remaining trees and blobs with --missing=error |
| 321 | git -C dst rev-list --missing=error --objects main >fetched_objects && |
| 322 | test_line_count = 88 fetched_objects && |
| 323 | |
| 324 | awk -f print_1.awk fetched_objects | |
| 325 | xargs -n1 git -C dst cat-file -t >fetched_types && |
| 326 | |
| 327 | sort -u fetched_types >unique_types.observed && |
| 328 | test_write_lines blob commit tree >unique_types.expected && |
| 329 | test_cmp unique_types.expected unique_types.observed |
| 330 | ' |
| 331 | |
| 332 | test_expect_success 'implicitly construct combine: filter with repeated flags' ' |
| 333 | GIT_TRACE=$(pwd)/trace git clone --bare \ |
| 334 | --filter=blob:none --filter=tree:1 \ |
| 335 | "file://$(pwd)/srv.bare" pc2 && |
| 336 | grep "trace:.* git pack-objects .*--filter=combine:blob:none+tree:1" \ |
| 337 | trace && |
| 338 | git -C pc2 rev-list --objects --missing=allow-any HEAD >objects && |
| 339 | |
| 340 | # We should have gotten some root trees. |
| 341 | grep " $" objects && |
| 342 | # Should not have gotten any non-root trees or blobs. |
| 343 | ! grep " ." objects && |
| 344 | |
| 345 | xargs -n 1 git -C pc2 cat-file -t <objects >types && |
| 346 | sort -u types >unique_types.actual && |
| 347 | test_write_lines commit tree >unique_types.expected && |
| 348 | test_cmp unique_types.expected unique_types.actual |
| 349 | ' |
| 350 | |
| 351 | test_expect_success 'upload-pack complains of bogus filter config' ' |
| 352 | printf 0000 | |
| 353 | test_must_fail git \ |
| 354 | -c uploadpackfilter.tree.maxdepth \ |
| 355 | upload-pack . >/dev/null 2>err && |
| 356 | test_grep "unable to parse.*tree.maxdepth" err |
| 357 | ' |
| 358 | |
| 359 | test_expect_success 'upload-pack fails banned object filters' ' |
| 360 | test_config -C srv.bare uploadpackfilter.blob:none.allow false && |
| 361 | test_must_fail ok=sigpipe git clone --no-checkout --filter=blob:none \ |
| 362 | "file://$(pwd)/srv.bare" pc3 2>err && |
| 363 | test_grep "filter '\''blob:none'\'' not supported" err |
| 364 | ' |
| 365 | |
| 366 | test_expect_success 'upload-pack fails banned combine object filters' ' |
| 367 | test_config -C srv.bare uploadpackfilter.allow false && |
| 368 | test_config -C srv.bare uploadpackfilter.combine.allow true && |
| 369 | test_config -C srv.bare uploadpackfilter.tree.allow true && |
| 370 | test_config -C srv.bare uploadpackfilter.blob:none.allow false && |
| 371 | test_must_fail ok=sigpipe git clone --no-checkout --filter=tree:1 \ |
| 372 | --filter=blob:none "file://$(pwd)/srv.bare" pc3 2>err && |
| 373 | test_grep "filter '\''blob:none'\'' not supported" err |
| 374 | ' |
| 375 | |
| 376 | test_expect_success 'upload-pack fails banned object filters with fallback' ' |
| 377 | test_config -C srv.bare uploadpackfilter.allow false && |
| 378 | test_must_fail ok=sigpipe git clone --no-checkout --filter=blob:none \ |
| 379 | "file://$(pwd)/srv.bare" pc3 2>err && |
| 380 | test_grep "filter '\''blob:none'\'' not supported" err |
| 381 | ' |
| 382 | |
| 383 | test_expect_success 'upload-pack limits tree depth filters' ' |
| 384 | test_config -C srv.bare uploadpackfilter.allow false && |
| 385 | test_config -C srv.bare uploadpackfilter.tree.allow true && |
| 386 | test_config -C srv.bare uploadpackfilter.tree.maxDepth 0 && |
| 387 | test_must_fail ok=sigpipe git clone --no-checkout --filter=tree:1 \ |
| 388 | "file://$(pwd)/srv.bare" pc3 2>err && |
| 389 | test_grep "tree filter allows max depth 0, but got 1" err && |
| 390 | |
| 391 | git clone --no-checkout --filter=tree:0 "file://$(pwd)/srv.bare" pc4 && |
| 392 | |
| 393 | test_config -C srv.bare uploadpackfilter.tree.maxDepth 5 && |
| 394 | git clone --no-checkout --filter=tree:5 "file://$(pwd)/srv.bare" pc5 && |
| 395 | test_must_fail ok=sigpipe git clone --no-checkout --filter=tree:6 \ |
| 396 | "file://$(pwd)/srv.bare" pc6 2>err && |
| 397 | test_grep "tree filter allows max depth 5, but got 6" err |
| 398 | ' |
| 399 | |
| 400 | test_expect_success 'partial clone fetches blobs pointed to by refs even if normally filtered out' ' |
| 401 | rm -rf src dst && |
| 402 | git init src && |
| 403 | test_commit -C src x && |
| 404 | test_config -C src uploadpack.allowfilter 1 && |
| 405 | test_config -C src uploadpack.allowanysha1inwant 1 && |
| 406 | |
| 407 | # Create a tag pointing to a blob. |
| 408 | BLOB=$(echo blob-contents | git -C src hash-object --stdin -w) && |
| 409 | git -C src tag myblob "$BLOB" && |
| 410 | |
| 411 | git clone --filter="blob:none" "file://$(pwd)/src" dst 2>err && |
| 412 | ! grep "does not point to a valid object" err && |
| 413 | git -C dst fsck |
| 414 | ' |
| 415 | |
| 416 | test_expect_success 'fetch what is specified on CLI even if already promised' ' |
| 417 | rm -rf src dst.git && |
| 418 | git init src && |
| 419 | test_commit -C src foo && |
| 420 | test_config -C src uploadpack.allowfilter 1 && |
| 421 | test_config -C src uploadpack.allowanysha1inwant 1 && |
| 422 | |
| 423 | git hash-object --stdin <src/foo.t >blob && |
| 424 | |
| 425 | git clone --bare --filter=blob:none "file://$(pwd)/src" dst.git && |
| 426 | git -C dst.git rev-list --objects --quiet --missing=print HEAD >missing_before && |
| 427 | grep "?$(cat blob)" missing_before && |
| 428 | git -C dst.git fetch origin $(cat blob) && |
| 429 | git -C dst.git rev-list --objects --quiet --missing=print HEAD >missing_after && |
| 430 | ! grep "?$(cat blob)" missing_after |
| 431 | ' |
| 432 | |
| 433 | test_expect_success 'setup src repo for sparse filter' ' |
| 434 | git init sparse-src && |
| 435 | git -C sparse-src config --local uploadpack.allowfilter 1 && |
| 436 | git -C sparse-src config --local uploadpack.allowanysha1inwant 1 && |
| 437 | test_commit -C sparse-src one && |
| 438 | test_commit -C sparse-src two && |
| 439 | echo /one.t >sparse-src/only-one && |
| 440 | git -C sparse-src add . && |
| 441 | git -C sparse-src commit -m "add sparse checkout files" |
| 442 | ' |
| 443 | |
| 444 | test_expect_success 'partial clone with sparse filter succeeds' ' |
| 445 | rm -rf dst.git && |
| 446 | git clone --no-local --bare \ |
| 447 | --filter=sparse:oid=main:only-one \ |
| 448 | sparse-src dst.git && |
| 449 | ( |
| 450 | cd dst.git && |
| 451 | git rev-list --objects --missing=print HEAD >out && |
| 452 | grep "^$(git rev-parse HEAD:one.t)" out && |
| 453 | grep "^?$(git rev-parse HEAD:two.t)" out |
| 454 | ) |
| 455 | ' |
| 456 | |
| 457 | test_expect_success 'partial clone with unresolvable sparse filter fails cleanly' ' |
| 458 | rm -rf dst.git && |
| 459 | test_must_fail git clone --no-local --bare \ |
| 460 | --filter=sparse:oid=main:no-such-name \ |
| 461 | sparse-src dst.git 2>err && |
| 462 | test_grep "unable to access sparse blob in .main:no-such-name" err && |
| 463 | test_must_fail git clone --no-local --bare \ |
| 464 | --filter=sparse:oid=main \ |
| 465 | sparse-src dst.git 2>err && |
| 466 | test_grep "unable to parse sparse filter data in" err |
| 467 | ' |
| 468 | |
| 469 | setup_triangle () { |
| 470 | rm -rf big-blob.txt server client promisor-remote && |
| 471 | |
| 472 | printf "line %d\n" $(test_seq 1 100) >big-blob.txt && |
| 473 | |
| 474 | # Create a server with 2 commits: a commit with a big tree and a child |
| 475 | # commit with an incremental change. Also, create a partial clone |
| 476 | # client that only contains the first commit. |
| 477 | git init server && |
| 478 | git -C server config --local uploadpack.allowfilter 1 && |
| 479 | for i in $(test_seq 1 100) |
| 480 | do |
| 481 | echo "make the tree big" >server/file$i && |
| 482 | git -C server add file$i || return 1 |
| 483 | done && |
| 484 | git -C server commit -m "initial" && |
| 485 | git clone --bare --filter=tree:0 "file://$(pwd)/server" client && |
| 486 | echo another line >>server/file1 && |
| 487 | git -C server commit -am "incremental change" && |
| 488 | |
| 489 | # Create a promisor remote that only contains the tree and blob from |
| 490 | # the first commit. |
| 491 | git init promisor-remote && |
| 492 | git -C server config --local uploadpack.allowanysha1inwant 1 && |
| 493 | TREE_HASH=$(git -C server rev-parse HEAD~1^{tree}) && |
| 494 | git -C promisor-remote fetch --keep "file://$(pwd)/server" "$TREE_HASH" && |
| 495 | git -C promisor-remote count-objects -v >object-count && |
| 496 | test_grep "count: 0" object-count && |
| 497 | test_grep "in-pack: 2" object-count && |
| 498 | |
| 499 | # Set it as the promisor remote of client. Thus, whenever |
| 500 | # the client lazy fetches, the lazy fetch will succeed only if it is |
| 501 | # for this tree or blob. |
| 502 | test_commit -C promisor-remote one && # so that ref advertisement is not empty |
| 503 | git -C promisor-remote config --local uploadpack.allowanysha1inwant 1 && |
| 504 | git -C client remote set-url origin "file://$(pwd)/promisor-remote" |
| 505 | } |
| 506 | |
| 507 | # NEEDSWORK: The tests beginning with "fetch lazy-fetches" below only |
| 508 | # test that "fetch" avoid fetching trees and blobs, but not commits or |
| 509 | # tags. Revisit this if Git is ever taught to support partial clones |
| 510 | # with commits and/or tags filtered out. |
| 511 | |
| 512 | test_expect_success 'fetch lazy-fetches only to resolve deltas' ' |
| 513 | setup_triangle && |
| 514 | |
| 515 | # Exercise to make sure it works. Git will not fetch anything from the |
| 516 | # promisor remote other than for the big tree (because it needs to |
| 517 | # resolve the delta). |
| 518 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client \ |
| 519 | fetch "file://$(pwd)/server" main && |
| 520 | |
| 521 | # Verify the assumption that the client needed to fetch the delta base |
| 522 | # to resolve the delta. |
| 523 | git -C server rev-parse HEAD~1^{tree} >hash && |
| 524 | grep "want $(cat hash)" trace |
| 525 | ' |
| 526 | |
| 527 | test_expect_success 'fetch lazy-fetches only to resolve deltas, protocol v2' ' |
| 528 | setup_triangle && |
| 529 | |
| 530 | git -C server config --local protocol.version 2 && |
| 531 | git -C client config --local protocol.version 2 && |
| 532 | git -C promisor-remote config --local protocol.version 2 && |
| 533 | |
| 534 | # Exercise to make sure it works. Git will not fetch anything from the |
| 535 | # promisor remote other than for the big blob (because it needs to |
| 536 | # resolve the delta). |
| 537 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client \ |
| 538 | fetch "file://$(pwd)/server" main && |
| 539 | |
| 540 | # Verify that protocol version 2 was used. |
| 541 | grep "fetch< version 2" trace && |
| 542 | |
| 543 | # Verify the assumption that the client needed to fetch the delta base |
| 544 | # to resolve the delta. |
| 545 | git -C server rev-parse HEAD~1^{tree} >hash && |
| 546 | grep "want $(cat hash)" trace |
| 547 | ' |
| 548 | |
| 549 | test_expect_success 'fetch does not lazy-fetch missing targets of its refs' ' |
| 550 | rm -rf server client trace && |
| 551 | |
| 552 | test_create_repo server && |
| 553 | test_config -C server uploadpack.allowfilter 1 && |
| 554 | test_config -C server uploadpack.allowanysha1inwant 1 && |
| 555 | test_commit -C server foo && |
| 556 | |
| 557 | git clone --filter=blob:none "file://$(pwd)/server" client && |
| 558 | # Make all refs point to nothing by deleting all objects. |
| 559 | rm client/.git/objects/pack/* && |
| 560 | |
| 561 | test_commit -C server bar && |
| 562 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ |
| 563 | --no-tags --recurse-submodules=no \ |
| 564 | origin refs/tags/bar && |
| 565 | FOO_HASH=$(git -C server rev-parse foo) && |
| 566 | ! grep "want $FOO_HASH" trace |
| 567 | ' |
| 568 | |
| 569 | # The following two tests must be in this order. It is important that |
| 570 | # the srv.bare repository did not have tags during clone, but has tags |
| 571 | # in the fetch. |
| 572 | |
| 573 | test_expect_success 'verify fetch succeeds when asking for new tags' ' |
| 574 | git clone --filter=blob:none "file://$(pwd)/srv.bare" tag-test && |
| 575 | for i in I J K |
| 576 | do |
| 577 | test_commit -C src $i && |
| 578 | git -C src branch $i || return 1 |
| 579 | done && |
| 580 | git -C srv.bare fetch --tags origin +refs/heads/*:refs/heads/* && |
| 581 | git -C tag-test -c protocol.version=2 fetch --tags origin |
| 582 | ' |
| 583 | |
| 584 | test_expect_success 'verify fetch downloads only one pack when updating refs' ' |
| 585 | git clone --filter=blob:none "file://$(pwd)/srv.bare" pack-test && |
| 586 | ls pack-test/.git/objects/pack/*pack >pack-list && |
| 587 | test_line_count = 2 pack-list && |
| 588 | test_config -C pack-test maintenance.auto false && |
| 589 | for i in A B C |
| 590 | do |
| 591 | test_commit -C src $i && |
| 592 | git -C src branch $i || return 1 |
| 593 | done && |
| 594 | git -C srv.bare fetch origin +refs/heads/*:refs/heads/* && |
| 595 | git -C pack-test fetch origin && |
| 596 | ls pack-test/.git/objects/pack/*pack >pack-list && |
| 597 | test_line_count = 3 pack-list |
| 598 | ' |
| 599 | |
| 600 | test_expect_success 'single-branch tag following respects partial clone' ' |
| 601 | git clone --single-branch -b B --filter=blob:none \ |
| 602 | "file://$(pwd)/srv.bare" single && |
| 603 | git -C single rev-parse --verify refs/tags/B && |
| 604 | git -C single rev-parse --verify refs/tags/A && |
| 605 | test_must_fail git -C single rev-parse --verify refs/tags/C |
| 606 | ' |
| 607 | |
| 608 | test_expect_success 'fetch from a partial clone, protocol v0' ' |
| 609 | rm -rf server client trace && |
| 610 | |
| 611 | # Pretend that the server is a partial clone |
| 612 | git init server && |
| 613 | git -C server remote add a_remote "file://$(pwd)/" && |
| 614 | test_config -C server core.repositoryformatversion 1 && |
| 615 | test_config -C server extensions.partialclone a_remote && |
| 616 | test_config -C server protocol.version 0 && |
| 617 | test_commit -C server foo && |
| 618 | |
| 619 | # Fetch from the server |
| 620 | git init client && |
| 621 | test_config -C client protocol.version 0 && |
| 622 | test_commit -C client bar && |
| 623 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "file://$(pwd)/server" && |
| 624 | ! grep "version 2" trace |
| 625 | ' |
| 626 | |
| 627 | test_expect_success 'fetch from a partial clone, protocol v2' ' |
| 628 | rm -rf server client trace && |
| 629 | |
| 630 | # Pretend that the server is a partial clone |
| 631 | git init server && |
| 632 | git -C server remote add a_remote "file://$(pwd)/" && |
| 633 | test_config -C server core.repositoryformatversion 1 && |
| 634 | test_config -C server extensions.partialclone a_remote && |
| 635 | test_config -C server protocol.version 2 && |
| 636 | test_commit -C server foo && |
| 637 | |
| 638 | # Fetch from the server |
| 639 | git init client && |
| 640 | test_config -C client protocol.version 2 && |
| 641 | test_commit -C client bar && |
| 642 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "file://$(pwd)/server" && |
| 643 | grep "version 2" trace |
| 644 | ' |
| 645 | |
| 646 | test_expect_success 'repack does not loosen promisor objects' ' |
| 647 | rm -rf client trace && |
| 648 | git clone --bare --filter=blob:none "file://$(pwd)/srv.bare" client && |
| 649 | test_when_finished "rm -rf client trace" && |
| 650 | GIT_TRACE2_PERF="$(pwd)/trace" git -C client repack -A -d && |
| 651 | grep "loosen_unused_packed_objects/loosened:0" trace |
| 652 | ' |
| 653 | |
| 654 | test_expect_success 'lazy-fetch in submodule succeeds' ' |
| 655 | # setup |
| 656 | test_config_global protocol.file.allow always && |
| 657 | |
| 658 | test_when_finished "rm -rf src-sub" && |
| 659 | git init src-sub && |
| 660 | git -C src-sub config uploadpack.allowfilter 1 && |
| 661 | git -C src-sub config uploadpack.allowanysha1inwant 1 && |
| 662 | |
| 663 | # This blob must be missing in the subsequent commit. |
| 664 | echo foo >src-sub/file && |
| 665 | git -C src-sub add file && |
| 666 | git -C src-sub commit -m "submodule one" && |
| 667 | SUB_ONE=$(git -C src-sub rev-parse HEAD) && |
| 668 | |
| 669 | echo bar >src-sub/file && |
| 670 | git -C src-sub add file && |
| 671 | git -C src-sub commit -m "submodule two" && |
| 672 | SUB_TWO=$(git -C src-sub rev-parse HEAD) && |
| 673 | |
| 674 | test_when_finished "rm -rf src-super" && |
| 675 | git init src-super && |
| 676 | git -C src-super config uploadpack.allowfilter 1 && |
| 677 | git -C src-super config uploadpack.allowanysha1inwant 1 && |
| 678 | git -C src-super submodule add ../src-sub src-sub && |
| 679 | |
| 680 | git -C src-super/src-sub checkout $SUB_ONE && |
| 681 | git -C src-super add src-sub && |
| 682 | git -C src-super commit -m "superproject one" && |
| 683 | |
| 684 | git -C src-super/src-sub checkout $SUB_TWO && |
| 685 | git -C src-super add src-sub && |
| 686 | git -C src-super commit -m "superproject two" && |
| 687 | |
| 688 | # the fetch |
| 689 | test_when_finished "rm -rf client" && |
| 690 | git clone --filter=blob:none --also-filter-submodules \ |
| 691 | --recurse-submodules "file://$(pwd)/src-super" client && |
| 692 | |
| 693 | # Trigger lazy-fetch from the superproject |
| 694 | git -C client restore --recurse-submodules --source=HEAD^ :/ |
| 695 | ' |
| 696 | |
| 697 | test_expect_success 'after fetching descendants of non-promisor commits, gc works' ' |
| 698 | # Setup |
| 699 | git init full && |
| 700 | git -C full config uploadpack.allowfilter 1 && |
| 701 | git -C full config uploadpack.allowanysha1inwant 1 && |
| 702 | touch full/foo && |
| 703 | git -C full add foo && |
| 704 | git -C full commit -m "commit 1" && |
| 705 | git -C full checkout --detach && |
| 706 | |
| 707 | # Partial clone and push commit to remote |
| 708 | git clone "file://$(pwd)/full" --filter=blob:none partial && |
| 709 | echo "hello" > partial/foo && |
| 710 | git -C partial commit -a -m "commit 2" && |
| 711 | git -C partial push && |
| 712 | |
| 713 | # gc in partial repo |
| 714 | git -C partial gc --prune=now && |
| 715 | |
| 716 | # Create another commit in normal repo |
| 717 | git -C full checkout main && |
| 718 | echo " world" >> full/foo && |
| 719 | git -C full commit -a -m "commit 3" && |
| 720 | |
| 721 | # Pull from remote in partial repo, and run gc again |
| 722 | git -C partial pull && |
| 723 | git -C partial gc --prune=now |
| 724 | ' |
| 725 | |
| 726 | |
| 727 | . "$TEST_DIRECTORY"/lib-httpd.sh |
| 728 | start_httpd |
| 729 | |
| 730 | # Converts bytes into their hexadecimal representation. For example, |
| 731 | # "printf 'ab\r\n' | hex_unpack" results in '61620d0a'. |
| 732 | hex_unpack () { |
| 733 | perl -e '$/ = undef; $input = <>; print unpack("H2" x length($input), $input)' |
| 734 | } |
| 735 | |
| 736 | # Inserts $1 at the start of the string and every 2 characters thereafter. |
| 737 | intersperse () { |
| 738 | sed 's/\(..\)/'$1'\1/g' |
| 739 | } |
| 740 | |
| 741 | # Create a one-time-script command to replace the existing packfile with $1. |
| 742 | replace_packfile () { |
| 743 | cp "$1" one-time-pack && |
| 744 | write_script "$HTTPD_ROOT_PATH/one-time-script" <<-EOF |
| 745 | if grep packfile "\$1" >/dev/null |
| 746 | then |
| 747 | sed '/packfile/q' "\$1" && |
| 748 | # The protocol requires that the packfile be sent in sideband |
| 749 | # 1, hence the extra \001 byte at the beginning. |
| 750 | printf "%04x\001" \$((\$(wc -c <"$PWD/one-time-pack") + 5)) && |
| 751 | cat "$PWD/one-time-pack" && |
| 752 | printf "0000" |
| 753 | else |
| 754 | cat "\$1" |
| 755 | fi |
| 756 | EOF |
| 757 | } |
| 758 | |
| 759 | test_expect_success 'upon cloning, check that all refs point to objects' ' |
| 760 | SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && |
| 761 | rm -rf "$SERVER" repo && |
| 762 | test_create_repo "$SERVER" && |
| 763 | test_commit -C "$SERVER" foo && |
| 764 | test_config -C "$SERVER" uploadpack.allowfilter 1 && |
| 765 | test_config -C "$SERVER" uploadpack.allowanysha1inwant 1 && |
| 766 | |
| 767 | # Create a tag pointing to a blob. |
| 768 | BLOB=$(echo blob-contents | git -C "$SERVER" hash-object --stdin -w) && |
| 769 | git -C "$SERVER" tag myblob "$BLOB" && |
| 770 | |
| 771 | # Craft a packfile not including that blob. |
| 772 | git -C "$SERVER" rev-parse HEAD | |
| 773 | git -C "$SERVER" pack-objects --stdout >incomplete.pack && |
| 774 | |
| 775 | # Replace the existing packfile with the crafted one. The protocol |
| 776 | # requires that the packfile be sent in sideband 1, hence the extra |
| 777 | # \x01 byte at the beginning. |
| 778 | replace_packfile incomplete.pack && |
| 779 | |
| 780 | # Use protocol v2 because the perl command looks for the "packfile" |
| 781 | # section header. |
| 782 | test_config -C "$SERVER" protocol.version 2 && |
| 783 | test_must_fail git -c protocol.version=2 clone \ |
| 784 | --filter=blob:none $HTTPD_URL/one_time_script/server repo 2>err && |
| 785 | |
| 786 | test_grep "did not send all necessary objects" err && |
| 787 | |
| 788 | # Ensure that the one-time-script script was used. |
| 789 | ! test -e "$HTTPD_ROOT_PATH/one-time-script" |
| 790 | ' |
| 791 | |
| 792 | test_expect_success 'when partial cloning, tolerate server not sending target of tag' ' |
| 793 | SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && |
| 794 | rm -rf "$SERVER" repo && |
| 795 | test_create_repo "$SERVER" && |
| 796 | test_commit -C "$SERVER" foo && |
| 797 | test_config -C "$SERVER" uploadpack.allowfilter 1 && |
| 798 | test_config -C "$SERVER" uploadpack.allowanysha1inwant 1 && |
| 799 | |
| 800 | # Create an annotated tag pointing to a blob. |
| 801 | BLOB=$(echo blob-contents | git -C "$SERVER" hash-object --stdin -w) && |
| 802 | git -C "$SERVER" tag -m message -a myblob "$BLOB" && |
| 803 | |
| 804 | # Craft a packfile including the tag, but not the blob it points to. |
| 805 | # Also, omit objects referenced from HEAD in order to force a second |
| 806 | # fetch (to fetch missing objects) upon the automatic checkout that |
| 807 | # happens after a clone. |
| 808 | printf "%s\n%s\n--not\n%s\n%s\n" \ |
| 809 | $(git -C "$SERVER" rev-parse HEAD) \ |
| 810 | $(git -C "$SERVER" rev-parse myblob) \ |
| 811 | $(git -C "$SERVER" rev-parse HEAD^{tree}) \ |
| 812 | $(git -C "$SERVER" rev-parse myblob^{blob}) | |
| 813 | git -C "$SERVER" pack-objects --thin --stdout >incomplete.pack && |
| 814 | |
| 815 | # Replace the existing packfile with the crafted one. The protocol |
| 816 | # requires that the packfile be sent in sideband 1, hence the extra |
| 817 | # \x01 byte at the beginning. |
| 818 | replace_packfile incomplete.pack && |
| 819 | |
| 820 | # Use protocol v2 because the perl command looks for the "packfile" |
| 821 | # section header. |
| 822 | test_config -C "$SERVER" protocol.version 2 && |
| 823 | |
| 824 | # Exercise to make sure it works. |
| 825 | git -c protocol.version=2 clone \ |
| 826 | --filter=blob:none $HTTPD_URL/one_time_script/server repo 2> err && |
| 827 | ! grep "missing object referenced by" err && |
| 828 | |
| 829 | # Ensure that the one-time-script script was used. |
| 830 | ! test -e "$HTTPD_ROOT_PATH/one-time-script" |
| 831 | ' |
| 832 | |
| 833 | test_expect_success PERL_TEST_HELPERS 'tolerate server sending REF_DELTA against missing promisor objects' ' |
| 834 | SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && |
| 835 | rm -rf "$SERVER" repo && |
| 836 | test_create_repo "$SERVER" && |
| 837 | test_config -C "$SERVER" uploadpack.allowfilter 1 && |
| 838 | test_config -C "$SERVER" uploadpack.allowanysha1inwant 1 && |
| 839 | |
| 840 | # Create a commit with 2 blobs to be used as delta bases. |
| 841 | for i in $(test_seq 10) |
| 842 | do |
| 843 | echo "this is a line" >>"$SERVER/foo.txt" && |
| 844 | echo "this is another line" >>"$SERVER/have.txt" || return 1 |
| 845 | done && |
| 846 | git -C "$SERVER" add foo.txt have.txt && |
| 847 | git -C "$SERVER" commit -m bar && |
| 848 | git -C "$SERVER" rev-parse HEAD:foo.txt >deltabase_missing && |
| 849 | git -C "$SERVER" rev-parse HEAD:have.txt >deltabase_have && |
| 850 | |
| 851 | # Clone. The client has deltabase_have but not deltabase_missing. |
| 852 | git -c protocol.version=2 clone --no-checkout \ |
| 853 | --filter=blob:none $HTTPD_URL/one_time_script/server repo && |
| 854 | git -C repo hash-object -w -- "$SERVER/have.txt" && |
| 855 | |
| 856 | # Sanity check to ensure that the client does not have |
| 857 | # deltabase_missing. |
| 858 | git -C repo rev-list --objects --ignore-missing \ |
| 859 | -- $(cat deltabase_missing) >objlist && |
| 860 | test_line_count = 0 objlist && |
| 861 | |
| 862 | # Another commit. This commit will be fetched by the client. |
| 863 | echo "abcdefghijklmnopqrstuvwxyz" >>"$SERVER/foo.txt" && |
| 864 | echo "abcdefghijklmnopqrstuvwxyz" >>"$SERVER/have.txt" && |
| 865 | git -C "$SERVER" add foo.txt have.txt && |
| 866 | git -C "$SERVER" commit -m baz && |
| 867 | |
| 868 | # Pack a thin pack containing, among other things, HEAD:foo.txt |
| 869 | # delta-ed against HEAD^:foo.txt and HEAD:have.txt delta-ed against |
| 870 | # HEAD^:have.txt. |
| 871 | printf "%s\n--not\n%s\n" \ |
| 872 | $(git -C "$SERVER" rev-parse HEAD) \ |
| 873 | $(git -C "$SERVER" rev-parse HEAD^) | |
| 874 | git -C "$SERVER" pack-objects --thin --stdout >thin.pack && |
| 875 | |
| 876 | # Ensure that the pack contains one delta against HEAD^:foo.txt. Since |
| 877 | # the delta contains at least 26 novel characters, the size cannot be |
| 878 | # contained in 4 bits, so the object header will take up 2 bytes. The |
| 879 | # most significant nybble of the first byte is 0b1111 (0b1 to indicate |
| 880 | # that the header continues, and 0b111 to indicate REF_DELTA), followed |
| 881 | # by any 3 nybbles, then the OID of the delta base. |
| 882 | printf "f.,..%s" $(intersperse "," <deltabase_missing) >want && |
| 883 | hex_unpack <thin.pack | intersperse "," >have && |
| 884 | grep $(cat want) have && |
| 885 | |
| 886 | # Ensure that the pack contains one delta against HEAD^:have.txt, |
| 887 | # similar to the above. |
| 888 | printf "f.,..%s" $(intersperse "," <deltabase_have) >want && |
| 889 | hex_unpack <thin.pack | intersperse "," >have && |
| 890 | grep $(cat want) have && |
| 891 | |
| 892 | replace_packfile thin.pack && |
| 893 | |
| 894 | # Use protocol v2 because the perl command looks for the "packfile" |
| 895 | # section header. |
| 896 | test_config -C "$SERVER" protocol.version 2 && |
| 897 | |
| 898 | # Fetch the thin pack and ensure that index-pack is able to handle the |
| 899 | # REF_DELTA object with a missing promisor delta base. |
| 900 | GIT_TRACE_PACKET="$(pwd)/trace" git -C repo -c protocol.version=2 fetch && |
| 901 | |
| 902 | # Ensure that the missing delta base was directly fetched, but not the |
| 903 | # one that the client has. |
| 904 | grep "want $(cat deltabase_missing)" trace && |
| 905 | ! grep "want $(cat deltabase_have)" trace && |
| 906 | |
| 907 | # Ensure that the one-time-script script was used. |
| 908 | ! test -e "$HTTPD_ROOT_PATH/one-time-script" |
| 909 | ' |
| 910 | |
| 911 | # DO NOT add non-httpd-specific tests here, because the last part of this |
| 912 | # test script is only executed when httpd is available and enabled. |
| 913 | |
| 914 | test_done |