| 1 | # Helpers for scripts testing bitmap functionality; see t5310 for |
| 2 | # example usage. |
| 3 | |
| 4 | . "$TEST_DIRECTORY"/lib-midx.sh |
| 5 | |
| 6 | objdir=.git/objects |
| 7 | midx=$objdir/pack/multi-pack-index |
| 8 | |
| 9 | # Compare a file containing rev-list bitmap traversal output to its non-bitmap |
| 10 | # counterpart. You can't just use test_cmp for this, because the two produce |
| 11 | # subtly different output: |
| 12 | # |
| 13 | # - regular output is in traversal order, whereas bitmap is split by type, |
| 14 | # with non-packed objects at the end |
| 15 | # |
| 16 | # - regular output has a space and the pathname appended to non-commit |
| 17 | # objects; bitmap output omits this |
| 18 | # |
| 19 | # This function normalizes and compares the two. The second file should |
| 20 | # always be the bitmap output. |
| 21 | test_bitmap_traversal () { |
| 22 | if test "$1" = "--no-confirm-bitmaps" |
| 23 | then |
| 24 | shift |
| 25 | elif cmp "$1" "$2" |
| 26 | then |
| 27 | echo >&2 "identical raw outputs; are you sure bitmaps were used?" |
| 28 | return 1 |
| 29 | fi && |
| 30 | cut -d' ' -f1 "$1" | sort >"$1.normalized" && |
| 31 | sort "$2" >"$2.normalized" && |
| 32 | test_cmp "$1.normalized" "$2.normalized" && |
| 33 | rm -f "$1.normalized" "$2.normalized" |
| 34 | } |
| 35 | |
| 36 | # To ensure the logic for "maximal commits" is exercised, make |
| 37 | # the repository a bit more complicated. |
| 38 | # |
| 39 | # other second |
| 40 | # * * |
| 41 | # (99 commits) (99 commits) |
| 42 | # * * |
| 43 | # |\ /| |
| 44 | # | * octo-other octo-second * | |
| 45 | # |/|\_________ ____________/|\| |
| 46 | # | \ \/ __________/ | |
| 47 | # | | ________/\ / | |
| 48 | # * |/ * merge-right * |
| 49 | # | _|__________/ \____________ | |
| 50 | # |/ | \| |
| 51 | # (l1) * * merge-left * (r1) |
| 52 | # | / \________________________ | |
| 53 | # |/ \| |
| 54 | # (l2) * * (r2) |
| 55 | # \___________________________ | |
| 56 | # \| |
| 57 | # * (base) |
| 58 | # |
| 59 | # We only push bits down the first-parent history, which |
| 60 | # makes some of these commits unimportant! |
| 61 | # |
| 62 | # The important part for the maximal commit algorithm is how |
| 63 | # the bitmasks are extended. Assuming starting bit positions |
| 64 | # for second (bit 0) and other (bit 1), the bitmasks at the |
| 65 | # end should be: |
| 66 | # |
| 67 | # second: 1 (maximal, selected) |
| 68 | # other: 01 (maximal, selected) |
| 69 | # (base): 11 (maximal) |
| 70 | # |
| 71 | # This complicated history was important for a previous |
| 72 | # version of the walk that guarantees never walking a |
| 73 | # commit multiple times. That goal might be important |
| 74 | # again, so preserve this complicated case. For now, this |
| 75 | # test will guarantee that the bitmaps are computed |
| 76 | # correctly, even with the repeat calculations. |
| 77 | setup_bitmap_history() { |
| 78 | test_expect_success 'setup repo with moderate-sized history' ' |
| 79 | test_commit_bulk --id=file 10 && |
| 80 | git branch -M second && |
| 81 | git checkout -b other HEAD~5 && |
| 82 | test_commit_bulk --id=side 10 && |
| 83 | |
| 84 | # add complicated history setup, including merges and |
| 85 | # ambiguous merge-bases |
| 86 | |
| 87 | git checkout -b merge-left other~2 && |
| 88 | git merge second~2 -m "merge-left" && |
| 89 | |
| 90 | git checkout -b merge-right second~1 && |
| 91 | git merge other~1 -m "merge-right" && |
| 92 | |
| 93 | git checkout -b octo-second second && |
| 94 | git merge merge-left merge-right -m "octopus-second" && |
| 95 | |
| 96 | git checkout -b octo-other other && |
| 97 | git merge merge-left merge-right -m "octopus-other" && |
| 98 | |
| 99 | git checkout other && |
| 100 | git merge octo-other -m "pull octopus" && |
| 101 | |
| 102 | git checkout second && |
| 103 | git merge octo-second -m "pull octopus" && |
| 104 | |
| 105 | # Remove these branches so they are not selected |
| 106 | # as bitmap tips |
| 107 | git branch -D merge-left && |
| 108 | git branch -D merge-right && |
| 109 | git branch -D octo-other && |
| 110 | git branch -D octo-second && |
| 111 | |
| 112 | # add padding to make these merges less interesting |
| 113 | # and avoid having them selected for bitmaps |
| 114 | test_commit_bulk --id=file 100 && |
| 115 | git checkout other && |
| 116 | test_commit_bulk --id=side 100 && |
| 117 | git checkout second && |
| 118 | |
| 119 | bitmaptip=$(git rev-parse second) && |
| 120 | blob=$(echo tagged-blob | git hash-object -w --stdin) && |
| 121 | git tag tagged-blob $blob |
| 122 | ' |
| 123 | } |
| 124 | |
| 125 | rev_list_tests_head () { |
| 126 | test_expect_success "counting commits via bitmap ($state, $branch)" ' |
| 127 | git rev-list --count $branch >expect && |
| 128 | git rev-list --use-bitmap-index --count $branch >actual && |
| 129 | test_cmp expect actual |
| 130 | ' |
| 131 | |
| 132 | test_expect_success "counting partial commits via bitmap ($state, $branch)" ' |
| 133 | git rev-list --count $branch~5..$branch >expect && |
| 134 | git rev-list --use-bitmap-index --count $branch~5..$branch >actual && |
| 135 | test_cmp expect actual |
| 136 | ' |
| 137 | |
| 138 | test_expect_success "counting commits with limit ($state, $branch)" ' |
| 139 | git rev-list --count -n 1 $branch >expect && |
| 140 | git rev-list --use-bitmap-index --count -n 1 $branch >actual && |
| 141 | test_cmp expect actual |
| 142 | ' |
| 143 | |
| 144 | test_expect_success "counting non-linear history ($state, $branch)" ' |
| 145 | git rev-list --count other...second >expect && |
| 146 | git rev-list --use-bitmap-index --count other...second >actual && |
| 147 | test_cmp expect actual |
| 148 | ' |
| 149 | |
| 150 | test_expect_success "counting commits with limiting ($state, $branch)" ' |
| 151 | git rev-list --count $branch -- 1.t >expect && |
| 152 | git rev-list --use-bitmap-index --count $branch -- 1.t >actual && |
| 153 | test_cmp expect actual |
| 154 | ' |
| 155 | |
| 156 | test_expect_success "counting objects via bitmap ($state, $branch)" ' |
| 157 | git rev-list --count --objects $branch >expect && |
| 158 | git rev-list --use-bitmap-index --count --objects $branch >actual && |
| 159 | test_cmp expect actual |
| 160 | ' |
| 161 | |
| 162 | test_expect_success "enumerate commits ($state, $branch)" ' |
| 163 | git rev-list --use-bitmap-index $branch >actual && |
| 164 | git rev-list $branch >expect && |
| 165 | test_bitmap_traversal --no-confirm-bitmaps expect actual |
| 166 | ' |
| 167 | |
| 168 | test_expect_success "enumerate --objects ($state, $branch)" ' |
| 169 | git rev-list --objects --use-bitmap-index $branch >actual && |
| 170 | git rev-list --objects $branch >expect && |
| 171 | test_bitmap_traversal expect actual |
| 172 | ' |
| 173 | |
| 174 | test_expect_success "bitmap --objects handles non-commit objects ($state, $branch)" ' |
| 175 | git rev-list --objects --use-bitmap-index $branch tagged-blob >actual && |
| 176 | grep $blob actual |
| 177 | ' |
| 178 | } |
| 179 | |
| 180 | rev_list_tests () { |
| 181 | state=$1 |
| 182 | |
| 183 | for branch in "second" "other" |
| 184 | do |
| 185 | rev_list_tests_head |
| 186 | done |
| 187 | } |
| 188 | |
| 189 | basic_bitmap_tests () { |
| 190 | tip="$1" |
| 191 | test_expect_success 'rev-list --test-bitmap verifies bitmaps' " |
| 192 | git rev-list --test-bitmap "${tip:-HEAD}" |
| 193 | " |
| 194 | |
| 195 | rev_list_tests 'full bitmap' |
| 196 | |
| 197 | test_expect_success 'clone from bitmapped repository' ' |
| 198 | rm -fr clone.git && |
| 199 | git clone --no-local --bare . clone.git && |
| 200 | git rev-parse HEAD >expect && |
| 201 | git --git-dir=clone.git rev-parse HEAD >actual && |
| 202 | test_cmp expect actual |
| 203 | ' |
| 204 | |
| 205 | test_expect_success 'partial clone from bitmapped repository' ' |
| 206 | test_config uploadpack.allowfilter true && |
| 207 | rm -fr partial-clone.git && |
| 208 | git clone --no-local --bare --filter=blob:none . partial-clone.git && |
| 209 | ( |
| 210 | cd partial-clone.git && |
| 211 | pack=$(echo objects/pack/*.pack) && |
| 212 | git verify-pack -v "$pack" >have && |
| 213 | awk "/blob/ { print \$1 }" <have >blobs && |
| 214 | # we expect this single blob because of the direct ref |
| 215 | git rev-parse refs/tags/tagged-blob >expect && |
| 216 | test_cmp expect blobs |
| 217 | ) |
| 218 | ' |
| 219 | |
| 220 | test_expect_success 'setup further non-bitmapped commits' ' |
| 221 | test_commit_bulk --id=further 10 |
| 222 | ' |
| 223 | |
| 224 | rev_list_tests 'partial bitmap' |
| 225 | |
| 226 | test_expect_success 'fetch (partial bitmap)' ' |
| 227 | git --git-dir=clone.git fetch origin second:second && |
| 228 | git rev-parse HEAD >expect && |
| 229 | git --git-dir=clone.git rev-parse HEAD >actual && |
| 230 | test_cmp expect actual |
| 231 | ' |
| 232 | |
| 233 | test_expect_success 'enumerating progress counts pack-reused objects' ' |
| 234 | count=$(git rev-list --objects --all --count) && |
| 235 | git repack -adb && |
| 236 | |
| 237 | # check first with only reused objects; confirm that our |
| 238 | # progress showed the right number, and also that we did |
| 239 | # pack-reuse as expected. Check only the final "done" |
| 240 | # line of the meter (there may be an arbitrary number of |
| 241 | # intermediate lines ending with CR). |
| 242 | GIT_PROGRESS_DELAY=0 \ |
| 243 | git pack-objects --all --stdout --progress \ |
| 244 | </dev/null >/dev/null 2>stderr && |
| 245 | grep "Enumerating objects: $count, done" stderr && |
| 246 | grep "pack-reused $count" stderr && |
| 247 | |
| 248 | # now the same but with one non-reused object |
| 249 | git commit --allow-empty -m "an extra commit object" && |
| 250 | GIT_PROGRESS_DELAY=0 \ |
| 251 | git pack-objects --all --stdout --progress \ |
| 252 | </dev/null >/dev/null 2>stderr && |
| 253 | grep "Enumerating objects: $((count+1)), done" stderr && |
| 254 | grep "pack-reused $count" stderr |
| 255 | ' |
| 256 | } |
| 257 | |
| 258 | # have_delta <obj> <expected_base> |
| 259 | # |
| 260 | # Note that because this relies on cat-file, it might find _any_ copy of an |
| 261 | # object in the repository. The caller is responsible for making sure |
| 262 | # there's only one (e.g., via "repack -ad", or having just fetched a copy). |
| 263 | have_delta () { |
| 264 | echo $2 >expect && |
| 265 | echo $1 | git cat-file --batch-check="%(deltabase)" >actual && |
| 266 | test_cmp expect actual |
| 267 | } |
| 268 | |
| 269 | # midx_pack_source <obj> |
| 270 | midx_pack_source () { |
| 271 | test-tool read-midx --show-objects .git/objects | grep "^$1 " | cut -f2 |
| 272 | } |
| 273 | |
| 274 | test_rev_exists () { |
| 275 | commit="$1" |
| 276 | kind="$2" |
| 277 | |
| 278 | test_expect_success "reverse index exists ($kind)" ' |
| 279 | GIT_TRACE2_EVENT=$(pwd)/event.trace \ |
| 280 | git rev-list --test-bitmap "$commit" && |
| 281 | |
| 282 | if test "rev" = "$kind" |
| 283 | then |
| 284 | test_path_is_file $midx-$(midx_checksum $objdir).rev |
| 285 | fi && |
| 286 | grep "\"category\":\"load_midx_revindex\",\"key\":\"source\",\"value\":\"$kind\"" event.trace |
| 287 | ' |
| 288 | } |
| 289 | |
| 290 | midx_bitmap_core () { |
| 291 | rev_kind="${1:-midx}" |
| 292 | |
| 293 | setup_bitmap_history |
| 294 | |
| 295 | test_expect_success 'create single-pack midx with bitmaps' ' |
| 296 | git repack -ad && |
| 297 | git multi-pack-index write --bitmap && |
| 298 | test_path_is_file $midx && |
| 299 | test_path_is_file $midx-$(midx_checksum $objdir).bitmap |
| 300 | ' |
| 301 | |
| 302 | test_rev_exists HEAD "$rev_kind" |
| 303 | |
| 304 | basic_bitmap_tests |
| 305 | |
| 306 | test_expect_success 'create new additional packs' ' |
| 307 | for i in $(test_seq 1 16) |
| 308 | do |
| 309 | test_commit "$i" && |
| 310 | git repack -d || return 1 |
| 311 | done && |
| 312 | |
| 313 | git checkout -b other2 HEAD~8 && |
| 314 | for i in $(test_seq 1 8) |
| 315 | do |
| 316 | test_commit "side-$i" && |
| 317 | git repack -d || return 1 |
| 318 | done && |
| 319 | git checkout second |
| 320 | ' |
| 321 | |
| 322 | test_expect_success 'create multi-pack midx with bitmaps' ' |
| 323 | git multi-pack-index write --bitmap && |
| 324 | |
| 325 | ls $objdir/pack/pack-*.pack >packs && |
| 326 | test_line_count = 25 packs && |
| 327 | |
| 328 | test_path_is_file $midx && |
| 329 | test_path_is_file $midx-$(midx_checksum $objdir).bitmap |
| 330 | ' |
| 331 | |
| 332 | test_rev_exists HEAD "$rev_kind" |
| 333 | |
| 334 | basic_bitmap_tests |
| 335 | |
| 336 | test_expect_success '--no-bitmap is respected when bitmaps exist' ' |
| 337 | git multi-pack-index write --bitmap && |
| 338 | |
| 339 | test_commit respect--no-bitmap && |
| 340 | git repack -d && |
| 341 | |
| 342 | test_path_is_file $midx && |
| 343 | test_path_is_file $midx-$(midx_checksum $objdir).bitmap && |
| 344 | |
| 345 | git multi-pack-index write --no-bitmap && |
| 346 | |
| 347 | test_path_is_file $midx && |
| 348 | test_path_is_missing $midx-$(midx_checksum $objdir).bitmap && |
| 349 | test_path_is_missing $midx-$(midx_checksum $objdir).rev |
| 350 | ' |
| 351 | |
| 352 | test_expect_success 'setup midx with base from later pack' ' |
| 353 | # Write a and b so that "a" is a delta on top of base "b", since Git |
| 354 | # prefers to delete contents out of a base rather than add to a shorter |
| 355 | # object. |
| 356 | test_seq 1 128 >a && |
| 357 | test_seq 1 130 >b && |
| 358 | |
| 359 | git add a b && |
| 360 | git commit -m "initial commit" && |
| 361 | |
| 362 | a=$(git rev-parse HEAD:a) && |
| 363 | b=$(git rev-parse HEAD:b) && |
| 364 | |
| 365 | # In the first pack, "a" is stored as a delta to "b". |
| 366 | p1=$(git pack-objects .git/objects/pack/pack <<-EOF |
| 367 | $a |
| 368 | $b |
| 369 | EOF |
| 370 | ) && |
| 371 | |
| 372 | # In the second pack, "a" is missing, and "b" is not a delta nor base to |
| 373 | # any other object. |
| 374 | p2=$(git pack-objects .git/objects/pack/pack <<-EOF |
| 375 | $b |
| 376 | $(git rev-parse HEAD) |
| 377 | $(git rev-parse HEAD^{tree}) |
| 378 | EOF |
| 379 | ) && |
| 380 | |
| 381 | git prune-packed && |
| 382 | # Use the second pack as the preferred source, so that "b" occurs |
| 383 | # earlier in the MIDX object order, rendering "a" unusable for pack |
| 384 | # reuse. |
| 385 | git multi-pack-index write --bitmap --preferred-pack=pack-$p2.idx && |
| 386 | |
| 387 | have_delta $a $b && |
| 388 | test $(midx_pack_source $a) != $(midx_pack_source $b) |
| 389 | ' |
| 390 | |
| 391 | rev_list_tests 'full bitmap with backwards delta' |
| 392 | |
| 393 | test_expect_success 'clone with bitmaps enabled' ' |
| 394 | git clone --no-local --bare . clone-reverse-delta.git && |
| 395 | test_when_finished "rm -fr clone-reverse-delta.git" && |
| 396 | |
| 397 | git rev-parse HEAD >expect && |
| 398 | git --git-dir=clone-reverse-delta.git rev-parse HEAD >actual && |
| 399 | test_cmp expect actual |
| 400 | ' |
| 401 | |
| 402 | test_expect_success 'changing the preferred pack does not corrupt bitmaps' ' |
| 403 | rm -fr repo && |
| 404 | git init repo && |
| 405 | test_when_finished "rm -fr repo" && |
| 406 | ( |
| 407 | cd repo && |
| 408 | |
| 409 | test_commit A && |
| 410 | test_commit B && |
| 411 | |
| 412 | git rev-list --objects --no-object-names HEAD^ >A.objects && |
| 413 | git rev-list --objects --no-object-names HEAD^.. >B.objects && |
| 414 | |
| 415 | A=$(git pack-objects $objdir/pack/pack <A.objects) && |
| 416 | B=$(git pack-objects $objdir/pack/pack <B.objects) && |
| 417 | |
| 418 | cat >indexes <<-EOF && |
| 419 | pack-$A.idx |
| 420 | pack-$B.idx |
| 421 | EOF |
| 422 | |
| 423 | git multi-pack-index write --bitmap --stdin-packs \ |
| 424 | --preferred-pack=pack-$A.pack <indexes && |
| 425 | git rev-list --test-bitmap A && |
| 426 | |
| 427 | git multi-pack-index write --bitmap --stdin-packs \ |
| 428 | --preferred-pack=pack-$B.pack <indexes && |
| 429 | git rev-list --test-bitmap A |
| 430 | ) |
| 431 | ' |
| 432 | } |
| 433 | |
| 434 | midx_bitmap_partial_tests () { |
| 435 | rev_kind="${1:-midx}" |
| 436 | |
| 437 | test_expect_success 'setup partial bitmaps' ' |
| 438 | test_commit packed && |
| 439 | git repack && |
| 440 | test_commit loose && |
| 441 | git multi-pack-index write --bitmap && |
| 442 | test_path_is_file $midx && |
| 443 | test_path_is_file $midx-$(midx_checksum $objdir).bitmap |
| 444 | ' |
| 445 | |
| 446 | test_rev_exists HEAD~ "$rev_kind" |
| 447 | |
| 448 | basic_bitmap_tests HEAD~ |
| 449 | } |