| 1 | #!/bin/sh |
| 2 | |
| 3 | test_description='git repack --write-midx=incremental' |
| 4 | |
| 5 | . ./test-lib.sh |
| 6 | |
| 7 | GIT_TEST_MULTI_PACK_INDEX=0 |
| 8 | GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 |
| 9 | GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0 |
| 10 | |
| 11 | objdir=.git/objects |
| 12 | packdir=$objdir/pack |
| 13 | midxdir=$packdir/multi-pack-index.d |
| 14 | midx_chain=$midxdir/multi-pack-index-chain |
| 15 | |
| 16 | # incrementally_repack N |
| 17 | # |
| 18 | # Make "N" new commits, each stored in their own pack, and then repacked |
| 19 | # with the --write-midx=incremental strategy. |
| 20 | incrementally_repack () { |
| 21 | for i in $(test_seq 1 "$1") |
| 22 | do |
| 23 | test_commit "$i" && |
| 24 | |
| 25 | git repack --geometric=2 -d --write-midx=incremental \ |
| 26 | --write-bitmap-index && |
| 27 | git multi-pack-index verify || return 1 |
| 28 | done |
| 29 | } |
| 30 | |
| 31 | # Create packs with geometrically increasing sizes so that they |
| 32 | # satisfy the geometric progression and survive a --geometric=2 |
| 33 | # repack without being rolled up. Creates 3 packs containing 1, |
| 34 | # 2, and 6 commits (3, 6, and 18 objects) respectively. |
| 35 | create_geometric_packs () { |
| 36 | test_commit "small" && |
| 37 | git repack -d && |
| 38 | |
| 39 | test_commit_bulk --message="medium" 2 && |
| 40 | test_commit_bulk --message="large" 6 && |
| 41 | |
| 42 | git repack --geometric=2 -d --write-midx=incremental \ |
| 43 | --write-bitmap-index |
| 44 | } |
| 45 | |
| 46 | # create_layer <test_commit_bulk args> |
| 47 | # |
| 48 | # Creates a new MIDX layer with the contents of "test_commit_bulk $@". |
| 49 | create_layer () { |
| 50 | test_commit_bulk "$@" && |
| 51 | |
| 52 | git multi-pack-index write --incremental --bitmap |
| 53 | } |
| 54 | |
| 55 | # create_layers |
| 56 | # |
| 57 | # Reads lines of "<message> <nr>" from stdin and creates a new MIDX |
| 58 | # layer for each line. See create_layer above for more. |
| 59 | create_layers () { |
| 60 | while read msg nr |
| 61 | do |
| 62 | create_layer --message="$msg" "$nr" || return 1 |
| 63 | done |
| 64 | } |
| 65 | |
| 66 | test_expect_success '--write-midx=incremental without --geometric' ' |
| 67 | git init incremental-without-geometric && |
| 68 | ( |
| 69 | cd incremental-without-geometric && |
| 70 | |
| 71 | git config maintenance.auto false && |
| 72 | |
| 73 | test_commit first && |
| 74 | git repack -d && |
| 75 | |
| 76 | test_commit second && |
| 77 | git repack --write-midx=incremental && |
| 78 | |
| 79 | git multi-pack-index verify && |
| 80 | test_line_count = 1 $midx_chain && |
| 81 | cp $midx_chain $midx_chain.before && |
| 82 | |
| 83 | # A second repack appends a new layer without |
| 84 | # disturbing the existing one. |
| 85 | test_commit third && |
| 86 | git repack --write-midx=incremental && |
| 87 | |
| 88 | git multi-pack-index verify && |
| 89 | test_line_count = 2 $midx_chain && |
| 90 | head -n 1 $midx_chain.before >expect && |
| 91 | head -n 1 $midx_chain >actual && |
| 92 | test_cmp expect actual && |
| 93 | |
| 94 | git fsck |
| 95 | ) |
| 96 | ' |
| 97 | |
| 98 | test_expect_success 'below layer threshold, tip packs excluded' ' |
| 99 | git init below-layer-threshold-tip-packs-excluded && |
| 100 | ( |
| 101 | cd below-layer-threshold-tip-packs-excluded && |
| 102 | |
| 103 | git config maintenance.auto false && |
| 104 | git config repack.midxnewlayerthreshold 4 && |
| 105 | git config repack.midxsplitfactor 2 && |
| 106 | |
| 107 | # Create 3 packs forming a geometric progression by |
| 108 | # object count such that they are unmodified by the |
| 109 | # initial repack. The MIDX chain thusly contains a |
| 110 | # single layer with three packs. |
| 111 | create_geometric_packs && |
| 112 | ls $packdir/pack-*.idx | sort >packs.before && |
| 113 | test_line_count = 1 $midx_chain && |
| 114 | cp $midx_chain $midx_chain.before && |
| 115 | |
| 116 | # Repack a new commit. Since the layer threshold is |
| 117 | # unmet, a new MIDX layer is added on top of the |
| 118 | # existing one. |
| 119 | test_commit extra && |
| 120 | git repack --geometric=2 -d --write-midx=incremental \ |
| 121 | --write-bitmap-index && |
| 122 | git multi-pack-index verify && |
| 123 | |
| 124 | ls $packdir/pack-*.idx | sort >packs.after && |
| 125 | comm -13 packs.before packs.after >packs.new && |
| 126 | test_line_count = 1 packs.new && |
| 127 | |
| 128 | test_line_count = 2 "$midx_chain" && |
| 129 | head -n 1 "$midx_chain.before" >expect && |
| 130 | head -n 1 "$midx_chain" >actual && |
| 131 | test_cmp expect actual |
| 132 | ) |
| 133 | ' |
| 134 | |
| 135 | test_expect_success 'above layer threshold, tip packs repacked' ' |
| 136 | git init above-layer-threshold-tip-packs-repacked && |
| 137 | ( |
| 138 | cd above-layer-threshold-tip-packs-repacked && |
| 139 | |
| 140 | git config maintenance.auto false && |
| 141 | git config repack.midxnewlayerthreshold 2 && |
| 142 | git config repack.midxsplitfactor 2 && |
| 143 | |
| 144 | # Same setup, but with the layer threshold set to 2. |
| 145 | # Since the tip MIDX layer meets that threshold, its |
| 146 | # packs are considered repack candidates. |
| 147 | create_geometric_packs && |
| 148 | cp $midx_chain $midx_chain.before && |
| 149 | |
| 150 | # Perturb the existing progression such that it is |
| 151 | # rolled up into a single new pack, invalidating the |
| 152 | # existing MIDX layer and replacing it with a new one. |
| 153 | test_commit extra && |
| 154 | git repack -d && |
| 155 | git repack --geometric=2 -d --write-midx=incremental \ |
| 156 | --write-bitmap-index && |
| 157 | |
| 158 | ! test_cmp $midx_chain.before $midx_chain && |
| 159 | test_line_count = 1 $midx_chain && |
| 160 | |
| 161 | git multi-pack-index verify |
| 162 | ) |
| 163 | ' |
| 164 | |
| 165 | test_expect_success 'above layer threshold, tip layer preserved' ' |
| 166 | git init above-layer-threshold-tip-layer-preserved && |
| 167 | ( |
| 168 | cd above-layer-threshold-tip-layer-preserved && |
| 169 | |
| 170 | git config maintenance.auto false && |
| 171 | git config repack.midxnewlayerthreshold 2 && |
| 172 | git config repack.midxsplitfactor 2 && |
| 173 | |
| 174 | test_commit_bulk --message="medium" 2 && |
| 175 | test_commit_bulk --message="large" 6 && |
| 176 | |
| 177 | git repack --geometric=2 -d --write-midx=incremental \ |
| 178 | --write-bitmap-index && |
| 179 | |
| 180 | test_line_count = 1 "$midx_chain" && |
| 181 | ls $packdir/pack-*.idx | sort >packs.before && |
| 182 | cp $midx_chain $midx_chain.before && |
| 183 | |
| 184 | # Create objects to form a pack satisfying the geometric |
| 185 | # progression (thus preserving the tip layer), but not |
| 186 | # so large that it meets the layer merging condition. |
| 187 | test_commit_bulk --message="small" 1 && |
| 188 | git repack --geometric=2 -d --write-midx=incremental \ |
| 189 | --write-bitmap-index && |
| 190 | |
| 191 | ls $packdir/pack-*.idx | sort >packs.after && |
| 192 | comm -13 packs.before packs.after >packs.new && |
| 193 | |
| 194 | test_line_count = 1 packs.new && |
| 195 | test_line_count = 3 packs.after && |
| 196 | test_line_count = 2 "$midx_chain" && |
| 197 | head -n 1 "$midx_chain.before" >expect && |
| 198 | head -n 1 "$midx_chain" >actual && |
| 199 | test_cmp expect actual && |
| 200 | |
| 201 | git multi-pack-index verify |
| 202 | ) |
| 203 | ' |
| 204 | |
| 205 | test_expect_success 'above layer threshold, tip packs preserved' ' |
| 206 | git init above-layer-threshold-tip-packs-preserved && |
| 207 | ( |
| 208 | cd above-layer-threshold-tip-packs-preserved && |
| 209 | |
| 210 | git config maintenance.auto false && |
| 211 | git config repack.midxnewlayerthreshold 2 && |
| 212 | git config repack.midxsplitfactor 2 && |
| 213 | |
| 214 | create_geometric_packs && |
| 215 | ls $packdir/pack-*.idx | sort >packs.before && |
| 216 | cp $midx_chain $midx_chain.before && |
| 217 | |
| 218 | # Same setup as above, but this time the new objects do |
| 219 | # not satisfy the new layer merging condition, resulting |
| 220 | # in a new tip layer. |
| 221 | test_commit_bulk --message="huge" 18 && |
| 222 | git repack --geometric=2 -d --write-midx=incremental \ |
| 223 | --write-bitmap-index && |
| 224 | |
| 225 | ls $packdir/pack-*.idx | sort >packs.after && |
| 226 | comm -13 packs.before packs.after >packs.new && |
| 227 | |
| 228 | ! test_cmp $midx_chain.before $midx_chain && |
| 229 | test_line_count = 1 $midx_chain && |
| 230 | test_line_count = 1 packs.new && |
| 231 | |
| 232 | git multi-pack-index verify |
| 233 | ) |
| 234 | ' |
| 235 | |
| 236 | test_expect_success 'new tip absorbs multiple layers' ' |
| 237 | git init new-tip-absorbs-multiple-layers && |
| 238 | ( |
| 239 | cd new-tip-absorbs-multiple-layers && |
| 240 | |
| 241 | git config maintenance.auto false && |
| 242 | git config repack.midxnewlayerthreshold 1 && |
| 243 | git config repack.midxsplitfactor 2 && |
| 244 | |
| 245 | # Build a 4-layer chain where each layer is too small to |
| 246 | # absorb the one below it. The sizes must satisfy L(n) < |
| 247 | # L(n-1)/2 for each adjacent pair: |
| 248 | # |
| 249 | # L0 (oldest): 75 obj (25 commits) |
| 250 | # L1: 21 obj (7 commits, 21 < 75/2) |
| 251 | # L2: 9 obj (3 commits, 9 < 21/2) |
| 252 | # L3 (tip): 3 obj (1 commit, 3 < 9/2) |
| 253 | create_layers <<-\EOF && |
| 254 | L0 25 |
| 255 | L1 7 |
| 256 | L2 3 |
| 257 | L3 1 |
| 258 | EOF |
| 259 | |
| 260 | test_line_count = 4 "$midx_chain" && |
| 261 | cp $midx_chain $midx_chain.before && |
| 262 | |
| 263 | # Now add a new commit. The merging condition is |
| 264 | # satisfied between L3-L1, but violated at L0, which is |
| 265 | # too large relative to the accumulated size. |
| 266 | # |
| 267 | # As a result, the chain shrinks from 4 to 2 layers. |
| 268 | test_commit new && |
| 269 | git repack --geometric=2 -d --write-midx=incremental \ |
| 270 | --write-bitmap-index && |
| 271 | |
| 272 | ! test_cmp $midx_chain.before $midx_chain && |
| 273 | test_line_count = 2 "$midx_chain" && |
| 274 | git multi-pack-index verify |
| 275 | ) |
| 276 | ' |
| 277 | |
| 278 | test_expect_success 'compaction of older layers' ' |
| 279 | git init compaction-of-older-layers && |
| 280 | ( |
| 281 | cd compaction-of-older-layers && |
| 282 | |
| 283 | git config maintenance.auto false && |
| 284 | git config repack.midxnewlayerthreshold 1 && |
| 285 | git config repack.midxsplitfactor 2 && |
| 286 | |
| 287 | # Build a chain with two small layers at the bottom |
| 288 | # and a larger barrier layer on top, producing a |
| 289 | # chain that violates the compaction invariant, since |
| 290 | # the two small layers would normally have been merged. |
| 291 | create_layers <<-\EOF && |
| 292 | one 2 |
| 293 | two 4 |
| 294 | barrier 54 |
| 295 | EOF |
| 296 | |
| 297 | cp $midx_chain $midx_chain.before && |
| 298 | |
| 299 | # Running an incremental repack compacts the two |
| 300 | # small layers at the bottom of the chain as a |
| 301 | # separate step in the compaction plan. |
| 302 | test_commit another && |
| 303 | git repack --geometric=2 -d --write-midx=incremental \ |
| 304 | --write-bitmap-index && |
| 305 | |
| 306 | test_line_count = 2 "$midx_chain" && |
| 307 | git multi-pack-index verify |
| 308 | ) |
| 309 | ' |
| 310 | |
| 311 | test_expect_success 'geometric rollup with surviving tip packs' ' |
| 312 | git init geometric-rollup-with-surviving-tip-packs && |
| 313 | ( |
| 314 | cd geometric-rollup-with-surviving-tip-packs && |
| 315 | |
| 316 | git config maintenance.auto false && |
| 317 | git config repack.midxnewlayerthreshold 1 && |
| 318 | git config repack.midxsplitfactor 2 && |
| 319 | |
| 320 | # Create a pack large enough to anchor the geometric |
| 321 | # progression when small packs are added alongside it. |
| 322 | create_layer --message="big" 5 && |
| 323 | |
| 324 | test_line_count = 1 "$midx_chain" && |
| 325 | cp $midx_chain $midx_chain.before && |
| 326 | |
| 327 | # Repack a small number of objects such that the |
| 328 | # progression is unbothered. Note that the existing pack |
| 329 | # is considered a repack candidate as the new layer |
| 330 | # threshold is set to 1. |
| 331 | test_commit small-1 && |
| 332 | git repack -d && |
| 333 | git repack --geometric=2 -d --write-midx=incremental \ |
| 334 | --write-bitmap-index && |
| 335 | |
| 336 | ! test_cmp $midx_chain.before $midx_chain && |
| 337 | cp $midx_chain $midx_chain.before |
| 338 | ) |
| 339 | ' |
| 340 | |
| 341 | test_expect_success 'kept packs are excluded from repack' ' |
| 342 | git init kept-packs-excluded-from-repack && |
| 343 | ( |
| 344 | cd kept-packs-excluded-from-repack && |
| 345 | |
| 346 | git config maintenance.auto false && |
| 347 | git config repack.midxnewlayerthreshold 1 && |
| 348 | git config repack.midxsplitfactor 2 && |
| 349 | |
| 350 | # Create two equal-sized packs, marking one as kept. |
| 351 | for i in A B |
| 352 | do |
| 353 | test_commit "$i" && git repack -d || return 1 |
| 354 | done && |
| 355 | |
| 356 | keep=$(ls $packdir/pack-*.idx | head -n 1) && |
| 357 | touch "${keep%.idx}.keep" && |
| 358 | |
| 359 | # The kept pack is excluded as a repacking candidate |
| 360 | # entirely, so no rollup occurs as there is only one |
| 361 | # non-kept pack. A new MIDX layer is written containing |
| 362 | # that pack. |
| 363 | git repack --geometric=2 -d --write-midx=incremental && |
| 364 | |
| 365 | test-tool read-midx $objdir >actual && |
| 366 | grep "^pack-.*\.idx$" actual >actual.packs && |
| 367 | test_line_count = 1 actual.packs && |
| 368 | test_grep ! "$keep" actual.packs && |
| 369 | |
| 370 | git multi-pack-index verify && |
| 371 | |
| 372 | # All objects (from both kept and non-kept packs) |
| 373 | # must still be accessible. |
| 374 | git fsck |
| 375 | ) |
| 376 | ' |
| 377 | |
| 378 | test_expect_success 'incremental MIDX with --max-pack-size' ' |
| 379 | git init incremental-midx-with--max-pack-size && |
| 380 | ( |
| 381 | cd incremental-midx-with--max-pack-size && |
| 382 | |
| 383 | git config maintenance.auto false && |
| 384 | git config repack.midxnewlayerthreshold 1 && |
| 385 | git config repack.midxsplitfactor 2 && |
| 386 | |
| 387 | create_layer --message="base" 1 && |
| 388 | |
| 389 | # Now add enough data that a small --max-pack-size will |
| 390 | # cause pack-objects to split its output. Create objects |
| 391 | # large enough to fill multiple packs. |
| 392 | test-tool genrandom foo 1M >big1 && |
| 393 | test-tool genrandom bar 1M >big2 && |
| 394 | git add big1 big2 && |
| 395 | test_tick && |
| 396 | git commit -a -m "big blobs" && |
| 397 | git repack -d && |
| 398 | |
| 399 | git repack --geometric=2 -d --write-midx=incremental \ |
| 400 | --write-bitmap-index --max-pack-size=1M && |
| 401 | |
| 402 | test_line_count = 1 "$midx_chain" && |
| 403 | test-tool read-midx $objdir >actual && |
| 404 | grep "^pack-.*\.idx$" actual >actual.packs && |
| 405 | test_line_count -gt 1 actual.packs && |
| 406 | |
| 407 | git multi-pack-index verify |
| 408 | ) |
| 409 | ' |
| 410 | |
| 411 | test_expect_success 'noop repack preserves valid MIDX chain' ' |
| 412 | git init noop-repack-preserves-valid-midx-chain && |
| 413 | ( |
| 414 | cd noop-repack-preserves-valid-midx-chain && |
| 415 | |
| 416 | git config maintenance.auto false && |
| 417 | git config repack.midxnewlayerthreshold 1 && |
| 418 | git config repack.midxsplitfactor 2 && |
| 419 | |
| 420 | create_layer --message="base" 1 && |
| 421 | |
| 422 | git multi-pack-index verify && |
| 423 | cp $midx_chain $midx_chain.before && |
| 424 | |
| 425 | # Running again with no new objects should not break |
| 426 | # the MIDX chain. It produces "Nothing new to pack." |
| 427 | git repack --geometric=2 -d --write-midx=incremental \ |
| 428 | --write-bitmap-index && |
| 429 | |
| 430 | test_cmp $midx_chain.before $midx_chain && |
| 431 | |
| 432 | git multi-pack-index verify && |
| 433 | git fsck |
| 434 | ) |
| 435 | ' |
| 436 | |
| 437 | test_expect_success 'repack -ad removes stale incremental chain' ' |
| 438 | git init repack--ad-removes-stale-incremental-chain && |
| 439 | ( |
| 440 | cd repack--ad-removes-stale-incremental-chain && |
| 441 | |
| 442 | git config maintenance.auto false && |
| 443 | git config repack.midxnewlayerthreshold 1 && |
| 444 | git config repack.midxsplitfactor 2 && |
| 445 | |
| 446 | create_layers <<-\EOF && |
| 447 | one 1 |
| 448 | two 1 |
| 449 | EOF |
| 450 | |
| 451 | test_path_is_file $midx_chain && |
| 452 | test_line_count = 2 $midx_chain && |
| 453 | |
| 454 | git repack -ad && |
| 455 | |
| 456 | test_path_is_missing $packdir/multi-pack-index && |
| 457 | test_dir_is_empty $midxdir |
| 458 | ) |
| 459 | ' |
| 460 | |
| 461 | test_expect_success 'repack -ad --write-midx=incremental is safe' ' |
| 462 | git init ad-incremental-midx && |
| 463 | ( |
| 464 | cd ad-incremental-midx && |
| 465 | |
| 466 | git config maintenance.auto false && |
| 467 | |
| 468 | # Build a MIDX chain with multiple layers referencing |
| 469 | # distinct packs. |
| 470 | test_commit first && |
| 471 | git repack -d && |
| 472 | |
| 473 | test_commit second && |
| 474 | git repack -d --write-midx=incremental && |
| 475 | |
| 476 | git multi-pack-index verify && |
| 477 | test_line_count = 1 $midx_chain && |
| 478 | |
| 479 | # Now do a full -ad repack. The new pack contains all |
| 480 | # objects, but any retained MIDX layers still reference |
| 481 | # the now-deleted packs. |
| 482 | test_commit third && |
| 483 | git repack -ad --write-midx=incremental && |
| 484 | |
| 485 | git multi-pack-index verify && |
| 486 | git fsck && |
| 487 | git rev-list --all --objects >/dev/null |
| 488 | ) |
| 489 | ' |
| 490 | |
| 491 | test_expect_success 'repack rejects invalid midxSplitFactor' ' |
| 492 | test_when_finished "rm -fr bad-split-factor" && |
| 493 | git init bad-split-factor && |
| 494 | ( |
| 495 | cd bad-split-factor && |
| 496 | test_commit base && |
| 497 | |
| 498 | for v in 0 1 -1 |
| 499 | do |
| 500 | test_must_fail git -c repack.midxSplitFactor=$v \ |
| 501 | repack -d --geometric=2 --write-midx=incremental 2>err && |
| 502 | test_grep "invalid value for --midx-split-factor" err || |
| 503 | return 1 |
| 504 | done |
| 505 | ) |
| 506 | ' |
| 507 | |
| 508 | test_expect_success 'repack rejects invalid midxNewLayerThreshold' ' |
| 509 | test_when_finished "rm -fr bad-layer-threshold" && |
| 510 | git init bad-layer-threshold && |
| 511 | ( |
| 512 | cd bad-layer-threshold && |
| 513 | test_commit base && |
| 514 | |
| 515 | for v in 0 -1 |
| 516 | do |
| 517 | test_must_fail git -c repack.midxNewLayerThreshold=$v \ |
| 518 | repack -d --geometric=2 --write-midx=incremental 2>err && |
| 519 | test_grep "invalid value for --midx-new-layer-threshold" err || |
| 520 | return 1 |
| 521 | done |
| 522 | ) |
| 523 | ' |
| 524 | |
| 525 | test_done |