Raw
1 #!/bin/sh
2
3 test_description='exercise basic multi-pack bitmap functionality'
4
5 . ./test-lib.sh
6 . "${TEST_DIRECTORY}/lib-bitmap.sh"
7
8 # We'll be writing our own MIDX, so avoid getting confused by the
9 # automatic ones.
10 GIT_TEST_MULTI_PACK_INDEX=0
11 GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
12
13 # This test exercise multi-pack bitmap functionality where the object order is
14 # stored and read from a special chunk within the MIDX, so use the default
15 # behavior here.
16 sane_unset GIT_TEST_MIDX_WRITE_REV
17 sane_unset GIT_TEST_MIDX_READ_RIDX
18
19 bitmap_reuse_tests() {
20 from=$1
21 to=$2
22 writeLookupTable=false
23
24 for i in $3-${$#}
25 do
26 case $i in
27 "pack.writeBitmapLookupTable") writeLookupTable=true;;
28 esac
29 done
30
31 test_expect_success "setup pack reuse tests ($from -> $to)" '
32 rm -fr repo &&
33 git init repo &&
34 (
35 cd repo &&
36 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
37 test_commit_bulk 16 &&
38 git tag old-tip &&
39
40 git config core.multiPackIndex true &&
41 if test "MIDX" = "$from"
42 then
43 git repack -Ad &&
44 git multi-pack-index write --bitmap
45 else
46 git repack -Adb
47 fi
48 )
49 '
50
51 test_expect_success "build bitmap from existing ($from -> $to)" '
52 (
53 cd repo &&
54 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
55 test_commit_bulk --id=further 16 &&
56 git tag new-tip &&
57
58 if test "MIDX" = "$to"
59 then
60 git repack -d &&
61 git multi-pack-index write --bitmap
62 else
63 git repack -Adb
64 fi
65 )
66 '
67
68 test_expect_success "verify resulting bitmaps ($from -> $to)" '
69 (
70 cd repo &&
71 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
72 git for-each-ref &&
73 git rev-list --test-bitmap refs/tags/old-tip &&
74 git rev-list --test-bitmap refs/tags/new-tip
75 )
76 '
77 }
78
79 test_midx_bitmap_cases () {
80 writeLookupTable=false
81 writeBitmapLookupTable=
82
83 for i in "$@"
84 do
85 case $i in
86 "pack.writeBitmapLookupTable")
87 writeLookupTable=true
88 writeBitmapLookupTable="$i"
89 ;;
90 esac
91 done
92
93 test_expect_success 'setup test_repository' '
94 rm -rf * .git &&
95 git init &&
96 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
97 git config maintenance.auto false
98 '
99
100 midx_bitmap_core
101
102 bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
103 bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
104 bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
105
106 test_expect_success 'missing object closure fails gracefully' '
107 rm -fr repo &&
108 git init repo &&
109 test_when_finished "rm -fr repo" &&
110 (
111 cd repo &&
112 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
113
114 test_commit loose &&
115 test_commit packed &&
116
117 # Do not pass "--revs"; we want a pack without the "loose"
118 # commit.
119 git pack-objects $objdir/pack/pack <<-EOF &&
120 $(git rev-parse packed)
121 EOF
122
123 test_must_fail git multi-pack-index write --bitmap 2>err &&
124 grep "doesn.t have full closure" err &&
125 test_path_is_missing $midx
126 )
127 '
128
129 midx_bitmap_partial_tests
130
131 test_expect_success 'removing a MIDX clears stale bitmaps' '
132 rm -fr repo &&
133 git init repo &&
134 test_when_finished "rm -fr repo" &&
135 (
136 cd repo &&
137 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
138 test_commit base &&
139 git repack &&
140 git multi-pack-index write --bitmap &&
141
142 # Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
143 stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
144 rm $midx &&
145
146 # Then write a new MIDX.
147 test_commit new &&
148 git repack &&
149 git multi-pack-index write --bitmap &&
150
151 test_path_is_file $midx &&
152 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
153 test_path_is_missing $stale_bitmap
154 )
155 '
156
157 test_expect_success 'pack.preferBitmapTips' '
158 git init repo &&
159 test_when_finished "rm -fr repo" &&
160 (
161 cd repo &&
162 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
163
164 test_commit_bulk --message="%s" 103 &&
165
166 git log --format="%H" >commits.raw &&
167 sort <commits.raw >commits &&
168
169 git log --format="create refs/tags/%s %H" HEAD >refs &&
170 git update-ref --stdin <refs &&
171
172 git multi-pack-index write --bitmap &&
173 test_path_is_file $midx &&
174 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
175
176 test-tool bitmap list-commits | sort >bitmaps &&
177 comm -13 bitmaps commits >before &&
178 test_line_count = 1 before &&
179
180 sed "s|\(.*\)|create refs/tags/include/\1 \1|" before |
181 git update-ref --stdin &&
182
183 rm -fr $midx-$(midx_checksum $objdir).bitmap &&
184 rm -fr $midx &&
185
186 git -c pack.preferBitmapTips=refs/tags/include \
187 multi-pack-index write --bitmap &&
188 test-tool bitmap list-commits | sort >bitmaps &&
189 comm -13 bitmaps commits >after &&
190
191 ! test_cmp before after
192 )
193 '
194
195 test_expect_success 'writing a bitmap with --refs-snapshot' '
196 git init repo &&
197 test_when_finished "rm -fr repo" &&
198 (
199 cd repo &&
200 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
201
202 test_commit one &&
203 test_commit two &&
204
205 git rev-parse one >snapshot &&
206
207 git repack -ad &&
208
209 # First, write a MIDX which see both refs/tags/one and
210 # refs/tags/two (causing both of those commits to receive
211 # bitmaps).
212 git multi-pack-index write --bitmap &&
213
214 test_path_is_file $midx &&
215 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
216
217 test-tool bitmap list-commits | sort >bitmaps &&
218 grep "$(git rev-parse one)" bitmaps &&
219 grep "$(git rev-parse two)" bitmaps &&
220
221 rm -fr $midx-$(midx_checksum $objdir).bitmap &&
222 rm -fr $midx &&
223
224 # Then again, but with a refs snapshot which only sees
225 # refs/tags/one.
226 git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
227
228 test_path_is_file $midx &&
229 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
230
231 test-tool bitmap list-commits | sort >bitmaps &&
232 grep "$(git rev-parse one)" bitmaps &&
233 ! grep "$(git rev-parse two)" bitmaps
234 )
235 '
236
237 test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
238 git init repo &&
239 test_when_finished "rm -fr repo" &&
240 (
241 cd repo &&
242 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
243
244 test_commit_bulk --message="%s" 103 &&
245
246 git log --format="%H" >commits.raw &&
247 sort <commits.raw >commits &&
248
249 git log --format="create refs/tags/%s %H" HEAD >refs &&
250 git update-ref --stdin <refs &&
251
252 git multi-pack-index write --bitmap &&
253 test_path_is_file $midx &&
254 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
255
256 test-tool bitmap list-commits | sort >bitmaps &&
257 comm -13 bitmaps commits >before &&
258 test_line_count = 1 before &&
259
260 (
261 grep -vf before commits.raw &&
262 # mark missing commits as preferred
263 sed "s/^/+/" before
264 ) >snapshot &&
265
266 rm -fr $midx-$(midx_checksum $objdir).bitmap &&
267 rm -fr $midx &&
268
269 git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
270 test-tool bitmap list-commits | sort >bitmaps &&
271 comm -13 bitmaps commits >after &&
272
273 ! test_cmp before after
274 )
275 '
276
277 test_expect_success 'hash-cache values are propagated from pack bitmaps' '
278 rm -fr repo &&
279 git init repo &&
280 test_when_finished "rm -fr repo" &&
281 (
282 cd repo &&
283 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
284
285 test_commit base &&
286 test_commit base2 &&
287 git repack -adb &&
288
289 test-tool bitmap dump-hashes >pack.raw &&
290 test_file_not_empty pack.raw &&
291 sort pack.raw >pack.hashes &&
292
293 test_commit new &&
294 git repack &&
295 git multi-pack-index write --bitmap &&
296
297 test-tool bitmap dump-hashes >midx.raw &&
298 sort midx.raw >midx.hashes &&
299
300 # ensure that every namehash in the pack bitmap can be found in
301 # the midx bitmap (i.e., that there are no oid-namehash pairs
302 # unique to the pack bitmap).
303 comm -23 pack.hashes midx.hashes >dropped.hashes &&
304 test_must_be_empty dropped.hashes
305 )
306 '
307
308 test_expect_success 'no .bitmap is written without any objects' '
309 rm -fr repo &&
310 git init repo &&
311 test_when_finished "rm -fr repo" &&
312 (
313 cd repo &&
314 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
315
316 empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
317 cat >packs <<-EOF &&
318 pack-$empty.idx
319 EOF
320
321 git multi-pack-index write --bitmap --stdin-packs \
322 <packs 2>err &&
323
324 grep "bitmap without any objects" err &&
325
326 test_path_is_file $midx &&
327 test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
328 )
329 '
330
331 test_expect_success 'graceful fallback when missing reverse index' '
332 rm -fr repo &&
333 git init repo &&
334 test_when_finished "rm -fr repo" &&
335 (
336 cd repo &&
337 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
338
339 test_commit base &&
340
341 # write a pack and MIDX bitmap containing base
342 git repack -adb &&
343 git multi-pack-index write --bitmap &&
344
345 GIT_TEST_MIDX_READ_RIDX=0 \
346 git rev-list --use-bitmap-index HEAD 2>err &&
347 ! grep "ignoring extra bitmap file" err
348 )
349 '
350 }
351
352 test_midx_bitmap_cases
353
354 test_midx_bitmap_cases "pack.writeBitmapLookupTable"
355
356 test_expect_success 'multi-pack-index write writes lookup table if enabled' '
357 rm -fr repo &&
358 git init repo &&
359 test_when_finished "rm -fr repo" &&
360 (
361 cd repo &&
362 test_commit base &&
363 git config pack.writeBitmapLookupTable true &&
364 git repack -ad &&
365 GIT_TRACE2_EVENT="$(pwd)/trace" \
366 git multi-pack-index write --bitmap &&
367 grep "\"label\":\"writing_lookup_table\"" trace
368 )
369 '
370
371 test_expect_success 'preferred pack change with existing MIDX bitmap' '
372 git init preferred-pack-with-existing &&
373 (
374 cd preferred-pack-with-existing &&
375
376 test_commit base &&
377 test_commit other &&
378
379 git rev-list --objects --no-object-names base >p1.objects &&
380 git rev-list --objects --no-object-names other >p2.objects &&
381
382 p1="$(git pack-objects "$objdir/pack/pack" \
383 --delta-base-offset <p1.objects)" &&
384 p2="$(git pack-objects "$objdir/pack/pack" \
385 --delta-base-offset <p2.objects)" &&
386
387 # Generate a MIDX containing the first two packs,
388 # marking p1 as preferred, and ensure that it can be
389 # successfully cloned.
390 git multi-pack-index write --bitmap \
391 --preferred-pack="pack-$p1.pack" &&
392 test_path_is_file $midx &&
393 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
394 git clone --no-local . clone1 &&
395
396 # Then generate a new pack which sorts ahead of any
397 # existing pack (by tweaking the pack prefix).
398 test_commit foo &&
399 git pack-objects --all --unpacked $objdir/pack/pack0 &&
400
401 # Generate a new MIDX which changes the preferred pack
402 # to a pack contained in the existing MIDX.
403 git multi-pack-index write --bitmap \
404 --preferred-pack="pack-$p2.pack" &&
405 test_path_is_file $midx &&
406 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
407
408 # When the above circumstances are met, the preferred
409 # pack should change appropriately and clones should
410 # (still) succeed.
411 git clone --no-local . clone2
412 )
413 '
414
415 test_expect_success 'tagged commits are selected for bitmapping' '
416 rm -fr repo &&
417 git init repo &&
418 test_when_finished "rm -fr repo" &&
419 (
420 cd repo &&
421
422 test_commit --annotate base &&
423 git repack -d &&
424
425 # Remove refs/heads/main which points at the commit directly,
426 # leaving only a reference to the annotated tag.
427 git branch -M main &&
428 git checkout base &&
429 git branch -d main &&
430
431 git multi-pack-index write --bitmap &&
432
433 git rev-parse HEAD >want &&
434 test-tool bitmap list-commits >actual &&
435 grep $(cat want) actual
436 )
437 '
438
439 test_expect_success 'do not follow replace objects for MIDX bitmap' '
440 rm -fr repo &&
441 git init repo &&
442 test_when_finished "rm -fr repo" &&
443 (
444 cd repo &&
445
446 test_commit A &&
447 test_commit B &&
448 git checkout --orphan=orphan A &&
449 test_commit orphan &&
450
451 git replace A HEAD &&
452 git repack -ad --write-midx --write-bitmap-index &&
453
454 # generating reachability bitmaps with replace refs
455 # enabled will result in broken clones
456 git clone --no-local --bare . clone.git
457 )
458 '
459
460 corrupt_file () {
461 chmod a+w "$1" &&
462 printf "bogus" | dd of="$1" bs=1 seek="12" conv=notrunc
463 }
464
465 test_expect_success 'git fsck correctly identifies good and bad bitmaps' '
466 git init valid &&
467 test_when_finished rm -rf valid &&
468
469 test_commit_bulk 20 &&
470 git repack -adbf &&
471
472 # Move pack-bitmap aside so it is not deleted
473 # in next repack.
474 packbitmap=$(ls .git/objects/pack/pack-*.bitmap) &&
475 mv "$packbitmap" "$packbitmap.bak" &&
476
477 test_commit_bulk 10 &&
478 git repack -b --write-midx &&
479 midxbitmap=$(ls .git/objects/pack/multi-pack-index-*.bitmap) &&
480
481 # Copy MIDX bitmap to backup. Copy pack bitmap from backup.
482 cp "$midxbitmap" "$midxbitmap.bak" &&
483 cp "$packbitmap.bak" "$packbitmap" &&
484
485 # fsck works at first
486 git fsck 2>err &&
487 test_must_be_empty err &&
488
489 corrupt_file "$packbitmap" &&
490 test_must_fail git fsck 2>err &&
491 grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err &&
492
493 cp "$packbitmap.bak" "$packbitmap" &&
494 corrupt_file "$midxbitmap" &&
495 test_must_fail git fsck 2>err &&
496 grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
497
498 corrupt_file "$packbitmap" &&
499 test_must_fail git fsck 2>err &&
500 grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
501 grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err
502 '
503
504 test_expect_success 'corrupt MIDX with bitmap causes fallback' '
505 git init corrupt-midx-bitmap &&
506 (
507 cd corrupt-midx-bitmap &&
508
509 test_commit first &&
510 git repack -d &&
511 test_commit second &&
512 git repack -d &&
513
514 git multi-pack-index write --bitmap &&
515 checksum=$(midx_checksum $objdir) &&
516 for f in $midx $midx-$checksum.bitmap
517 do
518 mv $f $f.bak || return 1
519 done &&
520
521 # pack everything together, invalidating the MIDX
522 git repack -ad &&
523 # then restore the now-stale MIDX
524 for f in $midx $midx-$checksum.bitmap
525 do
526 mv $f.bak $f || return 1
527 done &&
528
529 git rev-list --count --objects --use-bitmap-index HEAD >out 2>err &&
530 # should attempt opening the broken pack twice (once
531 # from the attempt to load it via the stale bitmap, and
532 # again when attempting to load it from the stale MIDX)
533 # before falling back to the non-MIDX case
534 test 2 -eq $(grep -c "could not open pack" err) &&
535 test 6 -eq $(cat out)
536 )
537 '
538
539 for allow_pack_reuse in single multi
540 do
541 test_expect_success "reading MIDX without BTMP chunk does not complain with $allow_pack_reuse pack reuse" '
542 test_when_finished "rm -rf midx-without-btmp" &&
543 git init midx-without-btmp &&
544 (
545 cd midx-without-btmp &&
546 test_commit initial &&
547
548 git repack -Adbl --write-bitmap-index --write-midx &&
549 GIT_TEST_MIDX_READ_BTMP=false git -c pack.allowPackReuse=$allow_pack_reuse \
550 pack-objects --all --use-bitmap-index --stdout </dev/null >/dev/null 2>err &&
551 test_must_be_empty err
552 )
553 '
554 done
555
556 test_expect_success 'remove one packfile between MIDX bitmap writes' '
557 git init remove-pack-between-writes &&
558 (
559 cd remove-pack-between-writes &&
560
561 test_commit A &&
562 test_commit B &&
563 test_commit C &&
564
565 # Create packs with the prefix "pack-A", "pack-B",
566 # "pack-C" to impose a lexicographic order on these
567 # packs so the pack being removed is always from the
568 # middle.
569 packdir=.git/objects/pack &&
570 A="$(echo A | git pack-objects $packdir/pack-A --revs)" &&
571 B="$(echo B | git pack-objects $packdir/pack-B --revs)" &&
572 C="$(echo C | git pack-objects $packdir/pack-C --revs)" &&
573
574 git multi-pack-index write --bitmap &&
575
576 cat >in <<-EOF &&
577 pack-A-$A.idx
578 pack-C-$C.idx
579 EOF
580 git multi-pack-index write --bitmap --stdin-packs <in &&
581
582 git rev-list --test-bitmap HEAD
583 )
584 '
585
586 test_done