builtin/repack.c: add cruft packs to MIDX during geometric repack

When using cruft packs, the following race can occur when a geometric repack that writes a MIDX bitmap takes place afterwords: - First, create an unreachable object and do an all-into-one cruft repack which stores that object in the repository's cruft pack. - Then make that object reachable. - Finally, do a geometric repack and write a MIDX bitmap. Assuming that we are sufficiently unlucky as to select a commit from the MIDX which reaches that object for bitmapping, then the `git multi-pack-index` process will complain that that object is missing. The reason is because we don't include cruft packs in the MIDX when doing a geometric repack. Since the "make that object reachable" doesn't necessarily mean that we'll create a new copy of that object in one of the packs that will get rolled up as part of a geometric repack, it's possible that the MIDX won't see any copies of that now-reachable object. Of course, it's desirable to avoid including cruft packs in the MIDX because it causes the MIDX to store a bunch of objects which are likely to get thrown away. But excluding that pack does open us up to the above race. This patch demonstrates the bug, and resolves it by including cruft packs in the MIDX even when doing a geometric repack. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed May 20, 2022 at 19:18 UTC ddee3703b36e96056f11ddc4621707b6054bab48
2 files changed +46 -3
builtin/repack.c
+20 -3
@@ -23,6 +23,7 @@
23 #define PACK_CRUFT 4
24
25 #define DELETE_PACK 1
26 +#define CRUFT_PACK 2
27
28 static int pack_everything;
29 static int delta_base_offset = 1;
@@ -159,10 +160,15 @@ static void collect_pack_filenames(struct string_list *fname_nonkept_list,
160 fname = xmemdupz(e->d_name, len);
161
162 if ((extra_keep->nr > 0 && i < extra_keep->nr) ||
162 - (file_exists(mkpath("%s/%s.keep", packdir, fname))))
163 + (file_exists(mkpath("%s/%s.keep", packdir, fname)))) {
164 string_list_append_nodup(fname_kept_list, fname);
164 - else
165 - string_list_append_nodup(fname_nonkept_list, fname);
165 + } else {
166 + struct string_list_item *item;
167 + item = string_list_append_nodup(fname_nonkept_list,
168 + fname);
169 + if (file_exists(mkpath("%s/%s.mtimes", packdir, fname)))
170 + item->util = (void*)(uintptr_t)CRUFT_PACK;
171 + }
172 }
173 closedir(dir);
174 }
@@ -564,6 +570,17 @@ static void midx_included_packs(struct string_list *include,
570
571 string_list_insert(include, strbuf_detach(&buf, NULL));
572 }
573 +
574 + for_each_string_list_item(item, existing_nonkept_packs) {
575 + if (!((uintptr_t)item->util & CRUFT_PACK)) {
576 + /*
577 + * no need to check DELETE_PACK, since we're not
578 + * doing an ALL_INTO_ONE repack
579 + */
580 + continue;
581 + }
582 + string_list_insert(include, xstrfmt("%s.idx", item->string));
583 + }
584 } else {
585 for_each_string_list_item(item, existing_nonkept_packs) {
586 if ((uintptr_t)item->util & DELETE_PACK)
t/t5329-pack-objects-cruft.sh
+26
@@ -648,4 +648,30 @@ test_expect_success 'cruft --local drops unreachable objects' '
648 )
649 '
650
651 +test_expect_success 'MIDX bitmaps tolerate reachable cruft objects' '
652 + git init repo &&
653 + test_when_finished "rm -fr repo" &&
654 + (
655 + cd repo &&
656 +
657 + test_commit reachable &&
658 + test_commit cruft &&
659 + unreachable="$(git rev-parse cruft)" &&
660 +
661 + git reset --hard $unreachable^ &&
662 + git tag -d cruft &&
663 + git reflog expire --all --expire=all &&
664 +
665 + git repack --cruft -d &&
666 +
667 + # resurrect the unreachable object via a new commit. the
668 + # new commit will get selected for a bitmap, but be
669 + # missing one of its parents from the selected packs.
670 + git reset --hard $unreachable &&
671 + test_commit resurrect &&
672 +
673 + git repack --write-midx --write-bitmap-index --geometric=2 -d
674 + )
675 +'
676 +
677 test_done