repack: support combining '--geometric' with '--cruft'

index 72c42015e2..e9df771327 100644 --- a/Documentation/git-repack.adoc +++ b/Documentation/git-repack.adoc @@ -70,6 +70,11 @@ to the new separate pack will be written. are packed into a separate cruft pack. Unreachable objects can be pruned using the normal expiry rules with the next `git gc` invocation (see linkgit:git-gc[1]). Incompatible with `-k`. ++ +When combined with `--geometric`, `--cruft` does not imply `-a`. Instead, +the geometric repack rolls up packs as usual, and a separate cruft pack is +written to collect unreachable objects. Only reachable objects from the +rolled-up packs are included in the resulting geometric pack. --cruft-expiration=<approxidate>:: Expire unreachable objects older than `<approxidate>` @@ -245,6 +250,12 @@ progression. Loose objects are implicitly included in this "roll-up", without respect to their reachability. This is subject to change in the future. + +When combined with `--cruft`, only reachable objects from rolled-up packs +are included in the geometric pack, along with any reachable loose objects. +Unreachable objects (both from rolled-up packs and loose) are collected +into a separate cruft pack. Existing cruft packs are retained. See +`--cruft` above for details. ++ When writing a multi-pack bitmap, `git repack` selects the largest resulting pack as the preferred pack for object selection by the MIDX (see linkgit:git-multi-pack-index[1]).

index dfb6fed231..165cfff75c 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -260,7 +260,7 @@ int cmd_repack(int argc, keep_unreachable, "-k/--keep-unreachable", pack_everything & PACK_CRUFT, "--cruft"); - if (pack_everything & PACK_CRUFT) + if (pack_everything & PACK_CRUFT && !geometry.split_factor) pack_everything |= ALL_INTO_ONE; if (write_bitmaps < 0) { @@ -296,7 +296,8 @@ int cmd_repack(int argc, die(_("invalid value for %s: %d"), "--midx-new-layer-threshold", config_ctx.midx_new_layer_threshold); - if (write_midx != REPACK_WRITE_MIDX_NONE && write_bitmaps) { + if ((write_midx != REPACK_WRITE_MIDX_NONE && write_bitmaps) || + (geometry.split_factor && (pack_everything & PACK_CRUFT))) { struct strbuf path = STRBUF_INIT; strbuf_addf(&path, "%s/%s_XXXXXX", @@ -317,7 +318,7 @@ int cmd_repack(int argc, existing_packs_collect(&existing, &keep_pack_list); if (geometry.split_factor) { - if (pack_everything) + if (pack_everything & ~PACK_CRUFT) die(_("options '%s' and '%s' cannot be used together"), "--geometric", "-A/-a"); if (write_midx == REPACK_WRITE_MIDX_INCREMENTAL) { geometry.midx_layer_threshold = config_ctx.midx_new_layer_threshold; @@ -393,10 +394,16 @@ int cmd_repack(int argc, pack_geometry_repack_promisors(repo, &po_args, &geometry, &names, packtmp); - if (midx_must_contain_cruft) + if (pack_everything & PACK_CRUFT) { + strvec_push(&cmd.args, "--stdin-packs=follow-reachable"); + if (refs_snapshot) + strvec_pushf(&cmd.args, "--refs-snapshot=%s", + get_tempfile_path(refs_snapshot)); + } else if (midx_must_contain_cruft) strvec_push(&cmd.args, "--stdin-packs"); else strvec_push(&cmd.args, "--stdin-packs=follow"); + strvec_push(&cmd.args, "--unpacked"); } else { strvec_push(&cmd.args, "--unpacked"); @@ -431,7 +438,8 @@ int cmd_repack(int argc, const char *basename = pack_basename(geometry.pack[i]); char marker = '^'; - if (!midx_must_contain_cruft && + if ((pack_everything & PACK_CRUFT || + !midx_must_contain_cruft) && !string_list_has_string(&existing.midx_packs, basename)) { /* @@ -505,7 +513,8 @@ int cmd_repack(int argc, ret = write_cruft_pack(&opts, cruft_expiration, combine_cruft_below_size, &names, - &existing); + &existing, + geometry.split_factor ? &geometry : NULL); if (ret) goto cleanup; @@ -540,7 +549,7 @@ int cmd_repack(int argc, */ opts.destination = expire_to; ret = write_cruft_pack(&opts, NULL, 0ul, &names, - &existing); + &existing, NULL); if (ret) goto cleanup; }

index 6a040e9801..6c553bbb0b 100644 --- a/repack-cruft.c +++ b/repack-cruft.c @@ -36,7 +36,8 @@ int write_cruft_pack(const struct write_pack_opts *opts, const char *cruft_expiration, unsigned long combine_cruft_below_size, struct string_list *names, - struct existing_packs *existing) + struct existing_packs *existing, + struct pack_geometry *geometry) { struct child_process cmd = CHILD_PROCESS_INIT; struct string_list_item *item; @@ -81,8 +82,24 @@ int write_cruft_pack(const struct write_pack_opts *opts, else for_each_string_list_item(item, &existing->cruft_packs) fprintf(in, "-%s.pack\n", item->string); - for_each_string_list_item(item, &existing->non_kept_packs) - fprintf(in, "-%s.pack\n", item->string); + if (geometry) { + uint32_t j; + for (j = 0; j < geometry->split; j++) + fprintf(in, "-%s\n", + pack_basename(geometry->pack[j])); + for (; j < geometry->pack_nr; j++) + fprintf(in, "%s\n", + pack_basename(geometry->pack[j])); + for (j = 0; j < geometry->promisor_split; j++) + fprintf(in, "-%s\n", + pack_basename(geometry->promisor_pack[j])); + for (; j < geometry->promisor_pack_nr; j++) + fprintf(in, "%s\n", + pack_basename(geometry->promisor_pack[j])); + } else { + for_each_string_list_item(item, &existing->non_kept_packs) + fprintf(in, "-%s.pack\n", item->string); + } for_each_string_list_item(item, &existing->kept_packs) fprintf(in, "%s.pack\n", item->string); fclose(in);

index 4295829cea..872a503fbd 100644 --- a/repack.h +++ b/repack.h @@ -169,6 +169,7 @@ int write_cruft_pack(const struct write_pack_opts *opts, const char *cruft_expiration, unsigned long combine_cruft_below_size, struct string_list *names, - struct existing_packs *existing); + struct existing_packs *existing, + struct pack_geometry *geometry); #endif /* REPACK_H */

index 9e03b04315..5e2b776e7b 100755 --- a/t/t7704-repack-cruft.sh +++ b/t/t7704-repack-cruft.sh @@ -891,4 +891,255 @@ test_expect_success 'repack rescues once-cruft objects above geometric split' ' git repack --geometric=2 -d --write-midx --write-bitmap-index ' +test_expect_success 'repack --geometric --cruft combines packs and writes cruft' ' + git init geometric-cruft-basic && + ( + cd geometric-cruft-basic && + + test_commit A && + test_commit B && + + B="$(git rev-parse B)" && + + git reset --hard $B^ && + git tag -d B && + git reflog expire --all --expire=all && + + # Initial state: one non-cruft pack, one cruft pack. + git repack -d --cruft && + + ls $packdir/pack-*.mtimes >cruft.before && + test_line_count = 1 cruft.before && + + test_commit C && + git repack && + + # At this point we have three packs: + # - the non-cruft pack from A + # - the cruft pack from B + # - a new non-cruft pack from C + # + # The two non-cruft packs are not in a geometric + # progression, so they should be rolled up. + git repack -d --geometric=2 --cruft && + + # The old cruft pack for B is retained, since the + # geometric repack does not touch cruft packs. + ls $packdir/pack-*.mtimes >cruft.after && + test_line_count = 1 cruft.after && + + # Ensure that all reachable objects are present. + git fsck + ) +' + +test_expect_success 'repack --geometric --cruft writes new cruft for loose unreachable' ' + git init geometric-cruft-new-cruft && + ( + cd geometric-cruft-new-cruft && + + git config set maintenance.auto false && + + test_commit A && + git repack && + + test_commit B && + git repack && + + # Create an unreachable commit whose objects are + # still loose (never packed). + test_commit C && + C="$(git rev-parse C)" && + git reset --hard $C^ && + git tag -d C && + git reflog expire --all --expire=all && + + # At this point we have two non-cruft packs of + # similar size that are not in geometric progression, + # and loose unreachable objects from commit C. + ls $packdir/pack-*.idx >packs.before && + test_line_count = 2 packs.before && + + # Geometric+cruft repack should roll up the two + # non-cruft packs and write a new cruft pack for C + # (whose objects are loose and unreachable). + git repack -d --geometric=2 --cruft && + + ls $packdir/pack-*.mtimes >cruft.after && + test_line_count = 1 cruft.after && + + git fsck + ) +' + +test_expect_success 'repack --geometric --cruft -d deletes rolled-up packs' ' + git init geometric-cruft-delete && + ( + cd geometric-cruft-delete && + + test_commit A && + git repack -d && + + test_commit B && + git repack -d && + + ls $packdir/pack-*.idx >before && + + git repack -d --geometric=2 --cruft && + + # Two packs should have been rolled into one. No cruft + # pack is written because there are no unreachable objects. + ls $packdir/pack-*.idx >after && + test_line_count = 1 after && + + # The rolled-up packs should be gone. + ! test_cmp before after + ) +' + +test_expect_success 'repack --geometric --cruft collects loose unreachable objects' ' + git init geometric-cruft-loose && + ( + cd geometric-cruft-loose && + + test_commit A && + git repack -d && + + test_commit B && + git repack && + + # Create a loose unreachable object by making it + # orphaned (not in any pack). + loose="$(echo "cruft object" | git hash-object -w --stdin)" && + + # We have two non-cruft packs and a loose unreachable + # object. The geometric+cruft repack should roll up + # the packs AND write a cruft pack for the loose + # unreachable object. + git repack -d --geometric=2 --cruft && + + ls $packdir/pack-*.mtimes >cruft.packs && + test_line_count = 1 cruft.packs && + + git fsck + ) +' + +test_expect_success 'repack --geometric --cruft accumulates cruft packs' ' + git init geometric-cruft-accumulate && + ( + cd geometric-cruft-accumulate && + + git config set maintenance.auto false && + + test_commit A && + git repack && + + # First round: create unreachable objects and do a + # geometric+cruft repack. + unreachable_1="$(echo "cruft 1" | git hash-object -w --stdin)" && + git repack -d --geometric=2 --cruft && + + ls $packdir/pack-*.mtimes >cruft.1 && + test_line_count = 1 cruft.1 && + + test_commit B && + git repack && + + # Second round: create more unreachable objects and + # repack again. The old cruft pack should be retained + # and a new one written. + unreachable_2="$(echo "cruft 2" | git hash-object -w --stdin)" && + git repack -d --geometric=2 --cruft && + + ls $packdir/pack-*.mtimes >cruft.2 && + test_line_count = 2 cruft.2 && + + git fsck + ) +' + +test_expect_success 'repack --geometric --cruft --combine-cruft-below-size' ' + git init geometric-cruft-combine && + ( + cd geometric-cruft-combine && + + git config set maintenance.auto false && + + test_commit A && + git repack && + + # Create a small cruft pack. + unreachable_1="$(echo "cruft 1" | git hash-object -w --stdin)" && + git repack -d --geometric=2 --cruft && + + ls $packdir/pack-*.mtimes >cruft.before && + test_line_count = 1 cruft.before && + + test_commit B && + git repack && + + # Create another small cruft pack. + unreachable_2="$(echo "cruft 2" | git hash-object -w --stdin)" && + git repack -d --geometric=2 --cruft && + + ls $packdir/pack-*.mtimes >cruft.mid && + test_line_count = 2 cruft.mid && + + test_commit C && + git repack && + + # With --combine-cruft-below-size, the two small cruft + # packs should be combined into one. + unreachable_3="$(echo "cruft 3" | git hash-object -w --stdin)" && + git repack -d --geometric=2 --cruft \ + --combine-cruft-below-size=10M && + + ls $packdir/pack-*.mtimes >cruft.after && + test_line_count = 1 cruft.after && + + git fsck + ) +' + +test_expect_success 'repack --geometric --cruft --expire-to' ' + git init geometric-cruft-expire-to && + ( + cd geometric-cruft-expire-to && + + git config set maintenance.auto false && + + test_commit A && + git repack && + + test_commit B && + git repack && + + # Create unreachable objects and record them. + test_commit C && + C="$(git rev-parse C)" && + git rev-list --objects --no-object-names B..C >unreachable.raw && + sort unreachable.raw >unreachable.want && + + git reset --hard $C^ && + git tag -d C && + git reflog expire --all --expire=all && + + git init --bare expired.git && + git repack -d --geometric=2 --cruft \ + --cruft-expiration=now \ + --expire-to="expired.git/objects/pack/pack" && + + # The expired objects should appear in the + # expire-to location. + expired="$(ls expired.git/objects/pack/pack-*.idx)" && + test_path_is_file "${expired%.idx}.mtimes" && + git show-index <"$expired" >expired.raw && + cut -d" " -f2 expired.raw | sort >expired.objects && + test_cmp unreachable.want expired.objects && + + git fsck + ) +' + test_done