pack-objects: introduce '--stdin-packs=follow-reachable'

index 8a27aa19fd..d7b2e39e76 100644 --- a/Documentation/git-pack-objects.adoc +++ b/Documentation/git-pack-objects.adoc @@ -113,6 +113,23 @@ This mode is useful, for example, to resurrect once-unreachable objects found in cruft packs to generate packs which are closed under reachability up to the boundary set by the excluded packs. + +When `mode` is "follow-reachable", the same pack prefixes are recognized +as in "follow" (`!` for excluded-open, `^` for excluded-closed). However, +instead of including all objects from included packs, only objects that +are reachable from reference tips AND belong to an included pack (or are +reachable from a commit in one) are included. Objects in excluded-open +packs are traversed but not included; objects in excluded-closed packs +halt the traversal. ++ +This mode is designed for geometric repacking with cruft packs, where +the output pack should contain only reachable objects so that unreachable +ones can be collected separately. ++ +When `--unpacked` is given alongside `--stdin-packs=follow-reachable`, +reachable loose objects are also included in the output pack, while +unreachable loose objects are left alone. This includes both loose +commits and annotated tag objects. ++ Incompatible with `--revs`, or options that imply `--revs` (such as `--all`), with the exception of `--unpacked`, which is compatible.

index 29e43abb51..5d96757b64 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -290,6 +290,7 @@ enum stdin_packs_mode { STDIN_PACKS_MODE_NONE, STDIN_PACKS_MODE_STANDARD, STDIN_PACKS_MODE_FOLLOW, + STDIN_PACKS_MODE_FOLLOW_REACHABLE, }; /** @@ -3835,7 +3836,8 @@ static void show_object_pack_hint(struct object *object, const char *name, void *data) { enum stdin_packs_mode mode = *(enum stdin_packs_mode *)data; - if (mode == STDIN_PACKS_MODE_FOLLOW) { + if (mode == STDIN_PACKS_MODE_FOLLOW || + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) { if (object->type == OBJ_BLOB && !odb_has_object(the_repository->objects, &object->oid, 0)) return; @@ -3866,7 +3868,8 @@ static void show_commit_pack_hint(struct commit *commit, void *data) { enum stdin_packs_mode mode = *(enum stdin_packs_mode *)data; - if (mode == STDIN_PACKS_MODE_FOLLOW) { + if (mode == STDIN_PACKS_MODE_FOLLOW || + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) { show_object_pack_hint((struct object *)commit, "", data); return; } @@ -3933,6 +3936,156 @@ static int stdin_packs_include_check(struct commit *commit, void *data) return stdin_packs_include_check_obj((struct object *)commit, data); } +/* + * Flag bit set on commits that belong to an included pack during + * '--stdin-packs=follow-reachable'. Used by the pre-walk to + * identify which reachable commits should be tips for the main + * object traversal. + */ +#define IN_INCLUDED_PACK (1u<<11) + +static int mark_included_pack_tip(const struct object_id *oid, + struct packed_git *p, + uint32_t pos, + void *data) +{ + struct rev_info *main_revs = data; + off_t ofs = nth_packed_object_offset(p, pos); + enum object_type type; + struct object_info oi = OBJECT_INFO_INIT; + struct object *obj; + + oi.typep = &type; + if (packed_object_info(p, ofs, &oi) < 0) + return 0; + if (type != OBJ_COMMIT && type != OBJ_TAG) + return 0; + + obj = parse_object(the_repository, oid); + if (!obj) + return 0; + + obj->flags |= IN_INCLUDED_PACK; + + if (type == OBJ_TAG && main_revs) + add_pending_object(main_revs, obj, ""); + return 0; +} + +static int mark_loose_object_tip(const struct object_id *oid, + struct object_info *oi UNUSED, + void *data) +{ + struct rev_info *main_revs = data; + struct object *obj; + enum object_type type; + + type = odb_read_object_info(the_repository->objects, oid, NULL); + if (type != OBJ_COMMIT && type != OBJ_TAG) + return 0; + + obj = parse_object(the_repository, oid); + if (!obj) + return 0; + + obj->flags |= IN_INCLUDED_PACK; + + if (type == OBJ_TAG && main_revs) + add_pending_object(main_revs, obj, ""); + + return 0; +} + +static int add_ref_to_pending(const struct reference *ref, void *cb_data) +{ + struct rev_info *revs = cb_data; + struct object *object; + + object = parse_object(the_repository, ref->oid); + if (!object) + return 0; + + add_pending_object(revs, object, ""); + return 0; +} + +static void stdin_packs_add_reachable_pack_entries(struct string_list *keys, + struct rev_info *revs, + int rev_list_unpacked) +{ + struct rev_info pre_walk; + struct commit *commit; + struct string_list_item *item; + + /* + * Phase 1: mark commits in included packs, then walk from + * ref tips to discover which of them are reachable. The walk + * halts at excluded-closed packs (via no_kept_objects) and + * continues through excluded-open ones. + * + * Also set include_check on the outer revs so that phase 2 + * (the main object traversal) halts at closed packs. + */ + revs->include_check = stdin_packs_include_check; + revs->include_check_obj = stdin_packs_include_check_obj; + + for_each_string_list_item(item, keys) { + struct stdin_pack_info *info = item->util; + if (info->kind & STDIN_PACK_INCLUDE) + for_each_object_in_pack(info->p, + mark_included_pack_tip, + revs, + ODB_FOR_EACH_OBJECT_PACK_ORDER); + } + + if (rev_list_unpacked) { + /* + * With '--stdin-packs=follow-reachable', specifying + * '--unpacked' instructs pack-objects to pack any loose + * objects which are reachable. + * + * Pretend as if all loose objects are in an included + * pack in order to make them eligible for packing. + */ + struct odb_source *source = revs->repo->objects->sources; + for (; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + struct odb_for_each_object_options opts = { 0 }; + if (local) + opts.flags |= ODB_FOR_EACH_OBJECT_LOCAL_ONLY; + + odb_source_for_each_object(&files->loose->base, NULL, + mark_loose_object_tip, + revs, &opts); + } + } + + repo_init_revisions(the_repository, &pre_walk, NULL); + pre_walk.no_kept_objects = 1; + pre_walk.keep_pack_cache_flags |= KEPT_PACK_IN_CORE; + pre_walk.ignore_missing_links = 1; + + refs_for_each_ref(get_main_ref_store(the_repository), + add_ref_to_pending, &pre_walk); + + if (prepare_revision_walk(&pre_walk)) + die(_("revision walk setup failed")); + + /* + * Phase 2 tips: every reachable commit that is in an + * included pack becomes a starting point for the main + * object traversal. + */ + while ((commit = get_revision(&pre_walk)) != NULL) { + if (commit->object.flags & IN_INCLUDED_PACK) + add_pending_oid(revs, NULL, + &commit->object.oid, 0); + } + + reset_revision_walk(); + release_revisions(&pre_walk); +} + static void stdin_packs_add_all_pack_entries(struct string_list *keys, struct rev_info *revs) { @@ -3962,7 +4115,9 @@ static void stdin_packs_add_all_pack_entries(struct string_list *keys, } static void stdin_packs_add_pack_entries(struct strmap *packs, - struct rev_info *revs) + struct rev_info *revs, + enum stdin_packs_mode mode, + int rev_list_unpacked) { struct string_list keys = STRING_LIST_INIT_NODUP; struct hashmap_iter iter; @@ -3983,13 +4138,18 @@ static void stdin_packs_add_pack_entries(struct strmap *packs, */ QSORT(keys.items, keys.nr, pack_mtime_cmp); - stdin_packs_add_all_pack_entries(&keys, revs); + if (mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) + stdin_packs_add_reachable_pack_entries(&keys, revs, + rev_list_unpacked); + else + stdin_packs_add_all_pack_entries(&keys, revs); string_list_clear(&keys, 0); } static void stdin_packs_read_input(struct rev_info *revs, - enum stdin_packs_mode mode) + enum stdin_packs_mode mode, + int rev_list_unpacked) { struct strbuf buf = STRBUF_INIT; struct strmap packs = STRMAP_INIT; @@ -4004,7 +4164,9 @@ static void stdin_packs_read_input(struct rev_info *revs, continue; else if (*key == '^') kind = STDIN_PACK_EXCLUDE_CLOSED; - else if (*key == '!' && mode == STDIN_PACKS_MODE_FOLLOW) + else if (*key == '!' && + (mode == STDIN_PACKS_MODE_FOLLOW || + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE)) kind = STDIN_PACK_EXCLUDE_OPEN; if (kind != STDIN_PACK_INCLUDE) @@ -4069,7 +4231,7 @@ static void stdin_packs_read_input(struct rev_info *revs, info->p = p; } - stdin_packs_add_pack_entries(&packs, revs); + stdin_packs_add_pack_entries(&packs, revs, mode, rev_list_unpacked); strbuf_release(&buf); strmap_clear(&packs, 1); @@ -4109,7 +4271,8 @@ static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked) /* avoids adding objects in excluded packs */ ignore_packed_keep_in_core = 1; - if (mode == STDIN_PACKS_MODE_FOLLOW) { + if (mode == STDIN_PACKS_MODE_FOLLOW || + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) { /* * In '--stdin-packs=follow' mode, additionally ignore * objects in excluded-open packs to prevent them from @@ -4117,8 +4280,8 @@ static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked) */ ignore_packed_keep_in_core_open = 1; } - stdin_packs_read_input(&revs, mode); - if (rev_list_unpacked) + stdin_packs_read_input(&revs, mode, rev_list_unpacked); + if (rev_list_unpacked && mode != STDIN_PACKS_MODE_FOLLOW_REACHABLE) add_unreachable_loose_objects(&revs); if (prepare_revision_walk(&revs)) @@ -5027,6 +5190,8 @@ static int parse_stdin_packs_mode(const struct option *opt, const char *arg, *mode = STDIN_PACKS_MODE_STANDARD; else if (!strcmp(arg, "follow")) *mode = STDIN_PACKS_MODE_FOLLOW; + else if (!strcmp(arg, "follow-reachable")) + *mode = STDIN_PACKS_MODE_FOLLOW_REACHABLE; else die(_("invalid value for '%s': '%s'"), opt->long_name, arg);

index c74b5861af..443d855291 100755 --- a/t/t5331-pack-objects-stdin.sh +++ b/t/t5331-pack-objects-stdin.sh @@ -520,4 +520,205 @@ test_expect_success '--stdin-packs with !-delimited pack without follow' ' ) ' +test_expect_success '--stdin-packs=follow-reachable excludes unreachable objects' ' + test_when_finished "rm -fr repo" && + + git init repo && + ( + cd repo && + git config set maintenance.auto false && + + git branch -M main && + + # Create the following commit structure: + # + # A <-- B <-- C (main) + # ^ + # \ + # U (unreachable, no ref) + test_commit A && + test_commit B && + test_commit U && + U_TIP="$(git rev-parse HEAD)" && + git reset --hard HEAD^ && + git tag -d U && + git reflog expire --all --expire=all && + + test_commit C && + + A="$(echo A | git pack-objects --revs $packdir/pack)" && + B="$(echo A..B | git pack-objects --revs $packdir/pack)" && + C="$(echo B..C | git pack-objects --revs $packdir/pack)" && + U="$(echo "$U_TIP" | git pack-objects $packdir/pack)" && + + git prune-packed && + + # Include packs A and C, exclude B as open (since B + # may not have closure), leave U as unknown. + # + # With follow-reachable: + # - objects from A and C are included (reachable from + # main, through excluded-open B, and in included + # packs) + # - objects from B are excluded (excluded-open) + # - objects from U are NOT included (not reachable + # from any ref, even though the pack exists) + P=$(git pack-objects --stdin-packs=follow-reachable \ + $packdir/pack <<-EOF + pack-$A.pack + !pack-$B.pack + pack-$C.pack + EOF + ) && + + objects_in_packs $A $C >expect && + objects_in_packs $P >actual && + test_cmp expect actual + ) +' + +test_expect_success '--stdin-packs=follow-reachable with open-excluded packs' ' + test_when_finished "rm -fr repo" && + + git init repo && + ( + cd repo && + git config set maintenance.auto false && + + git branch -M main && + + # Create the following commit structure: + # + # A <-- B <-- C <-- D (main) + # + # Pack each commit separately, then use follow-reachable + # with B excluded-open and A excluded-closed. Since B is + # open, the traversal continues through it, but since A + # is closed, it halts there. + test_commit A && + test_commit B && + test_commit C && + test_commit D && + + A="$(echo A | git pack-objects --revs $packdir/pack)" && + B="$(echo A..B | git pack-objects --revs $packdir/pack)" && + C="$(echo B..C | git pack-objects --revs $packdir/pack)" && + D="$(echo C..D | git pack-objects --revs $packdir/pack)" && + + git prune-packed && + + # Include C and D, B excluded-open, A excluded-closed. + # + # The traversal starts at main (D), walks: + # D (included) -> C (included) -> B (open, continue + # but do not include) -> A (closed, halt). + # + # Objects from C and D are in the output (reachable, + # included). B.t is also rescued (reachable via + # C^{tree} or similar). A and its objects are NOT + # (behind the closed boundary). + P=$(git pack-objects --stdin-packs=follow-reachable \ + $packdir/pack <<-EOF + pack-$C.pack + pack-$D.pack + !pack-$B.pack + ^pack-$A.pack + EOF + ) && + + objects_in_packs $C $D >expect && + objects_in_packs $P >actual && + test_cmp expect actual + ) +' + +test_expect_success '--stdin-packs=follow-reachable with --unpacked and loose objects' ' + test_when_finished "rm -fr repo" && + + git init repo && + ( + cd repo && + git config set maintenance.auto false && + + git branch -M main && + + test_commit A && + test_commit B && + + A="$(echo A | git pack-objects --revs $packdir/pack)" && + B="$(echo A..B | git pack-objects --revs $packdir/pack)" && + + git prune-packed && + + # Create a reachable loose commit on top of B. + test_commit C && + + # Create an unreachable loose object. + unreachable="$(echo "unreachable" | git hash-object -w --stdin)" && + + # Include A and B, no excluded packs. With --unpacked, + # the reachable loose objects from C should be included + # in the output but the unreachable blob should not. + P=$(git pack-objects --stdin-packs=follow-reachable \ + --unpacked $packdir/pack <<-EOF + pack-$A.pack + pack-$B.pack + EOF + ) && + + # The output should contain objects from A, B, and C. + { + objects_in_packs $A $B && + git rev-list --objects --no-object-names B..C + } >expect.raw && + sort expect.raw >expect && + + objects_in_packs $P >actual && + + # The unreachable blob should NOT be in the output. + ! grep $unreachable actual && + + test_cmp expect actual + ) +' + +test_expect_success '--stdin-packs=follow-reachable with --unpacked and loose annotated tag' ' + test_when_finished "rm -fr repo" && + + git init repo && + ( + cd repo && + git config set maintenance.auto false && + + git branch -M main && + + test_commit A && + + A="$(echo A | git pack-objects --revs $packdir/pack)" && + + git prune-packed && + + # Create a loose annotated tag pointing at A. + git tag -a -m "annotated" annotated-tag A && + tag_oid="$(git rev-parse annotated-tag)" && + + P=$(git pack-objects --stdin-packs=follow-reachable \ + --unpacked $packdir/pack <<-EOF + pack-$A.pack + EOF + ) && + + # The output should contain objects from A plus the + # loose annotated tag object. + { + objects_in_packs $A && + echo $tag_oid + } >expect.raw && + sort expect.raw >expect && + + objects_in_packs $P >actual && + test_cmp expect actual + ) +' + test_done