@matheusbsbs / matheus-bagliano / commits / ec0165a0cd

pack-objects: introduce '--stdin-packs=follow-reachable'

Introduce a new '--stdin-packs=follow-reachable' mode. Like '--stdin-packs=follow', this mode recognizes the '!' (excluded-open) pack prefix and halts at '^' (excluded-closed) packs. Unlike 'follow', which eagerly includes all objects from listed packs and then walks reachability to rescue additional objects, the new 'follow-reachable' mode uses reference tips as its traversal starting points and only includes objects that are both reachable AND belong to an included pack (or are reachable from a commit or tag in one): - Objects in included packs: added to the output if reachable. - Objects reachable from included-pack commits but in unknown packs: added to the output (rescued). - Objects in excluded-open ('!') packs: not included, but the traversal continues through them. - Objects in excluded-closed ('^') packs: not included, and the traversal halts. The implementation uses a two-phase approach: 1. In the first phase, commits and tags in included packs (and loose, when --unpacked is given) are marked with a flag bit (IN_INCLUDED_PACK). A commit-only walk from ref tips then identifies which marked objects are reachable, halting at excluded-closed packs. 2. In the second phase, every reachable marked object (from the previous step) becomes a tip for a full object traversal whose `show_object_pack_hint()` and `show_commit_pack_hint()` callbacks add discovered objects (obeying the usual constraints imposed by `want_object_in_pack()`). When '--unpacked' is given, reachable loose objects are included in the output while unreachable loose objects are left alone. This is achieved by marking loose commits and tags with IN_INCLUDED_PACK during the first phase, so the pre-walk discovers them naturally. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Jun 26, 2026 at 15:02 UTC ec0165a0cd79f6391dbb546a065db1695db0aa91

3 files changed +393 -10

Documentation/git-pack-objects.adoc

+17

@@ -113,6 +113,23 @@ This mode is useful, for example, to resurrect once-unreachable

113 objects found in cruft packs to generate packs which are closed under

114 reachability up to the boundary set by the excluded packs.

115 +

116 +When `mode` is "follow-reachable", the same pack prefixes are recognized

117 +as in "follow" (`!` for excluded-open, `^` for excluded-closed). However,

118 +instead of including all objects from included packs, only objects that

119 +are reachable from reference tips AND belong to an included pack (or are

120 +reachable from a commit in one) are included. Objects in excluded-open

121 +packs are traversed but not included; objects in excluded-closed packs

122 +halt the traversal.

123 ++

124 +This mode is designed for geometric repacking with cruft packs, where

125 +the output pack should contain only reachable objects so that unreachable

126 +ones can be collected separately.

127 ++

128 +When `--unpacked` is given alongside `--stdin-packs=follow-reachable`,

129 +reachable loose objects are also included in the output pack, while

130 +unreachable loose objects are left alone. This includes both loose

131 +commits and annotated tag objects.

132 ++

133 Incompatible with `--revs`, or options that imply `--revs` (such as

134 `--all`), with the exception of `--unpacked`, which is compatible.

135

builtin/pack-objects.c

+175 -10

@@ -290,6 +290,7 @@ enum stdin_packs_mode {

290 STDIN_PACKS_MODE_NONE,

291 STDIN_PACKS_MODE_STANDARD,

292 STDIN_PACKS_MODE_FOLLOW,

293 + STDIN_PACKS_MODE_FOLLOW_REACHABLE,

294 };

295

296 /**

@@ -3835,7 +3836,8 @@ static void show_object_pack_hint(struct object *object, const char *name,

3836 void *data)

3837 {

3838 enum stdin_packs_mode mode = *(enum stdin_packs_mode *)data;

3838 - if (mode == STDIN_PACKS_MODE_FOLLOW) {

3839 + if (mode == STDIN_PACKS_MODE_FOLLOW ||

3840 + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) {

3841 if (object->type == OBJ_BLOB &&

3842 !odb_has_object(the_repository->objects, &object->oid, 0))

3843 return;

@@ -3866,7 +3868,8 @@ static void show_commit_pack_hint(struct commit *commit, void *data)

3868 {

3869 enum stdin_packs_mode mode = *(enum stdin_packs_mode *)data;

3870

3869 - if (mode == STDIN_PACKS_MODE_FOLLOW) {

3871 + if (mode == STDIN_PACKS_MODE_FOLLOW ||

3872 + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) {

3873 show_object_pack_hint((struct object *)commit, "", data);

3874 return;

3875 }

@@ -3933,6 +3936,156 @@ static int stdin_packs_include_check(struct commit *commit, void *data)

3936 return stdin_packs_include_check_obj((struct object *)commit, data);

3937 }

3938

3939 +/*

3940 + * Flag bit set on commits that belong to an included pack during

3941 + * '--stdin-packs=follow-reachable'. Used by the pre-walk to

3942 + * identify which reachable commits should be tips for the main

3943 + * object traversal.

3944 + */

3945 +#define IN_INCLUDED_PACK (1u<<11)

3946 +

3947 +static int mark_included_pack_tip(const struct object_id *oid,

3948 + struct packed_git *p,

3949 + uint32_t pos,

3950 + void *data)

3951 +{

3952 + struct rev_info *main_revs = data;

3953 + off_t ofs = nth_packed_object_offset(p, pos);

3954 + enum object_type type;

3955 + struct object_info oi = OBJECT_INFO_INIT;

3956 + struct object *obj;

3957 +

3958 + oi.typep = &type;

3959 + if (packed_object_info(p, ofs, &oi) < 0)

3960 + return 0;

3961 + if (type != OBJ_COMMIT && type != OBJ_TAG)

3962 + return 0;

3963 +

3964 + obj = parse_object(the_repository, oid);

3965 + if (!obj)

3966 + return 0;

3967 +

3968 + obj->flags |= IN_INCLUDED_PACK;

3969 +

3970 + if (type == OBJ_TAG && main_revs)

3971 + add_pending_object(main_revs, obj, "");

3972 + return 0;

3973 +}

3974 +

3975 +static int mark_loose_object_tip(const struct object_id *oid,

3976 + struct object_info *oi UNUSED,

3977 + void *data)

3978 +{

3979 + struct rev_info *main_revs = data;

3980 + struct object *obj;

3981 + enum object_type type;

3982 +

3983 + type = odb_read_object_info(the_repository->objects, oid, NULL);

3984 + if (type != OBJ_COMMIT && type != OBJ_TAG)

3985 + return 0;

3986 +

3987 + obj = parse_object(the_repository, oid);

3988 + if (!obj)

3989 + return 0;

3990 +

3991 + obj->flags |= IN_INCLUDED_PACK;

3992 +

3993 + if (type == OBJ_TAG && main_revs)

3994 + add_pending_object(main_revs, obj, "");

3995 +

3996 + return 0;

3997 +}

3998 +

3999 +static int add_ref_to_pending(const struct reference *ref, void *cb_data)

4000 +{

4001 + struct rev_info *revs = cb_data;

4002 + struct object *object;

4003 +

4004 + object = parse_object(the_repository, ref->oid);

4005 + if (!object)

4006 + return 0;

4007 +

4008 + add_pending_object(revs, object, "");

4009 + return 0;

4010 +}

4011 +

4012 +static void stdin_packs_add_reachable_pack_entries(struct string_list *keys,

4013 + struct rev_info *revs,

4014 + int rev_list_unpacked)

4015 +{

4016 + struct rev_info pre_walk;

4017 + struct commit *commit;

4018 + struct string_list_item *item;

4019 +

4020 + /*

4021 + * Phase 1: mark commits in included packs, then walk from

4022 + * ref tips to discover which of them are reachable. The walk

4023 + * halts at excluded-closed packs (via no_kept_objects) and

4024 + * continues through excluded-open ones.

4025 + *

4026 + * Also set include_check on the outer revs so that phase 2

4027 + * (the main object traversal) halts at closed packs.

4028 + */

4029 + revs->include_check = stdin_packs_include_check;

4030 + revs->include_check_obj = stdin_packs_include_check_obj;

4031 +

4032 + for_each_string_list_item(item, keys) {

4033 + struct stdin_pack_info *info = item->util;

4034 + if (info->kind & STDIN_PACK_INCLUDE)

4035 + for_each_object_in_pack(info->p,

4036 + mark_included_pack_tip,

4037 + revs,

4038 + ODB_FOR_EACH_OBJECT_PACK_ORDER);

4039 + }

4040 +

4041 + if (rev_list_unpacked) {

4042 + /*

4043 + * With '--stdin-packs=follow-reachable', specifying

4044 + * '--unpacked' instructs pack-objects to pack any loose

4045 + * objects which are reachable.

4046 + *

4047 + * Pretend as if all loose objects are in an included

4048 + * pack in order to make them eligible for packing.

4049 + */

4050 + struct odb_source *source = revs->repo->objects->sources;

4051 + for (; source; source = source->next) {

4052 + struct odb_source_files *files = odb_source_files_downcast(source);

4053 + struct odb_for_each_object_options opts = { 0 };

4054 + if (local)

4055 + opts.flags |= ODB_FOR_EACH_OBJECT_LOCAL_ONLY;

4056 +

4057 + odb_source_for_each_object(&files->loose->base, NULL,

4058 + mark_loose_object_tip,

4059 + revs, &opts);

4060 + }

4061 + }

4062 +

4063 + repo_init_revisions(the_repository, &pre_walk, NULL);

4064 + pre_walk.no_kept_objects = 1;

4065 + pre_walk.keep_pack_cache_flags |= KEPT_PACK_IN_CORE;

4066 + pre_walk.ignore_missing_links = 1;

4067 +

4068 + refs_for_each_ref(get_main_ref_store(the_repository),

4069 + add_ref_to_pending, &pre_walk);

4070 +

4071 + if (prepare_revision_walk(&pre_walk))

4072 + die(_("revision walk setup failed"));

4073 +

4074 + /*

4075 + * Phase 2 tips: every reachable commit that is in an

4076 + * included pack becomes a starting point for the main

4077 + * object traversal.

4078 + */

4079 + while ((commit = get_revision(&pre_walk)) != NULL) {

4080 + if (commit->object.flags & IN_INCLUDED_PACK)

4081 + add_pending_oid(revs, NULL,

4082 + &commit->object.oid, 0);

4083 + }

4084 +

4085 + reset_revision_walk();

4086 + release_revisions(&pre_walk);

4087 +}

4088 +

4089 static void stdin_packs_add_all_pack_entries(struct string_list *keys,

4090 struct rev_info *revs)

4091 {

@@ -3962,7 +4115,9 @@ static void stdin_packs_add_all_pack_entries(struct string_list *keys,

4115 }

4116

4117 static void stdin_packs_add_pack_entries(struct strmap *packs,

3965 - struct rev_info *revs)

4118 + struct rev_info *revs,

4119 + enum stdin_packs_mode mode,

4120 + int rev_list_unpacked)

4121 {

4122 struct string_list keys = STRING_LIST_INIT_NODUP;

4123 struct hashmap_iter iter;

@@ -3983,13 +4138,18 @@ static void stdin_packs_add_pack_entries(struct strmap *packs,

4138 */

4139 QSORT(keys.items, keys.nr, pack_mtime_cmp);

4140

3986 - stdin_packs_add_all_pack_entries(&keys, revs);

4141 + if (mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE)

4142 + stdin_packs_add_reachable_pack_entries(&keys, revs,

4143 + rev_list_unpacked);

4144 + else

4145 + stdin_packs_add_all_pack_entries(&keys, revs);

4146

4147 string_list_clear(&keys, 0);

4148 }

4149

4150 static void stdin_packs_read_input(struct rev_info *revs,

3992 - enum stdin_packs_mode mode)

4151 + enum stdin_packs_mode mode,

4152 + int rev_list_unpacked)

4153 {

4154 struct strbuf buf = STRBUF_INIT;

4155 struct strmap packs = STRMAP_INIT;

@@ -4004,7 +4164,9 @@ static void stdin_packs_read_input(struct rev_info *revs,

4164 continue;

4165 else if (*key == '^')

4166 kind = STDIN_PACK_EXCLUDE_CLOSED;

4007 - else if (*key == '!' && mode == STDIN_PACKS_MODE_FOLLOW)

4167 + else if (*key == '!' &&

4168 + (mode == STDIN_PACKS_MODE_FOLLOW ||

4169 + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE))

4170 kind = STDIN_PACK_EXCLUDE_OPEN;

4171

4172 if (kind != STDIN_PACK_INCLUDE)

@@ -4069,7 +4231,7 @@ static void stdin_packs_read_input(struct rev_info *revs,

4231 info->p = p;

4232 }

4233

4072 - stdin_packs_add_pack_entries(&packs, revs);

4234 + stdin_packs_add_pack_entries(&packs, revs, mode, rev_list_unpacked);

4235

4236 strbuf_release(&buf);

4237 strmap_clear(&packs, 1);

@@ -4109,7 +4271,8 @@ static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked)

4271

4272 /* avoids adding objects in excluded packs */

4273 ignore_packed_keep_in_core = 1;

4112 - if (mode == STDIN_PACKS_MODE_FOLLOW) {

4274 + if (mode == STDIN_PACKS_MODE_FOLLOW ||

4275 + mode == STDIN_PACKS_MODE_FOLLOW_REACHABLE) {

4276 /*

4277 * In '--stdin-packs=follow' mode, additionally ignore

4278 * objects in excluded-open packs to prevent them from

@@ -4117,8 +4280,8 @@ static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked)

4280 */

4281 ignore_packed_keep_in_core_open = 1;

4282 }

4120 - stdin_packs_read_input(&revs, mode);

4121 - if (rev_list_unpacked)

4283 + stdin_packs_read_input(&revs, mode, rev_list_unpacked);

4284 + if (rev_list_unpacked && mode != STDIN_PACKS_MODE_FOLLOW_REACHABLE)

4285 add_unreachable_loose_objects(&revs);

4286

4287 if (prepare_revision_walk(&revs))

@@ -5027,6 +5190,8 @@ static int parse_stdin_packs_mode(const struct option *opt, const char *arg,

5190 *mode = STDIN_PACKS_MODE_STANDARD;

5191 else if (!strcmp(arg, "follow"))

5192 *mode = STDIN_PACKS_MODE_FOLLOW;

5193 + else if (!strcmp(arg, "follow-reachable"))

5194 + *mode = STDIN_PACKS_MODE_FOLLOW_REACHABLE;

5195 else

5196 die(_("invalid value for '%s': '%s'"), opt->long_name, arg);

5197

t/t5331-pack-objects-stdin.sh

+201

@@ -520,4 +520,205 @@ test_expect_success '--stdin-packs with !-delimited pack without follow' '

520 )

521 '

522

523 +test_expect_success '--stdin-packs=follow-reachable excludes unreachable objects' '

524 + test_when_finished "rm -fr repo" &&

525 +

526 + git init repo &&

527 + (

528 + cd repo &&

529 + git config set maintenance.auto false &&

530 +

531 + git branch -M main &&

532 +

533 + # Create the following commit structure:

534 + #

535 + # A <-- B <-- C (main)

536 + # ^

537 + # \

538 + # U (unreachable, no ref)

539 + test_commit A &&

540 + test_commit B &&

541 + test_commit U &&

542 + U_TIP="$(git rev-parse HEAD)" &&

543 + git reset --hard HEAD^ &&

544 + git tag -d U &&

545 + git reflog expire --all --expire=all &&

546 +

547 + test_commit C &&

548 +

549 + A="$(echo A | git pack-objects --revs $packdir/pack)" &&

550 + B="$(echo A..B | git pack-objects --revs $packdir/pack)" &&

551 + C="$(echo B..C | git pack-objects --revs $packdir/pack)" &&

552 + U="$(echo "$U_TIP" | git pack-objects $packdir/pack)" &&

553 +

554 + git prune-packed &&

555 +

556 + # Include packs A and C, exclude B as open (since B

557 + # may not have closure), leave U as unknown.

558 + #

559 + # With follow-reachable:

560 + # - objects from A and C are included (reachable from

561 + # main, through excluded-open B, and in included

562 + # packs)

563 + # - objects from B are excluded (excluded-open)

564 + # - objects from U are NOT included (not reachable

565 + # from any ref, even though the pack exists)

566 + P=$(git pack-objects --stdin-packs=follow-reachable \

567 + $packdir/pack <<-EOF

568 + pack-$A.pack

569 + !pack-$B.pack

570 + pack-$C.pack

571 + EOF

572 + ) &&

573 +

574 + objects_in_packs $A $C >expect &&

575 + objects_in_packs $P >actual &&

576 + test_cmp expect actual

577 + )

578 +'

579 +

580 +test_expect_success '--stdin-packs=follow-reachable with open-excluded packs' '

581 + test_when_finished "rm -fr repo" &&

582 +

583 + git init repo &&

584 + (

585 + cd repo &&

586 + git config set maintenance.auto false &&

587 +

588 + git branch -M main &&

589 +

590 + # Create the following commit structure:

591 + #

592 + # A <-- B <-- C <-- D (main)

593 + #

594 + # Pack each commit separately, then use follow-reachable

595 + # with B excluded-open and A excluded-closed. Since B is

596 + # open, the traversal continues through it, but since A

597 + # is closed, it halts there.

598 + test_commit A &&

599 + test_commit B &&

600 + test_commit C &&

601 + test_commit D &&

602 +

603 + A="$(echo A | git pack-objects --revs $packdir/pack)" &&

604 + B="$(echo A..B | git pack-objects --revs $packdir/pack)" &&

605 + C="$(echo B..C | git pack-objects --revs $packdir/pack)" &&

606 + D="$(echo C..D | git pack-objects --revs $packdir/pack)" &&

607 +

608 + git prune-packed &&

609 +

610 + # Include C and D, B excluded-open, A excluded-closed.

611 + #

612 + # The traversal starts at main (D), walks:

613 + # D (included) -> C (included) -> B (open, continue

614 + # but do not include) -> A (closed, halt).

615 + #

616 + # Objects from C and D are in the output (reachable,

617 + # included). B.t is also rescued (reachable via

618 + # C^{tree} or similar). A and its objects are NOT

619 + # (behind the closed boundary).

620 + P=$(git pack-objects --stdin-packs=follow-reachable \

621 + $packdir/pack <<-EOF

622 + pack-$C.pack

623 + pack-$D.pack

624 + !pack-$B.pack

625 + ^pack-$A.pack

626 + EOF

627 + ) &&

628 +

629 + objects_in_packs $C $D >expect &&

630 + objects_in_packs $P >actual &&

631 + test_cmp expect actual

632 + )

633 +'

634 +

635 +test_expect_success '--stdin-packs=follow-reachable with --unpacked and loose objects' '

636 + test_when_finished "rm -fr repo" &&

637 +

638 + git init repo &&

639 + (

640 + cd repo &&

641 + git config set maintenance.auto false &&

642 +

643 + git branch -M main &&

644 +

645 + test_commit A &&

646 + test_commit B &&

647 +

648 + A="$(echo A | git pack-objects --revs $packdir/pack)" &&

649 + B="$(echo A..B | git pack-objects --revs $packdir/pack)" &&

650 +

651 + git prune-packed &&

652 +

653 + # Create a reachable loose commit on top of B.

654 + test_commit C &&

655 +

656 + # Create an unreachable loose object.

657 + unreachable="$(echo "unreachable" | git hash-object -w --stdin)" &&

658 +

659 + # Include A and B, no excluded packs. With --unpacked,

660 + # the reachable loose objects from C should be included

661 + # in the output but the unreachable blob should not.

662 + P=$(git pack-objects --stdin-packs=follow-reachable \

663 + --unpacked $packdir/pack <<-EOF

664 + pack-$A.pack

665 + pack-$B.pack

666 + EOF

667 + ) &&

668 +

669 + # The output should contain objects from A, B, and C.

670 + {

671 + objects_in_packs $A $B &&

672 + git rev-list --objects --no-object-names B..C

673 + } >expect.raw &&

674 + sort expect.raw >expect &&

675 +

676 + objects_in_packs $P >actual &&

677 +

678 + # The unreachable blob should NOT be in the output.

679 + ! grep $unreachable actual &&

680 +

681 + test_cmp expect actual

682 + )

683 +'

684 +

685 +test_expect_success '--stdin-packs=follow-reachable with --unpacked and loose annotated tag' '

686 + test_when_finished "rm -fr repo" &&

687 +

688 + git init repo &&

689 + (

690 + cd repo &&

691 + git config set maintenance.auto false &&

692 +

693 + git branch -M main &&

694 +

695 + test_commit A &&

696 +

697 + A="$(echo A | git pack-objects --revs $packdir/pack)" &&

698 +

699 + git prune-packed &&

700 +

701 + # Create a loose annotated tag pointing at A.

702 + git tag -a -m "annotated" annotated-tag A &&

703 + tag_oid="$(git rev-parse annotated-tag)" &&

704 +

705 + P=$(git pack-objects --stdin-packs=follow-reachable \

706 + --unpacked $packdir/pack <<-EOF

707 + pack-$A.pack

708 + EOF

709 + ) &&

710 +

711 + # The output should contain objects from A plus the

712 + # loose annotated tag object.

713 + {

714 + objects_in_packs $A &&

715 + echo $tag_oid

716 + } >expect.raw &&

717 + sort expect.raw >expect &&

718 +

719 + objects_in_packs $P >actual &&

720 + test_cmp expect actual

721 + )

722 +'

723 +

724 test_done