builtin/pack-objects.c: --cruft without expiration

Teach `pack-objects` how to generate a cruft pack when no objects are dropped (i.e., `--cruft-expiration=never`). Later patches will teach `pack-objects` how to generate a cruft pack that prunes objects. When generating a cruft pack which does not prune objects, we want to collect all unreachable objects into a single pack (noting and updating their mtimes as we accumulate them). Ordinary use will pass the result of a `git repack -A` as a kept pack, so when this patch says "kept pack", readers should think "reachable objects". Generating a non-expiring cruft packs works as follows: - Callers provide a list of every pack they know about, and indicate which packs are about to be removed. - All packs which are going to be removed (we'll call these the redundant ones) are marked as kept in-core. Any packs the caller did not mention (but are known to the `pack-objects` process) are also marked as kept in-core. Packs not mentioned by the caller are assumed to be unknown to them, i.e., they entered the repository after the caller decided which packs should be kept and which should be discarded. Since we do not want to include objects in these "unknown" packs (because we don't know which of their objects are or aren't reachable), these are also marked as kept in-core. - Then, we enumerate all objects in the repository, and add them to our packing list if they do not appear in an in-core kept pack. This results in a new cruft pack which contains all known objects that aren't included in the kept packs. When the kept pack is the result of `git repack -A`, the resulting pack contains all unreachable objects. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed May 20, 2022 at 19:17 UTC b757353676d6ffe1ac275366fa8b5b42b5d9727d
5 files changed +448 -5
Documentation/git-pack-objects.txt
+30
@@ -13,6 +13,7 @@ SYNOPSIS
13 [--no-reuse-delta] [--delta-base-offset] [--non-empty]
14 [--local] [--incremental] [--window=<n>] [--depth=<n>]
15 [--revs [--unpacked | --all]] [--keep-pack=<pack-name>]
16 + [--cruft] [--cruft-expiration=<time>]
17 [--stdout [--filter=<filter-spec>] | <base-name>]
18 [--shallow] [--keep-true-parents] [--[no-]sparse] < <object-list>
19
@@ -95,6 +96,35 @@ base-name::
96 Incompatible with `--revs`, or options that imply `--revs` (such as
97 `--all`), with the exception of `--unpacked`, which is compatible.
98
99 +--cruft::
100 + Packs unreachable objects into a separate "cruft" pack, denoted
101 + by the existence of a `.mtimes` file. Typically used by `git
102 + repack --cruft`. Callers provide a list of pack names and
103 + indicate which packs will remain in the repository, along with
104 + which packs will be deleted (indicated by the `-` prefix). The
105 + contents of the cruft pack are all objects not contained in the
106 + surviving packs which have not exceeded the grace period (see
107 + `--cruft-expiration` below), or which have exceeded the grace
108 + period, but are reachable from an other object which hasn't.
109 ++
110 +When the input lists a pack containing all reachable objects (and lists
111 +all other packs as pending deletion), the corresponding cruft pack will
112 +contain all unreachable objects (with mtime newer than the
113 +`--cruft-expiration`) along with any unreachable objects whose mtime is
114 +older than the `--cruft-expiration`, but are reachable from an
115 +unreachable object whose mtime is newer than the `--cruft-expiration`).
116 ++
117 +Incompatible with `--unpack-unreachable`, `--keep-unreachable`,
118 +`--pack-loose-unreachable`, `--stdin-packs`, as well as any other
119 +options which imply `--revs`. Also incompatible with `--max-pack-size`;
120 +when this option is set, the maximum pack size is not inferred from
121 +`pack.packSizeLimit`.
122 +
123 +--cruft-expiration=<approxidate>::
124 + If specified, objects are eliminated from the cruft pack if they
125 + have an mtime older than `<approxidate>`. If unspecified (and
126 + given `--cruft`), then no objects are eliminated.
127 +
128 --window=<n>::
129 --depth=<n>::
130 These two options affect how the objects contained in
builtin/pack-objects.c
+197 -4
@@ -36,6 +36,7 @@
36 #include "trace2.h"
37 #include "shallow.h"
38 #include "promisor-remote.h"
39 +#include "pack-mtimes.h"
40
41 /*
42 * Objects we are going to pack are collected in the `to_pack` structure.
@@ -194,6 +195,8 @@ static int reuse_delta = 1, reuse_object = 1;
195 static int keep_unreachable, unpack_unreachable, include_tag;
196 static timestamp_t unpack_unreachable_expiration;
197 static int pack_loose_unreachable;
198 +static int cruft;
199 +static timestamp_t cruft_expiration;
200 static int local;
201 static int have_non_local_packs;
202 static int incremental;
@@ -1260,6 +1263,9 @@ static void write_pack_file(void)
1263 &to_pack, written_list, nr_written);
1264 }
1265
1266 + if (cruft)
1267 + pack_idx_opts.flags |= WRITE_MTIMES;
1268 +
1269 stage_tmp_packfiles(&tmpname, pack_tmp_name,
1270 written_list, nr_written,
1271 &to_pack, &pack_idx_opts, hash,
@@ -3397,6 +3403,135 @@ static void read_packs_list_from_stdin(void)
3403 string_list_clear(&exclude_packs, 0);
3404 }
3405
3406 +static void add_cruft_object_entry(const struct object_id *oid, enum object_type type,
3407 + struct packed_git *pack, off_t offset,
3408 + const char *name, uint32_t mtime)
3409 +{
3410 + struct object_entry *entry;
3411 +
3412 + display_progress(progress_state, ++nr_seen);
3413 +
3414 + entry = packlist_find(&to_pack, oid);
3415 + if (entry) {
3416 + if (name) {
3417 + entry->hash = pack_name_hash(name);
3418 + entry->no_try_delta = no_try_delta(name);
3419 + }
3420 + } else {
3421 + if (!want_object_in_pack(oid, 0, &pack, &offset))
3422 + return;
3423 + if (!pack && type == OBJ_BLOB && !has_loose_object(oid)) {
3424 + /*
3425 + * If a traversed tree has a missing blob then we want
3426 + * to avoid adding that missing object to our pack.
3427 + *
3428 + * This only applies to missing blobs, not trees,
3429 + * because the traversal needs to parse sub-trees but
3430 + * not blobs.
3431 + *
3432 + * Note we only perform this check when we couldn't
3433 + * already find the object in a pack, so we're really
3434 + * limited to "ensure non-tip blobs which don't exist in
3435 + * packs do exist via loose objects". Confused?
3436 + */
3437 + return;
3438 + }
3439 +
3440 + entry = create_object_entry(oid, type, pack_name_hash(name),
3441 + 0, name && no_try_delta(name),
3442 + pack, offset);
3443 + }
3444 +
3445 + if (mtime > oe_cruft_mtime(&to_pack, entry))
3446 + oe_set_cruft_mtime(&to_pack, entry, mtime);
3447 + return;
3448 +}
3449 +
3450 +static void mark_pack_kept_in_core(struct string_list *packs, unsigned keep)
3451 +{
3452 + struct string_list_item *item = NULL;
3453 + for_each_string_list_item(item, packs) {
3454 + struct packed_git *p = item->util;
3455 + if (!p)
3456 + die(_("could not find pack '%s'"), item->string);
3457 + p->pack_keep_in_core = keep;
3458 + }
3459 +}
3460 +
3461 +static void add_unreachable_loose_objects(void);
3462 +static void add_objects_in_unpacked_packs(void);
3463 +
3464 +static void enumerate_cruft_objects(void)
3465 +{
3466 + if (progress)
3467 + progress_state = start_progress(_("Enumerating cruft objects"), 0);
3468 +
3469 + add_objects_in_unpacked_packs();
3470 + add_unreachable_loose_objects();
3471 +
3472 + stop_progress(&progress_state);
3473 +}
3474 +
3475 +static void read_cruft_objects(void)
3476 +{
3477 + struct strbuf buf = STRBUF_INIT;
3478 + struct string_list discard_packs = STRING_LIST_INIT_DUP;
3479 + struct string_list fresh_packs = STRING_LIST_INIT_DUP;
3480 + struct packed_git *p;
3481 +
3482 + ignore_packed_keep_in_core = 1;
3483 +
3484 + while (strbuf_getline(&buf, stdin) != EOF) {
3485 + if (!buf.len)
3486 + continue;
3487 +
3488 + if (*buf.buf == '-')
3489 + string_list_append(&discard_packs, buf.buf + 1);
3490 + else
3491 + string_list_append(&fresh_packs, buf.buf);
3492 + strbuf_reset(&buf);
3493 + }
3494 +
3495 + string_list_sort(&discard_packs);
3496 + string_list_sort(&fresh_packs);
3497 +
3498 + for (p = get_all_packs(the_repository); p; p = p->next) {
3499 + const char *pack_name = pack_basename(p);
3500 + struct string_list_item *item;
3501 +
3502 + item = string_list_lookup(&fresh_packs, pack_name);
3503 + if (!item)
3504 + item = string_list_lookup(&discard_packs, pack_name);
3505 +
3506 + if (item) {
3507 + item->util = p;
3508 + } else {
3509 + /*
3510 + * This pack wasn't mentioned in either the "fresh" or
3511 + * "discard" list, so the caller didn't know about it.
3512 + *
3513 + * Mark it as kept so that its objects are ignored by
3514 + * add_unseen_recent_objects_to_traversal(). We'll
3515 + * unmark it before starting the traversal so it doesn't
3516 + * halt the traversal early.
3517 + */
3518 + p->pack_keep_in_core = 1;
3519 + }
3520 + }
3521 +
3522 + mark_pack_kept_in_core(&fresh_packs, 1);
3523 + mark_pack_kept_in_core(&discard_packs, 0);
3524 +
3525 + if (cruft_expiration)
3526 + die("--cruft-expiration not yet implemented");
3527 + else
3528 + enumerate_cruft_objects();
3529 +
3530 + strbuf_release(&buf);
3531 + string_list_clear(&discard_packs, 0);
3532 + string_list_clear(&fresh_packs, 0);
3533 +}
3534 +
3535 static void read_object_list_from_stdin(void)
3536 {
3537 char line[GIT_MAX_HEXSZ + 1 + PATH_MAX + 2];
@@ -3529,7 +3664,24 @@ static int add_object_in_unpacked_pack(const struct object_id *oid,
3664 uint32_t pos,
3665 void *_data)
3666 {
3532 - add_object_entry(oid, OBJ_NONE, "", 0);
3667 + if (cruft) {
3668 + off_t offset;
3669 + time_t mtime;
3670 +
3671 + if (pack->is_cruft) {
3672 + if (load_pack_mtimes(pack) < 0)
3673 + die(_("could not load cruft pack .mtimes"));
3674 + mtime = nth_packed_mtime(pack, pos);
3675 + } else {
3676 + mtime = pack->mtime;
3677 + }
3678 + offset = nth_packed_object_offset(pack, pos);
3679 +
3680 + add_cruft_object_entry(oid, OBJ_NONE, pack, offset,
3681 + NULL, mtime);
3682 + } else {
3683 + add_object_entry(oid, OBJ_NONE, "", 0);
3684 + }
3685 return 0;
3686 }
3687
@@ -3553,7 +3705,19 @@ static int add_loose_object(const struct object_id *oid, const char *path,
3705 return 0;
3706 }
3707
3556 - add_object_entry(oid, type, "", 0);
3708 + if (cruft) {
3709 + struct stat st;
3710 + if (stat(path, &st) < 0) {
3711 + if (errno == ENOENT)
3712 + return 0;
3713 + return error_errno("unable to stat %s", oid_to_hex(oid));
3714 + }
3715 +
3716 + add_cruft_object_entry(oid, type, NULL, 0, NULL,
3717 + st.st_mtime);
3718 + } else {
3719 + add_object_entry(oid, type, "", 0);
3720 + }
3721 return 0;
3722 }
3723
@@ -3870,6 +4034,20 @@ static int option_parse_unpack_unreachable(const struct option *opt,
4034 return 0;
4035 }
4036
4037 +static int option_parse_cruft_expiration(const struct option *opt,
4038 + const char *arg, int unset)
4039 +{
4040 + if (unset) {
4041 + cruft = 0;
4042 + cruft_expiration = 0;
4043 + } else {
4044 + cruft = 1;
4045 + if (arg)
4046 + cruft_expiration = approxidate(arg);
4047 + }
4048 + return 0;
4049 +}
4050 +
4051 struct po_filter_data {
4052 unsigned have_revs:1;
4053 struct rev_info revs;
@@ -3959,6 +4137,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
4137 OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"),
4138 N_("unpack unreachable objects newer than <time>"),
4139 PARSE_OPT_OPTARG, option_parse_unpack_unreachable),
4140 + OPT_BOOL(0, "cruft", &cruft, N_("create a cruft pack")),
4141 + OPT_CALLBACK_F(0, "cruft-expiration", NULL, N_("time"),
4142 + N_("expire cruft objects older than <time>"),
4143 + PARSE_OPT_OPTARG, option_parse_cruft_expiration),
4144 OPT_BOOL(0, "sparse", &sparse,
4145 N_("use the sparse reachability algorithm")),
4146 OPT_BOOL(0, "thin", &thin,
@@ -4085,7 +4267,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
4267
4268 if (!HAVE_THREADS && delta_search_threads != 1)
4269 warning(_("no threads support, ignoring --threads"));
4088 - if (!pack_to_stdout && !pack_size_limit)
4270 + if (!pack_to_stdout && !pack_size_limit && !cruft)
4271 pack_size_limit = pack_size_limit_cfg;
4272 if (pack_to_stdout && pack_size_limit)
4273 die(_("--max-pack-size cannot be used to build a pack for transfer"));
@@ -4112,6 +4294,15 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
4294 if (stdin_packs && use_internal_rev_list)
4295 die(_("cannot use internal rev list with --stdin-packs"));
4296
4297 + if (cruft) {
4298 + if (use_internal_rev_list)
4299 + die(_("cannot use internal rev list with --cruft"));
4300 + if (stdin_packs)
4301 + die(_("cannot use --stdin-packs with --cruft"));
4302 + if (pack_size_limit)
4303 + die(_("cannot use --max-pack-size with --cruft"));
4304 + }
4305 +
4306 /*
4307 * "soft" reasons not to use bitmaps - for on-disk repack by default we want
4308 *
@@ -4168,7 +4359,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
4359 the_repository);
4360 prepare_packing_data(the_repository, &to_pack);
4361
4171 - if (progress)
4362 + if (progress && !cruft)
4363 progress_state = start_progress(_("Enumerating objects"), 0);
4364 if (stdin_packs) {
4365 /* avoids adding objects in excluded packs */
@@ -4176,6 +4367,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
4367 read_packs_list_from_stdin();
4368 if (rev_list_unpacked)
4369 add_unreachable_loose_objects();
4370 + } else if (cruft) {
4371 + read_cruft_objects();
4372 } else if (!use_internal_rev_list) {
4373 read_object_list_from_stdin();
4374 } else if (pfd.have_revs) {
object-file.c
+1 -1
@@ -997,7 +997,7 @@ int has_loose_object_nonlocal(const struct object_id *oid)
997 return check_and_freshen_nonlocal(oid, 0);
998 }
999
1000 -static int has_loose_object(const struct object_id *oid)
1000 +int has_loose_object(const struct object_id *oid)
1001 {
1002 return check_and_freshen(oid, 0);
1003 }
object-store.h
+2
@@ -339,6 +339,8 @@ int repo_has_object_file_with_flags(struct repository *r,
339 */
340 int has_loose_object_nonlocal(const struct object_id *);
341
342 +int has_loose_object(const struct object_id *);
343 +
344 /**
345 * format_object_header() is a thin wrapper around s xsnprintf() that
346 * writes the initial "<type> <obj-len>" part of the loose object
t/t5329-pack-objects-cruft.sh new
+218
@@ -0,0 +1,218 @@
1 +#!/bin/sh
2 +
3 +test_description='cruft pack related pack-objects tests'
4 +. ./test-lib.sh
5 +
6 +objdir=.git/objects
7 +packdir=$objdir/pack
8 +
9 +basic_cruft_pack_tests () {
10 + expire="$1"
11 +
12 + test_expect_success "unreachable loose objects are packed (expire $expire)" '
13 + git init repo &&
14 + test_when_finished "rm -fr repo" &&
15 + (
16 + cd repo &&
17 +
18 + test_commit base &&
19 + git repack -Ad &&
20 + test_commit loose &&
21 +
22 + test-tool chmtime +2000 "$objdir/$(test_oid_to_path \
23 + $(git rev-parse loose:loose.t))" &&
24 + test-tool chmtime +1000 "$objdir/$(test_oid_to_path \
25 + $(git rev-parse loose^{tree}))" &&
26 +
27 + (
28 + git rev-list --objects --no-object-names base..loose |
29 + while read oid
30 + do
31 + path="$objdir/$(test_oid_to_path "$oid")" &&
32 + printf "%s %d\n" "$oid" "$(test-tool chmtime --get "$path")"
33 + done |
34 + sort -k1
35 + ) >expect &&
36 +
37 + keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
38 + cruft="$(echo $keep | git pack-objects --cruft \
39 + --cruft-expiration="$expire" $packdir/pack)" &&
40 + test-tool pack-mtimes "pack-$cruft.mtimes" >actual &&
41 +
42 + test_cmp expect actual
43 + )
44 + '
45 +
46 + test_expect_success "unreachable packed objects are packed (expire $expire)" '
47 + git init repo &&
48 + test_when_finished "rm -fr repo" &&
49 + (
50 + cd repo &&
51 +
52 + test_commit packed &&
53 + git repack -Ad &&
54 + test_commit other &&
55 +
56 + git rev-list --objects --no-object-names packed.. >objects &&
57 + keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
58 + other="$(git pack-objects --delta-base-offset \
59 + $packdir/pack <objects)" &&
60 + git prune-packed &&
61 +
62 + test-tool chmtime --get -100 "$packdir/pack-$other.pack" >expect &&
63 +
64 + cruft="$(git pack-objects --cruft --cruft-expiration="$expire" $packdir/pack <<-EOF
65 + $keep
66 + -pack-$other.pack
67 + EOF
68 + )" &&
69 + test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
70 +
71 + cut -d" " -f2 <actual.raw | sort -u >actual &&
72 +
73 + test_cmp expect actual
74 + )
75 + '
76 +
77 + test_expect_success "unreachable cruft objects are repacked (expire $expire)" '
78 + git init repo &&
79 + test_when_finished "rm -fr repo" &&
80 + (
81 + cd repo &&
82 +
83 + test_commit packed &&
84 + git repack -Ad &&
85 + test_commit other &&
86 +
87 + git rev-list --objects --no-object-names packed.. >objects &&
88 + keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
89 +
90 + cruft_a="$(echo $keep | git pack-objects --cruft --cruft-expiration="$expire" $packdir/pack)" &&
91 + git prune-packed &&
92 + cruft_b="$(git pack-objects --cruft --cruft-expiration="$expire" $packdir/pack <<-EOF
93 + $keep
94 + -pack-$cruft_a.pack
95 + EOF
96 + )" &&
97 +
98 + test-tool pack-mtimes "pack-$cruft_a.mtimes" >expect.raw &&
99 + test-tool pack-mtimes "pack-$cruft_b.mtimes" >actual.raw &&
100 +
101 + sort <expect.raw >expect &&
102 + sort <actual.raw >actual &&
103 +
104 + test_cmp expect actual
105 + )
106 + '
107 +
108 + test_expect_success "multiple cruft packs (expire $expire)" '
109 + git init repo &&
110 + test_when_finished "rm -fr repo" &&
111 + (
112 + cd repo &&
113 +
114 + test_commit reachable &&
115 + git repack -Ad &&
116 + keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
117 +
118 + test_commit cruft &&
119 + loose="$objdir/$(test_oid_to_path $(git rev-parse cruft))" &&
120 +
121 + # generate three copies of the cruft object in different
122 + # cruft packs, each with a unique mtime:
123 + # - one expired (1000 seconds ago)
124 + # - two non-expired (one 1000 seconds in the future,
125 + # one 1500 seconds in the future)
126 + test-tool chmtime =-1000 "$loose" &&
127 + git pack-objects --cruft $packdir/pack-A <<-EOF &&
128 + $keep
129 + EOF
130 + test-tool chmtime =+1000 "$loose" &&
131 + git pack-objects --cruft $packdir/pack-B <<-EOF &&
132 + $keep
133 + -$(basename $(ls $packdir/pack-A-*.pack))
134 + EOF
135 + test-tool chmtime =+1500 "$loose" &&
136 + git pack-objects --cruft $packdir/pack-C <<-EOF &&
137 + $keep
138 + -$(basename $(ls $packdir/pack-A-*.pack))
139 + -$(basename $(ls $packdir/pack-B-*.pack))
140 + EOF
141 +
142 + # ensure the resulting cruft pack takes the most recent
143 + # mtime among all copies
144 + cruft="$(git pack-objects --cruft \
145 + --cruft-expiration="$expire" \
146 + $packdir/pack <<-EOF
147 + $keep
148 + -$(basename $(ls $packdir/pack-A-*.pack))
149 + -$(basename $(ls $packdir/pack-B-*.pack))
150 + -$(basename $(ls $packdir/pack-C-*.pack))
151 + EOF
152 + )" &&
153 +
154 + test-tool pack-mtimes "$(basename $(ls $packdir/pack-C-*.mtimes))" >expect.raw &&
155 + test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
156 +
157 + sort expect.raw >expect &&
158 + sort actual.raw >actual &&
159 + test_cmp expect actual
160 + )
161 + '
162 +
163 + test_expect_success "cruft packs tolerate missing trees (expire $expire)" '
164 + git init repo &&
165 + test_when_finished "rm -fr repo" &&
166 + (
167 + cd repo &&
168 +
169 + test_commit reachable &&
170 + test_commit cruft &&
171 +
172 + tree="$(git rev-parse cruft^{tree})" &&
173 +
174 + git reset --hard reachable &&
175 + git tag -d cruft &&
176 + git reflog expire --all --expire=all &&
177 +
178 + # remove the unreachable tree, but leave the commit
179 + # which has it as its root tree intact
180 + rm -fr "$objdir/$(test_oid_to_path "$tree")" &&
181 +
182 + git repack -Ad &&
183 + basename $(ls $packdir/pack-*.pack) >in &&
184 + git pack-objects --cruft --cruft-expiration="$expire" \
185 + $packdir/pack <in
186 + )
187 + '
188 +
189 + test_expect_success "cruft packs tolerate missing blobs (expire $expire)" '
190 + git init repo &&
191 + test_when_finished "rm -fr repo" &&
192 + (
193 + cd repo &&
194 +
195 + test_commit reachable &&
196 + test_commit cruft &&
197 +
198 + blob="$(git rev-parse cruft:cruft.t)" &&
199 +
200 + git reset --hard reachable &&
201 + git tag -d cruft &&
202 + git reflog expire --all --expire=all &&
203 +
204 + # remove the unreachable blob, but leave the commit (and
205 + # the root tree of that commit) intact
206 + rm -fr "$objdir/$(test_oid_to_path "$blob")" &&
207 +
208 + git repack -Ad &&
209 + basename $(ls $packdir/pack-*.pack) >in &&
210 + git pack-objects --cruft --cruft-expiration="$expire" \
211 + $packdir/pack <in
212 + )
213 + '
214 +}
215 +
216 +basic_cruft_pack_tests never
217 +
218 +test_done