midx: enable reachability bitmaps during MIDX compaction
Enable callers to generate reachability bitmaps when performing MIDX layer compaction by combining all existing bitmaps from the compacted layers. Note that because of the object/pack ordering described by the previous commit, the pseudo-pack order for the compacted MIDX is the same as concatenating the individual pseudo-pack orderings for each layer in the compaction range. As a result, the only non-test or documentation change necessary is to treat all objects as non-preferred during compaction so as not to disturb the object ordering. In the future, we may want to adjust which commit(s) receive reachability bitmaps when compacting multiple .bitmap files into one, or even generate new bitmaps (e.g., if the references have moved significantly since the .bitmap was generated). This commit only implements combining all existing bitmaps in range together in order to demonstrate and lay the groundwork for more exotic strategies. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Feb 24, 2026 at 14:00 UTC
d54da84bd9de09fc339accff553f1fc8a5539154
4 files changed
+128
-5
Documentation/git-multi-pack-index.adoc
+4
-1
@@ -13,7 +13,7 @@ SYNOPSIS
13
[--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]
14
[--refs-snapshot=<path>]
15
'git multi-pack-index' [<options>] compact [--[no-]incremental]
16
- <from> <to>
16
+ [--[no-]bitmap] <from> <to>
17
'git multi-pack-index' [<options>] verify
18
'git multi-pack-index' [<options>] expire
19
'git multi-pack-index' [<options>] repack [--batch-size=<size>]
@@ -94,6 +94,9 @@ compact::
94
--incremental::
95
Write the result to a MIDX chain instead of writing a
96
stand-alone MIDX.
97
+
98
+ --[no-]bitmap::
99
+ Control whether or not a multi-pack bitmap is written.
100
--
101
102
verify::
builtin/multi-pack-index.c
+3
-1
@@ -19,7 +19,7 @@
19
20
#define BUILTIN_MIDX_COMPACT_USAGE \
21
N_("git multi-pack-index [<options>] compact [--[no-]incremental]\n" \
22
- " <from> <to>")
22
+ " [--[no-]bitmap] <from> <to>")
23
24
#define BUILTIN_MIDX_VERIFY_USAGE \
25
N_("git multi-pack-index [<options>] verify")
@@ -216,6 +216,8 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
216
217
struct option *options;
218
static struct option builtin_multi_pack_index_compact_options[] = {
219
+ OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
220
+ MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
221
OPT_BIT(0, "incremental", &opts.flags,
222
N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
223
OPT_END(),
midx-write.c
+1
-1
@@ -676,7 +676,7 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx)
676
struct pack_midx_entry *e = &ctx->entries[i];
677
data[i].nr = i;
678
data[i].pack = midx_pack_perm(ctx, e->pack_int_id);
679
- if (!e->preferred)
679
+ if (!e->preferred || ctx->compact)
680
data[i].pack |= (1U << 31);
681
data[i].offset = e->offset;
682
}
t/t5335-compact-multi-pack-index.sh
+120
-2
@@ -67,7 +67,7 @@ test_expect_success 'MIDX compaction with lex-ordered pack names' '
67
write_packs A B C D E &&
68
test_line_count = 5 $midx_chain &&
69
70
- git multi-pack-index compact --incremental \
70
+ git multi-pack-index compact --incremental --bitmap \
71
"$(nth_line 2 "$midx_chain")" \
72
"$(nth_line 4 "$midx_chain")" &&
73
test_line_count = 3 $midx_chain &&
@@ -90,7 +90,7 @@ test_expect_success 'MIDX compaction with non-lex-ordered pack names' '
90
write_packs D C A B E &&
91
test_line_count = 5 $midx_chain &&
92
93
- git multi-pack-index compact --incremental \
93
+ git multi-pack-index compact --incremental --bitmap \
94
"$(nth_line 2 "$midx_chain")" \
95
"$(nth_line 4 "$midx_chain")" &&
96
test_line_count = 3 $midx_chain &&
@@ -172,4 +172,122 @@ test_expect_success 'MIDX compaction with midx.version=1' '
172
)
173
'
174
175
+midx_objs_by_pack () {
176
+ awk '/\.pack$/ { split($3, a, "-"); print a[2], $1 }' | sort
177
+}
178
+
179
+tag_objs_from_pack () {
180
+ objs="$(git rev-list --objects --no-object-names "$2")" &&
181
+ printf "$1 %s\n" $objs | sort
182
+}
183
+
184
+test_expect_success 'MIDX compaction preserves pack object selection' '
185
+ git init midx-compact-preserve-selection &&
186
+ (
187
+ cd midx-compact-preserve-selection &&
188
+
189
+ git config maintenance.auto false &&
190
+
191
+ test_commit A &&
192
+ test_commit B &&
193
+
194
+ # Create two packs, one containing just the objects from
195
+ # A, and another containing all objects from the
196
+ # repository.
197
+ p1="$(echo A | git pack-objects --revs --delta-base-offset \
198
+ $packdir/pack-1)" &&
199
+ p0="$(echo B | git pack-objects --revs --delta-base-offset \
200
+ $packdir/pack-0)" &&
201
+
202
+ echo "pack-1-$p1.idx" | git multi-pack-index write \
203
+ --incremental --bitmap --stdin-packs &&
204
+ echo "pack-0-$p0.idx" | git multi-pack-index write \
205
+ --incremental --bitmap --stdin-packs &&
206
+
207
+ write_packs C &&
208
+
209
+ git multi-pack-index compact --incremental --bitmap \
210
+ "$(nth_line 1 "$midx_chain")" \
211
+ "$(nth_line 2 "$midx_chain")" &&
212
+
213
+
214
+ test-tool read-midx --show-objects $objdir \
215
+ "$(nth_line 1 "$midx_chain")" >AB.info &&
216
+ test-tool read-midx --show-objects $objdir \
217
+ "$(nth_line 2 "$midx_chain")" >C.info &&
218
+
219
+ midx_objs_by_pack <AB.info >AB.actual &&
220
+ midx_objs_by_pack <C.info >C.actual &&
221
+
222
+ {
223
+ tag_objs_from_pack 1 A &&
224
+ tag_objs_from_pack 0 A..B
225
+ } | sort >AB.expect &&
226
+ tag_objs_from_pack C B..C >C.expect &&
227
+
228
+ test_cmp AB.expect AB.actual &&
229
+ test_cmp C.expect C.actual
230
+ )
231
+'
232
+
233
+test_expect_success 'MIDX compaction with bitmaps' '
234
+ git init midx-compact-with-bitmaps &&
235
+ (
236
+ cd midx-compact-with-bitmaps &&
237
+
238
+ git config maintenance.auto false &&
239
+
240
+ write_packs foo bar baz quux woot &&
241
+
242
+ test-tool read-midx --bitmap $objdir >bitmap.expect &&
243
+ git multi-pack-index compact --incremental --bitmap \
244
+ "$(nth_line 2 "$midx_chain")" \
245
+ "$(nth_line 4 "$midx_chain")" &&
246
+ test-tool read-midx --bitmap $objdir >bitmap.actual &&
247
+
248
+ test_cmp bitmap.expect bitmap.actual &&
249
+
250
+ true
251
+ )
252
+'
253
+
254
+test_expect_success 'MIDX compaction with bitmaps (non-trivial)' '
255
+ git init midx-compact-with-bitmaps-non-trivial &&
256
+ (
257
+ cd midx-compact-with-bitmaps-non-trivial &&
258
+
259
+ git config maintenance.auto false &&
260
+
261
+ git branch -m main &&
262
+
263
+ # D(4)
264
+ # /
265
+ # A(1) --- B(2) --- C(3) --- G(7)
266
+ # \
267
+ # E(5) --- F(6)
268
+ write_packs A B C &&
269
+ git checkout -b side &&
270
+ write_packs D &&
271
+ git checkout -b other B &&
272
+ write_packs E F &&
273
+ git checkout main &&
274
+ write_packs G &&
275
+
276
+ # Compact layers 2-4, leaving us with:
277
+ #
278
+ # [A, [B, C, D], E, F, G]
279
+ git multi-pack-index compact --incremental --bitmap \
280
+ "$(nth_line 2 "$midx_chain")" \
281
+ "$(nth_line 4 "$midx_chain")" &&
282
+
283
+ # Then compact the top two layers, condensing the above
284
+ # such that the new 4th layer contains F and G.
285
+ #
286
+ # [A, [B, C, D], E, [F, G]]
287
+ git multi-pack-index compact --incremental --bitmap \
288
+ "$(nth_line 4 "$midx_chain")" \
289
+ "$(nth_line 5 "$midx_chain")"
290
+ )
291
+'
292
+
293
test_done