midx: support custom `--base` for incremental MIDX writes
Both `compact` and `write --incremental` fix the base of the resulting MIDX layer: `compact` always places the compacted result on top of "from's" immediate parent in the chain, and `write --incremental` always appends a new layer to the existing tip. In both cases the base is not configurable. Future callers need additional flexibility. For instance, the incremental MIDX-based repacking code may wish to write a layer based on some intermediate ancestor rather than the current tip, or produce a root layer when replacing the bottommost entries in the chain. Introduce a new `--base` option for both subcommands to specify the checksum of the MIDX layer to use as the base. The given checksum must refer to a valid layer in the MIDX chain that is an ancestor of the topmost layer being written or compacted. The special value "none" is accepted to produce a root layer with no parent. This will be needed when the incremental repacking machinery determines that the bottommost layers of the chain should be replaced. If no `--base` is given, behavior is unchanged: `compact` uses "from's" immediate parent in the chain, and `write` appends to the existing tip. For the `write` subcommand, `--base` requires `--no-write-chain-file`. A plain `write --incremental` appends a new layer to the live chain tip with no mechanism to atomically replace it; overriding the base would produce a layer that does not extend the tip, breaking chain invariants. With `--no-write-chain-file` the chain is left unmodified and the caller is responsible for assembling a valid chain. For `compact`, no such restriction applies. The compaction operation atomically replaces the compacted range in the chain file, so writing the result on top of any valid ancestor preserves chain invariants. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 19, 2026 at 11:57 UTC
0cd2255e64b4775520a6acbbb1868437fc26662d
6 files changed
+178
-9
Documentation/git-multi-pack-index.adoc
+16
-1
@@ -12,8 +12,10 @@ SYNOPSIS
12
'git multi-pack-index' [<options>] write [--preferred-pack=<pack>]
13
[--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]
14
[--refs-snapshot=<path>] [--[no-]write-chain-file]
15
+ [--base=<checksum>]
16
'git multi-pack-index' [<options>] compact [--[no-]incremental]
16
- [--[no-]bitmap] [--[no-]write-chain-file] <from> <to>
17
+ [--[no-]bitmap] [--base=<checksum>] [--[no-]write-chain-file]
18
+ <from> <to>
19
'git multi-pack-index' [<options>] verify
20
'git multi-pack-index' [<options>] expire
21
'git multi-pack-index' [<options>] repack [--batch-size=<size>]
@@ -90,6 +92,13 @@ marker).
92
The checksum of the new layer is printed to standard
93
output, allowing the caller to assemble and write the
94
chain itself. Requires `--incremental`.
95
+
96
+ --base=<checksum>::
97
+ Specify the checksum of an existing MIDX layer to use
98
+ as the base when writing a new incremental layer.
99
+ The special value `none` indicates that the new layer
100
+ should have no base (i.e., it becomes a root layer).
101
+ Requires `--no-write-chain-file`.
102
--
103
104
compact::
@@ -110,6 +119,12 @@ compact::
119
MIDX layer but do not update the multi-pack-index-chain
120
file. The checksum of the new layer is printed to
121
standard output. Requires `--incremental`.
122
+
123
+ --base=<checksum>::
124
+ Specify the checksum of an existing MIDX layer to use
125
+ as the base for the compacted result, instead of using
126
+ the immediate parent of `<from>`. The special value
127
+ `none` indicates that the result should have no base.
128
--
129
+
130
Note that the compact command requires writing a version-2 midx that
builtin/multi-pack-index.c
+20
-4
@@ -16,11 +16,13 @@
16
#define BUILTIN_MIDX_WRITE_USAGE \
17
N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]\n" \
18
" [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]\n" \
19
- " [--refs-snapshot=<path>] [--[no-]write-chain-file]")
19
+ " [--refs-snapshot=<path>] [--[no-]write-chain-file]\n" \
20
+ " [--base=<checksum>]")
21
22
#define BUILTIN_MIDX_COMPACT_USAGE \
23
N_("git multi-pack-index [<options>] compact [--[no-]incremental]\n" \
23
- " [--[no-]bitmap] [--[no-]write-chain-file] <from> <to>")
24
+ " [--[no-]bitmap] [--base=<checksum>] [--[no-]write-chain-file]\n" \
25
+ " <from> <to>")
26
27
#define BUILTIN_MIDX_VERIFY_USAGE \
28
N_("git multi-pack-index [<options>] verify")
@@ -63,6 +65,7 @@ static char const * const builtin_multi_pack_index_usage[] = {
65
static struct opts_multi_pack_index {
66
char *object_dir;
67
const char *preferred_pack;
68
+ const char *incremental_base;
69
char *refs_snapshot;
70
unsigned long batch_size;
71
unsigned flags;
@@ -151,6 +154,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
154
N_("pack for reuse when computing a multi-pack bitmap")),
155
OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
156
MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
157
+ OPT_STRING(0, "base", &opts.incremental_base, N_("checksum"),
158
+ N_("base MIDX for incremental writes")),
159
OPT_BIT(0, "incremental", &opts.flags,
160
N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
161
OPT_NEGBIT(0, "write-chain-file", &opts.flags,
@@ -190,6 +195,13 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
195
options);
196
}
197
198
+ if (opts.incremental_base &&
199
+ !(opts.flags & MIDX_WRITE_NO_CHAIN)) {
200
+ error(_("cannot use --base without --no-write-chain-file"));
201
+ usage_with_options(builtin_multi_pack_index_write_usage,
202
+ options);
203
+ }
204
+
205
source = handle_object_dir_option(repo);
206
207
FREE_AND_NULL(options);
@@ -201,7 +213,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
213
214
ret = write_midx_file_only(source, &packs,
215
opts.preferred_pack,
204
- opts.refs_snapshot, opts.flags);
216
+ opts.refs_snapshot,
217
+ opts.incremental_base, opts.flags);
218
219
string_list_clear(&packs, 0);
220
free(opts.refs_snapshot);
@@ -229,6 +242,8 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
242
243
struct option *options;
244
static struct option builtin_multi_pack_index_compact_options[] = {
245
+ OPT_STRING(0, "base", &opts.incremental_base, N_("checksum"),
246
+ N_("base MIDX for incremental writes")),
247
OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
248
MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
249
OPT_BIT(0, "incremental", &opts.flags,
@@ -290,7 +305,8 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
305
die(_("MIDX %s must be an ancestor of %s"), argv[0], argv[1]);
306
}
307
293
- ret = write_midx_file_compact(source, from_midx, to_midx, opts.flags);
308
+ ret = write_midx_file_compact(source, from_midx, to_midx,
309
+ opts.incremental_base, opts.flags);
310
311
return ret;
312
}
midx-write.c
+31
-3
@@ -1247,6 +1247,7 @@ struct write_midx_opts {
1247
1248
const char *preferred_pack_name;
1249
const char *refs_snapshot;
1250
+ const char *incremental_base;
1251
unsigned flags;
1252
};
1253
@@ -1330,11 +1331,32 @@ static int write_midx_internal(struct write_midx_opts *opts)
1331
1332
/*
1333
* If compacting MIDX layer(s) in the range [from, to], then the
1333
- * compacted MIDX will share the same base MIDX as 'from'.
1334
+ * compacted MIDX will share the same base MIDX as 'from',
1335
+ * unless a custom --base is specified (see below).
1336
*/
1337
if (ctx.compact)
1338
ctx.base_midx = ctx.compact_from->base_midx;
1339
1340
+ if (opts->incremental_base) {
1341
+ if (!strcmp(opts->incremental_base, "none")) {
1342
+ ctx.base_midx = NULL;
1343
+ } else {
1344
+ while (ctx.base_midx) {
1345
+ const char *cmp = midx_get_checksum_hex(ctx.base_midx);
1346
+ if (!strcmp(opts->incremental_base, cmp))
1347
+ break;
1348
+
1349
+ ctx.base_midx = ctx.base_midx->base_midx;
1350
+ }
1351
+
1352
+ if (!ctx.base_midx) {
1353
+ error(_("could not find base MIDX '%s'"),
1354
+ opts->incremental_base);
1355
+ goto cleanup;
1356
+ }
1357
+ }
1358
+ }
1359
+
1360
ctx.nr = 0;
1361
ctx.alloc = ctx.m ? ctx.m->num_packs + ctx.m->num_packs_in_base : 16;
1362
ctx.info = NULL;
@@ -1827,7 +1849,8 @@ cleanup:
1849
1850
int write_midx_file(struct odb_source *source,
1851
const char *preferred_pack_name,
1830
- const char *refs_snapshot, unsigned flags)
1852
+ const char *refs_snapshot,
1853
+ unsigned flags)
1854
{
1855
struct write_midx_opts opts = {
1856
.source = source,
@@ -1842,13 +1865,16 @@ int write_midx_file(struct odb_source *source,
1865
int write_midx_file_only(struct odb_source *source,
1866
struct string_list *packs_to_include,
1867
const char *preferred_pack_name,
1845
- const char *refs_snapshot, unsigned flags)
1868
+ const char *refs_snapshot,
1869
+ const char *incremental_base,
1870
+ unsigned flags)
1871
{
1872
struct write_midx_opts opts = {
1873
.source = source,
1874
.packs_to_include = packs_to_include,
1875
.preferred_pack_name = preferred_pack_name,
1876
.refs_snapshot = refs_snapshot,
1877
+ .incremental_base = incremental_base,
1878
.flags = flags,
1879
};
1880
@@ -1858,12 +1884,14 @@ int write_midx_file_only(struct odb_source *source,
1884
int write_midx_file_compact(struct odb_source *source,
1885
struct multi_pack_index *from,
1886
struct multi_pack_index *to,
1887
+ const char *incremental_base,
1888
unsigned flags)
1889
{
1890
struct write_midx_opts opts = {
1891
.source = source,
1892
.compact_from = from,
1893
.compact_to = to,
1894
+ .incremental_base = incremental_base,
1895
.flags = flags | MIDX_WRITE_COMPACT,
1896
};
1897
midx.h
+4
-1
@@ -132,10 +132,13 @@ int write_midx_file(struct odb_source *source,
132
int write_midx_file_only(struct odb_source *source,
133
struct string_list *packs_to_include,
134
const char *preferred_pack_name,
135
- const char *refs_snapshot, unsigned flags);
135
+ const char *refs_snapshot,
136
+ const char *incremental_base,
137
+ unsigned flags);
138
int write_midx_file_compact(struct odb_source *source,
139
struct multi_pack_index *from,
140
struct multi_pack_index *to,
141
+ const char *incremental_base,
142
unsigned flags);
143
void clear_midx_file(struct repository *r);
144
int verify_midx_file(struct odb_source *source, unsigned flags);
t/t5334-incremental-multi-pack-index.sh
+30
@@ -113,6 +113,36 @@ test_expect_success 'write non-incremental MIDX layer with --no-write-chain-file
113
test_grep "cannot use --no-write-chain-file without --incremental" err
114
'
115
116
+test_expect_success 'write MIDX layer with --base without --no-write-chain-file' '
117
+ test_must_fail git multi-pack-index write --bitmap --incremental \
118
+ --base=none 2>err &&
119
+ test_grep "cannot use --base without --no-write-chain-file" err
120
+'
121
+
122
+test_expect_success 'write MIDX layer with --base=none and --no-write-chain-file' '
123
+ test_commit base-none &&
124
+ git repack -d &&
125
+
126
+ cp "$midx_chain" "$midx_chain.bak" &&
127
+ layer="$(git multi-pack-index write --bitmap --incremental \
128
+ --no-write-chain-file --base=none)" &&
129
+
130
+ test_cmp "$midx_chain.bak" "$midx_chain" &&
131
+ test_path_is_file "$midxdir/multi-pack-index-$layer.midx"
132
+'
133
+
134
+test_expect_success 'write MIDX layer with --base=<hash> and --no-write-chain-file' '
135
+ test_commit base-hash &&
136
+ git repack -d &&
137
+
138
+ cp "$midx_chain" "$midx_chain.bak" &&
139
+ layer="$(git multi-pack-index write --bitmap --incremental \
140
+ --no-write-chain-file --base="$(nth_line 1 "$midx_chain")")" &&
141
+
142
+ test_cmp "$midx_chain.bak" "$midx_chain" &&
143
+ test_path_is_file "$midxdir/multi-pack-index-$layer.midx"
144
+'
145
+
146
for reuse in false single multi
147
do
148
test_expect_success "full clone (pack.allowPackReuse=$reuse)" '
t/t5335-compact-multi-pack-index.sh
+77
@@ -304,6 +304,7 @@ test_expect_success 'MIDX compaction with --no-write-chain-file' '
304
305
layer="$(git multi-pack-index compact --incremental \
306
--no-write-chain-file \
307
+ --base="$(nth_line 1 "$midx_chain")" \
308
"$(nth_line 2 "$midx_chain")" \
309
"$(nth_line 3 "$midx_chain")")" &&
310
@@ -326,4 +327,80 @@ test_expect_success 'MIDX compaction with --no-write-chain-file' '
327
)
328
'
329
330
+test_expect_success 'MIDX compaction with --base' '
331
+ git init midx-compact-with--base &&
332
+ (
333
+ cd midx-compact-with--base &&
334
+
335
+ git config maintenance.auto false &&
336
+
337
+ write_packs A B C D &&
338
+
339
+ test_line_count = 4 "$midx_chain" &&
340
+
341
+ cp "$midx_chain" "$midx_chain.bak" &&
342
+
343
+ git multi-pack-index compact --incremental \
344
+ --base="$(nth_line 1 "$midx_chain")" \
345
+ "$(nth_line 3 "$midx_chain")" \
346
+ "$(nth_line 4 "$midx_chain")" &&
347
+ test_line_count = 2 $midx_chain &&
348
+
349
+ nth_line 1 "$midx_chain.bak" >expect &&
350
+ nth_line 1 "$midx_chain" >actual &&
351
+
352
+ test_cmp expect actual
353
+ )
354
+'
355
+
356
+test_expect_success 'MIDX compaction with --base=none' '
357
+ git init midx-compact-base-none &&
358
+ (
359
+ cd midx-compact-base-none &&
360
+
361
+ git config maintenance.auto false &&
362
+
363
+ write_packs A B C D &&
364
+
365
+ test_line_count = 4 $midx_chain &&
366
+
367
+ cp "$midx_chain" "$midx_chain".bak &&
368
+
369
+ # Compact the two bottommost layers (A and B) into a new
370
+ # root layer with no parent.
371
+ git multi-pack-index compact --incremental \
372
+ --base=none \
373
+ "$(nth_line 1 "$midx_chain")" \
374
+ "$(nth_line 2 "$midx_chain")" &&
375
+
376
+ test_line_count = 3 $midx_chain &&
377
+
378
+ # The upper layers (C and D) should be preserved
379
+ # unchanged.
380
+ nth_line 3 "$midx_chain.bak" >expect &&
381
+ nth_line 4 "$midx_chain.bak" >>expect &&
382
+ nth_line 2 "$midx_chain" >actual &&
383
+ nth_line 3 "$midx_chain" >>actual &&
384
+
385
+ test_cmp expect actual
386
+ )
387
+'
388
+
389
+test_expect_success 'MIDX compaction with bogus --base checksum' '
390
+ git init midx-compact-bogus-base &&
391
+ (
392
+ cd midx-compact-bogus-base &&
393
+
394
+ git config maintenance.auto false &&
395
+
396
+ write_packs A B C &&
397
+
398
+ test_must_fail git multi-pack-index compact --incremental \
399
+ --base=deadbeef \
400
+ "$(nth_line 2 "$midx_chain")" \
401
+ "$(nth_line 3 "$midx_chain")" 2>err &&
402
+ test_grep "could not find base MIDX" err
403
+ )
404
+'
405
+
406
test_done