midx: introduce `--no-write-chain-file` for incremental MIDX writes
When writing an incremental MIDX layer, the MIDX machinery writes the new layer into the multi-pack-index.d directory and then updates the multi-pack-index-chain file to include the freshly written layer. Future callers however may not wish to immediately update the MIDX chain itself, preferring instead to write out new layer(s) themselves before atomically updating the chain. Concretely, the new incremental MIDX-based repacking strategy will want to do exactly this (that is, assemble the new MIDX chain itself before writing a new chain file and atomically linking it into place). Introduce a `--no-write-chain-file` flag that: * writes the new MIDX layer into the multi-pack-index.d directory * prints its checksum * does not update the multi-pack-index-chain file. The MIDX chain file (and thus, the lock protecting it) remain untouched, allowing callers to assemble the chain themselves. This flag requires `--incremental`, since the notion of a separate layer only makes sense for incremental MIDXs. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 19, 2026 at 11:57 UTC
8d342ed4b5dfbaa16f8bbc4537907d1e1224358e
6 files changed
+123
-18
Documentation/git-multi-pack-index.adoc
+15
-2
@@ -11,9 +11,9 @@ SYNOPSIS
11
[verse]
12
'git multi-pack-index' [<options>] write [--preferred-pack=<pack>]
13
[--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]
14
- [--refs-snapshot=<path>]
14
+ [--refs-snapshot=<path>] [--[no-]write-chain-file]
15
'git multi-pack-index' [<options>] compact [--[no-]incremental]
16
- [--[no-]bitmap] <from> <to>
16
+ [--[no-]bitmap] [--[no-]write-chain-file] <from> <to>
17
'git multi-pack-index' [<options>] verify
18
'git multi-pack-index' [<options>] expire
19
'git multi-pack-index' [<options>] repack [--batch-size=<size>]
@@ -83,6 +83,13 @@ marker).
83
and packs not present in an existing MIDX layer.
84
Migrates non-incremental MIDXs to incremental ones when
85
necessary.
86
+
87
+ --[no-]write-chain-file::
88
+ When used with `--incremental`, write a new MIDX layer
89
+ but do not update the multi-pack-index-chain file.
90
+ The checksum of the new layer is printed to standard
91
+ output, allowing the caller to assemble and write the
92
+ chain itself. Requires `--incremental`.
93
--
94
95
compact::
@@ -97,6 +104,12 @@ compact::
104
105
--[no-]bitmap::
106
Control whether or not a multi-pack bitmap is written.
107
+
108
+ --[no-]write-chain-file::
109
+ When used with `--incremental`, write a new compacted
110
+ MIDX layer but do not update the multi-pack-index-chain
111
+ file. The checksum of the new layer is printed to
112
+ standard output. Requires `--incremental`.
113
--
114
+
115
Note that the compact command requires writing a version-2 midx that
builtin/multi-pack-index.c
+26
-2
@@ -16,11 +16,11 @@
16
#define BUILTIN_MIDX_WRITE_USAGE \
17
N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]\n" \
18
" [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]\n" \
19
- " [--refs-snapshot=<path>]")
19
+ " [--refs-snapshot=<path>] [--[no-]write-chain-file]")
20
21
#define BUILTIN_MIDX_COMPACT_USAGE \
22
N_("git multi-pack-index [<options>] compact [--[no-]incremental]\n" \
23
- " [--[no-]bitmap] <from> <to>")
23
+ " [--[no-]bitmap] [--[no-]write-chain-file] <from> <to>")
24
25
#define BUILTIN_MIDX_VERIFY_USAGE \
26
N_("git multi-pack-index [<options>] verify")
@@ -153,6 +153,9 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
153
MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
154
OPT_BIT(0, "incremental", &opts.flags,
155
N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
156
+ OPT_NEGBIT(0, "write-chain-file", &opts.flags,
157
+ N_("write the multi-pack-index chain file"),
158
+ MIDX_WRITE_NO_CHAIN),
159
OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,
160
N_("write multi-pack index containing only given indexes")),
161
OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot,
@@ -178,6 +181,15 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
181
if (argc)
182
usage_with_options(builtin_multi_pack_index_write_usage,
183
options);
184
+
185
+ if (opts.flags & MIDX_WRITE_NO_CHAIN &&
186
+ !(opts.flags & MIDX_WRITE_INCREMENTAL)) {
187
+ error(_("cannot use %s without %s"),
188
+ "--no-write-chain-file", "--incremental");
189
+ usage_with_options(builtin_multi_pack_index_write_usage,
190
+ options);
191
+ }
192
+
193
source = handle_object_dir_option(repo);
194
195
FREE_AND_NULL(options);
@@ -221,6 +233,9 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
233
MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
234
OPT_BIT(0, "incremental", &opts.flags,
235
N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
236
+ OPT_NEGBIT(0, "write-chain-file", &opts.flags,
237
+ N_("write the multi-pack-index chain file"),
238
+ MIDX_WRITE_NO_CHAIN),
239
OPT_END(),
240
};
241
@@ -239,6 +254,15 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
254
if (argc != 2)
255
usage_with_options(builtin_multi_pack_index_compact_usage,
256
options);
257
+
258
+ if (opts.flags & MIDX_WRITE_NO_CHAIN &&
259
+ !(opts.flags & MIDX_WRITE_INCREMENTAL)) {
260
+ error(_("cannot use %s without %s"),
261
+ "--no-write-chain-file", "--incremental");
262
+ usage_with_options(builtin_multi_pack_index_compact_usage,
263
+ options);
264
+ }
265
+
266
source = handle_object_dir_option(the_repository);
267
268
FREE_AND_NULL(options);
midx-write.c
+28
-14
@@ -1257,7 +1257,7 @@ static int write_midx_internal(struct write_midx_opts *opts)
1257
unsigned char midx_hash[GIT_MAX_RAWSZ];
1258
uint32_t start_pack;
1259
struct hashfile *f = NULL;
1260
- struct lock_file lk;
1260
+ struct lock_file lk = LOCK_INIT;
1261
struct tempfile *incr;
1262
struct write_midx_context ctx = {
1263
.preferred_pack_idx = NO_PREFERRED_PACK,
@@ -1601,11 +1601,14 @@ static int write_midx_internal(struct write_midx_opts *opts)
1601
}
1602
1603
if (ctx.incremental) {
1604
- struct strbuf lock_name = STRBUF_INIT;
1604
+ if (!(opts->flags & MIDX_WRITE_NO_CHAIN)) {
1605
+ struct strbuf lock_name = STRBUF_INIT;
1606
1606
- get_midx_chain_filename(opts->source, &lock_name);
1607
- hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR);
1608
- strbuf_release(&lock_name);
1607
+ get_midx_chain_filename(opts->source, &lock_name);
1608
+ hold_lock_file_for_update(&lk, lock_name.buf,
1609
+ LOCK_DIE_ON_ERROR);
1610
+ strbuf_release(&lock_name);
1611
+ }
1612
1613
incr = mks_tempfile_m(midx_name.buf, 0444);
1614
if (!incr) {
@@ -1707,16 +1710,23 @@ static int write_midx_internal(struct write_midx_opts *opts)
1710
if (ctx.num_multi_pack_indexes_before == UINT32_MAX)
1711
die(_("too many multi-pack-indexes"));
1712
1713
+ if (!is_lock_file_locked(&lk))
1714
+ printf("%s\n", hash_to_hex_algop(midx_hash, r->hash_algo));
1715
+ else if (opts->flags & MIDX_WRITE_NO_CHAIN)
1716
+ BUG("lockfile held with MIDX_WRITE_NO_CHAIN set?");
1717
+
1718
if (ctx.incremental) {
1711
- FILE *chainf = fdopen_lock_file(&lk, "w");
1719
struct strbuf final_midx_name = STRBUF_INIT;
1720
struct multi_pack_index *m = ctx.base_midx;
1721
struct multi_pack_index **layers = NULL;
1722
size_t layers_nr = 0, layers_alloc = 0;
1723
1717
- if (!chainf) {
1718
- error_errno(_("unable to open multi-pack-index chain file"));
1719
- goto cleanup;
1724
+ if (is_lock_file_locked(&lk)){
1725
+ FILE *chainf = fdopen_lock_file(&lk, "w");
1726
+ if (!chainf) {
1727
+ error_errno(_("unable to open multi-pack-index chain file"));
1728
+ goto cleanup;
1729
+ }
1730
}
1731
1732
if (link_midx_to_chain(ctx.base_midx) < 0)
@@ -1773,8 +1783,10 @@ static int write_midx_internal(struct write_midx_opts *opts)
1783
1784
free(layers);
1785
1776
- for (size_t i = 0; i < keep_hashes.nr; i++)
1777
- fprintf(get_lock_file_fp(&lk), "%s\n", keep_hashes.v[i]);
1786
+ if (is_lock_file_locked(&lk))
1787
+ for (size_t i = 0; i < keep_hashes.nr; i++)
1788
+ fprintf(get_lock_file_fp(&lk), "%s\n",
1789
+ keep_hashes.v[i]);
1790
} else {
1791
strvec_push(&keep_hashes,
1792
hash_to_hex_algop(midx_hash, r->hash_algo));
@@ -1783,10 +1795,12 @@ static int write_midx_internal(struct write_midx_opts *opts)
1795
if (ctx.m || ctx.base_midx)
1796
odb_close(ctx.repo->objects);
1797
1786
- if (commit_lock_file(&lk) < 0)
1787
- die_errno(_("could not write multi-pack-index"));
1798
+ if (is_lock_file_locked(&lk)) {
1799
+ if (commit_lock_file(&lk) < 0)
1800
+ die_errno(_("could not write multi-pack-index"));
1801
1789
- clear_midx_files(opts->source, &keep_hashes, ctx.incremental);
1802
+ clear_midx_files(opts->source, &keep_hashes, ctx.incremental);
1803
+ }
1804
result = 0;
1805
1806
cleanup:
midx.h
+1
@@ -83,6 +83,7 @@ struct multi_pack_index {
83
#define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
84
#define MIDX_WRITE_INCREMENTAL (1 << 5)
85
#define MIDX_WRITE_COMPACT (1 << 6)
86
+#define MIDX_WRITE_NO_CHAIN (1 << 7)
87
88
#define MIDX_EXT_REV "rev"
89
#define MIDX_EXT_BITMAP "bitmap"
t/t5334-incremental-multi-pack-index.sh
+17
@@ -96,6 +96,23 @@ test_expect_success 'show object from second pack' '
96
git cat-file -p 2.2
97
'
98
99
+test_expect_success 'write MIDX layer with --no-write-chain-file' '
100
+ test_commit no-write-chain-file &&
101
+ git repack -d &&
102
+
103
+ cp "$midx_chain" "$midx_chain.bak" &&
104
+ layer="$(git multi-pack-index write --bitmap --incremental \
105
+ --no-write-chain-file)" &&
106
+
107
+ test_cmp "$midx_chain.bak" "$midx_chain" &&
108
+ test_path_is_file "$midxdir/multi-pack-index-$layer.midx"
109
+'
110
+
111
+test_expect_success 'write non-incremental MIDX layer with --no-write-chain-file' '
112
+ test_must_fail git multi-pack-index write --bitmap --no-write-chain-file 2>err &&
113
+ test_grep "cannot use --no-write-chain-file without --incremental" err
114
+'
115
+
116
for reuse in false single multi
117
do
118
test_expect_success "full clone (pack.allowPackReuse=$reuse)" '
t/t5335-compact-multi-pack-index.sh
+36
@@ -290,4 +290,40 @@ test_expect_success 'MIDX compaction with bitmaps (non-trivial)' '
290
)
291
'
292
293
+test_expect_success 'MIDX compaction with --no-write-chain-file' '
294
+ git init midx-compact-with--no-write-chain-file &&
295
+ (
296
+ cd midx-compact-with--no-write-chain-file &&
297
+
298
+ git config maintenance.auto false &&
299
+
300
+ write_packs A B C D &&
301
+
302
+ test_line_count = 4 $midx_chain &&
303
+ cp "$midx_chain" "$midx_chain".bak &&
304
+
305
+ layer="$(git multi-pack-index compact --incremental \
306
+ --no-write-chain-file \
307
+ "$(nth_line 2 "$midx_chain")" \
308
+ "$(nth_line 3 "$midx_chain")")" &&
309
+
310
+ test_cmp "$midx_chain.bak" "$midx_chain" &&
311
+
312
+ # After writing the new layer, insert it into the chain
313
+ # manually. This is done in order to make $layer visible
314
+ # to the read-midx test helper below, and matches what
315
+ # the MIDX command would do without --no-write-chain-file.
316
+ {
317
+ nth_line 1 "$midx_chain.bak" &&
318
+ echo $layer &&
319
+ nth_line 4 "$midx_chain.bak"
320
+ } >$midx_chain &&
321
+
322
+ test-tool read-midx $objdir $layer >midx.data &&
323
+ grep "^pack-B-.*\.idx" midx.data &&
324
+ grep "^pack-C-.*\.idx" midx.data
325
+
326
+ )
327
+'
328
+
329
test_done