midx: load multi-pack indices via their source
To load a multi-pack index the caller is expected to pass both the repository and the object directory where the multi-pack index is located. While this works, this layout has a couple of downsides: - We need to pass in information reduntant with the owning source, namely its object directory and whether the source is local or not. - We don't have access to the source when loading the multi-pack index. If we had that access, we could store a pointer to the owning source in the MIDX and thus deduplicate some information. - Multi-pack indices are inherently specific to the object source and its format. With the goal of pluggable object backends in mind we will eventually want the backends to own the logic of reading and writing multi-pack indices. Making the logic work on top of object sources is a step into that direction. Refactor loading of multi-pack indices accordingly. This surfaces one small problem though: git-multi-pack-index(1) and our MIDX test helper both know to read and write multi-pack-indices located in a different object directory. This issue is addressed by adding the user-provided object directory as an in-memory alternate. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Patrick Steinhardt committed
Aug 11, 2025 at 15:46 UTC
017db7bb14246dea55b678fc20e34ce91c28968a
5 files changed
+62
-52
builtin/multi-pack-index.c
+16
-2
@@ -64,12 +64,20 @@ static int parse_object_dir(const struct option *opt, const char *arg,
64
char **value = opt->value;
65
free(*value);
66
if (unset)
67
- *value = xstrdup(repo_get_object_directory(the_repository));
67
+ *value = xstrdup(the_repository->objects->sources->path);
68
else
69
*value = real_pathdup(arg, 1);
70
return 0;
71
}
72
73
+static struct odb_source *handle_object_dir_option(struct repository *repo)
74
+{
75
+ struct odb_source *source = odb_find_source(repo->objects, opts.object_dir);
76
+ if (!source)
77
+ source = odb_add_to_alternates_memory(repo->objects, opts.object_dir);
78
+ return source;
79
+}
80
+
81
static struct option common_opts[] = {
82
OPT_CALLBACK(0, "object-dir", &opts.object_dir,
83
N_("directory"),
@@ -157,6 +165,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
165
if (argc)
166
usage_with_options(builtin_multi_pack_index_write_usage,
167
options);
168
+ handle_object_dir_option(repo);
169
170
FREE_AND_NULL(options);
171
@@ -193,6 +202,8 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv,
202
N_("force progress reporting"), MIDX_PROGRESS),
203
OPT_END(),
204
};
205
+ struct odb_source *source;
206
+
207
options = add_common_options(builtin_multi_pack_index_verify_options);
208
209
trace2_cmd_mode(argv[0]);
@@ -205,10 +216,11 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv,
216
if (argc)
217
usage_with_options(builtin_multi_pack_index_verify_usage,
218
options);
219
+ source = handle_object_dir_option(the_repository);
220
221
FREE_AND_NULL(options);
222
211
- return verify_midx_file(the_repository, opts.object_dir, opts.flags);
223
+ return verify_midx_file(source, opts.flags);
224
}
225
226
static int cmd_multi_pack_index_expire(int argc, const char **argv,
@@ -233,6 +245,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv,
245
if (argc)
246
usage_with_options(builtin_multi_pack_index_expire_usage,
247
options);
248
+ handle_object_dir_option(the_repository);
249
250
FREE_AND_NULL(options);
251
@@ -265,6 +278,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv,
278
if (argc)
279
usage_with_options(builtin_multi_pack_index_repack_usage,
280
options);
281
+ handle_object_dir_option(the_repository);
282
283
FREE_AND_NULL(options);
284
midx.c
+25
-32
@@ -95,11 +95,10 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,
95
return 0;
96
}
97
98
-static struct multi_pack_index *load_multi_pack_index_one(struct repository *r,
99
- const char *object_dir,
100
- const char *midx_name,
101
- int local)
98
+static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,
99
+ const char *midx_name)
100
{
101
+ struct repository *r = source->odb->repo;
102
struct multi_pack_index *m = NULL;
103
int fd;
104
struct stat st;
@@ -129,10 +128,10 @@ static struct multi_pack_index *load_multi_pack_index_one(struct repository *r,
128
midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
129
close(fd);
130
132
- FLEX_ALLOC_STR(m, object_dir, object_dir);
131
+ FLEX_ALLOC_STR(m, object_dir, source->path);
132
m->data = midx_map;
133
m->data_len = midx_size;
135
- m->local = local;
134
+ m->local = source->local;
135
m->repo = r;
136
137
m->signature = get_be32(m->data);
@@ -297,19 +296,18 @@ static int add_midx_to_chain(struct multi_pack_index *midx,
296
return 1;
297
}
298
300
-static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r,
301
- const char *object_dir,
302
- int local,
299
+static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source,
300
int fd, struct stat *st,
301
int *incomplete_chain)
302
{
303
+ const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo;
304
struct multi_pack_index *midx_chain = NULL;
305
struct strbuf buf = STRBUF_INIT;
306
int valid = 1;
307
uint32_t i, count;
308
FILE *fp = xfdopen(fd, "r");
309
312
- count = st->st_size / (r->hash_algo->hexsz + 1);
310
+ count = st->st_size / (hash_algo->hexsz + 1);
311
312
for (i = 0; i < count; i++) {
313
struct multi_pack_index *m;
@@ -318,7 +316,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r,
316
if (strbuf_getline_lf(&buf, fp) == EOF)
317
break;
318
321
- if (get_oid_hex_algop(buf.buf, &layer, r->hash_algo)) {
319
+ if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) {
320
warning(_("invalid multi-pack-index chain: line '%s' "
321
"not a hash"),
322
buf.buf);
@@ -329,9 +327,9 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r,
327
valid = 0;
328
329
strbuf_reset(&buf);
332
- get_split_midx_filename_ext(r->hash_algo, &buf, object_dir,
330
+ get_split_midx_filename_ext(hash_algo, &buf, source->path,
331
layer.hash, MIDX_EXT_MIDX);
334
- m = load_multi_pack_index_one(r, object_dir, buf.buf, local);
332
+ m = load_multi_pack_index_one(source, buf.buf);
333
334
if (m) {
335
if (add_midx_to_chain(m, midx_chain)) {
@@ -354,40 +352,35 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r,
352
return midx_chain;
353
}
354
357
-static struct multi_pack_index *load_multi_pack_index_chain(struct repository *r,
358
- const char *object_dir,
359
- int local)
355
+static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source)
356
{
357
struct strbuf chain_file = STRBUF_INIT;
358
struct stat st;
359
int fd;
360
struct multi_pack_index *m = NULL;
361
366
- get_midx_chain_filename(&chain_file, object_dir);
367
- if (open_multi_pack_index_chain(r->hash_algo, chain_file.buf, &fd, &st)) {
362
+ get_midx_chain_filename(&chain_file, source->path);
363
+ if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) {
364
int incomplete;
365
/* ownership of fd is taken over by load function */
370
- m = load_midx_chain_fd_st(r, object_dir, local, fd, &st,
371
- &incomplete);
366
+ m = load_midx_chain_fd_st(source, fd, &st, &incomplete);
367
}
368
369
strbuf_release(&chain_file);
370
return m;
371
}
372
378
-struct multi_pack_index *load_multi_pack_index(struct repository *r,
379
- const char *object_dir,
380
- int local)
373
+struct multi_pack_index *load_multi_pack_index(struct odb_source *source)
374
{
375
struct strbuf midx_name = STRBUF_INIT;
376
struct multi_pack_index *m;
377
385
- get_midx_filename(r->hash_algo, &midx_name, object_dir);
378
+ get_midx_filename(source->odb->repo->hash_algo, &midx_name,
379
+ source->path);
380
387
- m = load_multi_pack_index_one(r, object_dir,
388
- midx_name.buf, local);
381
+ m = load_multi_pack_index_one(source, midx_name.buf);
382
if (!m)
390
- m = load_multi_pack_index_chain(r, object_dir, local);
383
+ m = load_multi_pack_index_chain(source);
384
385
strbuf_release(&midx_name);
386
@@ -734,8 +727,7 @@ int prepare_multi_pack_index_one(struct odb_source *source)
727
if (source->midx)
728
return 1;
729
737
- source->midx = load_multi_pack_index(r, source->path,
738
- source->local);
730
+ source->midx = load_multi_pack_index(source);
731
732
return !!source->midx;
733
}
@@ -880,12 +872,13 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b)
872
display_progress(progress, _n); \
873
} while (0)
874
883
-int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags)
875
+int verify_midx_file(struct odb_source *source, unsigned flags)
876
{
877
+ struct repository *r = source->odb->repo;
878
struct pair_pos_vs_id *pairs = NULL;
879
uint32_t i;
880
struct progress *progress = NULL;
888
- struct multi_pack_index *m = load_multi_pack_index(r, object_dir, 1);
881
+ struct multi_pack_index *m = load_multi_pack_index(source);
882
struct multi_pack_index *curr;
883
verify_midx_error = 0;
884
@@ -894,7 +887,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
887
struct stat sb;
888
struct strbuf filename = STRBUF_INIT;
889
897
- get_midx_filename(r->hash_algo, &filename, object_dir);
890
+ get_midx_filename(r->hash_algo, &filename, source->path);
891
892
if (!stat(filename.buf, &sb)) {
893
error(_("multi-pack-index file exists, but failed to parse"));
midx.h
+2
-4
@@ -100,9 +100,7 @@ void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo,
100
struct strbuf *buf, const char *object_dir,
101
const unsigned char *hash, const char *ext);
102
103
-struct multi_pack_index *load_multi_pack_index(struct repository *r,
104
- const char *object_dir,
105
- int local);
103
+struct multi_pack_index *load_multi_pack_index(struct odb_source *source);
104
int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id);
105
struct packed_git *nth_midxed_pack(struct multi_pack_index *m,
106
uint32_t pack_int_id);
@@ -136,7 +134,7 @@ int write_midx_file_only(struct repository *r, const char *object_dir,
134
const char *preferred_pack_name,
135
const char *refs_snapshot, unsigned flags);
136
void clear_midx_file(struct repository *r);
139
-int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags);
137
+int verify_midx_file(struct odb_source *source, unsigned flags);
138
int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags);
139
int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags);
140
t/helper/test-read-midx.c
+15
-10
@@ -11,14 +11,24 @@
11
#include "gettext.h"
12
#include "pack-revindex.h"
13
14
+static struct multi_pack_index *setup_midx(const char *object_dir)
15
+{
16
+ struct odb_source *source;
17
+ setup_git_directory();
18
+ source = odb_find_source(the_repository->objects, object_dir);
19
+ if (!source)
20
+ source = odb_add_to_alternates_memory(the_repository->objects,
21
+ object_dir);
22
+ return load_multi_pack_index(source);
23
+}
24
+
25
static int read_midx_file(const char *object_dir, const char *checksum,
26
int show_objects)
27
{
28
uint32_t i;
29
struct multi_pack_index *m;
30
20
- setup_git_directory();
21
- m = load_multi_pack_index(the_repository, object_dir, 1);
31
+ m = setup_midx(object_dir);
32
33
if (!m)
34
return 1;
@@ -81,8 +91,7 @@ static int read_midx_checksum(const char *object_dir)
91
{
92
struct multi_pack_index *m;
93
84
- setup_git_directory();
85
- m = load_multi_pack_index(the_repository, object_dir, 1);
94
+ m = setup_midx(object_dir);
95
if (!m)
96
return 1;
97
printf("%s\n", hash_to_hex(get_midx_checksum(m)));
@@ -96,9 +105,7 @@ static int read_midx_preferred_pack(const char *object_dir)
105
struct multi_pack_index *midx = NULL;
106
uint32_t preferred_pack;
107
99
- setup_git_directory();
100
-
101
- midx = load_multi_pack_index(the_repository, object_dir, 1);
108
+ midx = setup_midx(object_dir);
109
if (!midx)
110
return 1;
111
@@ -119,9 +126,7 @@ static int read_midx_bitmapped_packs(const char *object_dir)
126
struct bitmapped_pack pack;
127
uint32_t i;
128
122
- setup_git_directory();
123
-
124
- midx = load_multi_pack_index(the_repository, object_dir, 1);
129
+ midx = setup_midx(object_dir);
130
if (!midx)
131
return 1;
132
t/t5319-multi-pack-index.sh
+4
-4
@@ -28,11 +28,11 @@ midx_read_expect () {
28
EOF
29
if test $NUM_PACKS -ge 1
30
then
31
- ls $OBJECT_DIR/pack/ | grep idx | sort
31
+ ls "$OBJECT_DIR"/pack/ | grep idx | sort
32
fi &&
33
printf "object-dir: $OBJECT_DIR\n"
34
} >expect &&
35
- test-tool read-midx $OBJECT_DIR >actual &&
35
+ test-tool read-midx "$OBJECT_DIR" >actual &&
36
test_cmp expect actual
37
}
38
@@ -305,7 +305,7 @@ test_expect_success 'midx picks objects from preferred pack' '
305
306
ofs=$(git show-index <objects/pack/test-BC-$bc.idx | grep $b |
307
cut -d" " -f1) &&
308
- printf "%s %s\tobjects/pack/test-BC-%s.pack\n" \
308
+ printf "%s %s\t./objects/pack/test-BC-%s.pack\n" \
309
"$b" "$ofs" "$bc" >expect &&
310
grep ^$b out >actual &&
311
@@ -639,7 +639,7 @@ test_expect_success 'force some 64-bit offsets with pack-objects' '
639
( cd ../objects64 && pwd ) >.git/objects/info/alternates &&
640
midx64=$(git multi-pack-index --object-dir=../objects64 write)
641
) &&
642
- midx_read_expect 1 63 5 objects64 " large-offsets"
642
+ midx_read_expect 1 63 5 "$(pwd)/objects64" " large-offsets"
643
'
644
645
test_expect_success 'verify multi-pack-index with 64-bit offsets' '