pack-revindex: prepare for incremental MIDX bitmaps
Prepare the reverse index machinery to handle object lookups in an incremental MIDX bitmap. These changes are broken out across a few functions: - load_midx_revindex() learns to use the appropriate MIDX filename depending on whether the given 'struct multi_pack_index *' is incremental or not. - pack_pos_to_midx() and midx_to_pack_pos() now both take in a global object position in the MIDX pseudo-pack order, and find the earliest containing MIDX (similar to midx.c::midx_for_object(). - midx_pack_order_cmp() adjusts its call to pack_pos_to_midx() by the number of objects in the base (since 'vb - midx->revindx_data' is relative to the containing MIDX, and pack_pos_to_midx() expects a global position). Likewise, this function adjusts its output by adding m->num_objects_in_base to return a global position out through the `*pos` pointer. Together, these changes are sufficient to use the multi-pack index's reverse index format for incremental multi-pack reachability bitmaps. Signed-off-by: Taylor Blau <me@ttaylorr.com> Acked-by: Elijah Newren <newren@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Mar 20, 2025 at 13:56 UTC
8331c17b7961736ac69b74d26af579048ad77866
2 files changed
+56
-21
pack-bitmap.c
+31
-12
@@ -170,6 +170,15 @@ static struct ewah_bitmap *read_bitmap_1(struct bitmap_index *index)
170
return read_bitmap(index->map, index->map_size, &index->map_pos);
171
}
172
173
+static uint32_t bitmap_num_objects_total(struct bitmap_index *index)
174
+{
175
+ if (index->midx) {
176
+ struct multi_pack_index *m = index->midx;
177
+ return m->num_objects + m->num_objects_in_base;
178
+ }
179
+ return index->pack->num_objects;
180
+}
181
+
182
static uint32_t bitmap_num_objects(struct bitmap_index *index)
183
{
184
if (index->midx)
@@ -924,7 +933,7 @@ static inline int bitmap_position_extended(struct bitmap_index *bitmap_git,
933
934
if (pos < kh_end(positions)) {
935
int bitmap_pos = kh_value(positions, pos);
927
- return bitmap_pos + bitmap_num_objects(bitmap_git);
936
+ return bitmap_pos + bitmap_num_objects_total(bitmap_git);
937
}
938
939
return -1;
@@ -992,7 +1001,7 @@ static int ext_index_add_object(struct bitmap_index *bitmap_git,
1001
bitmap_pos = kh_value(eindex->positions, hash_pos);
1002
}
1003
995
- return bitmap_pos + bitmap_num_objects(bitmap_git);
1004
+ return bitmap_pos + bitmap_num_objects_total(bitmap_git);
1005
}
1006
1007
struct bitmap_show_data {
@@ -1342,11 +1351,17 @@ struct ewah_bitmap *pseudo_merge_bitmap_for_commit(struct bitmap_index *bitmap_g
1351
if (pos < 0 || pos >= bitmap_num_objects(bitmap_git))
1352
goto done;
1353
1354
+ /*
1355
+ * Use bitmap-relative positions instead of offsetting
1356
+ * by bitmap_git->num_objects_in_base because we use
1357
+ * this to find a match in pseudo_merge_for_parents(),
1358
+ * and pseudo-merge groups cannot span multiple bitmap
1359
+ * layers.
1360
+ */
1361
bitmap_set(parents, pos);
1362
}
1363
1348
- match = pseudo_merge_for_parents(&bitmap_git->pseudo_merges,
1349
- parents);
1364
+ match = pseudo_merge_for_parents(&bitmap_git->pseudo_merges, parents);
1365
1366
done:
1367
bitmap_free(parents);
@@ -1500,7 +1515,9 @@ static void show_extended_objects(struct bitmap_index *bitmap_git,
1515
for (i = 0; i < eindex->count; ++i) {
1516
struct object *obj;
1517
1503
- if (!bitmap_get(objects, st_add(bitmap_num_objects(bitmap_git), i)))
1518
+ if (!bitmap_get(objects,
1519
+ st_add(bitmap_num_objects_total(bitmap_git),
1520
+ i)))
1521
continue;
1522
1523
obj = eindex->objects[i];
@@ -1679,7 +1696,7 @@ static void filter_bitmap_exclude_type(struct bitmap_index *bitmap_git,
1696
* them individually.
1697
*/
1698
for (i = 0; i < eindex->count; i++) {
1682
- size_t pos = st_add(i, bitmap_num_objects(bitmap_git));
1699
+ size_t pos = st_add(i, bitmap_num_objects_total(bitmap_git));
1700
if (eindex->objects[i]->type == type &&
1701
bitmap_get(to_filter, pos) &&
1702
!bitmap_get(tips, pos))
@@ -1705,7 +1722,7 @@ static unsigned long get_size_by_pos(struct bitmap_index *bitmap_git,
1722
1723
oi.sizep = &size;
1724
1708
- if (pos < bitmap_num_objects(bitmap_git)) {
1725
+ if (pos < bitmap_num_objects_total(bitmap_git)) {
1726
struct packed_git *pack;
1727
off_t ofs;
1728
@@ -1728,8 +1745,9 @@ static unsigned long get_size_by_pos(struct bitmap_index *bitmap_git,
1745
die(_("unable to get size of %s"), oid_to_hex(&oid));
1746
}
1747
} else {
1748
+ size_t eindex_pos = pos - bitmap_num_objects_total(bitmap_git);
1749
struct eindex *eindex = &bitmap_git->ext_index;
1732
- struct object *obj = eindex->objects[pos - bitmap_num_objects(bitmap_git)];
1750
+ struct object *obj = eindex->objects[eindex_pos];
1751
if (oid_object_info_extended(bitmap_repo(bitmap_git), &obj->oid,
1752
&oi, 0) < 0)
1753
die(_("unable to get size of %s"), oid_to_hex(&obj->oid));
@@ -1882,7 +1900,7 @@ static void filter_packed_objects_from_bitmap(struct bitmap_index *bitmap_git,
1900
uint32_t objects_nr;
1901
size_t i, pos;
1902
1885
- objects_nr = bitmap_num_objects(bitmap_git);
1903
+ objects_nr = bitmap_num_objects_total(bitmap_git);
1904
pos = objects_nr / BITS_IN_EWORD;
1905
1906
if (pos > result->word_alloc)
@@ -2419,7 +2437,7 @@ static uint32_t count_object_type(struct bitmap_index *bitmap_git,
2437
for (i = 0; i < eindex->count; ++i) {
2438
if (eindex->objects[i]->type == type &&
2439
bitmap_get(objects,
2422
- st_add(bitmap_num_objects(bitmap_git), i)))
2440
+ st_add(bitmap_num_objects_total(bitmap_git), i)))
2441
count++;
2442
}
2443
@@ -2820,7 +2838,7 @@ uint32_t *create_bitmap_mapping(struct bitmap_index *bitmap_git,
2838
BUG("rebuild_existing_bitmaps: missing required rev-cache "
2839
"extension");
2840
2823
- num_objects = bitmap_num_objects(bitmap_git);
2841
+ num_objects = bitmap_num_objects_total(bitmap_git);
2842
CALLOC_ARRAY(reposition, num_objects);
2843
2844
for (i = 0; i < num_objects; ++i) {
@@ -2963,7 +2981,8 @@ static off_t get_disk_usage_for_extended(struct bitmap_index *bitmap_git)
2981
struct object *obj = eindex->objects[i];
2982
2983
if (!bitmap_get(result,
2966
- st_add(bitmap_num_objects(bitmap_git), i)))
2984
+ st_add(bitmap_num_objects_total(bitmap_git),
2985
+ i)))
2986
continue;
2987
2988
if (oid_object_info_extended(bitmap_repo(bitmap_git), &obj->oid,
pack-revindex.c
+25
-9
@@ -383,8 +383,14 @@ int load_midx_revindex(struct multi_pack_index *m)
383
trace2_data_string("load_midx_revindex", the_repository,
384
"source", "rev");
385
386
- get_midx_filename_ext(m->repo->hash_algo, &revindex_name, m->object_dir,
387
- get_midx_checksum(m), MIDX_EXT_REV);
386
+ if (m->has_chain)
387
+ get_split_midx_filename_ext(m->repo->hash_algo, &revindex_name,
388
+ m->object_dir, get_midx_checksum(m),
389
+ MIDX_EXT_REV);
390
+ else
391
+ get_midx_filename_ext(m->repo->hash_algo, &revindex_name,
392
+ m->object_dir, get_midx_checksum(m),
393
+ MIDX_EXT_REV);
394
395
ret = load_revindex_from_disk(revindex_name.buf,
396
m->num_objects,
@@ -471,11 +477,15 @@ off_t pack_pos_to_offset(struct packed_git *p, uint32_t pos)
477
478
uint32_t pack_pos_to_midx(struct multi_pack_index *m, uint32_t pos)
479
{
480
+ while (m && pos < m->num_objects_in_base)
481
+ m = m->base_midx;
482
+ if (!m)
483
+ BUG("NULL multi-pack-index for object position: %"PRIu32, pos);
484
if (!m->revindex_data)
485
BUG("pack_pos_to_midx: reverse index not yet loaded");
476
- if (m->num_objects <= pos)
486
+ if (m->num_objects + m->num_objects_in_base <= pos)
487
BUG("pack_pos_to_midx: out-of-bounds object at %"PRIu32, pos);
478
- return get_be32(m->revindex_data + pos);
488
+ return get_be32(m->revindex_data + pos - m->num_objects_in_base);
489
}
490
491
struct midx_pack_key {
@@ -491,7 +501,8 @@ static int midx_pack_order_cmp(const void *va, const void *vb)
501
const struct midx_pack_key *key = va;
502
struct multi_pack_index *midx = key->midx;
503
494
- uint32_t versus = pack_pos_to_midx(midx, (uint32_t*)vb - (const uint32_t *)midx->revindex_data);
504
+ size_t pos = (uint32_t *)vb - (const uint32_t *)midx->revindex_data;
505
+ uint32_t versus = pack_pos_to_midx(midx, pos + midx->num_objects_in_base);
506
uint32_t versus_pack = nth_midxed_pack_int_id(midx, versus);
507
off_t versus_offset;
508
@@ -529,9 +540,9 @@ static int midx_key_to_pack_pos(struct multi_pack_index *m,
540
{
541
uint32_t *found;
542
532
- if (key->pack >= m->num_packs)
543
+ if (key->pack >= m->num_packs + m->num_packs_in_base)
544
BUG("MIDX pack lookup out of bounds (%"PRIu32" >= %"PRIu32")",
534
- key->pack, m->num_packs);
545
+ key->pack, m->num_packs + m->num_packs_in_base);
546
/*
547
* The preferred pack sorts first, so determine its identifier by
548
* looking at the first object in pseudo-pack order.
@@ -551,7 +562,8 @@ static int midx_key_to_pack_pos(struct multi_pack_index *m,
562
if (!found)
563
return -1;
564
554
- *pos = found - m->revindex_data;
565
+ *pos = (found - m->revindex_data) + m->num_objects_in_base;
566
+
567
return 0;
568
}
569
@@ -559,9 +571,13 @@ int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos)
571
{
572
struct midx_pack_key key;
573
574
+ while (m && at < m->num_objects_in_base)
575
+ m = m->base_midx;
576
+ if (!m)
577
+ BUG("NULL multi-pack-index for object position: %"PRIu32, at);
578
if (!m->revindex_data)
579
BUG("midx_to_pack_pos: reverse index not yet loaded");
564
- if (m->num_objects <= at)
580
+ if (m->num_objects + m->num_objects_in_base <= at)
581
BUG("midx_to_pack_pos: out-of-bounds object at %"PRIu32, at);
582
583
key.pack = nth_midxed_pack_int_id(m, at);