midx: implement verification support for incremental MIDXs
Teach the verification implementation used by `git multi-pack-index verify` to perform verification for incremental MIDX chains by independently validating each layer within the chain. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Aug 6, 2024 at 11:37 UTC
3592796d0a176f51bb257b31dd9afe12a5af7932
2 files changed
+32
-17
midx.c
+30
-17
@@ -470,6 +470,13 @@ int prepare_midx_pack(struct repository *r, struct multi_pack_index *m,
470
return 0;
471
}
472
473
+struct packed_git *nth_midxed_pack(struct multi_pack_index *m,
474
+ uint32_t pack_int_id)
475
+{
476
+ uint32_t local_pack_int_id = midx_for_pack(&m, pack_int_id);
477
+ return m->packs[local_pack_int_id];
478
+}
479
+
480
#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t))
481
482
int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m,
@@ -818,6 +825,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
825
uint32_t i;
826
struct progress *progress = NULL;
827
struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
828
+ struct multi_pack_index *curr;
829
verify_midx_error = 0;
830
831
if (!m) {
@@ -840,8 +848,8 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
848
849
if (flags & MIDX_PROGRESS)
850
progress = start_delayed_progress(_("Looking for referenced packfiles"),
843
- m->num_packs);
844
- for (i = 0; i < m->num_packs; i++) {
851
+ m->num_packs + m->num_packs_in_base);
852
+ for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) {
853
if (prepare_midx_pack(r, m, i))
854
midx_report("failed to load pack in position %d", i);
855
@@ -861,17 +869,20 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
869
if (flags & MIDX_PROGRESS)
870
progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"),
871
m->num_objects - 1);
864
- for (i = 0; i < m->num_objects - 1; i++) {
865
- struct object_id oid1, oid2;
872
867
- nth_midxed_object_oid(&oid1, m, i);
868
- nth_midxed_object_oid(&oid2, m, i + 1);
873
+ for (curr = m; curr; curr = curr->base_midx) {
874
+ for (i = 0; i < m->num_objects - 1; i++) {
875
+ struct object_id oid1, oid2;
876
870
- if (oidcmp(&oid1, &oid2) >= 0)
871
- midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
872
- i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
877
+ nth_midxed_object_oid(&oid1, m, m->num_objects_in_base + i);
878
+ nth_midxed_object_oid(&oid2, m, m->num_objects_in_base + i + 1);
879
874
- midx_display_sparse_progress(progress, i + 1);
880
+ if (oidcmp(&oid1, &oid2) >= 0)
881
+ midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
882
+ i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
883
+
884
+ midx_display_sparse_progress(progress, i + 1);
885
+ }
886
}
887
stop_progress(&progress);
888
@@ -881,8 +892,8 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
892
* each of the objects and only require 1 packfile to be open at a
893
* time.
894
*/
884
- ALLOC_ARRAY(pairs, m->num_objects);
885
- for (i = 0; i < m->num_objects; i++) {
895
+ ALLOC_ARRAY(pairs, m->num_objects + m->num_objects_in_base);
896
+ for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) {
897
pairs[i].pos = i;
898
pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
899
}
@@ -896,16 +907,18 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
907
908
if (flags & MIDX_PROGRESS)
909
progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects);
899
- for (i = 0; i < m->num_objects; i++) {
910
+ for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) {
911
struct object_id oid;
912
struct pack_entry e;
913
off_t m_offset, p_offset;
914
915
if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
905
- m->packs[pairs[i-1].pack_int_id])
906
- {
907
- close_pack_fd(m->packs[pairs[i-1].pack_int_id]);
908
- close_pack_index(m->packs[pairs[i-1].pack_int_id]);
916
+ nth_midxed_pack(m, pairs[i-1].pack_int_id)) {
917
+ uint32_t pack_int_id = pairs[i-1].pack_int_id;
918
+ struct packed_git *p = nth_midxed_pack(m, pack_int_id);
919
+
920
+ close_pack_fd(p);
921
+ close_pack_index(p);
922
}
923
924
nth_midxed_object_oid(&oid, m, pairs[i].pos);
midx.h
+2
@@ -95,6 +95,8 @@ void get_split_midx_filename_ext(struct strbuf *buf, const char *object_dir,
95
96
struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local);
97
int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id);
98
+struct packed_git *nth_midxed_pack(struct multi_pack_index *m,
99
+ uint32_t pack_int_id);
100
int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m,
101
struct bitmapped_pack *bp, uint32_t pack_int_id);
102
int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m,