midx: verify duplicate pack entries by OID and offset
A MIDX retains one entry per OID, while a non-strict pack index can contain the same OID at several offsets. verify_midx_file() compares the recorded offset with find_pack_entry_one(), which may return a different member of that duplicate run and falsely report a valid MIDX as corrupt. Check instead that the exact OID/offset pair exists anywhere in the contiguous duplicate run in the source pack. That is the relevant invariant: same-pack duplicates have no canonical representation, and every matching physical copy is valid, including those recorded by existing MIDXs. This matches reader behavior. The OOFF chunk records the selected (pack, offset), and midx_to_pack_pos() reconstructs its RIDX position from that pair. If midx_pair_to_pack_pos() is given an unselected duplicate offset, the lookup misses and its sole caller falls back from partial reuse to normal packing. Readers therefore remain consistent with whichever representation the MIDX records. Write a MIDX over the existing duplicate-pack fixture, assert that its selected offset differs from find_pack_entry_one(), and verify it. Then run fsck as an application-level check. Signed-off-by: Taylor Blau <ttaylorr@openai.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Jul 24, 2026 at 16:06 UTC
e3ffc236a8c7780d22442c7ebf1fca757bdccdf4
3 files changed
+80
-11
midx.c
+52
-7
@@ -906,6 +906,48 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b)
906
return b->pack_int_id - a->pack_int_id;
907
}
908
909
+/*
910
+ * Return whether the pack index contains an entry with both "oid" and
911
+ * "offset". A pack index may contain duplicate OIDs, so an arbitrary
912
+ * OID lookup is not enough to validate a particular offset.
913
+ *
914
+ * Do not use offset_to_pack_pos() here: it may consult an optional '.rev'
915
+ * file, which is verified separately, or build an in-memory reverse index
916
+ * that remains attached to the pack. Verify the MIDX directly against its
917
+ * source pack index instead.
918
+ */
919
+static int pack_index_has_oid_at_offset(struct packed_git *p,
920
+ const struct object_id *oid,
921
+ off_t offset)
922
+{
923
+ struct object_id candidate;
924
+ uint32_t pos, i;
925
+
926
+ if (!bsearch_pack(oid, p, &pos))
927
+ return 0;
928
+
929
+ if (nth_packed_object_offset(p, pos) == offset)
930
+ return 1;
931
+
932
+ for (i = pos; i > 0; i--) {
933
+ if (nth_packed_object_id(&candidate, p, i - 1) ||
934
+ !oideq(&candidate, oid))
935
+ break;
936
+ if (nth_packed_object_offset(p, i - 1) == offset)
937
+ return 1;
938
+ }
939
+
940
+ for (i = pos + 1; i < p->num_objects; i++) {
941
+ if (nth_packed_object_id(&candidate, p, i) ||
942
+ !oideq(&candidate, oid))
943
+ break;
944
+ if (nth_packed_object_offset(p, i) == offset)
945
+ return 1;
946
+ }
947
+
948
+ return 0;
949
+}
950
+
951
/*
952
* Limit calls to display_progress() for performance reasons.
953
* The interval here was arbitrarily chosen.
@@ -1015,7 +1057,6 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags)
1057
for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) {
1058
struct object_id oid;
1059
struct pack_entry e;
1018
- off_t m_offset, p_offset;
1060
1061
if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
1062
nth_midxed_pack(m, pairs[i-1].pack_int_id)) {
@@ -1040,12 +1081,16 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags)
1081
break;
1082
}
1083
1043
- m_offset = e.offset;
1044
- p_offset = find_pack_entry_one(&oid, e.p);
1045
-
1046
- if (m_offset != p_offset)
1047
- midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
1048
- pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
1084
+ /*
1085
+ * Check that the exact offset recorded in the MIDX
1086
+ * belongs to this OID. A pack index may contain
1087
+ * duplicate OIDs, in which case an arbitrary OID lookup
1088
+ * can return a different, equally valid copy than the
1089
+ * one selected by the MIDX writer.
1090
+ */
1091
+ if (!pack_index_has_oid_at_offset(e.p, &oid, e.offset))
1092
+ midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64),
1093
+ pairs[i].pos, oid_to_hex(&oid), e.offset);
1094
1095
midx_display_sparse_progress(progress, i + 1);
1096
}
t/helper/test-find-pack.c
+14
-4
@@ -11,12 +11,15 @@
11
* Display the path(s), one per line, of the packfile(s) containing
12
* the given object.
13
*
14
+ * With '--show-offset', display the offset selected by
15
+ * find_pack_entry_one() instead of the packfile path.
16
+ *
17
* If '--check-count <n>' is passed, then error out if the number of
18
* packfiles containing the object is not <n>.
19
*/
20
21
static const char *const find_pack_usage[] = {
19
- "test-tool find-pack [--check-count <n>] <object>",
22
+ "test-tool find-pack [--check-count <n>] [--show-offset] <object>",
23
NULL
24
};
25
@@ -24,11 +27,13 @@ int cmd__find_pack(int argc, const char **argv)
27
{
28
struct object_id oid;
29
struct packed_git *p;
27
- int count = -1, actual_count = 0;
30
+ int count = -1, actual_count = 0, show_offset = 0;
31
const char *prefix = setup_git_directory(the_repository);
32
33
struct option options[] = {
34
OPT_INTEGER('c', "check-count", &count, "expected number of packs"),
35
+ OPT_BOOL(0, "show-offset", &show_offset,
36
+ "show matching pack offsets"),
37
OPT_END(),
38
};
39
@@ -40,8 +45,13 @@ int cmd__find_pack(int argc, const char **argv)
45
die("cannot parse %s as an object name", argv[0]);
46
47
repo_for_each_pack(the_repository, p) {
43
- if (find_pack_entry_one(&oid, p)) {
44
- printf("%s\n", p->pack_name);
48
+ off_t offset = find_pack_entry_one(&oid, p);
49
+
50
+ if (offset) {
51
+ if (show_offset)
52
+ printf("%"PRIuMAX"\n", (uintmax_t)offset);
53
+ else
54
+ printf("%s\n", p->pack_name);
55
actual_count++;
56
}
57
}
t/t5308-pack-detect-duplicates.sh
+14
@@ -77,6 +77,20 @@ test_expect_success 'lookup in duplicated pack' '
77
test_cmp expect actual
78
'
79
80
+test_expect_success 'verify MIDX containing duplicated pack objects' '
81
+ git multi-pack-index write &&
82
+ test-tool read-midx --show-objects .git/objects >midx-objects &&
83
+ midx_offset=$(
84
+ awk -v oid="$LO_SHA1" "\$1 == oid { print \$2 }" <midx-objects
85
+ ) &&
86
+ lookup_offset=$(
87
+ test-tool find-pack --check-count=1 --show-offset "$LO_SHA1"
88
+ ) &&
89
+ test "$midx_offset" -ne "$lookup_offset" &&
90
+ git multi-pack-index verify &&
91
+ git fsck --full
92
+'
93
+
94
test_expect_success 'duplicate entries remain in pack reverse index' '
95
clear_packs &&
96
{