midx: write object id fanout chunk
Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Jul 12, 2018 at 15:39 UTC
d7cacf29ccfcb2a33bcd8468f83daf822430f19a
5 files changed
+68
-11
Documentation/technical/pack-format.txt
+5
@@ -302,6 +302,11 @@ CHUNK DATA:
302
name. This is the only chunk not guaranteed to be a multiple of four
303
bytes in length, so should be the last chunk for alignment reasons.
304
305
+ OID Fanout (ID: {'O', 'I', 'D', 'F'})
306
+ The ith entry, F[i], stores the number of OIDs with first
307
+ byte at most i. Thus F[255] stores the total
308
+ number of objects.
309
+
310
OID Lookup (ID: {'O', 'I', 'D', 'L'})
311
The OIDs for all objects in the MIDX are stored in lexicographic
312
order in this chunk.
midx.c
+50
-3
@@ -18,11 +18,13 @@
18
#define MIDX_HASH_LEN 20
19
#define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + MIDX_HASH_LEN)
20
21
-#define MIDX_MAX_CHUNKS 2
21
+#define MIDX_MAX_CHUNKS 3
22
#define MIDX_CHUNK_ALIGNMENT 4
23
#define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
24
+#define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
25
#define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
26
#define MIDX_CHUNKLOOKUP_WIDTH (sizeof(uint32_t) + sizeof(uint64_t))
27
+#define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
28
29
static char *get_midx_filename(const char *object_dir)
30
{
@@ -102,6 +104,10 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir)
104
m->chunk_pack_names = m->data + chunk_offset;
105
break;
106
107
+ case MIDX_CHUNKID_OIDFANOUT:
108
+ m->chunk_oid_fanout = (uint32_t *)(m->data + chunk_offset);
109
+ break;
110
+
111
case MIDX_CHUNKID_OIDLOOKUP:
112
m->chunk_oid_lookup = m->data + chunk_offset;
113
break;
@@ -121,9 +127,13 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir)
127
128
if (!m->chunk_pack_names)
129
die(_("multi-pack-index missing required pack-name chunk"));
130
+ if (!m->chunk_oid_fanout)
131
+ die(_("multi-pack-index missing required OID fanout chunk"));
132
if (!m->chunk_oid_lookup)
133
die(_("multi-pack-index missing required OID lookup chunk"));
134
135
+ m->num_objects = ntohl(m->chunk_oid_fanout[255]);
136
+
137
m->pack_names = xcalloc(m->num_packs, sizeof(*m->pack_names));
138
139
cur_pack_name = (const char *)m->chunk_pack_names;
@@ -389,6 +399,35 @@ static size_t write_midx_pack_names(struct hashfile *f,
399
return written;
400
}
401
402
+static size_t write_midx_oid_fanout(struct hashfile *f,
403
+ struct pack_midx_entry *objects,
404
+ uint32_t nr_objects)
405
+{
406
+ struct pack_midx_entry *list = objects;
407
+ struct pack_midx_entry *last = objects + nr_objects;
408
+ uint32_t count = 0;
409
+ uint32_t i;
410
+
411
+ /*
412
+ * Write the first-level table (the list is sorted,
413
+ * but we use a 256-entry lookup to be able to avoid
414
+ * having to do eight extra binary search iterations).
415
+ */
416
+ for (i = 0; i < 256; i++) {
417
+ struct pack_midx_entry *next = list;
418
+
419
+ while (next < last && next->oid.hash[0] == i) {
420
+ count++;
421
+ next++;
422
+ }
423
+
424
+ hashwrite_be32(f, count);
425
+ list = next;
426
+ }
427
+
428
+ return MIDX_CHUNK_FANOUT_SIZE;
429
+}
430
+
431
static size_t write_midx_oid_lookup(struct hashfile *f, unsigned char hash_len,
432
struct pack_midx_entry *objects,
433
uint32_t nr_objects)
@@ -461,7 +500,7 @@ int write_midx_file(const char *object_dir)
500
FREE_AND_NULL(midx_name);
501
502
cur_chunk = 0;
464
- num_chunks = 2;
503
+ num_chunks = 3;
504
505
written = write_midx_header(f, num_chunks, packs.nr);
506
@@ -469,9 +508,13 @@ int write_midx_file(const char *object_dir)
508
chunk_offsets[cur_chunk] = written + (num_chunks + 1) * MIDX_CHUNKLOOKUP_WIDTH;
509
510
cur_chunk++;
472
- chunk_ids[cur_chunk] = MIDX_CHUNKID_OIDLOOKUP;
511
+ chunk_ids[cur_chunk] = MIDX_CHUNKID_OIDFANOUT;
512
chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + packs.pack_name_concat_len;
513
514
+ cur_chunk++;
515
+ chunk_ids[cur_chunk] = MIDX_CHUNKID_OIDLOOKUP;
516
+ chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + MIDX_CHUNK_FANOUT_SIZE;
517
+
518
cur_chunk++;
519
chunk_ids[cur_chunk] = 0;
520
chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + nr_entries * MIDX_HASH_LEN;
@@ -505,6 +548,10 @@ int write_midx_file(const char *object_dir)
548
written += write_midx_pack_names(f, packs.names, packs.nr);
549
break;
550
551
+ case MIDX_CHUNKID_OIDFANOUT:
552
+ written += write_midx_oid_fanout(f, entries, nr_entries);
553
+ break;
554
+
555
case MIDX_CHUNKID_OIDLOOKUP:
556
written += write_midx_oid_lookup(f, MIDX_HASH_LEN, entries, nr_entries);
557
break;
midx.h
+1
@@ -15,6 +15,7 @@ struct multi_pack_index {
15
uint32_t num_objects;
16
17
const unsigned char *chunk_pack_names;
18
+ const uint32_t *chunk_oid_fanout;
19
const unsigned char *chunk_oid_lookup;
20
21
const char **pack_names;
t/helper/test-read-midx.c
+3
-1
@@ -22,10 +22,12 @@ static int read_midx_file(const char *object_dir)
22
23
if (m->chunk_pack_names)
24
printf(" pack-names");
25
+ if (m->chunk_oid_fanout)
26
+ printf(" oid-fanout");
27
if (m->chunk_oid_lookup)
28
printf(" oid-lookup");
29
28
- printf("\n");
30
+ printf("\nnum_objects: %d\n", m->num_objects);
31
32
printf("packs:\n");
33
for (i = 0; i < m->num_packs; i++)
t/t5319-multi-pack-index.sh
+9
-7
@@ -5,10 +5,12 @@ test_description='multi-pack-indexes'
5
6
midx_read_expect () {
7
NUM_PACKS=$1
8
+ NUM_OBJECTS=$2
9
{
10
cat <<-EOF &&
10
- header: 4d494458 1 2 $NUM_PACKS
11
- chunks: pack-names oid-lookup
11
+ header: 4d494458 1 3 $NUM_PACKS
12
+ chunks: pack-names oid-fanout oid-lookup
13
+ num_objects: $NUM_OBJECTS
14
packs:
15
EOF
16
if test $NUM_PACKS -ge 1
@@ -24,7 +26,7 @@ midx_read_expect () {
26
test_expect_success 'write midx with no packs' '
27
test_when_finished rm -f pack/multi-pack-index &&
28
git multi-pack-index --object-dir=. write &&
27
- midx_read_expect 0
29
+ midx_read_expect 0 0
30
'
31
32
generate_objects () {
@@ -74,13 +76,13 @@ test_expect_success 'write midx with one v1 pack' '
76
pack=$(git pack-objects --index-version=1 pack/test <obj-list) &&
77
test_when_finished rm pack/test-$pack.pack pack/test-$pack.idx pack/multi-pack-index &&
78
git multi-pack-index --object-dir=. write &&
77
- midx_read_expect 1
79
+ midx_read_expect 1 18
80
'
81
82
test_expect_success 'write midx with one v2 pack' '
83
git pack-objects --index-version=2,0x40 pack/test <obj-list &&
84
git multi-pack-index --object-dir=. write &&
83
- midx_read_expect 1
85
+ midx_read_expect 1 18
86
'
87
88
test_expect_success 'add more objects' '
@@ -94,7 +96,7 @@ test_expect_success 'add more objects' '
96
test_expect_success 'write midx with two packs' '
97
git pack-objects --index-version=1 pack/test-2 <obj-list &&
98
git multi-pack-index --object-dir=. write &&
97
- midx_read_expect 2
99
+ midx_read_expect 2 34
100
'
101
102
test_expect_success 'add more packs' '
@@ -108,7 +110,7 @@ test_expect_success 'add more packs' '
110
111
test_expect_success 'write midx with twelve packs' '
112
git multi-pack-index --object-dir=. write &&
111
- midx_read_expect 12
113
+ midx_read_expect 12 74
114
'
115
116
test_done