midx: read objects from multi-pack-index
Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Jul 12, 2018 at 15:39 UTC
3715a6335c37367b4240b6bfa842dc64dedee34d
3 files changed
+100
-2
midx.c
+90
-1
@@ -5,7 +5,7 @@
5
#include "lockfile.h"
6
#include "packfile.h"
7
#include "object-store.h"
8
-#include "packfile.h"
8
+#include "sha1-lookup.h"
9
#include "midx.h"
10
11
#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
@@ -151,6 +151,7 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir)
151
m->num_objects = ntohl(m->chunk_oid_fanout[255]);
152
153
m->pack_names = xcalloc(m->num_packs, sizeof(*m->pack_names));
154
+ m->packs = xcalloc(m->num_packs, sizeof(*m->packs));
155
156
cur_pack_name = (const char *)m->chunk_pack_names;
157
for (i = 0; i < m->num_packs; i++) {
@@ -178,6 +179,94 @@ cleanup_fail:
179
return NULL;
180
}
181
182
+static int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id)
183
+{
184
+ struct strbuf pack_name = STRBUF_INIT;
185
+
186
+ if (pack_int_id >= m->num_packs)
187
+ BUG("bad pack-int-id");
188
+
189
+ if (m->packs[pack_int_id])
190
+ return 0;
191
+
192
+ strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir,
193
+ m->pack_names[pack_int_id]);
194
+
195
+ m->packs[pack_int_id] = add_packed_git(pack_name.buf, pack_name.len, 1);
196
+ strbuf_release(&pack_name);
197
+ return !m->packs[pack_int_id];
198
+}
199
+
200
+int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result)
201
+{
202
+ return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup,
203
+ MIDX_HASH_LEN, result);
204
+}
205
+
206
+static off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
207
+{
208
+ const unsigned char *offset_data;
209
+ uint32_t offset32;
210
+
211
+ offset_data = m->chunk_object_offsets + pos * MIDX_CHUNK_OFFSET_WIDTH;
212
+ offset32 = get_be32(offset_data + sizeof(uint32_t));
213
+
214
+ if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
215
+ if (sizeof(offset32) < sizeof(uint64_t))
216
+ die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
217
+
218
+ offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
219
+ return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32);
220
+ }
221
+
222
+ return offset32;
223
+}
224
+
225
+static uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos)
226
+{
227
+ return get_be32(m->chunk_object_offsets + pos * MIDX_CHUNK_OFFSET_WIDTH);
228
+}
229
+
230
+static int nth_midxed_pack_entry(struct multi_pack_index *m, struct pack_entry *e, uint32_t pos)
231
+{
232
+ uint32_t pack_int_id;
233
+ struct packed_git *p;
234
+
235
+ if (pos >= m->num_objects)
236
+ return 0;
237
+
238
+ pack_int_id = nth_midxed_pack_int_id(m, pos);
239
+
240
+ if (prepare_midx_pack(m, pack_int_id))
241
+ die(_("error preparing packfile from multi-pack-index"));
242
+ p = m->packs[pack_int_id];
243
+
244
+ /*
245
+ * We are about to tell the caller where they can locate the
246
+ * requested object. We better make sure the packfile is
247
+ * still here and can be accessed before supplying that
248
+ * answer, as it may have been deleted since the MIDX was
249
+ * loaded!
250
+ */
251
+ if (!is_pack_valid(p))
252
+ return 0;
253
+
254
+ e->offset = nth_midxed_offset(m, pos);
255
+ e->p = p;
256
+
257
+ return 1;
258
+}
259
+
260
+int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m)
261
+{
262
+ uint32_t pos;
263
+
264
+ if (!bsearch_midx(oid, m, &pos))
265
+ return 0;
266
+
267
+ return nth_midxed_pack_entry(m, e, pos);
268
+}
269
+
270
int prepare_multi_pack_index_one(struct repository *r, const char *object_dir)
271
{
272
struct multi_pack_index *m = r->objects->multi_pack_index;
midx.h
+3
@@ -25,10 +25,13 @@ struct multi_pack_index {
25
const unsigned char *chunk_large_offsets;
26
27
const char **pack_names;
28
+ struct packed_git **packs;
29
char object_dir[FLEX_ARRAY];
30
};
31
32
struct multi_pack_index *load_multi_pack_index(const char *object_dir);
33
+int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result);
34
+int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m);
35
int prepare_multi_pack_index_one(struct repository *r, const char *object_dir);
36
37
int write_midx_file(const char *object_dir);
packfile.c
+7
-1
@@ -1902,11 +1902,17 @@ static int fill_pack_entry(const struct object_id *oid,
1902
int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e)
1903
{
1904
struct list_head *pos;
1905
+ struct multi_pack_index *m;
1906
1907
prepare_packed_git(r);
1907
- if (!r->objects->packed_git)
1908
+ if (!r->objects->packed_git && !r->objects->multi_pack_index)
1909
return 0;
1910
1911
+ for (m = r->objects->multi_pack_index; m; m = m->next) {
1912
+ if (fill_midx_entry(oid, e, m))
1913
+ return 1;
1914
+ }
1915
+
1916
list_for_each(pos, &r->objects->packed_git_mru) {
1917
struct packed_git *p = list_entry(pos, struct packed_git, mru);
1918
if (fill_pack_entry(oid, e, p)) {