@matheusbsbs / matheus-bagliano / commits / 748b88a021

midx-write: move writing-related functions from midx.c

Introduce a new midx-write.c source file, which holds all of the functionality from the MIDX sub-system related to writing new MIDX files. Similar to the relationship between "pack-bitmap.c" and "pack-bitmap-write.c", this source file will hold code that is specific to writing MIDX files as opposed to reading them (the latter will remain in midx.c). Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Apr 1, 2024 at 17:16 UTC 748b88a0214d4f65fd56d4728234ea86dbcfffd6
4 files changed +1711 -1695
Makefile
+1
@@ -1075,6 +1075,7 @@ LIB_OBJS += merge-ort-wrappers.o
1075 LIB_OBJS += merge-recursive.o
1076 LIB_OBJS += merge.o
1077 LIB_OBJS += midx.o
1078 +LIB_OBJS += midx-write.o
1079 LIB_OBJS += name-hash.o
1080 LIB_OBJS += negotiator/default.o
1081 LIB_OBJS += negotiator/noop.o
midx-write.c new
+1521
@@ -0,0 +1,1521 @@
1 +#include "git-compat-util.h"
2 +#include "abspath.h"
3 +#include "config.h"
4 +#include "hex.h"
5 +#include "lockfile.h"
6 +#include "packfile.h"
7 +#include "object-file.h"
8 +#include "hash-lookup.h"
9 +#include "midx.h"
10 +#include "progress.h"
11 +#include "trace2.h"
12 +#include "run-command.h"
13 +#include "chunk-format.h"
14 +#include "pack-bitmap.h"
15 +#include "refs.h"
16 +#include "revision.h"
17 +#include "list-objects.h"
18 +
19 +#define PACK_EXPIRED UINT_MAX
20 +#define BITMAP_POS_UNKNOWN (~((uint32_t)0))
21 +#define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
22 +#define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
23 +
24 +extern int midx_checksum_valid(struct multi_pack_index *m);
25 +extern void clear_midx_files_ext(const char *object_dir, const char *ext,
26 + unsigned char *keep_hash);
27 +extern int cmp_idx_or_pack_name(const char *idx_or_pack_name,
28 + const char *idx_name);
29 +
30 +static size_t write_midx_header(struct hashfile *f,
31 + unsigned char num_chunks,
32 + uint32_t num_packs)
33 +{
34 + hashwrite_be32(f, MIDX_SIGNATURE);
35 + hashwrite_u8(f, MIDX_VERSION);
36 + hashwrite_u8(f, oid_version(the_hash_algo));
37 + hashwrite_u8(f, num_chunks);
38 + hashwrite_u8(f, 0); /* unused */
39 + hashwrite_be32(f, num_packs);
40 +
41 + return MIDX_HEADER_SIZE;
42 +}
43 +
44 +struct pack_info {
45 + uint32_t orig_pack_int_id;
46 + char *pack_name;
47 + struct packed_git *p;
48 +
49 + uint32_t bitmap_pos;
50 + uint32_t bitmap_nr;
51 +
52 + unsigned expired : 1;
53 +};
54 +
55 +static void fill_pack_info(struct pack_info *info,
56 + struct packed_git *p, const char *pack_name,
57 + uint32_t orig_pack_int_id)
58 +{
59 + memset(info, 0, sizeof(struct pack_info));
60 +
61 + info->orig_pack_int_id = orig_pack_int_id;
62 + info->pack_name = xstrdup(pack_name);
63 + info->p = p;
64 + info->bitmap_pos = BITMAP_POS_UNKNOWN;
65 +}
66 +
67 +static int pack_info_compare(const void *_a, const void *_b)
68 +{
69 + struct pack_info *a = (struct pack_info *)_a;
70 + struct pack_info *b = (struct pack_info *)_b;
71 + return strcmp(a->pack_name, b->pack_name);
72 +}
73 +
74 +static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
75 +{
76 + const char *pack_name = _va;
77 + const struct pack_info *compar = _vb;
78 +
79 + return cmp_idx_or_pack_name(pack_name, compar->pack_name);
80 +}
81 +
82 +struct write_midx_context {
83 + struct pack_info *info;
84 + size_t nr;
85 + size_t alloc;
86 + struct multi_pack_index *m;
87 + struct progress *progress;
88 + unsigned pack_paths_checked;
89 +
90 + struct pack_midx_entry *entries;
91 + size_t entries_nr;
92 +
93 + uint32_t *pack_perm;
94 + uint32_t *pack_order;
95 + unsigned large_offsets_needed:1;
96 + uint32_t num_large_offsets;
97 +
98 + int preferred_pack_idx;
99 +
100 + struct string_list *to_include;
101 +};
102 +
103 +static void add_pack_to_midx(const char *full_path, size_t full_path_len,
104 + const char *file_name, void *data)
105 +{
106 + struct write_midx_context *ctx = data;
107 + struct packed_git *p;
108 +
109 + if (ends_with(file_name, ".idx")) {
110 + display_progress(ctx->progress, ++ctx->pack_paths_checked);
111 + /*
112 + * Note that at most one of ctx->m and ctx->to_include are set,
113 + * so we are testing midx_contains_pack() and
114 + * string_list_has_string() independently (guarded by the
115 + * appropriate NULL checks).
116 + *
117 + * We could support passing to_include while reusing an existing
118 + * MIDX, but don't currently since the reuse process drags
119 + * forward all packs from an existing MIDX (without checking
120 + * whether or not they appear in the to_include list).
121 + *
122 + * If we added support for that, these next two conditional
123 + * should be performed independently (likely checking
124 + * to_include before the existing MIDX).
125 + */
126 + if (ctx->m && midx_contains_pack(ctx->m, file_name))
127 + return;
128 + else if (ctx->to_include &&
129 + !string_list_has_string(ctx->to_include, file_name))
130 + return;
131 +
132 + ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
133 +
134 + p = add_packed_git(full_path, full_path_len, 0);
135 + if (!p) {
136 + warning(_("failed to add packfile '%s'"),
137 + full_path);
138 + return;
139 + }
140 +
141 + if (open_pack_index(p)) {
142 + warning(_("failed to open pack-index '%s'"),
143 + full_path);
144 + close_pack(p);
145 + free(p);
146 + return;
147 + }
148 +
149 + fill_pack_info(&ctx->info[ctx->nr], p, file_name, ctx->nr);
150 + ctx->nr++;
151 + }
152 +}
153 +
154 +struct pack_midx_entry {
155 + struct object_id oid;
156 + uint32_t pack_int_id;
157 + time_t pack_mtime;
158 + uint64_t offset;
159 + unsigned preferred : 1;
160 +};
161 +
162 +static int midx_oid_compare(const void *_a, const void *_b)
163 +{
164 + const struct pack_midx_entry *a = (const struct pack_midx_entry *)_a;
165 + const struct pack_midx_entry *b = (const struct pack_midx_entry *)_b;
166 + int cmp = oidcmp(&a->oid, &b->oid);
167 +
168 + if (cmp)
169 + return cmp;
170 +
171 + /* Sort objects in a preferred pack first when multiple copies exist. */
172 + if (a->preferred > b->preferred)
173 + return -1;
174 + if (a->preferred < b->preferred)
175 + return 1;
176 +
177 + if (a->pack_mtime > b->pack_mtime)
178 + return -1;
179 + else if (a->pack_mtime < b->pack_mtime)
180 + return 1;
181 +
182 + return a->pack_int_id - b->pack_int_id;
183 +}
184 +
185 +static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
186 + struct pack_midx_entry *e,
187 + uint32_t pos)
188 +{
189 + if (pos >= m->num_objects)
190 + return 1;
191 +
192 + nth_midxed_object_oid(&e->oid, m, pos);
193 + e->pack_int_id = nth_midxed_pack_int_id(m, pos);
194 + e->offset = nth_midxed_offset(m, pos);
195 +
196 + /* consider objects in midx to be from "old" packs */
197 + e->pack_mtime = 0;
198 + return 0;
199 +}
200 +
201 +static void fill_pack_entry(uint32_t pack_int_id,
202 + struct packed_git *p,
203 + uint32_t cur_object,
204 + struct pack_midx_entry *entry,
205 + int preferred)
206 +{
207 + if (nth_packed_object_id(&entry->oid, p, cur_object) < 0)
208 + die(_("failed to locate object %d in packfile"), cur_object);
209 +
210 + entry->pack_int_id = pack_int_id;
211 + entry->pack_mtime = p->mtime;
212 +
213 + entry->offset = nth_packed_object_offset(p, cur_object);
214 + entry->preferred = !!preferred;
215 +}
216 +
217 +struct midx_fanout {
218 + struct pack_midx_entry *entries;
219 + size_t nr, alloc;
220 +};
221 +
222 +static void midx_fanout_grow(struct midx_fanout *fanout, size_t nr)
223 +{
224 + if (nr < fanout->nr)
225 + BUG("negative growth in midx_fanout_grow() (%"PRIuMAX" < %"PRIuMAX")",
226 + (uintmax_t)nr, (uintmax_t)fanout->nr);
227 + ALLOC_GROW(fanout->entries, nr, fanout->alloc);
228 +}
229 +
230 +static void midx_fanout_sort(struct midx_fanout *fanout)
231 +{
232 + QSORT(fanout->entries, fanout->nr, midx_oid_compare);
233 +}
234 +
235 +static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
236 + struct multi_pack_index *m,
237 + uint32_t cur_fanout,
238 + int preferred_pack)
239 +{
240 + uint32_t start = 0, end;
241 + uint32_t cur_object;
242 +
243 + if (cur_fanout)
244 + start = ntohl(m->chunk_oid_fanout[cur_fanout - 1]);
245 + end = ntohl(m->chunk_oid_fanout[cur_fanout]);
246 +
247 + for (cur_object = start; cur_object < end; cur_object++) {
248 + if ((preferred_pack > -1) &&
249 + (preferred_pack == nth_midxed_pack_int_id(m, cur_object))) {
250 + /*
251 + * Objects from preferred packs are added
252 + * separately.
253 + */
254 + continue;
255 + }
256 +
257 + midx_fanout_grow(fanout, fanout->nr + 1);
258 + nth_midxed_pack_midx_entry(m,
259 + &fanout->entries[fanout->nr],
260 + cur_object);
261 + fanout->entries[fanout->nr].preferred = 0;
262 + fanout->nr++;
263 + }
264 +}
265 +
266 +static void midx_fanout_add_pack_fanout(struct midx_fanout *fanout,
267 + struct pack_info *info,
268 + uint32_t cur_pack,
269 + int preferred,
270 + uint32_t cur_fanout)
271 +{
272 + struct packed_git *pack = info[cur_pack].p;
273 + uint32_t start = 0, end;
274 + uint32_t cur_object;
275 +
276 + if (cur_fanout)
277 + start = get_pack_fanout(pack, cur_fanout - 1);
278 + end = get_pack_fanout(pack, cur_fanout);
279 +
280 + for (cur_object = start; cur_object < end; cur_object++) {
281 + midx_fanout_grow(fanout, fanout->nr + 1);
282 + fill_pack_entry(cur_pack,
283 + info[cur_pack].p,
284 + cur_object,
285 + &fanout->entries[fanout->nr],
286 + preferred);
287 + fanout->nr++;
288 + }
289 +}
290 +
291 +/*
292 + * It is possible to artificially get into a state where there are many
293 + * duplicate copies of objects. That can create high memory pressure if
294 + * we are to create a list of all objects before de-duplication. To reduce
295 + * this memory pressure without a significant performance drop, automatically
296 + * group objects by the first byte of their object id. Use the IDX fanout
297 + * tables to group the data, copy to a local array, then sort.
298 + *
299 + * Copy only the de-duplicated entries (selected by most-recent modified time
300 + * of a packfile containing the object).
301 + */
302 +static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
303 + struct pack_info *info,
304 + uint32_t nr_packs,
305 + size_t *nr_objects,
306 + int preferred_pack)
307 +{
308 + uint32_t cur_fanout, cur_pack, cur_object;
309 + size_t alloc_objects, total_objects = 0;
310 + struct midx_fanout fanout = { 0 };
311 + struct pack_midx_entry *deduplicated_entries = NULL;
312 + uint32_t start_pack = m ? m->num_packs : 0;
313 +
314 + for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++)
315 + total_objects = st_add(total_objects,
316 + info[cur_pack].p->num_objects);
317 +
318 + /*
319 + * As we de-duplicate by fanout value, we expect the fanout
320 + * slices to be evenly distributed, with some noise. Hence,
321 + * allocate slightly more than one 256th.
322 + */
323 + alloc_objects = fanout.alloc = total_objects > 3200 ? total_objects / 200 : 16;
324 +
325 + ALLOC_ARRAY(fanout.entries, fanout.alloc);
326 + ALLOC_ARRAY(deduplicated_entries, alloc_objects);
327 + *nr_objects = 0;
328 +
329 + for (cur_fanout = 0; cur_fanout < 256; cur_fanout++) {
330 + fanout.nr = 0;
331 +
332 + if (m)
333 + midx_fanout_add_midx_fanout(&fanout, m, cur_fanout,
334 + preferred_pack);
335 +
336 + for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) {
337 + int preferred = cur_pack == preferred_pack;
338 + midx_fanout_add_pack_fanout(&fanout,
339 + info, cur_pack,
340 + preferred, cur_fanout);
341 + }
342 +
343 + if (-1 < preferred_pack && preferred_pack < start_pack)
344 + midx_fanout_add_pack_fanout(&fanout, info,
345 + preferred_pack, 1,
346 + cur_fanout);
347 +
348 + midx_fanout_sort(&fanout);
349 +
350 + /*
351 + * The batch is now sorted by OID and then mtime (descending).
352 + * Take only the first duplicate.
353 + */
354 + for (cur_object = 0; cur_object < fanout.nr; cur_object++) {
355 + if (cur_object && oideq(&fanout.entries[cur_object - 1].oid,
356 + &fanout.entries[cur_object].oid))
357 + continue;
358 +
359 + ALLOC_GROW(deduplicated_entries, st_add(*nr_objects, 1),
360 + alloc_objects);
361 + memcpy(&deduplicated_entries[*nr_objects],
362 + &fanout.entries[cur_object],
363 + sizeof(struct pack_midx_entry));
364 + (*nr_objects)++;
365 + }
366 + }
367 +
368 + free(fanout.entries);
369 + return deduplicated_entries;
370 +}
371 +
372 +static int write_midx_pack_names(struct hashfile *f, void *data)
373 +{
374 + struct write_midx_context *ctx = data;
375 + uint32_t i;
376 + unsigned char padding[MIDX_CHUNK_ALIGNMENT];
377 + size_t written = 0;
378 +
379 + for (i = 0; i < ctx->nr; i++) {
380 + size_t writelen;
381 +
382 + if (ctx->info[i].expired)
383 + continue;
384 +
385 + if (i && strcmp(ctx->info[i].pack_name, ctx->info[i - 1].pack_name) <= 0)
386 + BUG("incorrect pack-file order: %s before %s",
387 + ctx->info[i - 1].pack_name,
388 + ctx->info[i].pack_name);
389 +
390 + writelen = strlen(ctx->info[i].pack_name) + 1;
391 + hashwrite(f, ctx->info[i].pack_name, writelen);
392 + written += writelen;
393 + }
394 +
395 + /* add padding to be aligned */
396 + i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
397 + if (i < MIDX_CHUNK_ALIGNMENT) {
398 + memset(padding, 0, sizeof(padding));
399 + hashwrite(f, padding, i);
400 + }
401 +
402 + return 0;
403 +}
404 +
405 +static int write_midx_bitmapped_packs(struct hashfile *f, void *data)
406 +{
407 + struct write_midx_context *ctx = data;
408 + size_t i;
409 +
410 + for (i = 0; i < ctx->nr; i++) {
411 + struct pack_info *pack = &ctx->info[i];
412 + if (pack->expired)
413 + continue;
414 +
415 + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN && pack->bitmap_nr)
416 + BUG("pack '%s' has no bitmap position, but has %d bitmapped object(s)",
417 + pack->pack_name, pack->bitmap_nr);
418 +
419 + hashwrite_be32(f, pack->bitmap_pos);
420 + hashwrite_be32(f, pack->bitmap_nr);
421 + }
422 + return 0;
423 +}
424 +
425 +static int write_midx_oid_fanout(struct hashfile *f,
426 + void *data)
427 +{
428 + struct write_midx_context *ctx = data;
429 + struct pack_midx_entry *list = ctx->entries;
430 + struct pack_midx_entry *last = ctx->entries + ctx->entries_nr;
431 + uint32_t count = 0;
432 + uint32_t i;
433 +
434 + /*
435 + * Write the first-level table (the list is sorted,
436 + * but we use a 256-entry lookup to be able to avoid
437 + * having to do eight extra binary search iterations).
438 + */
439 + for (i = 0; i < 256; i++) {
440 + struct pack_midx_entry *next = list;
441 +
442 + while (next < last && next->oid.hash[0] == i) {
443 + count++;
444 + next++;
445 + }
446 +
447 + hashwrite_be32(f, count);
448 + list = next;
449 + }
450 +
451 + return 0;
452 +}
453 +
454 +static int write_midx_oid_lookup(struct hashfile *f,
455 + void *data)
456 +{
457 + struct write_midx_context *ctx = data;
458 + unsigned char hash_len = the_hash_algo->rawsz;
459 + struct pack_midx_entry *list = ctx->entries;
460 + uint32_t i;
461 +
462 + for (i = 0; i < ctx->entries_nr; i++) {
463 + struct pack_midx_entry *obj = list++;
464 +
465 + if (i < ctx->entries_nr - 1) {
466 + struct pack_midx_entry *next = list;
467 + if (oidcmp(&obj->oid, &next->oid) >= 0)
468 + BUG("OIDs not in order: %s >= %s",
469 + oid_to_hex(&obj->oid),
470 + oid_to_hex(&next->oid));
471 + }
472 +
473 + hashwrite(f, obj->oid.hash, (int)hash_len);
474 + }
475 +
476 + return 0;
477 +}
478 +
479 +static int write_midx_object_offsets(struct hashfile *f,
480 + void *data)
481 +{
482 + struct write_midx_context *ctx = data;
483 + struct pack_midx_entry *list = ctx->entries;
484 + uint32_t i, nr_large_offset = 0;
485 +
486 + for (i = 0; i < ctx->entries_nr; i++) {
487 + struct pack_midx_entry *obj = list++;
488 +
489 + if (ctx->pack_perm[obj->pack_int_id] == PACK_EXPIRED)
490 + BUG("object %s is in an expired pack with int-id %d",
491 + oid_to_hex(&obj->oid),
492 + obj->pack_int_id);
493 +
494 + hashwrite_be32(f, ctx->pack_perm[obj->pack_int_id]);
495 +
496 + if (ctx->large_offsets_needed && obj->offset >> 31)
497 + hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++);
498 + else if (!ctx->large_offsets_needed && obj->offset >> 32)
499 + BUG("object %s requires a large offset (%"PRIx64") but the MIDX is not writing large offsets!",
500 + oid_to_hex(&obj->oid),
501 + obj->offset);
502 + else
503 + hashwrite_be32(f, (uint32_t)obj->offset);
504 + }
505 +
506 + return 0;
507 +}
508 +
509 +static int write_midx_large_offsets(struct hashfile *f,
510 + void *data)
511 +{
512 + struct write_midx_context *ctx = data;
513 + struct pack_midx_entry *list = ctx->entries;
514 + struct pack_midx_entry *end = ctx->entries + ctx->entries_nr;
515 + uint32_t nr_large_offset = ctx->num_large_offsets;
516 +
517 + while (nr_large_offset) {
518 + struct pack_midx_entry *obj;
519 + uint64_t offset;
520 +
521 + if (list >= end)
522 + BUG("too many large-offset objects");
523 +
524 + obj = list++;
525 + offset = obj->offset;
526 +
527 + if (!(offset >> 31))
528 + continue;
529 +
530 + hashwrite_be64(f, offset);
531 +
532 + nr_large_offset--;
533 + }
534 +
535 + return 0;
536 +}
537 +
538 +static int write_midx_revindex(struct hashfile *f,
539 + void *data)
540 +{
541 + struct write_midx_context *ctx = data;
542 + uint32_t i;
543 +
544 + for (i = 0; i < ctx->entries_nr; i++)
545 + hashwrite_be32(f, ctx->pack_order[i]);
546 +
547 + return 0;
548 +}
549 +
550 +struct midx_pack_order_data {
551 + uint32_t nr;
552 + uint32_t pack;
553 + off_t offset;
554 +};
555 +
556 +static int midx_pack_order_cmp(const void *va, const void *vb)
557 +{
558 + const struct midx_pack_order_data *a = va, *b = vb;
559 + if (a->pack < b->pack)
560 + return -1;
561 + else if (a->pack > b->pack)
562 + return 1;
563 + else if (a->offset < b->offset)
564 + return -1;
565 + else if (a->offset > b->offset)
566 + return 1;
567 + else
568 + return 0;
569 +}
570 +
571 +static uint32_t *midx_pack_order(struct write_midx_context *ctx)
572 +{
573 + struct midx_pack_order_data *data;
574 + uint32_t *pack_order;
575 + uint32_t i;
576 +
577 + trace2_region_enter("midx", "midx_pack_order", the_repository);
578 +
579 + ALLOC_ARRAY(data, ctx->entries_nr);
580 + for (i = 0; i < ctx->entries_nr; i++) {
581 + struct pack_midx_entry *e = &ctx->entries[i];
582 + data[i].nr = i;
583 + data[i].pack = ctx->pack_perm[e->pack_int_id];
584 + if (!e->preferred)
585 + data[i].pack |= (1U << 31);
586 + data[i].offset = e->offset;
587 + }
588 +
589 + QSORT(data, ctx->entries_nr, midx_pack_order_cmp);
590 +
591 + ALLOC_ARRAY(pack_order, ctx->entries_nr);
592 + for (i = 0; i < ctx->entries_nr; i++) {
593 + struct pack_midx_entry *e = &ctx->entries[data[i].nr];
594 + struct pack_info *pack = &ctx->info[ctx->pack_perm[e->pack_int_id]];
595 + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
596 + pack->bitmap_pos = i;
597 + pack->bitmap_nr++;
598 + pack_order[i] = data[i].nr;
599 + }
600 + for (i = 0; i < ctx->nr; i++) {
601 + struct pack_info *pack = &ctx->info[ctx->pack_perm[i]];
602 + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
603 + pack->bitmap_pos = 0;
604 + }
605 + free(data);
606 +
607 + trace2_region_leave("midx", "midx_pack_order", the_repository);
608 +
609 + return pack_order;
610 +}
611 +
612 +static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
613 + struct write_midx_context *ctx)
614 +{
615 + struct strbuf buf = STRBUF_INIT;
616 + const char *tmp_file;
617 +
618 + trace2_region_enter("midx", "write_midx_reverse_index", the_repository);
619 +
620 + strbuf_addf(&buf, "%s-%s.rev", midx_name, hash_to_hex(midx_hash));
621 +
622 + tmp_file = write_rev_file_order(NULL, ctx->pack_order, ctx->entries_nr,
623 + midx_hash, WRITE_REV);
624 +
625 + if (finalize_object_file(tmp_file, buf.buf))
626 + die(_("cannot store reverse index file"));
627 +
628 + strbuf_release(&buf);
629 +
630 + trace2_region_leave("midx", "write_midx_reverse_index", the_repository);
631 +}
632 +
633 +static void prepare_midx_packing_data(struct packing_data *pdata,
634 + struct write_midx_context *ctx)
635 +{
636 + uint32_t i;
637 +
638 + trace2_region_enter("midx", "prepare_midx_packing_data", the_repository);
639 +
640 + memset(pdata, 0, sizeof(struct packing_data));
641 + prepare_packing_data(the_repository, pdata);
642 +
643 + for (i = 0; i < ctx->entries_nr; i++) {
644 + struct pack_midx_entry *from = &ctx->entries[ctx->pack_order[i]];
645 + struct object_entry *to = packlist_alloc(pdata, &from->oid);
646 +
647 + oe_set_in_pack(pdata, to,
648 + ctx->info[ctx->pack_perm[from->pack_int_id]].p);
649 + }
650 +
651 + trace2_region_leave("midx", "prepare_midx_packing_data", the_repository);
652 +}
653 +
654 +static int add_ref_to_pending(const char *refname,
655 + const struct object_id *oid,
656 + int flag, void *cb_data)
657 +{
658 + struct rev_info *revs = (struct rev_info*)cb_data;
659 + struct object_id peeled;
660 + struct object *object;
661 +
662 + if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) {
663 + warning("symbolic ref is dangling: %s", refname);
664 + return 0;
665 + }
666 +
667 + if (!peel_iterated_oid(oid, &peeled))
668 + oid = &peeled;
669 +
670 + object = parse_object_or_die(oid, refname);
671 + if (object->type != OBJ_COMMIT)
672 + return 0;
673 +
674 + add_pending_object(revs, object, "");
675 + if (bitmap_is_preferred_refname(revs->repo, refname))
676 + object->flags |= NEEDS_BITMAP;
677 + return 0;
678 +}
679 +
680 +struct bitmap_commit_cb {
681 + struct commit **commits;
682 + size_t commits_nr, commits_alloc;
683 +
684 + struct write_midx_context *ctx;
685 +};
686 +
687 +static const struct object_id *bitmap_oid_access(size_t index,
688 + const void *_entries)
689 +{
690 + const struct pack_midx_entry *entries = _entries;
691 + return &entries[index].oid;
692 +}
693 +
694 +static void bitmap_show_commit(struct commit *commit, void *_data)
695 +{
696 + struct bitmap_commit_cb *data = _data;
697 + int pos = oid_pos(&commit->object.oid, data->ctx->entries,
698 + data->ctx->entries_nr,
699 + bitmap_oid_access);
700 + if (pos < 0)
701 + return;
702 +
703 + ALLOC_GROW(data->commits, data->commits_nr + 1, data->commits_alloc);
704 + data->commits[data->commits_nr++] = commit;
705 +}
706 +
707 +static int read_refs_snapshot(const char *refs_snapshot,
708 + struct rev_info *revs)
709 +{
710 + struct strbuf buf = STRBUF_INIT;
711 + struct object_id oid;
712 + FILE *f = xfopen(refs_snapshot, "r");
713 +
714 + while (strbuf_getline(&buf, f) != EOF) {
715 + struct object *object;
716 + int preferred = 0;
717 + char *hex = buf.buf;
718 + const char *end = NULL;
719 +
720 + if (buf.len && *buf.buf == '+') {
721 + preferred = 1;
722 + hex = &buf.buf[1];
723 + }
724 +
725 + if (parse_oid_hex(hex, &oid, &end) < 0)
726 + die(_("could not parse line: %s"), buf.buf);
727 + if (*end)
728 + die(_("malformed line: %s"), buf.buf);
729 +
730 + object = parse_object_or_die(&oid, NULL);
731 + if (preferred)
732 + object->flags |= NEEDS_BITMAP;
733 +
734 + add_pending_object(revs, object, "");
735 + }
736 +
737 + fclose(f);
738 + strbuf_release(&buf);
739 + return 0;
740 +}
741 +static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,
742 + const char *refs_snapshot,
743 + struct write_midx_context *ctx)
744 +{
745 + struct rev_info revs;
746 + struct bitmap_commit_cb cb = {0};
747 +
748 + trace2_region_enter("midx", "find_commits_for_midx_bitmap",
749 + the_repository);
750 +
751 + cb.ctx = ctx;
752 +
753 + repo_init_revisions(the_repository, &revs, NULL);
754 + if (refs_snapshot) {
755 + read_refs_snapshot(refs_snapshot, &revs);
756 + } else {
757 + setup_revisions(0, NULL, &revs, NULL);
758 + for_each_ref(add_ref_to_pending, &revs);
759 + }
760 +
761 + /*
762 + * Skipping promisor objects here is intentional, since it only excludes
763 + * them from the list of reachable commits that we want to select from
764 + * when computing the selection of MIDX'd commits to receive bitmaps.
765 + *
766 + * Reachability bitmaps do require that their objects be closed under
767 + * reachability, but fetching any objects missing from promisors at this
768 + * point is too late. But, if one of those objects can be reached from
769 + * an another object that is included in the bitmap, then we will
770 + * complain later that we don't have reachability closure (and fail
771 + * appropriately).
772 + */
773 + fetch_if_missing = 0;
774 + revs.exclude_promisor_objects = 1;
775 +
776 + if (prepare_revision_walk(&revs))
777 + die(_("revision walk setup failed"));
778 +
779 + traverse_commit_list(&revs, bitmap_show_commit, NULL, &cb);
780 + if (indexed_commits_nr_p)
781 + *indexed_commits_nr_p = cb.commits_nr;
782 +
783 + release_revisions(&revs);
784 +
785 + trace2_region_leave("midx", "find_commits_for_midx_bitmap",
786 + the_repository);
787 +
788 + return cb.commits;
789 +}
790 +
791 +static int write_midx_bitmap(const char *midx_name,
792 + const unsigned char *midx_hash,
793 + struct packing_data *pdata,
794 + struct commit **commits,
795 + uint32_t commits_nr,
796 + uint32_t *pack_order,
797 + unsigned flags)
798 +{
799 + int ret, i;
800 + uint16_t options = 0;
801 + struct pack_idx_entry **index;
802 + char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name,
803 + hash_to_hex(midx_hash));
804 +
805 + trace2_region_enter("midx", "write_midx_bitmap", the_repository);
806 +
807 + if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
808 + options |= BITMAP_OPT_HASH_CACHE;
809 +
810 + if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
811 + options |= BITMAP_OPT_LOOKUP_TABLE;
812 +
813 + /*
814 + * Build the MIDX-order index based on pdata.objects (which is already
815 + * in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
816 + * this order).
817 + */
818 + ALLOC_ARRAY(index, pdata->nr_objects);
819 + for (i = 0; i < pdata->nr_objects; i++)
820 + index[i] = &pdata->objects[i].idx;
821 +
822 + bitmap_writer_show_progress(flags & MIDX_PROGRESS);
823 + bitmap_writer_build_type_index(pdata, index, pdata->nr_objects);
824 +
825 + /*
826 + * bitmap_writer_finish expects objects in lex order, but pack_order
827 + * gives us exactly that. use it directly instead of re-sorting the
828 + * array.
829 + *
830 + * This changes the order of objects in 'index' between
831 + * bitmap_writer_build_type_index and bitmap_writer_finish.
832 + *
833 + * The same re-ordering takes place in the single-pack bitmap code via
834 + * write_idx_file(), which is called by finish_tmp_packfile(), which
835 + * happens between bitmap_writer_build_type_index() and
836 + * bitmap_writer_finish().
837 + */
838 + for (i = 0; i < pdata->nr_objects; i++)
839 + index[pack_order[i]] = &pdata->objects[i].idx;
840 +
841 + bitmap_writer_select_commits(commits, commits_nr, -1);
842 + ret = bitmap_writer_build(pdata);
843 + if (ret < 0)
844 + goto cleanup;
845 +
846 + bitmap_writer_set_checksum(midx_hash);
847 + bitmap_writer_finish(index, pdata->nr_objects, bitmap_name, options);
848 +
849 +cleanup:
850 + free(index);
851 + free(bitmap_name);
852 +
853 + trace2_region_leave("midx", "write_midx_bitmap", the_repository);
854 +
855 + return ret;
856 +}
857 +
858 +static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
859 + const char *object_dir)
860 +{
861 + struct multi_pack_index *result = NULL;
862 + struct multi_pack_index *cur;
863 + char *obj_dir_real = real_pathdup(object_dir, 1);
864 + struct strbuf cur_path_real = STRBUF_INIT;
865 +
866 + /* Ensure the given object_dir is local, or a known alternate. */
867 + find_odb(r, obj_dir_real);
868 +
869 + for (cur = get_multi_pack_index(r); cur; cur = cur->next) {
870 + strbuf_realpath(&cur_path_real, cur->object_dir, 1);
871 + if (!strcmp(obj_dir_real, cur_path_real.buf)) {
872 + result = cur;
873 + goto cleanup;
874 + }
875 + }
876 +
877 +cleanup:
878 + free(obj_dir_real);
879 + strbuf_release(&cur_path_real);
880 + return result;
881 +}
882 +
883 +static int write_midx_internal(const char *object_dir,
884 + struct string_list *packs_to_include,
885 + struct string_list *packs_to_drop,
886 + const char *preferred_pack_name,
887 + const char *refs_snapshot,
888 + unsigned flags)
889 +{
890 + struct strbuf midx_name = STRBUF_INIT;
891 + unsigned char midx_hash[GIT_MAX_RAWSZ];
892 + uint32_t i;
893 + struct hashfile *f = NULL;
894 + struct lock_file lk;
895 + struct write_midx_context ctx = { 0 };
896 + int bitmapped_packs_concat_len = 0;
897 + int pack_name_concat_len = 0;
898 + int dropped_packs = 0;
899 + int result = 0;
900 + struct chunkfile *cf;
901 +
902 + trace2_region_enter("midx", "write_midx_internal", the_repository);
903 +
904 + get_midx_filename(&midx_name, object_dir);
905 + if (safe_create_leading_directories(midx_name.buf))
906 + die_errno(_("unable to create leading directories of %s"),
907 + midx_name.buf);
908 +
909 + if (!packs_to_include) {
910 + /*
911 + * Only reference an existing MIDX when not filtering which
912 + * packs to include, since all packs and objects are copied
913 + * blindly from an existing MIDX if one is present.
914 + */
915 + ctx.m = lookup_multi_pack_index(the_repository, object_dir);
916 + }
917 +
918 + if (ctx.m && !midx_checksum_valid(ctx.m)) {
919 + warning(_("ignoring existing multi-pack-index; checksum mismatch"));
920 + ctx.m = NULL;
921 + }
922 +
923 + ctx.nr = 0;
924 + ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
925 + ctx.info = NULL;
926 + ALLOC_ARRAY(ctx.info, ctx.alloc);
927 +
928 + if (ctx.m) {
929 + for (i = 0; i < ctx.m->num_packs; i++) {
930 + ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc);
931 +
932 + if (flags & MIDX_WRITE_REV_INDEX) {
933 + /*
934 + * If generating a reverse index, need to have
935 + * packed_git's loaded to compare their
936 + * mtimes and object count.
937 + */
938 + if (prepare_midx_pack(the_repository, ctx.m, i)) {
939 + error(_("could not load pack"));
940 + result = 1;
941 + goto cleanup;
942 + }
943 +
944 + if (open_pack_index(ctx.m->packs[i]))
945 + die(_("could not open index for %s"),
946 + ctx.m->packs[i]->pack_name);
947 + }
948 +
949 + fill_pack_info(&ctx.info[ctx.nr++], ctx.m->packs[i],
950 + ctx.m->pack_names[i], i);
951 + }
952 + }
953 +
954 + ctx.pack_paths_checked = 0;
955 + if (flags & MIDX_PROGRESS)
956 + ctx.progress = start_delayed_progress(_("Adding packfiles to multi-pack-index"), 0);
957 + else
958 + ctx.progress = NULL;
959 +
960 + ctx.to_include = packs_to_include;
961 +
962 + for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
963 + stop_progress(&ctx.progress);
964 +
965 + if ((ctx.m && ctx.nr == ctx.m->num_packs) &&
966 + !(packs_to_include || packs_to_drop)) {
967 + struct bitmap_index *bitmap_git;
968 + int bitmap_exists;
969 + int want_bitmap = flags & MIDX_WRITE_BITMAP;
970 +
971 + bitmap_git = prepare_midx_bitmap_git(ctx.m);
972 + bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
973 + free_bitmap_index(bitmap_git);
974 +
975 + if (bitmap_exists || !want_bitmap) {
976 + /*
977 + * The correct MIDX already exists, and so does a
978 + * corresponding bitmap (or one wasn't requested).
979 + */
980 + if (!want_bitmap)
981 + clear_midx_files_ext(object_dir, ".bitmap",
982 + NULL);
983 + goto cleanup;
984 + }
985 + }
986 +
987 + if (preferred_pack_name) {
988 + ctx.preferred_pack_idx = -1;
989 +
990 + for (i = 0; i < ctx.nr; i++) {
991 + if (!cmp_idx_or_pack_name(preferred_pack_name,
992 + ctx.info[i].pack_name)) {
993 + ctx.preferred_pack_idx = i;
994 + break;
995 + }
996 + }
997 +
998 + if (ctx.preferred_pack_idx == -1)
999 + warning(_("unknown preferred pack: '%s'"),
1000 + preferred_pack_name);
1001 + } else if (ctx.nr &&
1002 + (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) {
1003 + struct packed_git *oldest = ctx.info[ctx.preferred_pack_idx].p;
1004 + ctx.preferred_pack_idx = 0;
1005 +
1006 + if (packs_to_drop && packs_to_drop->nr)
1007 + BUG("cannot write a MIDX bitmap during expiration");
1008 +
1009 + /*
1010 + * set a preferred pack when writing a bitmap to ensure that
1011 + * the pack from which the first object is selected in pseudo
1012 + * pack-order has all of its objects selected from that pack
1013 + * (and not another pack containing a duplicate)
1014 + */
1015 + for (i = 1; i < ctx.nr; i++) {
1016 + struct packed_git *p = ctx.info[i].p;
1017 +
1018 + if (!oldest->num_objects || p->mtime < oldest->mtime) {
1019 + oldest = p;
1020 + ctx.preferred_pack_idx = i;
1021 + }
1022 + }
1023 +
1024 + if (!oldest->num_objects) {
1025 + /*
1026 + * If all packs are empty; unset the preferred index.
1027 + * This is acceptable since there will be no duplicate
1028 + * objects to resolve, so the preferred value doesn't
1029 + * matter.
1030 + */
1031 + ctx.preferred_pack_idx = -1;
1032 + }
1033 + } else {
1034 + /*
1035 + * otherwise don't mark any pack as preferred to avoid
1036 + * interfering with expiration logic below
1037 + */
1038 + ctx.preferred_pack_idx = -1;
1039 + }
1040 +
1041 + if (ctx.preferred_pack_idx > -1) {
1042 + struct packed_git *preferred = ctx.info[ctx.preferred_pack_idx].p;
1043 + if (!preferred->num_objects) {
1044 + error(_("cannot select preferred pack %s with no objects"),
1045 + preferred->pack_name);
1046 + result = 1;
1047 + goto cleanup;
1048 + }
1049 + }
1050 +
1051 + ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr,
1052 + ctx.preferred_pack_idx);
1053 +
1054 + ctx.large_offsets_needed = 0;
1055 + for (i = 0; i < ctx.entries_nr; i++) {
1056 + if (ctx.entries[i].offset > 0x7fffffff)
1057 + ctx.num_large_offsets++;
1058 + if (ctx.entries[i].offset > 0xffffffff)
1059 + ctx.large_offsets_needed = 1;
1060 + }
1061 +
1062 + QSORT(ctx.info, ctx.nr, pack_info_compare);
1063 +
1064 + if (packs_to_drop && packs_to_drop->nr) {
1065 + int drop_index = 0;
1066 + int missing_drops = 0;
1067 +
1068 + for (i = 0; i < ctx.nr && drop_index < packs_to_drop->nr; i++) {
1069 + int cmp = strcmp(ctx.info[i].pack_name,
1070 + packs_to_drop->items[drop_index].string);
1071 +
1072 + if (!cmp) {
1073 + drop_index++;
1074 + ctx.info[i].expired = 1;
1075 + } else if (cmp > 0) {
1076 + error(_("did not see pack-file %s to drop"),
1077 + packs_to_drop->items[drop_index].string);
1078 + drop_index++;
1079 + missing_drops++;
1080 + i--;
1081 + } else {
1082 + ctx.info[i].expired = 0;
1083 + }
1084 + }
1085 +
1086 + if (missing_drops) {
1087 + result = 1;
1088 + goto cleanup;
1089 + }
1090 + }
1091 +
1092 + /*
1093 + * pack_perm stores a permutation between pack-int-ids from the
1094 + * previous multi-pack-index to the new one we are writing:
1095 + *
1096 + * pack_perm[old_id] = new_id
1097 + */
1098 + ALLOC_ARRAY(ctx.pack_perm, ctx.nr);
1099 + for (i = 0; i < ctx.nr; i++) {
1100 + if (ctx.info[i].expired) {
1101 + dropped_packs++;
1102 + ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
1103 + } else {
1104 + ctx.pack_perm[ctx.info[i].orig_pack_int_id] = i - dropped_packs;
1105 + }
1106 + }
1107 +
1108 + for (i = 0; i < ctx.nr; i++) {
1109 + if (ctx.info[i].expired)
1110 + continue;
1111 + pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
1112 + bitmapped_packs_concat_len += 2 * sizeof(uint32_t);
1113 + }
1114 +
1115 + /* Check that the preferred pack wasn't expired (if given). */
1116 + if (preferred_pack_name) {
1117 + struct pack_info *preferred = bsearch(preferred_pack_name,
1118 + ctx.info, ctx.nr,
1119 + sizeof(*ctx.info),
1120 + idx_or_pack_name_cmp);
1121 + if (preferred) {
1122 + uint32_t perm = ctx.pack_perm[preferred->orig_pack_int_id];
1123 + if (perm == PACK_EXPIRED)
1124 + warning(_("preferred pack '%s' is expired"),
1125 + preferred_pack_name);
1126 + }
1127 + }
1128 +
1129 + if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
1130 + pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
1131 + (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
1132 +
1133 + hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR);
1134 + f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
1135 +
1136 + if (ctx.nr - dropped_packs == 0) {
1137 + error(_("no pack files to index."));
1138 + result = 1;
1139 + goto cleanup;
1140 + }
1141 +
1142 + if (!ctx.entries_nr) {
1143 + if (flags & MIDX_WRITE_BITMAP)
1144 + warning(_("refusing to write multi-pack .bitmap without any objects"));
1145 + flags &= ~(MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP);
1146 + }
1147 +
1148 + cf = init_chunkfile(f);
1149 +
1150 + add_chunk(cf, MIDX_CHUNKID_PACKNAMES, pack_name_concat_len,
1151 + write_midx_pack_names);
1152 + add_chunk(cf, MIDX_CHUNKID_OIDFANOUT, MIDX_CHUNK_FANOUT_SIZE,
1153 + write_midx_oid_fanout);
1154 + add_chunk(cf, MIDX_CHUNKID_OIDLOOKUP,
1155 + st_mult(ctx.entries_nr, the_hash_algo->rawsz),
1156 + write_midx_oid_lookup);
1157 + add_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS,
1158 + st_mult(ctx.entries_nr, MIDX_CHUNK_OFFSET_WIDTH),
1159 + write_midx_object_offsets);
1160 +
1161 + if (ctx.large_offsets_needed)
1162 + add_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS,
1163 + st_mult(ctx.num_large_offsets,
1164 + MIDX_CHUNK_LARGE_OFFSET_WIDTH),
1165 + write_midx_large_offsets);
1166 +
1167 + if (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP)) {
1168 + ctx.pack_order = midx_pack_order(&ctx);
1169 + add_chunk(cf, MIDX_CHUNKID_REVINDEX,
1170 + st_mult(ctx.entries_nr, sizeof(uint32_t)),
1171 + write_midx_revindex);
1172 + add_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS,
1173 + bitmapped_packs_concat_len,
1174 + write_midx_bitmapped_packs);
1175 + }
1176 +
1177 + write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs);
1178 + write_chunkfile(cf, &ctx);
1179 +
1180 + finalize_hashfile(f, midx_hash, FSYNC_COMPONENT_PACK_METADATA,
1181 + CSUM_FSYNC | CSUM_HASH_IN_STREAM);
1182 + free_chunkfile(cf);
1183 +
1184 + if (flags & MIDX_WRITE_REV_INDEX &&
1185 + git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
1186 + write_midx_reverse_index(midx_name.buf, midx_hash, &ctx);
1187 +
1188 + if (flags & MIDX_WRITE_BITMAP) {
1189 + struct packing_data pdata;
1190 + struct commit **commits;
1191 + uint32_t commits_nr;
1192 +
1193 + if (!ctx.entries_nr)
1194 + BUG("cannot write a bitmap without any objects");
1195 +
1196 + prepare_midx_packing_data(&pdata, &ctx);
1197 +
1198 + commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, &ctx);
1199 +
1200 + /*
1201 + * The previous steps translated the information from
1202 + * 'entries' into information suitable for constructing
1203 + * bitmaps. We no longer need that array, so clear it to
1204 + * reduce memory pressure.
1205 + */
1206 + FREE_AND_NULL(ctx.entries);
1207 + ctx.entries_nr = 0;
1208 +
1209 + if (write_midx_bitmap(midx_name.buf, midx_hash, &pdata,
1210 + commits, commits_nr, ctx.pack_order,
1211 + flags) < 0) {
1212 + error(_("could not write multi-pack bitmap"));
1213 + result = 1;
1214 + clear_packing_data(&pdata);
1215 + free(commits);
1216 + goto cleanup;
1217 + }
1218 +
1219 + clear_packing_data(&pdata);
1220 + free(commits);
1221 + }
1222 + /*
1223 + * NOTE: Do not use ctx.entries beyond this point, since it might
1224 + * have been freed in the previous if block.
1225 + */
1226 +
1227 + if (ctx.m)
1228 + close_object_store(the_repository->objects);
1229 +
1230 + if (commit_lock_file(&lk) < 0)
1231 + die_errno(_("could not write multi-pack-index"));
1232 +
1233 + clear_midx_files_ext(object_dir, ".bitmap", midx_hash);
1234 + clear_midx_files_ext(object_dir, ".rev", midx_hash);
1235 +
1236 +cleanup:
1237 + for (i = 0; i < ctx.nr; i++) {
1238 + if (ctx.info[i].p) {
1239 + close_pack(ctx.info[i].p);
1240 + free(ctx.info[i].p);
1241 + }
1242 + free(ctx.info[i].pack_name);
1243 + }
1244 +
1245 + free(ctx.info);
1246 + free(ctx.entries);
1247 + free(ctx.pack_perm);
1248 + free(ctx.pack_order);
1249 + strbuf_release(&midx_name);
1250 +
1251 + trace2_region_leave("midx", "write_midx_internal", the_repository);
1252 +
1253 + return result;
1254 +}
1255 +
1256 +int write_midx_file(const char *object_dir,
1257 + const char *preferred_pack_name,
1258 + const char *refs_snapshot,
1259 + unsigned flags)
1260 +{
1261 + return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name,
1262 + refs_snapshot, flags);
1263 +}
1264 +
1265 +int write_midx_file_only(const char *object_dir,
1266 + struct string_list *packs_to_include,
1267 + const char *preferred_pack_name,
1268 + const char *refs_snapshot,
1269 + unsigned flags)
1270 +{
1271 + return write_midx_internal(object_dir, packs_to_include, NULL,
1272 + preferred_pack_name, refs_snapshot, flags);
1273 +}
1274 +
1275 +int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags)
1276 +{
1277 + uint32_t i, *count, result = 0;
1278 + struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
1279 + struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
1280 + struct progress *progress = NULL;
1281 +
1282 + if (!m)
1283 + return 0;
1284 +
1285 + CALLOC_ARRAY(count, m->num_packs);
1286 +
1287 + if (flags & MIDX_PROGRESS)
1288 + progress = start_delayed_progress(_("Counting referenced objects"),
1289 + m->num_objects);
1290 + for (i = 0; i < m->num_objects; i++) {
1291 + int pack_int_id = nth_midxed_pack_int_id(m, i);
1292 + count[pack_int_id]++;
1293 + display_progress(progress, i + 1);
1294 + }
1295 + stop_progress(&progress);
1296 +
1297 + if (flags & MIDX_PROGRESS)
1298 + progress = start_delayed_progress(_("Finding and deleting unreferenced packfiles"),
1299 + m->num_packs);
1300 + for (i = 0; i < m->num_packs; i++) {
1301 + char *pack_name;
1302 + display_progress(progress, i + 1);
1303 +
1304 + if (count[i])
1305 + continue;
1306 +
1307 + if (prepare_midx_pack(r, m, i))
1308 + continue;
1309 +
1310 + if (m->packs[i]->pack_keep || m->packs[i]->is_cruft)
1311 + continue;
1312 +
1313 + pack_name = xstrdup(m->packs[i]->pack_name);
1314 + close_pack(m->packs[i]);
1315 +
1316 + string_list_insert(&packs_to_drop, m->pack_names[i]);
1317 + unlink_pack_path(pack_name, 0);
1318 + free(pack_name);
1319 + }
1320 + stop_progress(&progress);
1321 +
1322 + free(count);
1323 +
1324 + if (packs_to_drop.nr)
1325 + result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags);
1326 +
1327 + string_list_clear(&packs_to_drop, 0);
1328 +
1329 + return result;
1330 +}
1331 +
1332 +struct repack_info {
1333 + timestamp_t mtime;
1334 + uint32_t referenced_objects;
1335 + uint32_t pack_int_id;
1336 +};
1337 +
1338 +static int compare_by_mtime(const void *a_, const void *b_)
1339 +{
1340 + const struct repack_info *a, *b;
1341 +
1342 + a = (const struct repack_info *)a_;
1343 + b = (const struct repack_info *)b_;
1344 +
1345 + if (a->mtime < b->mtime)
1346 + return -1;
1347 + if (a->mtime > b->mtime)
1348 + return 1;
1349 + return 0;
1350 +}
1351 +
1352 +static int fill_included_packs_all(struct repository *r,
1353 + struct multi_pack_index *m,
1354 + unsigned char *include_pack)
1355 +{
1356 + uint32_t i, count = 0;
1357 + int pack_kept_objects = 0;
1358 +
1359 + repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
1360 +
1361 + for (i = 0; i < m->num_packs; i++) {
1362 + if (prepare_midx_pack(r, m, i))
1363 + continue;
1364 + if (!pack_kept_objects && m->packs[i]->pack_keep)
1365 + continue;
1366 + if (m->packs[i]->is_cruft)
1367 + continue;
1368 +
1369 + include_pack[i] = 1;
1370 + count++;
1371 + }
1372 +
1373 + return count < 2;
1374 +}
1375 +
1376 +static int fill_included_packs_batch(struct repository *r,
1377 + struct multi_pack_index *m,
1378 + unsigned char *include_pack,
1379 + size_t batch_size)
1380 +{
1381 + uint32_t i, packs_to_repack;
1382 + size_t total_size;
1383 + struct repack_info *pack_info;
1384 + int pack_kept_objects = 0;
1385 +
1386 + CALLOC_ARRAY(pack_info, m->num_packs);
1387 +
1388 + repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
1389 +
1390 + for (i = 0; i < m->num_packs; i++) {
1391 + pack_info[i].pack_int_id = i;
1392 +
1393 + if (prepare_midx_pack(r, m, i))
1394 + continue;
1395 +
1396 + pack_info[i].mtime = m->packs[i]->mtime;
1397 + }
1398 +
1399 + for (i = 0; i < m->num_objects; i++) {
1400 + uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
1401 + pack_info[pack_int_id].referenced_objects++;
1402 + }
1403 +
1404 + QSORT(pack_info, m->num_packs, compare_by_mtime);
1405 +
1406 + total_size = 0;
1407 + packs_to_repack = 0;
1408 + for (i = 0; total_size < batch_size && i < m->num_packs; i++) {
1409 + int pack_int_id = pack_info[i].pack_int_id;
1410 + struct packed_git *p = m->packs[pack_int_id];
1411 + size_t expected_size;
1412 +
1413 + if (!p)
1414 + continue;
1415 + if (!pack_kept_objects && p->pack_keep)
1416 + continue;
1417 + if (p->is_cruft)
1418 + continue;
1419 + if (open_pack_index(p) || !p->num_objects)
1420 + continue;
1421 +
1422 + expected_size = st_mult(p->pack_size,
1423 + pack_info[i].referenced_objects);
1424 + expected_size /= p->num_objects;
1425 +
1426 + if (expected_size >= batch_size)
1427 + continue;
1428 +
1429 + packs_to_repack++;
1430 + total_size += expected_size;
1431 + include_pack[pack_int_id] = 1;
1432 + }
1433 +
1434 + free(pack_info);
1435 +
1436 + if (packs_to_repack < 2)
1437 + return 1;
1438 +
1439 + return 0;
1440 +}
1441 +
1442 +int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags)
1443 +{
1444 + int result = 0;
1445 + uint32_t i;
1446 + unsigned char *include_pack;
1447 + struct child_process cmd = CHILD_PROCESS_INIT;
1448 + FILE *cmd_in;
1449 + struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
1450 +
1451 + /*
1452 + * When updating the default for these configuration
1453 + * variables in builtin/repack.c, these must be adjusted
1454 + * to match.
1455 + */
1456 + int delta_base_offset = 1;
1457 + int use_delta_islands = 0;
1458 +
1459 + if (!m)
1460 + return 0;
1461 +
1462 + CALLOC_ARRAY(include_pack, m->num_packs);
1463 +
1464 + if (batch_size) {
1465 + if (fill_included_packs_batch(r, m, include_pack, batch_size))
1466 + goto cleanup;
1467 + } else if (fill_included_packs_all(r, m, include_pack))
1468 + goto cleanup;
1469 +
1470 + repo_config_get_bool(r, "repack.usedeltabaseoffset", &delta_base_offset);
1471 + repo_config_get_bool(r, "repack.usedeltaislands", &use_delta_islands);
1472 +
1473 + strvec_push(&cmd.args, "pack-objects");
1474 +
1475 + strvec_pushf(&cmd.args, "%s/pack/pack", object_dir);
1476 +
1477 + if (delta_base_offset)
1478 + strvec_push(&cmd.args, "--delta-base-offset");
1479 + if (use_delta_islands)
1480 + strvec_push(&cmd.args, "--delta-islands");
1481 +
1482 + if (flags & MIDX_PROGRESS)
1483 + strvec_push(&cmd.args, "--progress");
1484 + else
1485 + strvec_push(&cmd.args, "-q");
1486 +
1487 + cmd.git_cmd = 1;
1488 + cmd.in = cmd.out = -1;
1489 +
1490 + if (start_command(&cmd)) {
1491 + error(_("could not start pack-objects"));
1492 + result = 1;
1493 + goto cleanup;
1494 + }
1495 +
1496 + cmd_in = xfdopen(cmd.in, "w");
1497 +
1498 + for (i = 0; i < m->num_objects; i++) {
1499 + struct object_id oid;
1500 + uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
1501 +
1502 + if (!include_pack[pack_int_id])
1503 + continue;
1504 +
1505 + nth_midxed_object_oid(&oid, m, i);
1506 + fprintf(cmd_in, "%s\n", oid_to_hex(&oid));
1507 + }
1508 + fclose(cmd_in);
1509 +
1510 + if (finish_command(&cmd)) {
1511 + error(_("could not finish pack-objects"));
1512 + result = 1;
1513 + goto cleanup;
1514 + }
1515 +
1516 + result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags);
1517 +
1518 +cleanup:
1519 + free(include_pack);
1520 + return result;
1521 +}
midx.c
+170 -1695
@@ -1,52 +1,22 @@
1 #include "git-compat-util.h"
2 -#include "abspath.h"
2 #include "config.h"
4 -#include "csum-file.h"
3 #include "dir.h"
6 -#include "gettext.h"
4 #include "hex.h"
8 -#include "lockfile.h"
5 #include "packfile.h"
6 #include "object-file.h"
11 -#include "object-store-ll.h"
7 #include "hash-lookup.h"
8 #include "midx.h"
9 #include "progress.h"
10 #include "trace2.h"
16 -#include "run-command.h"
17 -#include "repository.h"
11 #include "chunk-format.h"
19 -#include "pack.h"
12 #include "pack-bitmap.h"
21 -#include "refs.h"
22 -#include "revision.h"
23 -#include "list-objects.h"
13 #include "pack-revindex.h"
14
26 -#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
27 -#define MIDX_VERSION 1
28 -#define MIDX_BYTE_FILE_VERSION 4
29 -#define MIDX_BYTE_HASH_VERSION 5
30 -#define MIDX_BYTE_NUM_CHUNKS 6
31 -#define MIDX_BYTE_NUM_PACKS 8
32 -#define MIDX_HEADER_SIZE 12
33 -#define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz)
34 -
35 -#define MIDX_CHUNK_ALIGNMENT 4
36 -#define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
37 -#define MIDX_CHUNKID_BITMAPPEDPACKS 0x42544d50 /* "BTMP" */
38 -#define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
39 -#define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
40 -#define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
41 -#define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
42 -#define MIDX_CHUNKID_REVINDEX 0x52494458 /* "RIDX" */
43 -#define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
44 -#define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
45 -#define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
46 -#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t))
47 -#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
48 -
49 -#define PACK_EXPIRED UINT_MAX
15 +int midx_checksum_valid(struct multi_pack_index *m);
16 +void clear_midx_files_ext(const char *object_dir, const char *ext,
17 + unsigned char *keep_hash);
18 +int cmp_idx_or_pack_name(const char *idx_or_pack_name,
19 + const char *idx_name);
20
21 const unsigned char *get_midx_checksum(struct multi_pack_index *m)
22 {
@@ -115,6 +85,8 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,
85 return 0;
86 }
87
88 +#define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz)
89 +
90 struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local)
91 {
92 struct multi_pack_index *m = NULL;
@@ -294,6 +266,8 @@ int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t
266 return 0;
267 }
268
269 +#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t))
270 +
271 int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m,
272 struct bitmapped_pack *bp, uint32_t pack_int_id)
273 {
@@ -400,8 +374,8 @@ int fill_midx_entry(struct repository *r,
374 }
375
376 /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
403 -static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
404 - const char *idx_name)
377 +int cmp_idx_or_pack_name(const char *idx_or_pack_name,
378 + const char *idx_name)
379 {
380 /* Skip past any initial matching prefix. */
381 while (*idx_name && *idx_name == *idx_or_pack_name) {
@@ -508,1731 +482,232 @@ int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, i
482 return 0;
483 }
484
511 -static size_t write_midx_header(struct hashfile *f,
512 - unsigned char num_chunks,
513 - uint32_t num_packs)
514 -{
515 - hashwrite_be32(f, MIDX_SIGNATURE);
516 - hashwrite_u8(f, MIDX_VERSION);
517 - hashwrite_u8(f, oid_version(the_hash_algo));
518 - hashwrite_u8(f, num_chunks);
519 - hashwrite_u8(f, 0); /* unused */
520 - hashwrite_be32(f, num_packs);
521 -
522 - return MIDX_HEADER_SIZE;
523 -}
524 -
525 -#define BITMAP_POS_UNKNOWN (~((uint32_t)0))
526 -
527 -struct pack_info {
528 - uint32_t orig_pack_int_id;
529 - char *pack_name;
530 - struct packed_git *p;
531 -
532 - uint32_t bitmap_pos;
533 - uint32_t bitmap_nr;
534 -
535 - unsigned expired : 1;
536 -};
537 -
538 -static void fill_pack_info(struct pack_info *info,
539 - struct packed_git *p, const char *pack_name,
540 - uint32_t orig_pack_int_id)
541 -{
542 - memset(info, 0, sizeof(struct pack_info));
543 -
544 - info->orig_pack_int_id = orig_pack_int_id;
545 - info->pack_name = xstrdup(pack_name);
546 - info->p = p;
547 - info->bitmap_pos = BITMAP_POS_UNKNOWN;
548 -}
549 -
550 -static int pack_info_compare(const void *_a, const void *_b)
551 -{
552 - struct pack_info *a = (struct pack_info *)_a;
553 - struct pack_info *b = (struct pack_info *)_b;
554 - return strcmp(a->pack_name, b->pack_name);
555 -}
556 -
557 -static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
485 +int midx_checksum_valid(struct multi_pack_index *m)
486 {
559 - const char *pack_name = _va;
560 - const struct pack_info *compar = _vb;
561 -
562 - return cmp_idx_or_pack_name(pack_name, compar->pack_name);
487 + return hashfile_checksum_valid(m->data, m->data_len);
488 }
489
565 -struct write_midx_context {
566 - struct pack_info *info;
567 - size_t nr;
568 - size_t alloc;
569 - struct multi_pack_index *m;
570 - struct progress *progress;
571 - unsigned pack_paths_checked;
572 -
573 - struct pack_midx_entry *entries;
574 - size_t entries_nr;
575 -
576 - uint32_t *pack_perm;
577 - uint32_t *pack_order;
578 - unsigned large_offsets_needed:1;
579 - uint32_t num_large_offsets;
580 -
581 - int preferred_pack_idx;
582 -
583 - struct string_list *to_include;
490 +struct clear_midx_data {
491 + char *keep;
492 + const char *ext;
493 };
494
586 -static void add_pack_to_midx(const char *full_path, size_t full_path_len,
587 - const char *file_name, void *data)
495 +static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED,
496 + const char *file_name, void *_data)
497 {
589 - struct write_midx_context *ctx = data;
590 - struct packed_git *p;
591 -
592 - if (ends_with(file_name, ".idx")) {
593 - display_progress(ctx->progress, ++ctx->pack_paths_checked);
594 - /*
595 - * Note that at most one of ctx->m and ctx->to_include are set,
596 - * so we are testing midx_contains_pack() and
597 - * string_list_has_string() independently (guarded by the
598 - * appropriate NULL checks).
599 - *
600 - * We could support passing to_include while reusing an existing
601 - * MIDX, but don't currently since the reuse process drags
602 - * forward all packs from an existing MIDX (without checking
603 - * whether or not they appear in the to_include list).
604 - *
605 - * If we added support for that, these next two conditional
606 - * should be performed independently (likely checking
607 - * to_include before the existing MIDX).
608 - */
609 - if (ctx->m && midx_contains_pack(ctx->m, file_name))
610 - return;
611 - else if (ctx->to_include &&
612 - !string_list_has_string(ctx->to_include, file_name))
613 - return;
614 -
615 - ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
616 -
617 - p = add_packed_git(full_path, full_path_len, 0);
618 - if (!p) {
619 - warning(_("failed to add packfile '%s'"),
620 - full_path);
621 - return;
622 - }
498 + struct clear_midx_data *data = _data;
499
624 - if (open_pack_index(p)) {
625 - warning(_("failed to open pack-index '%s'"),
626 - full_path);
627 - close_pack(p);
628 - free(p);
629 - return;
630 - }
500 + if (!(starts_with(file_name, "multi-pack-index-") &&
501 + ends_with(file_name, data->ext)))
502 + return;
503 + if (data->keep && !strcmp(data->keep, file_name))
504 + return;
505
632 - fill_pack_info(&ctx->info[ctx->nr], p, file_name, ctx->nr);
633 - ctx->nr++;
634 - }
506 + if (unlink(full_path))
507 + die_errno(_("failed to remove %s"), full_path);
508 }
509
637 -struct pack_midx_entry {
638 - struct object_id oid;
639 - uint32_t pack_int_id;
640 - time_t pack_mtime;
641 - uint64_t offset;
642 - unsigned preferred : 1;
643 -};
644 -
645 -static int midx_oid_compare(const void *_a, const void *_b)
510 +void clear_midx_files_ext(const char *object_dir, const char *ext,
511 + unsigned char *keep_hash)
512 {
647 - const struct pack_midx_entry *a = (const struct pack_midx_entry *)_a;
648 - const struct pack_midx_entry *b = (const struct pack_midx_entry *)_b;
649 - int cmp = oidcmp(&a->oid, &b->oid);
650 -
651 - if (cmp)
652 - return cmp;
513 + struct clear_midx_data data;
514 + memset(&data, 0, sizeof(struct clear_midx_data));
515
654 - /* Sort objects in a preferred pack first when multiple copies exist. */
655 - if (a->preferred > b->preferred)
656 - return -1;
657 - if (a->preferred < b->preferred)
658 - return 1;
516 + if (keep_hash)
517 + data.keep = xstrfmt("multi-pack-index-%s%s",
518 + hash_to_hex(keep_hash), ext);
519 + data.ext = ext;
520
660 - if (a->pack_mtime > b->pack_mtime)
661 - return -1;
662 - else if (a->pack_mtime < b->pack_mtime)
663 - return 1;
521 + for_each_file_in_pack_dir(object_dir,
522 + clear_midx_file_ext,
523 + &data);
524
665 - return a->pack_int_id - b->pack_int_id;
525 + free(data.keep);
526 }
527
668 -static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
669 - struct pack_midx_entry *e,
670 - uint32_t pos)
528 +void clear_midx_file(struct repository *r)
529 {
672 - if (pos >= m->num_objects)
673 - return 1;
530 + struct strbuf midx = STRBUF_INIT;
531
675 - nth_midxed_object_oid(&e->oid, m, pos);
676 - e->pack_int_id = nth_midxed_pack_int_id(m, pos);
677 - e->offset = nth_midxed_offset(m, pos);
532 + get_midx_filename(&midx, r->objects->odb->path);
533
679 - /* consider objects in midx to be from "old" packs */
680 - e->pack_mtime = 0;
681 - return 0;
682 -}
534 + if (r->objects && r->objects->multi_pack_index) {
535 + close_midx(r->objects->multi_pack_index);
536 + r->objects->multi_pack_index = NULL;
537 + }
538
684 -static void fill_pack_entry(uint32_t pack_int_id,
685 - struct packed_git *p,
686 - uint32_t cur_object,
687 - struct pack_midx_entry *entry,
688 - int preferred)
689 -{
690 - if (nth_packed_object_id(&entry->oid, p, cur_object) < 0)
691 - die(_("failed to locate object %d in packfile"), cur_object);
539 + if (remove_path(midx.buf))
540 + die(_("failed to clear multi-pack-index at %s"), midx.buf);
541
693 - entry->pack_int_id = pack_int_id;
694 - entry->pack_mtime = p->mtime;
542 + clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL);
543 + clear_midx_files_ext(r->objects->odb->path, ".rev", NULL);
544
696 - entry->offset = nth_packed_object_offset(p, cur_object);
697 - entry->preferred = !!preferred;
545 + strbuf_release(&midx);
546 }
547
700 -struct midx_fanout {
701 - struct pack_midx_entry *entries;
702 - size_t nr, alloc;
703 -};
548 +static int verify_midx_error;
549
705 -static void midx_fanout_grow(struct midx_fanout *fanout, size_t nr)
550 +__attribute__((format (printf, 1, 2)))
551 +static void midx_report(const char *fmt, ...)
552 {
707 - if (nr < fanout->nr)
708 - BUG("negative growth in midx_fanout_grow() (%"PRIuMAX" < %"PRIuMAX")",
709 - (uintmax_t)nr, (uintmax_t)fanout->nr);
710 - ALLOC_GROW(fanout->entries, nr, fanout->alloc);
553 + va_list ap;
554 + verify_midx_error = 1;
555 + va_start(ap, fmt);
556 + vfprintf(stderr, fmt, ap);
557 + fprintf(stderr, "\n");
558 + va_end(ap);
559 }
560
713 -static void midx_fanout_sort(struct midx_fanout *fanout)
561 +struct pair_pos_vs_id
562 {
715 - QSORT(fanout->entries, fanout->nr, midx_oid_compare);
716 -}
563 + uint32_t pos;
564 + uint32_t pack_int_id;
565 +};
566
718 -static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
719 - struct multi_pack_index *m,
720 - uint32_t cur_fanout,
721 - int preferred_pack)
567 +static int compare_pair_pos_vs_id(const void *_a, const void *_b)
568 {
723 - uint32_t start = 0, end;
724 - uint32_t cur_object;
725 -
726 - if (cur_fanout)
727 - start = ntohl(m->chunk_oid_fanout[cur_fanout - 1]);
728 - end = ntohl(m->chunk_oid_fanout[cur_fanout]);
729 -
730 - for (cur_object = start; cur_object < end; cur_object++) {
731 - if ((preferred_pack > -1) &&
732 - (preferred_pack == nth_midxed_pack_int_id(m, cur_object))) {
733 - /*
734 - * Objects from preferred packs are added
735 - * separately.
736 - */
737 - continue;
738 - }
739 -
740 - midx_fanout_grow(fanout, fanout->nr + 1);
741 - nth_midxed_pack_midx_entry(m,
742 - &fanout->entries[fanout->nr],
743 - cur_object);
744 - fanout->entries[fanout->nr].preferred = 0;
745 - fanout->nr++;
746 - }
747 -}
569 + struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a;
570 + struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b;
571
749 -static void midx_fanout_add_pack_fanout(struct midx_fanout *fanout,
750 - struct pack_info *info,
751 - uint32_t cur_pack,
752 - int preferred,
753 - uint32_t cur_fanout)
754 -{
755 - struct packed_git *pack = info[cur_pack].p;
756 - uint32_t start = 0, end;
757 - uint32_t cur_object;
758 -
759 - if (cur_fanout)
760 - start = get_pack_fanout(pack, cur_fanout - 1);
761 - end = get_pack_fanout(pack, cur_fanout);
762 -
763 - for (cur_object = start; cur_object < end; cur_object++) {
764 - midx_fanout_grow(fanout, fanout->nr + 1);
765 - fill_pack_entry(cur_pack,
766 - info[cur_pack].p,
767 - cur_object,
768 - &fanout->entries[fanout->nr],
769 - preferred);
770 - fanout->nr++;
771 - }
572 + return b->pack_int_id - a->pack_int_id;
573 }
574
575 /*
775 - * It is possible to artificially get into a state where there are many
776 - * duplicate copies of objects. That can create high memory pressure if
777 - * we are to create a list of all objects before de-duplication. To reduce
778 - * this memory pressure without a significant performance drop, automatically
779 - * group objects by the first byte of their object id. Use the IDX fanout
780 - * tables to group the data, copy to a local array, then sort.
781 - *
782 - * Copy only the de-duplicated entries (selected by most-recent modified time
783 - * of a packfile containing the object).
576 + * Limit calls to display_progress() for performance reasons.
577 + * The interval here was arbitrarily chosen.
578 */
785 -static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
786 - struct pack_info *info,
787 - uint32_t nr_packs,
788 - size_t *nr_objects,
789 - int preferred_pack)
790 -{
791 - uint32_t cur_fanout, cur_pack, cur_object;
792 - size_t alloc_objects, total_objects = 0;
793 - struct midx_fanout fanout = { 0 };
794 - struct pack_midx_entry *deduplicated_entries = NULL;
795 - uint32_t start_pack = m ? m->num_packs : 0;
796 -
797 - for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++)
798 - total_objects = st_add(total_objects,
799 - info[cur_pack].p->num_objects);
800 -
801 - /*
802 - * As we de-duplicate by fanout value, we expect the fanout
803 - * slices to be evenly distributed, with some noise. Hence,
804 - * allocate slightly more than one 256th.
805 - */
806 - alloc_objects = fanout.alloc = total_objects > 3200 ? total_objects / 200 : 16;
807 -
808 - ALLOC_ARRAY(fanout.entries, fanout.alloc);
809 - ALLOC_ARRAY(deduplicated_entries, alloc_objects);
810 - *nr_objects = 0;
811 -
812 - for (cur_fanout = 0; cur_fanout < 256; cur_fanout++) {
813 - fanout.nr = 0;
814 -
815 - if (m)
816 - midx_fanout_add_midx_fanout(&fanout, m, cur_fanout,
817 - preferred_pack);
579 +#define SPARSE_PROGRESS_INTERVAL (1 << 12)
580 +#define midx_display_sparse_progress(progress, n) \
581 + do { \
582 + uint64_t _n = (n); \
583 + if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \
584 + display_progress(progress, _n); \
585 + } while (0)
586
819 - for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) {
820 - int preferred = cur_pack == preferred_pack;
821 - midx_fanout_add_pack_fanout(&fanout,
822 - info, cur_pack,
823 - preferred, cur_fanout);
824 - }
587 +int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags)
588 +{
589 + struct pair_pos_vs_id *pairs = NULL;
590 + uint32_t i;
591 + struct progress *progress = NULL;
592 + struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
593 + verify_midx_error = 0;
594
826 - if (-1 < preferred_pack && preferred_pack < start_pack)
827 - midx_fanout_add_pack_fanout(&fanout, info,
828 - preferred_pack, 1,
829 - cur_fanout);
595 + if (!m) {
596 + int result = 0;
597 + struct stat sb;
598 + struct strbuf filename = STRBUF_INIT;
599
831 - midx_fanout_sort(&fanout);
600 + get_midx_filename(&filename, object_dir);
601
833 - /*
834 - * The batch is now sorted by OID and then mtime (descending).
835 - * Take only the first duplicate.
836 - */
837 - for (cur_object = 0; cur_object < fanout.nr; cur_object++) {
838 - if (cur_object && oideq(&fanout.entries[cur_object - 1].oid,
839 - &fanout.entries[cur_object].oid))
840 - continue;
841 -
842 - ALLOC_GROW(deduplicated_entries, st_add(*nr_objects, 1),
843 - alloc_objects);
844 - memcpy(&deduplicated_entries[*nr_objects],
845 - &fanout.entries[cur_object],
846 - sizeof(struct pack_midx_entry));
847 - (*nr_objects)++;
602 + if (!stat(filename.buf, &sb)) {
603 + error(_("multi-pack-index file exists, but failed to parse"));
604 + result = 1;
605 }
606 + strbuf_release(&filename);
607 + return result;
608 }
609
851 - free(fanout.entries);
852 - return deduplicated_entries;
853 -}
854 -
855 -static int write_midx_pack_names(struct hashfile *f, void *data)
856 -{
857 - struct write_midx_context *ctx = data;
858 - uint32_t i;
859 - unsigned char padding[MIDX_CHUNK_ALIGNMENT];
860 - size_t written = 0;
861 -
862 - for (i = 0; i < ctx->nr; i++) {
863 - size_t writelen;
864 -
865 - if (ctx->info[i].expired)
866 - continue;
610 + if (!midx_checksum_valid(m))
611 + midx_report(_("incorrect checksum"));
612
868 - if (i && strcmp(ctx->info[i].pack_name, ctx->info[i - 1].pack_name) <= 0)
869 - BUG("incorrect pack-file order: %s before %s",
870 - ctx->info[i - 1].pack_name,
871 - ctx->info[i].pack_name);
613 + if (flags & MIDX_PROGRESS)
614 + progress = start_delayed_progress(_("Looking for referenced packfiles"),
615 + m->num_packs);
616 + for (i = 0; i < m->num_packs; i++) {
617 + if (prepare_midx_pack(r, m, i))
618 + midx_report("failed to load pack in position %d", i);
619
873 - writelen = strlen(ctx->info[i].pack_name) + 1;
874 - hashwrite(f, ctx->info[i].pack_name, writelen);
875 - written += writelen;
620 + display_progress(progress, i + 1);
621 }
622 + stop_progress(&progress);
623
878 - /* add padding to be aligned */
879 - i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
880 - if (i < MIDX_CHUNK_ALIGNMENT) {
881 - memset(padding, 0, sizeof(padding));
882 - hashwrite(f, padding, i);
624 + if (m->num_objects == 0) {
625 + midx_report(_("the midx contains no oid"));
626 + /*
627 + * Remaining tests assume that we have objects, so we can
628 + * return here.
629 + */
630 + goto cleanup;
631 }
632
885 - return 0;
886 -}
887 -
888 -static int write_midx_bitmapped_packs(struct hashfile *f, void *data)
889 -{
890 - struct write_midx_context *ctx = data;
891 - size_t i;
633 + if (flags & MIDX_PROGRESS)
634 + progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"),
635 + m->num_objects - 1);
636 + for (i = 0; i < m->num_objects - 1; i++) {
637 + struct object_id oid1, oid2;
638
893 - for (i = 0; i < ctx->nr; i++) {
894 - struct pack_info *pack = &ctx->info[i];
895 - if (pack->expired)
896 - continue;
639 + nth_midxed_object_oid(&oid1, m, i);
640 + nth_midxed_object_oid(&oid2, m, i + 1);
641
898 - if (pack->bitmap_pos == BITMAP_POS_UNKNOWN && pack->bitmap_nr)
899 - BUG("pack '%s' has no bitmap position, but has %d bitmapped object(s)",
900 - pack->pack_name, pack->bitmap_nr);
642 + if (oidcmp(&oid1, &oid2) >= 0)
643 + midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
644 + i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
645
902 - hashwrite_be32(f, pack->bitmap_pos);
903 - hashwrite_be32(f, pack->bitmap_nr);
646 + midx_display_sparse_progress(progress, i + 1);
647 }
905 - return 0;
906 -}
907 -
908 -static int write_midx_oid_fanout(struct hashfile *f,
909 - void *data)
910 -{
911 - struct write_midx_context *ctx = data;
912 - struct pack_midx_entry *list = ctx->entries;
913 - struct pack_midx_entry *last = ctx->entries + ctx->entries_nr;
914 - uint32_t count = 0;
915 - uint32_t i;
648 + stop_progress(&progress);
649
650 /*
918 - * Write the first-level table (the list is sorted,
919 - * but we use a 256-entry lookup to be able to avoid
920 - * having to do eight extra binary search iterations).
921 - */
922 - for (i = 0; i < 256; i++) {
923 - struct pack_midx_entry *next = list;
651 + * Create an array mapping each object to its packfile id. Sort it
652 + * to group the objects by packfile. Use this permutation to visit
653 + * each of the objects and only require 1 packfile to be open at a
654 + * time.
655 + */
656 + ALLOC_ARRAY(pairs, m->num_objects);
657 + for (i = 0; i < m->num_objects; i++) {
658 + pairs[i].pos = i;
659 + pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
660 + }
661
925 - while (next < last && next->oid.hash[0] == i) {
926 - count++;
927 - next++;
928 - }
662 + if (flags & MIDX_PROGRESS)
663 + progress = start_sparse_progress(_("Sorting objects by packfile"),
664 + m->num_objects);
665 + display_progress(progress, 0); /* TODO: Measure QSORT() progress */
666 + QSORT(pairs, m->num_objects, compare_pair_pos_vs_id);
667 + stop_progress(&progress);
668
930 - hashwrite_be32(f, count);
931 - list = next;
932 - }
669 + if (flags & MIDX_PROGRESS)
670 + progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects);
671 + for (i = 0; i < m->num_objects; i++) {
672 + struct object_id oid;
673 + struct pack_entry e;
674 + off_t m_offset, p_offset;
675
934 - return 0;
935 -}
676 + if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
677 + m->packs[pairs[i-1].pack_int_id])
678 + {
679 + close_pack_fd(m->packs[pairs[i-1].pack_int_id]);
680 + close_pack_index(m->packs[pairs[i-1].pack_int_id]);
681 + }
682
937 -static int write_midx_oid_lookup(struct hashfile *f,
938 - void *data)
939 -{
940 - struct write_midx_context *ctx = data;
941 - unsigned char hash_len = the_hash_algo->rawsz;
942 - struct pack_midx_entry *list = ctx->entries;
943 - uint32_t i;
683 + nth_midxed_object_oid(&oid, m, pairs[i].pos);
684
945 - for (i = 0; i < ctx->entries_nr; i++) {
946 - struct pack_midx_entry *obj = list++;
685 + if (!fill_midx_entry(r, &oid, &e, m)) {
686 + midx_report(_("failed to load pack entry for oid[%d] = %s"),
687 + pairs[i].pos, oid_to_hex(&oid));
688 + continue;
689 + }
690
948 - if (i < ctx->entries_nr - 1) {
949 - struct pack_midx_entry *next = list;
950 - if (oidcmp(&obj->oid, &next->oid) >= 0)
951 - BUG("OIDs not in order: %s >= %s",
952 - oid_to_hex(&obj->oid),
953 - oid_to_hex(&next->oid));
691 + if (open_pack_index(e.p)) {
692 + midx_report(_("failed to load pack-index for packfile %s"),
693 + e.p->pack_name);
694 + break;
695 }
696
956 - hashwrite(f, obj->oid.hash, (int)hash_len);
957 - }
697 + m_offset = e.offset;
698 + p_offset = find_pack_entry_one(oid.hash, e.p);
699
959 - return 0;
960 -}
700 + if (m_offset != p_offset)
701 + midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
702 + pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
703
962 -static int write_midx_object_offsets(struct hashfile *f,
963 - void *data)
964 -{
965 - struct write_midx_context *ctx = data;
966 - struct pack_midx_entry *list = ctx->entries;
967 - uint32_t i, nr_large_offset = 0;
968 -
969 - for (i = 0; i < ctx->entries_nr; i++) {
970 - struct pack_midx_entry *obj = list++;
971 -
972 - if (ctx->pack_perm[obj->pack_int_id] == PACK_EXPIRED)
973 - BUG("object %s is in an expired pack with int-id %d",
974 - oid_to_hex(&obj->oid),
975 - obj->pack_int_id);
976 -
977 - hashwrite_be32(f, ctx->pack_perm[obj->pack_int_id]);
978 -
979 - if (ctx->large_offsets_needed && obj->offset >> 31)
980 - hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++);
981 - else if (!ctx->large_offsets_needed && obj->offset >> 32)
982 - BUG("object %s requires a large offset (%"PRIx64") but the MIDX is not writing large offsets!",
983 - oid_to_hex(&obj->oid),
984 - obj->offset);
985 - else
986 - hashwrite_be32(f, (uint32_t)obj->offset);
704 + midx_display_sparse_progress(progress, i + 1);
705 }
706 + stop_progress(&progress);
707
989 - return 0;
990 -}
991 -
992 -static int write_midx_large_offsets(struct hashfile *f,
993 - void *data)
994 -{
995 - struct write_midx_context *ctx = data;
996 - struct pack_midx_entry *list = ctx->entries;
997 - struct pack_midx_entry *end = ctx->entries + ctx->entries_nr;
998 - uint32_t nr_large_offset = ctx->num_large_offsets;
999 -
1000 - while (nr_large_offset) {
1001 - struct pack_midx_entry *obj;
1002 - uint64_t offset;
1003 -
1004 - if (list >= end)
1005 - BUG("too many large-offset objects");
1006 -
1007 - obj = list++;
1008 - offset = obj->offset;
1009 -
1010 - if (!(offset >> 31))
1011 - continue;
1012 -
1013 - hashwrite_be64(f, offset);
1014 -
1015 - nr_large_offset--;
1016 - }
1017 -
1018 - return 0;
1019 -}
1020 -
1021 -static int write_midx_revindex(struct hashfile *f,
1022 - void *data)
1023 -{
1024 - struct write_midx_context *ctx = data;
1025 - uint32_t i;
1026 -
1027 - for (i = 0; i < ctx->entries_nr; i++)
1028 - hashwrite_be32(f, ctx->pack_order[i]);
1029 -
1030 - return 0;
1031 -}
1032 -
1033 -struct midx_pack_order_data {
1034 - uint32_t nr;
1035 - uint32_t pack;
1036 - off_t offset;
1037 -};
1038 -
1039 -static int midx_pack_order_cmp(const void *va, const void *vb)
1040 -{
1041 - const struct midx_pack_order_data *a = va, *b = vb;
1042 - if (a->pack < b->pack)
1043 - return -1;
1044 - else if (a->pack > b->pack)
1045 - return 1;
1046 - else if (a->offset < b->offset)
1047 - return -1;
1048 - else if (a->offset > b->offset)
1049 - return 1;
1050 - else
1051 - return 0;
1052 -}
1053 -
1054 -static uint32_t *midx_pack_order(struct write_midx_context *ctx)
1055 -{
1056 - struct midx_pack_order_data *data;
1057 - uint32_t *pack_order;
1058 - uint32_t i;
1059 -
1060 - trace2_region_enter("midx", "midx_pack_order", the_repository);
1061 -
1062 - ALLOC_ARRAY(data, ctx->entries_nr);
1063 - for (i = 0; i < ctx->entries_nr; i++) {
1064 - struct pack_midx_entry *e = &ctx->entries[i];
1065 - data[i].nr = i;
1066 - data[i].pack = ctx->pack_perm[e->pack_int_id];
1067 - if (!e->preferred)
1068 - data[i].pack |= (1U << 31);
1069 - data[i].offset = e->offset;
1070 - }
1071 -
1072 - QSORT(data, ctx->entries_nr, midx_pack_order_cmp);
1073 -
1074 - ALLOC_ARRAY(pack_order, ctx->entries_nr);
1075 - for (i = 0; i < ctx->entries_nr; i++) {
1076 - struct pack_midx_entry *e = &ctx->entries[data[i].nr];
1077 - struct pack_info *pack = &ctx->info[ctx->pack_perm[e->pack_int_id]];
1078 - if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
1079 - pack->bitmap_pos = i;
1080 - pack->bitmap_nr++;
1081 - pack_order[i] = data[i].nr;
1082 - }
1083 - for (i = 0; i < ctx->nr; i++) {
1084 - struct pack_info *pack = &ctx->info[ctx->pack_perm[i]];
1085 - if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
1086 - pack->bitmap_pos = 0;
1087 - }
1088 - free(data);
1089 -
1090 - trace2_region_leave("midx", "midx_pack_order", the_repository);
1091 -
1092 - return pack_order;
1093 -}
1094 -
1095 -static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
1096 - struct write_midx_context *ctx)
1097 -{
1098 - struct strbuf buf = STRBUF_INIT;
1099 - const char *tmp_file;
1100 -
1101 - trace2_region_enter("midx", "write_midx_reverse_index", the_repository);
1102 -
1103 - strbuf_addf(&buf, "%s-%s.rev", midx_name, hash_to_hex(midx_hash));
1104 -
1105 - tmp_file = write_rev_file_order(NULL, ctx->pack_order, ctx->entries_nr,
1106 - midx_hash, WRITE_REV);
1107 -
1108 - if (finalize_object_file(tmp_file, buf.buf))
1109 - die(_("cannot store reverse index file"));
1110 -
1111 - strbuf_release(&buf);
1112 -
1113 - trace2_region_leave("midx", "write_midx_reverse_index", the_repository);
1114 -}
1115 -
1116 -static void clear_midx_files_ext(const char *object_dir, const char *ext,
1117 - unsigned char *keep_hash);
1118 -
1119 -static int midx_checksum_valid(struct multi_pack_index *m)
1120 -{
1121 - return hashfile_checksum_valid(m->data, m->data_len);
1122 -}
1123 -
1124 -static void prepare_midx_packing_data(struct packing_data *pdata,
1125 - struct write_midx_context *ctx)
1126 -{
1127 - uint32_t i;
1128 -
1129 - trace2_region_enter("midx", "prepare_midx_packing_data", the_repository);
1130 -
1131 - memset(pdata, 0, sizeof(struct packing_data));
1132 - prepare_packing_data(the_repository, pdata);
1133 -
1134 - for (i = 0; i < ctx->entries_nr; i++) {
1135 - struct pack_midx_entry *from = &ctx->entries[ctx->pack_order[i]];
1136 - struct object_entry *to = packlist_alloc(pdata, &from->oid);
1137 -
1138 - oe_set_in_pack(pdata, to,
1139 - ctx->info[ctx->pack_perm[from->pack_int_id]].p);
1140 - }
1141 -
1142 - trace2_region_leave("midx", "prepare_midx_packing_data", the_repository);
1143 -}
1144 -
1145 -static int add_ref_to_pending(const char *refname,
1146 - const struct object_id *oid,
1147 - int flag, void *cb_data)
1148 -{
1149 - struct rev_info *revs = (struct rev_info*)cb_data;
1150 - struct object_id peeled;
1151 - struct object *object;
1152 -
1153 - if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) {
1154 - warning("symbolic ref is dangling: %s", refname);
1155 - return 0;
1156 - }
1157 -
1158 - if (!peel_iterated_oid(oid, &peeled))
1159 - oid = &peeled;
1160 -
1161 - object = parse_object_or_die(oid, refname);
1162 - if (object->type != OBJ_COMMIT)
1163 - return 0;
1164 -
1165 - add_pending_object(revs, object, "");
1166 - if (bitmap_is_preferred_refname(revs->repo, refname))
1167 - object->flags |= NEEDS_BITMAP;
1168 - return 0;
1169 -}
1170 -
1171 -struct bitmap_commit_cb {
1172 - struct commit **commits;
1173 - size_t commits_nr, commits_alloc;
1174 -
1175 - struct write_midx_context *ctx;
1176 -};
1177 -
1178 -static const struct object_id *bitmap_oid_access(size_t index,
1179 - const void *_entries)
1180 -{
1181 - const struct pack_midx_entry *entries = _entries;
1182 - return &entries[index].oid;
1183 -}
1184 -
1185 -static void bitmap_show_commit(struct commit *commit, void *_data)
1186 -{
1187 - struct bitmap_commit_cb *data = _data;
1188 - int pos = oid_pos(&commit->object.oid, data->ctx->entries,
1189 - data->ctx->entries_nr,
1190 - bitmap_oid_access);
1191 - if (pos < 0)
1192 - return;
1193 -
1194 - ALLOC_GROW(data->commits, data->commits_nr + 1, data->commits_alloc);
1195 - data->commits[data->commits_nr++] = commit;
1196 -}
1197 -
1198 -static int read_refs_snapshot(const char *refs_snapshot,
1199 - struct rev_info *revs)
1200 -{
1201 - struct strbuf buf = STRBUF_INIT;
1202 - struct object_id oid;
1203 - FILE *f = xfopen(refs_snapshot, "r");
1204 -
1205 - while (strbuf_getline(&buf, f) != EOF) {
1206 - struct object *object;
1207 - int preferred = 0;
1208 - char *hex = buf.buf;
1209 - const char *end = NULL;
1210 -
1211 - if (buf.len && *buf.buf == '+') {
1212 - preferred = 1;
1213 - hex = &buf.buf[1];
1214 - }
1215 -
1216 - if (parse_oid_hex(hex, &oid, &end) < 0)
1217 - die(_("could not parse line: %s"), buf.buf);
1218 - if (*end)
1219 - die(_("malformed line: %s"), buf.buf);
1220 -
1221 - object = parse_object_or_die(&oid, NULL);
1222 - if (preferred)
1223 - object->flags |= NEEDS_BITMAP;
1224 -
1225 - add_pending_object(revs, object, "");
1226 - }
1227 -
1228 - fclose(f);
1229 - strbuf_release(&buf);
1230 - return 0;
1231 -}
1232 -
1233 -static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,
1234 - const char *refs_snapshot,
1235 - struct write_midx_context *ctx)
1236 -{
1237 - struct rev_info revs;
1238 - struct bitmap_commit_cb cb = {0};
1239 -
1240 - trace2_region_enter("midx", "find_commits_for_midx_bitmap",
1241 - the_repository);
1242 -
1243 - cb.ctx = ctx;
1244 -
1245 - repo_init_revisions(the_repository, &revs, NULL);
1246 - if (refs_snapshot) {
1247 - read_refs_snapshot(refs_snapshot, &revs);
1248 - } else {
1249 - setup_revisions(0, NULL, &revs, NULL);
1250 - for_each_ref(add_ref_to_pending, &revs);
1251 - }
1252 -
1253 - /*
1254 - * Skipping promisor objects here is intentional, since it only excludes
1255 - * them from the list of reachable commits that we want to select from
1256 - * when computing the selection of MIDX'd commits to receive bitmaps.
1257 - *
1258 - * Reachability bitmaps do require that their objects be closed under
1259 - * reachability, but fetching any objects missing from promisors at this
1260 - * point is too late. But, if one of those objects can be reached from
1261 - * an another object that is included in the bitmap, then we will
1262 - * complain later that we don't have reachability closure (and fail
1263 - * appropriately).
1264 - */
1265 - fetch_if_missing = 0;
1266 - revs.exclude_promisor_objects = 1;
1267 -
1268 - if (prepare_revision_walk(&revs))
1269 - die(_("revision walk setup failed"));
1270 -
1271 - traverse_commit_list(&revs, bitmap_show_commit, NULL, &cb);
1272 - if (indexed_commits_nr_p)
1273 - *indexed_commits_nr_p = cb.commits_nr;
1274 -
1275 - release_revisions(&revs);
1276 -
1277 - trace2_region_leave("midx", "find_commits_for_midx_bitmap",
1278 - the_repository);
1279 -
1280 - return cb.commits;
1281 -}
1282 -
1283 -static int write_midx_bitmap(const char *midx_name,
1284 - const unsigned char *midx_hash,
1285 - struct packing_data *pdata,
1286 - struct commit **commits,
1287 - uint32_t commits_nr,
1288 - uint32_t *pack_order,
1289 - unsigned flags)
1290 -{
1291 - int ret, i;
1292 - uint16_t options = 0;
1293 - struct pack_idx_entry **index;
1294 - char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name,
1295 - hash_to_hex(midx_hash));
1296 -
1297 - trace2_region_enter("midx", "write_midx_bitmap", the_repository);
1298 -
1299 - if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
1300 - options |= BITMAP_OPT_HASH_CACHE;
1301 -
1302 - if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
1303 - options |= BITMAP_OPT_LOOKUP_TABLE;
1304 -
1305 - /*
1306 - * Build the MIDX-order index based on pdata.objects (which is already
1307 - * in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
1308 - * this order).
1309 - */
1310 - ALLOC_ARRAY(index, pdata->nr_objects);
1311 - for (i = 0; i < pdata->nr_objects; i++)
1312 - index[i] = &pdata->objects[i].idx;
1313 -
1314 - bitmap_writer_show_progress(flags & MIDX_PROGRESS);
1315 - bitmap_writer_build_type_index(pdata, index, pdata->nr_objects);
1316 -
1317 - /*
1318 - * bitmap_writer_finish expects objects in lex order, but pack_order
1319 - * gives us exactly that. use it directly instead of re-sorting the
1320 - * array.
1321 - *
1322 - * This changes the order of objects in 'index' between
1323 - * bitmap_writer_build_type_index and bitmap_writer_finish.
1324 - *
1325 - * The same re-ordering takes place in the single-pack bitmap code via
1326 - * write_idx_file(), which is called by finish_tmp_packfile(), which
1327 - * happens between bitmap_writer_build_type_index() and
1328 - * bitmap_writer_finish().
1329 - */
1330 - for (i = 0; i < pdata->nr_objects; i++)
1331 - index[pack_order[i]] = &pdata->objects[i].idx;
1332 -
1333 - bitmap_writer_select_commits(commits, commits_nr, -1);
1334 - ret = bitmap_writer_build(pdata);
1335 - if (ret < 0)
1336 - goto cleanup;
1337 -
1338 - bitmap_writer_set_checksum(midx_hash);
1339 - bitmap_writer_finish(index, pdata->nr_objects, bitmap_name, options);
1340 -
1341 -cleanup:
1342 - free(index);
1343 - free(bitmap_name);
1344 -
1345 - trace2_region_leave("midx", "write_midx_bitmap", the_repository);
1346 -
1347 - return ret;
1348 -}
1349 -
1350 -static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
1351 - const char *object_dir)
1352 -{
1353 - struct multi_pack_index *result = NULL;
1354 - struct multi_pack_index *cur;
1355 - char *obj_dir_real = real_pathdup(object_dir, 1);
1356 - struct strbuf cur_path_real = STRBUF_INIT;
1357 -
1358 - /* Ensure the given object_dir is local, or a known alternate. */
1359 - find_odb(r, obj_dir_real);
1360 -
1361 - for (cur = get_multi_pack_index(r); cur; cur = cur->next) {
1362 - strbuf_realpath(&cur_path_real, cur->object_dir, 1);
1363 - if (!strcmp(obj_dir_real, cur_path_real.buf)) {
1364 - result = cur;
1365 - goto cleanup;
1366 - }
1367 - }
1368 -
1369 -cleanup:
1370 - free(obj_dir_real);
1371 - strbuf_release(&cur_path_real);
1372 - return result;
1373 -}
1374 -
1375 -static int write_midx_internal(const char *object_dir,
1376 - struct string_list *packs_to_include,
1377 - struct string_list *packs_to_drop,
1378 - const char *preferred_pack_name,
1379 - const char *refs_snapshot,
1380 - unsigned flags)
1381 -{
1382 - struct strbuf midx_name = STRBUF_INIT;
1383 - unsigned char midx_hash[GIT_MAX_RAWSZ];
1384 - uint32_t i;
1385 - struct hashfile *f = NULL;
1386 - struct lock_file lk;
1387 - struct write_midx_context ctx = { 0 };
1388 - int bitmapped_packs_concat_len = 0;
1389 - int pack_name_concat_len = 0;
1390 - int dropped_packs = 0;
1391 - int result = 0;
1392 - struct chunkfile *cf;
1393 -
1394 - trace2_region_enter("midx", "write_midx_internal", the_repository);
1395 -
1396 - get_midx_filename(&midx_name, object_dir);
1397 - if (safe_create_leading_directories(midx_name.buf))
1398 - die_errno(_("unable to create leading directories of %s"),
1399 - midx_name.buf);
1400 -
1401 - if (!packs_to_include) {
1402 - /*
1403 - * Only reference an existing MIDX when not filtering which
1404 - * packs to include, since all packs and objects are copied
1405 - * blindly from an existing MIDX if one is present.
1406 - */
1407 - ctx.m = lookup_multi_pack_index(the_repository, object_dir);
1408 - }
1409 -
1410 - if (ctx.m && !midx_checksum_valid(ctx.m)) {
1411 - warning(_("ignoring existing multi-pack-index; checksum mismatch"));
1412 - ctx.m = NULL;
1413 - }
1414 -
1415 - ctx.nr = 0;
1416 - ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
1417 - ctx.info = NULL;
1418 - ALLOC_ARRAY(ctx.info, ctx.alloc);
1419 -
1420 - if (ctx.m) {
1421 - for (i = 0; i < ctx.m->num_packs; i++) {
1422 - ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc);
1423 -
1424 - if (flags & MIDX_WRITE_REV_INDEX) {
1425 - /*
1426 - * If generating a reverse index, need to have
1427 - * packed_git's loaded to compare their
1428 - * mtimes and object count.
1429 - */
1430 - if (prepare_midx_pack(the_repository, ctx.m, i)) {
1431 - error(_("could not load pack"));
1432 - result = 1;
1433 - goto cleanup;
1434 - }
1435 -
1436 - if (open_pack_index(ctx.m->packs[i]))
1437 - die(_("could not open index for %s"),
1438 - ctx.m->packs[i]->pack_name);
1439 - }
1440 -
1441 - fill_pack_info(&ctx.info[ctx.nr++], ctx.m->packs[i],
1442 - ctx.m->pack_names[i], i);
1443 - }
1444 - }
1445 -
1446 - ctx.pack_paths_checked = 0;
1447 - if (flags & MIDX_PROGRESS)
1448 - ctx.progress = start_delayed_progress(_("Adding packfiles to multi-pack-index"), 0);
1449 - else
1450 - ctx.progress = NULL;
1451 -
1452 - ctx.to_include = packs_to_include;
1453 -
1454 - for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
1455 - stop_progress(&ctx.progress);
1456 -
1457 - if ((ctx.m && ctx.nr == ctx.m->num_packs) &&
1458 - !(packs_to_include || packs_to_drop)) {
1459 - struct bitmap_index *bitmap_git;
1460 - int bitmap_exists;
1461 - int want_bitmap = flags & MIDX_WRITE_BITMAP;
1462 -
1463 - bitmap_git = prepare_midx_bitmap_git(ctx.m);
1464 - bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
1465 - free_bitmap_index(bitmap_git);
1466 -
1467 - if (bitmap_exists || !want_bitmap) {
1468 - /*
1469 - * The correct MIDX already exists, and so does a
1470 - * corresponding bitmap (or one wasn't requested).
1471 - */
1472 - if (!want_bitmap)
1473 - clear_midx_files_ext(object_dir, ".bitmap",
1474 - NULL);
1475 - goto cleanup;
1476 - }
1477 - }
1478 -
1479 - if (preferred_pack_name) {
1480 - ctx.preferred_pack_idx = -1;
1481 -
1482 - for (i = 0; i < ctx.nr; i++) {
1483 - if (!cmp_idx_or_pack_name(preferred_pack_name,
1484 - ctx.info[i].pack_name)) {
1485 - ctx.preferred_pack_idx = i;
1486 - break;
1487 - }
1488 - }
1489 -
1490 - if (ctx.preferred_pack_idx == -1)
1491 - warning(_("unknown preferred pack: '%s'"),
1492 - preferred_pack_name);
1493 - } else if (ctx.nr &&
1494 - (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) {
1495 - struct packed_git *oldest = ctx.info[ctx.preferred_pack_idx].p;
1496 - ctx.preferred_pack_idx = 0;
1497 -
1498 - if (packs_to_drop && packs_to_drop->nr)
1499 - BUG("cannot write a MIDX bitmap during expiration");
1500 -
1501 - /*
1502 - * set a preferred pack when writing a bitmap to ensure that
1503 - * the pack from which the first object is selected in pseudo
1504 - * pack-order has all of its objects selected from that pack
1505 - * (and not another pack containing a duplicate)
1506 - */
1507 - for (i = 1; i < ctx.nr; i++) {
1508 - struct packed_git *p = ctx.info[i].p;
1509 -
1510 - if (!oldest->num_objects || p->mtime < oldest->mtime) {
1511 - oldest = p;
1512 - ctx.preferred_pack_idx = i;
1513 - }
1514 - }
1515 -
1516 - if (!oldest->num_objects) {
1517 - /*
1518 - * If all packs are empty; unset the preferred index.
1519 - * This is acceptable since there will be no duplicate
1520 - * objects to resolve, so the preferred value doesn't
1521 - * matter.
1522 - */
1523 - ctx.preferred_pack_idx = -1;
1524 - }
1525 - } else {
1526 - /*
1527 - * otherwise don't mark any pack as preferred to avoid
1528 - * interfering with expiration logic below
1529 - */
1530 - ctx.preferred_pack_idx = -1;
1531 - }
1532 -
1533 - if (ctx.preferred_pack_idx > -1) {
1534 - struct packed_git *preferred = ctx.info[ctx.preferred_pack_idx].p;
1535 - if (!preferred->num_objects) {
1536 - error(_("cannot select preferred pack %s with no objects"),
1537 - preferred->pack_name);
1538 - result = 1;
1539 - goto cleanup;
1540 - }
1541 - }
1542 -
1543 - ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr,
1544 - ctx.preferred_pack_idx);
1545 -
1546 - ctx.large_offsets_needed = 0;
1547 - for (i = 0; i < ctx.entries_nr; i++) {
1548 - if (ctx.entries[i].offset > 0x7fffffff)
1549 - ctx.num_large_offsets++;
1550 - if (ctx.entries[i].offset > 0xffffffff)
1551 - ctx.large_offsets_needed = 1;
1552 - }
1553 -
1554 - QSORT(ctx.info, ctx.nr, pack_info_compare);
1555 -
1556 - if (packs_to_drop && packs_to_drop->nr) {
1557 - int drop_index = 0;
1558 - int missing_drops = 0;
1559 -
1560 - for (i = 0; i < ctx.nr && drop_index < packs_to_drop->nr; i++) {
1561 - int cmp = strcmp(ctx.info[i].pack_name,
1562 - packs_to_drop->items[drop_index].string);
1563 -
1564 - if (!cmp) {
1565 - drop_index++;
1566 - ctx.info[i].expired = 1;
1567 - } else if (cmp > 0) {
1568 - error(_("did not see pack-file %s to drop"),
1569 - packs_to_drop->items[drop_index].string);
1570 - drop_index++;
1571 - missing_drops++;
1572 - i--;
1573 - } else {
1574 - ctx.info[i].expired = 0;
1575 - }
1576 - }
1577 -
1578 - if (missing_drops) {
1579 - result = 1;
1580 - goto cleanup;
1581 - }
1582 - }
1583 -
1584 - /*
1585 - * pack_perm stores a permutation between pack-int-ids from the
1586 - * previous multi-pack-index to the new one we are writing:
1587 - *
1588 - * pack_perm[old_id] = new_id
1589 - */
1590 - ALLOC_ARRAY(ctx.pack_perm, ctx.nr);
1591 - for (i = 0; i < ctx.nr; i++) {
1592 - if (ctx.info[i].expired) {
1593 - dropped_packs++;
1594 - ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
1595 - } else {
1596 - ctx.pack_perm[ctx.info[i].orig_pack_int_id] = i - dropped_packs;
1597 - }
1598 - }
1599 -
1600 - for (i = 0; i < ctx.nr; i++) {
1601 - if (ctx.info[i].expired)
1602 - continue;
1603 - pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
1604 - bitmapped_packs_concat_len += 2 * sizeof(uint32_t);
1605 - }
1606 -
1607 - /* Check that the preferred pack wasn't expired (if given). */
1608 - if (preferred_pack_name) {
1609 - struct pack_info *preferred = bsearch(preferred_pack_name,
1610 - ctx.info, ctx.nr,
1611 - sizeof(*ctx.info),
1612 - idx_or_pack_name_cmp);
1613 - if (preferred) {
1614 - uint32_t perm = ctx.pack_perm[preferred->orig_pack_int_id];
1615 - if (perm == PACK_EXPIRED)
1616 - warning(_("preferred pack '%s' is expired"),
1617 - preferred_pack_name);
1618 - }
1619 - }
1620 -
1621 - if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
1622 - pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
1623 - (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
1624 -
1625 - hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR);
1626 - f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
1627 -
1628 - if (ctx.nr - dropped_packs == 0) {
1629 - error(_("no pack files to index."));
1630 - result = 1;
1631 - goto cleanup;
1632 - }
1633 -
1634 - if (!ctx.entries_nr) {
1635 - if (flags & MIDX_WRITE_BITMAP)
1636 - warning(_("refusing to write multi-pack .bitmap without any objects"));
1637 - flags &= ~(MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP);
1638 - }
1639 -
1640 - cf = init_chunkfile(f);
1641 -
1642 - add_chunk(cf, MIDX_CHUNKID_PACKNAMES, pack_name_concat_len,
1643 - write_midx_pack_names);
1644 - add_chunk(cf, MIDX_CHUNKID_OIDFANOUT, MIDX_CHUNK_FANOUT_SIZE,
1645 - write_midx_oid_fanout);
1646 - add_chunk(cf, MIDX_CHUNKID_OIDLOOKUP,
1647 - st_mult(ctx.entries_nr, the_hash_algo->rawsz),
1648 - write_midx_oid_lookup);
1649 - add_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS,
1650 - st_mult(ctx.entries_nr, MIDX_CHUNK_OFFSET_WIDTH),
1651 - write_midx_object_offsets);
1652 -
1653 - if (ctx.large_offsets_needed)
1654 - add_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS,
1655 - st_mult(ctx.num_large_offsets,
1656 - MIDX_CHUNK_LARGE_OFFSET_WIDTH),
1657 - write_midx_large_offsets);
1658 -
1659 - if (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP)) {
1660 - ctx.pack_order = midx_pack_order(&ctx);
1661 - add_chunk(cf, MIDX_CHUNKID_REVINDEX,
1662 - st_mult(ctx.entries_nr, sizeof(uint32_t)),
1663 - write_midx_revindex);
1664 - add_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS,
1665 - bitmapped_packs_concat_len,
1666 - write_midx_bitmapped_packs);
1667 - }
1668 -
1669 - write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs);
1670 - write_chunkfile(cf, &ctx);
1671 -
1672 - finalize_hashfile(f, midx_hash, FSYNC_COMPONENT_PACK_METADATA,
1673 - CSUM_FSYNC | CSUM_HASH_IN_STREAM);
1674 - free_chunkfile(cf);
1675 -
1676 - if (flags & MIDX_WRITE_REV_INDEX &&
1677 - git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
1678 - write_midx_reverse_index(midx_name.buf, midx_hash, &ctx);
1679 -
1680 - if (flags & MIDX_WRITE_BITMAP) {
1681 - struct packing_data pdata;
1682 - struct commit **commits;
1683 - uint32_t commits_nr;
1684 -
1685 - if (!ctx.entries_nr)
1686 - BUG("cannot write a bitmap without any objects");
1687 -
1688 - prepare_midx_packing_data(&pdata, &ctx);
1689 -
1690 - commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, &ctx);
1691 -
1692 - /*
1693 - * The previous steps translated the information from
1694 - * 'entries' into information suitable for constructing
1695 - * bitmaps. We no longer need that array, so clear it to
1696 - * reduce memory pressure.
1697 - */
1698 - FREE_AND_NULL(ctx.entries);
1699 - ctx.entries_nr = 0;
1700 -
1701 - if (write_midx_bitmap(midx_name.buf, midx_hash, &pdata,
1702 - commits, commits_nr, ctx.pack_order,
1703 - flags) < 0) {
1704 - error(_("could not write multi-pack bitmap"));
1705 - result = 1;
1706 - clear_packing_data(&pdata);
1707 - free(commits);
1708 - goto cleanup;
1709 - }
1710 -
1711 - clear_packing_data(&pdata);
1712 - free(commits);
1713 - }
1714 - /*
1715 - * NOTE: Do not use ctx.entries beyond this point, since it might
1716 - * have been freed in the previous if block.
1717 - */
1718 -
1719 - if (ctx.m)
1720 - close_object_store(the_repository->objects);
1721 -
1722 - if (commit_lock_file(&lk) < 0)
1723 - die_errno(_("could not write multi-pack-index"));
1724 -
1725 - clear_midx_files_ext(object_dir, ".bitmap", midx_hash);
1726 - clear_midx_files_ext(object_dir, ".rev", midx_hash);
1727 -
1728 -cleanup:
1729 - for (i = 0; i < ctx.nr; i++) {
1730 - if (ctx.info[i].p) {
1731 - close_pack(ctx.info[i].p);
1732 - free(ctx.info[i].p);
1733 - }
1734 - free(ctx.info[i].pack_name);
1735 - }
1736 -
1737 - free(ctx.info);
1738 - free(ctx.entries);
1739 - free(ctx.pack_perm);
1740 - free(ctx.pack_order);
1741 - strbuf_release(&midx_name);
1742 -
1743 - trace2_region_leave("midx", "write_midx_internal", the_repository);
1744 -
1745 - return result;
1746 -}
1747 -
1748 -int write_midx_file(const char *object_dir,
1749 - const char *preferred_pack_name,
1750 - const char *refs_snapshot,
1751 - unsigned flags)
1752 -{
1753 - return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name,
1754 - refs_snapshot, flags);
1755 -}
1756 -
1757 -int write_midx_file_only(const char *object_dir,
1758 - struct string_list *packs_to_include,
1759 - const char *preferred_pack_name,
1760 - const char *refs_snapshot,
1761 - unsigned flags)
1762 -{
1763 - return write_midx_internal(object_dir, packs_to_include, NULL,
1764 - preferred_pack_name, refs_snapshot, flags);
1765 -}
1766 -
1767 -struct clear_midx_data {
1768 - char *keep;
1769 - const char *ext;
1770 -};
1771 -
1772 -static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED,
1773 - const char *file_name, void *_data)
1774 -{
1775 - struct clear_midx_data *data = _data;
1776 -
1777 - if (!(starts_with(file_name, "multi-pack-index-") &&
1778 - ends_with(file_name, data->ext)))
1779 - return;
1780 - if (data->keep && !strcmp(data->keep, file_name))
1781 - return;
1782 -
1783 - if (unlink(full_path))
1784 - die_errno(_("failed to remove %s"), full_path);
1785 -}
1786 -
1787 -static void clear_midx_files_ext(const char *object_dir, const char *ext,
1788 - unsigned char *keep_hash)
1789 -{
1790 - struct clear_midx_data data;
1791 - memset(&data, 0, sizeof(struct clear_midx_data));
1792 -
1793 - if (keep_hash)
1794 - data.keep = xstrfmt("multi-pack-index-%s%s",
1795 - hash_to_hex(keep_hash), ext);
1796 - data.ext = ext;
1797 -
1798 - for_each_file_in_pack_dir(object_dir,
1799 - clear_midx_file_ext,
1800 - &data);
1801 -
1802 - free(data.keep);
1803 -}
1804 -
1805 -void clear_midx_file(struct repository *r)
1806 -{
1807 - struct strbuf midx = STRBUF_INIT;
1808 -
1809 - get_midx_filename(&midx, r->objects->odb->path);
1810 -
1811 - if (r->objects && r->objects->multi_pack_index) {
1812 - close_midx(r->objects->multi_pack_index);
1813 - r->objects->multi_pack_index = NULL;
1814 - }
1815 -
1816 - if (remove_path(midx.buf))
1817 - die(_("failed to clear multi-pack-index at %s"), midx.buf);
1818 -
1819 - clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL);
1820 - clear_midx_files_ext(r->objects->odb->path, ".rev", NULL);
1821 -
1822 - strbuf_release(&midx);
1823 -}
1824 -
1825 -static int verify_midx_error;
1826 -
1827 -__attribute__((format (printf, 1, 2)))
1828 -static void midx_report(const char *fmt, ...)
1829 -{
1830 - va_list ap;
1831 - verify_midx_error = 1;
1832 - va_start(ap, fmt);
1833 - vfprintf(stderr, fmt, ap);
1834 - fprintf(stderr, "\n");
1835 - va_end(ap);
1836 -}
1837 -
1838 -struct pair_pos_vs_id
1839 -{
1840 - uint32_t pos;
1841 - uint32_t pack_int_id;
1842 -};
1843 -
1844 -static int compare_pair_pos_vs_id(const void *_a, const void *_b)
1845 -{
1846 - struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a;
1847 - struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b;
1848 -
1849 - return b->pack_int_id - a->pack_int_id;
1850 -}
1851 -
1852 -/*
1853 - * Limit calls to display_progress() for performance reasons.
1854 - * The interval here was arbitrarily chosen.
1855 - */
1856 -#define SPARSE_PROGRESS_INTERVAL (1 << 12)
1857 -#define midx_display_sparse_progress(progress, n) \
1858 - do { \
1859 - uint64_t _n = (n); \
1860 - if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \
1861 - display_progress(progress, _n); \
1862 - } while (0)
1863 -
1864 -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags)
1865 -{
1866 - struct pair_pos_vs_id *pairs = NULL;
1867 - uint32_t i;
1868 - struct progress *progress = NULL;
1869 - struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
1870 - verify_midx_error = 0;
1871 -
1872 - if (!m) {
1873 - int result = 0;
1874 - struct stat sb;
1875 - struct strbuf filename = STRBUF_INIT;
1876 -
1877 - get_midx_filename(&filename, object_dir);
1878 -
1879 - if (!stat(filename.buf, &sb)) {
1880 - error(_("multi-pack-index file exists, but failed to parse"));
1881 - result = 1;
1882 - }
1883 - strbuf_release(&filename);
1884 - return result;
1885 - }
1886 -
1887 - if (!midx_checksum_valid(m))
1888 - midx_report(_("incorrect checksum"));
1889 -
1890 - if (flags & MIDX_PROGRESS)
1891 - progress = start_delayed_progress(_("Looking for referenced packfiles"),
1892 - m->num_packs);
1893 - for (i = 0; i < m->num_packs; i++) {
1894 - if (prepare_midx_pack(r, m, i))
1895 - midx_report("failed to load pack in position %d", i);
1896 -
1897 - display_progress(progress, i + 1);
1898 - }
1899 - stop_progress(&progress);
1900 -
1901 - if (m->num_objects == 0) {
1902 - midx_report(_("the midx contains no oid"));
1903 - /*
1904 - * Remaining tests assume that we have objects, so we can
1905 - * return here.
1906 - */
1907 - goto cleanup;
1908 - }
1909 -
1910 - if (flags & MIDX_PROGRESS)
1911 - progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"),
1912 - m->num_objects - 1);
1913 - for (i = 0; i < m->num_objects - 1; i++) {
1914 - struct object_id oid1, oid2;
1915 -
1916 - nth_midxed_object_oid(&oid1, m, i);
1917 - nth_midxed_object_oid(&oid2, m, i + 1);
1918 -
1919 - if (oidcmp(&oid1, &oid2) >= 0)
1920 - midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
1921 - i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
1922 -
1923 - midx_display_sparse_progress(progress, i + 1);
1924 - }
1925 - stop_progress(&progress);
1926 -
1927 - /*
1928 - * Create an array mapping each object to its packfile id. Sort it
1929 - * to group the objects by packfile. Use this permutation to visit
1930 - * each of the objects and only require 1 packfile to be open at a
1931 - * time.
1932 - */
1933 - ALLOC_ARRAY(pairs, m->num_objects);
1934 - for (i = 0; i < m->num_objects; i++) {
1935 - pairs[i].pos = i;
1936 - pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
1937 - }
1938 -
1939 - if (flags & MIDX_PROGRESS)
1940 - progress = start_sparse_progress(_("Sorting objects by packfile"),
1941 - m->num_objects);
1942 - display_progress(progress, 0); /* TODO: Measure QSORT() progress */
1943 - QSORT(pairs, m->num_objects, compare_pair_pos_vs_id);
1944 - stop_progress(&progress);
1945 -
1946 - if (flags & MIDX_PROGRESS)
1947 - progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects);
1948 - for (i = 0; i < m->num_objects; i++) {
1949 - struct object_id oid;
1950 - struct pack_entry e;
1951 - off_t m_offset, p_offset;
1952 -
1953 - if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
1954 - m->packs[pairs[i-1].pack_int_id])
1955 - {
1956 - close_pack_fd(m->packs[pairs[i-1].pack_int_id]);
1957 - close_pack_index(m->packs[pairs[i-1].pack_int_id]);
1958 - }
1959 -
1960 - nth_midxed_object_oid(&oid, m, pairs[i].pos);
1961 -
1962 - if (!fill_midx_entry(r, &oid, &e, m)) {
1963 - midx_report(_("failed to load pack entry for oid[%d] = %s"),
1964 - pairs[i].pos, oid_to_hex(&oid));
1965 - continue;
1966 - }
1967 -
1968 - if (open_pack_index(e.p)) {
1969 - midx_report(_("failed to load pack-index for packfile %s"),
1970 - e.p->pack_name);
1971 - break;
1972 - }
1973 -
1974 - m_offset = e.offset;
1975 - p_offset = find_pack_entry_one(oid.hash, e.p);
1976 -
1977 - if (m_offset != p_offset)
1978 - midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
1979 - pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
1980 -
1981 - midx_display_sparse_progress(progress, i + 1);
1982 - }
1983 - stop_progress(&progress);
1984 -
1985 -cleanup:
1986 - free(pairs);
1987 - close_midx(m);
708 +cleanup:
709 + free(pairs);
710 + close_midx(m);
711
712 return verify_midx_error;
713 }
1991 -
1992 -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags)
1993 -{
1994 - uint32_t i, *count, result = 0;
1995 - struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
1996 - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
1997 - struct progress *progress = NULL;
1998 -
1999 - if (!m)
2000 - return 0;
2001 -
2002 - CALLOC_ARRAY(count, m->num_packs);
2003 -
2004 - if (flags & MIDX_PROGRESS)
2005 - progress = start_delayed_progress(_("Counting referenced objects"),
2006 - m->num_objects);
2007 - for (i = 0; i < m->num_objects; i++) {
2008 - int pack_int_id = nth_midxed_pack_int_id(m, i);
2009 - count[pack_int_id]++;
2010 - display_progress(progress, i + 1);
2011 - }
2012 - stop_progress(&progress);
2013 -
2014 - if (flags & MIDX_PROGRESS)
2015 - progress = start_delayed_progress(_("Finding and deleting unreferenced packfiles"),
2016 - m->num_packs);
2017 - for (i = 0; i < m->num_packs; i++) {
2018 - char *pack_name;
2019 - display_progress(progress, i + 1);
2020 -
2021 - if (count[i])
2022 - continue;
2023 -
2024 - if (prepare_midx_pack(r, m, i))
2025 - continue;
2026 -
2027 - if (m->packs[i]->pack_keep || m->packs[i]->is_cruft)
2028 - continue;
2029 -
2030 - pack_name = xstrdup(m->packs[i]->pack_name);
2031 - close_pack(m->packs[i]);
2032 -
2033 - string_list_insert(&packs_to_drop, m->pack_names[i]);
2034 - unlink_pack_path(pack_name, 0);
2035 - free(pack_name);
2036 - }
2037 - stop_progress(&progress);
2038 -
2039 - free(count);
2040 -
2041 - if (packs_to_drop.nr)
2042 - result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags);
2043 -
2044 - string_list_clear(&packs_to_drop, 0);
2045 -
2046 - return result;
2047 -}
2048 -
2049 -struct repack_info {
2050 - timestamp_t mtime;
2051 - uint32_t referenced_objects;
2052 - uint32_t pack_int_id;
2053 -};
2054 -
2055 -static int compare_by_mtime(const void *a_, const void *b_)
2056 -{
2057 - const struct repack_info *a, *b;
2058 -
2059 - a = (const struct repack_info *)a_;
2060 - b = (const struct repack_info *)b_;
2061 -
2062 - if (a->mtime < b->mtime)
2063 - return -1;
2064 - if (a->mtime > b->mtime)
2065 - return 1;
2066 - return 0;
2067 -}
2068 -
2069 -static int fill_included_packs_all(struct repository *r,
2070 - struct multi_pack_index *m,
2071 - unsigned char *include_pack)
2072 -{
2073 - uint32_t i, count = 0;
2074 - int pack_kept_objects = 0;
2075 -
2076 - repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
2077 -
2078 - for (i = 0; i < m->num_packs; i++) {
2079 - if (prepare_midx_pack(r, m, i))
2080 - continue;
2081 - if (!pack_kept_objects && m->packs[i]->pack_keep)
2082 - continue;
2083 - if (m->packs[i]->is_cruft)
2084 - continue;
2085 -
2086 - include_pack[i] = 1;
2087 - count++;
2088 - }
2089 -
2090 - return count < 2;
2091 -}
2092 -
2093 -static int fill_included_packs_batch(struct repository *r,
2094 - struct multi_pack_index *m,
2095 - unsigned char *include_pack,
2096 - size_t batch_size)
2097 -{
2098 - uint32_t i, packs_to_repack;
2099 - size_t total_size;
2100 - struct repack_info *pack_info;
2101 - int pack_kept_objects = 0;
2102 -
2103 - CALLOC_ARRAY(pack_info, m->num_packs);
2104 -
2105 - repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
2106 -
2107 - for (i = 0; i < m->num_packs; i++) {
2108 - pack_info[i].pack_int_id = i;
2109 -
2110 - if (prepare_midx_pack(r, m, i))
2111 - continue;
2112 -
2113 - pack_info[i].mtime = m->packs[i]->mtime;
2114 - }
2115 -
2116 - for (i = 0; i < m->num_objects; i++) {
2117 - uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
2118 - pack_info[pack_int_id].referenced_objects++;
2119 - }
2120 -
2121 - QSORT(pack_info, m->num_packs, compare_by_mtime);
2122 -
2123 - total_size = 0;
2124 - packs_to_repack = 0;
2125 - for (i = 0; total_size < batch_size && i < m->num_packs; i++) {
2126 - int pack_int_id = pack_info[i].pack_int_id;
2127 - struct packed_git *p = m->packs[pack_int_id];
2128 - size_t expected_size;
2129 -
2130 - if (!p)
2131 - continue;
2132 - if (!pack_kept_objects && p->pack_keep)
2133 - continue;
2134 - if (p->is_cruft)
2135 - continue;
2136 - if (open_pack_index(p) || !p->num_objects)
2137 - continue;
2138 -
2139 - expected_size = st_mult(p->pack_size,
2140 - pack_info[i].referenced_objects);
2141 - expected_size /= p->num_objects;
2142 -
2143 - if (expected_size >= batch_size)
2144 - continue;
2145 -
2146 - packs_to_repack++;
2147 - total_size += expected_size;
2148 - include_pack[pack_int_id] = 1;
2149 - }
2150 -
2151 - free(pack_info);
2152 -
2153 - if (packs_to_repack < 2)
2154 - return 1;
2155 -
2156 - return 0;
2157 -}
2158 -
2159 -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags)
2160 -{
2161 - int result = 0;
2162 - uint32_t i;
2163 - unsigned char *include_pack;
2164 - struct child_process cmd = CHILD_PROCESS_INIT;
2165 - FILE *cmd_in;
2166 - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
2167 -
2168 - /*
2169 - * When updating the default for these configuration
2170 - * variables in builtin/repack.c, these must be adjusted
2171 - * to match.
2172 - */
2173 - int delta_base_offset = 1;
2174 - int use_delta_islands = 0;
2175 -
2176 - if (!m)
2177 - return 0;
2178 -
2179 - CALLOC_ARRAY(include_pack, m->num_packs);
2180 -
2181 - if (batch_size) {
2182 - if (fill_included_packs_batch(r, m, include_pack, batch_size))
2183 - goto cleanup;
2184 - } else if (fill_included_packs_all(r, m, include_pack))
2185 - goto cleanup;
2186 -
2187 - repo_config_get_bool(r, "repack.usedeltabaseoffset", &delta_base_offset);
2188 - repo_config_get_bool(r, "repack.usedeltaislands", &use_delta_islands);
2189 -
2190 - strvec_push(&cmd.args, "pack-objects");
2191 -
2192 - strvec_pushf(&cmd.args, "%s/pack/pack", object_dir);
2193 -
2194 - if (delta_base_offset)
2195 - strvec_push(&cmd.args, "--delta-base-offset");
2196 - if (use_delta_islands)
2197 - strvec_push(&cmd.args, "--delta-islands");
2198 -
2199 - if (flags & MIDX_PROGRESS)
2200 - strvec_push(&cmd.args, "--progress");
2201 - else
2202 - strvec_push(&cmd.args, "-q");
2203 -
2204 - cmd.git_cmd = 1;
2205 - cmd.in = cmd.out = -1;
2206 -
2207 - if (start_command(&cmd)) {
2208 - error(_("could not start pack-objects"));
2209 - result = 1;
2210 - goto cleanup;
2211 - }
2212 -
2213 - cmd_in = xfdopen(cmd.in, "w");
2214 -
2215 - for (i = 0; i < m->num_objects; i++) {
2216 - struct object_id oid;
2217 - uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
2218 -
2219 - if (!include_pack[pack_int_id])
2220 - continue;
2221 -
2222 - nth_midxed_object_oid(&oid, m, i);
2223 - fprintf(cmd_in, "%s\n", oid_to_hex(&oid));
2224 - }
2225 - fclose(cmd_in);
2226 -
2227 - if (finish_command(&cmd)) {
2228 - error(_("could not finish pack-objects"));
2229 - result = 1;
2230 - goto cleanup;
2231 - }
2232 -
2233 - result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags);
2234 -
2235 -cleanup:
2236 - free(include_pack);
2237 - return result;
2238 -}
midx.h
+19
@@ -8,6 +8,25 @@ struct pack_entry;
8 struct repository;
9 struct bitmapped_pack;
10
11 +#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
12 +#define MIDX_VERSION 1
13 +#define MIDX_BYTE_FILE_VERSION 4
14 +#define MIDX_BYTE_HASH_VERSION 5
15 +#define MIDX_BYTE_NUM_CHUNKS 6
16 +#define MIDX_BYTE_NUM_PACKS 8
17 +#define MIDX_HEADER_SIZE 12
18 +
19 +#define MIDX_CHUNK_ALIGNMENT 4
20 +#define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
21 +#define MIDX_CHUNKID_BITMAPPEDPACKS 0x42544d50 /* "BTMP" */
22 +#define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
23 +#define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
24 +#define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
25 +#define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
26 +#define MIDX_CHUNKID_REVINDEX 0x52494458 /* "RIDX" */
27 +#define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
28 +#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
29 +
30 #define GIT_TEST_MULTI_PACK_INDEX "GIT_TEST_MULTI_PACK_INDEX"
31 #define GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP \
32 "GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP"