@matheusbsbs / matheus-bagliano / commits / 0f533c7284

pack-bitmap: read multi-pack bitmaps

This prepares the code in pack-bitmap to interpret the new multi-pack bitmaps described in Documentation/technical/bitmap-format.txt, which mostly involves converting bit positions to accommodate looking them up in a MIDX. Note that there are currently no writers who write multi-pack bitmaps, and that this will be implemented in the subsequent commit. Note also that get_midx_checksum() and get_midx_filename() are made non-static so they can be called from pack-bitmap.c. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Aug 31, 2021 at 16:52 UTC 0f533c728418fd3ef6ebcae5240e8df566cdaa72

7 files changed +336 -42

builtin/pack-objects.c

@@ -1124,6 +1124,11 @@ static void write_reused_pack(struct hashfile *f)

1124 break;

1125

1126 offset += ewah_bit_ctz64(word >> offset);

1127 + /*

1128 + * Can use bit positions directly, even for MIDX

1129 + * bitmaps. See comment in try_partial_reuse()

1130 + * for why.

1131 + */

1132 write_reused_pack_one(pos + offset, f, &w_curs);

1133 display_progress(progress_state, ++written);

1134 }

midx.c

+2 -2

@@ -48,12 +48,12 @@ static uint8_t oid_version(void)

48 }

49 }

51 -static const unsigned char *get_midx_checksum(struct multi_pack_index *m)

51 +const unsigned char *get_midx_checksum(struct multi_pack_index *m)

52 {

53 return m->data + m->data_len - the_hash_algo->rawsz;

54 }

56 -static char *get_midx_filename(const char *object_dir)

56 +char *get_midx_filename(const char *object_dir)

57 {

58 return xstrfmt("%s/pack/multi-pack-index", object_dir);

59 }

midx.h

@@ -42,6 +42,8 @@ struct multi_pack_index {

42 #define MIDX_PROGRESS (1 << 0)

43 #define MIDX_WRITE_REV_INDEX (1 << 1)

45 +const unsigned char *get_midx_checksum(struct multi_pack_index *m);

46 +char *get_midx_filename(const char *object_dir);

47 char *get_midx_rev_filename(struct multi_pack_index *m);

49 struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local);

pack-bitmap-write.c

+1 -1

@@ -48,7 +48,7 @@ void bitmap_writer_show_progress(int show)

48 }

50 /**

51 - * Build the initial type index for the packfile

51 + * Build the initial type index for the packfile or multi-pack-index

52 */

53 void bitmap_writer_build_type_index(struct packing_data *to_pack,

54 struct pack_idx_entry **index,

pack-bitmap.c

+319 -38

@@ -13,6 +13,7 @@

13 #include "repository.h"

14 #include "object-store.h"

15 #include "list-objects-filter-options.h"

16 +#include "midx.h"

17 #include "config.h"

19 /*

@@ -35,8 +36,15 @@ struct stored_bitmap {

36 * the active bitmap index is the largest one.

37 */

38 struct bitmap_index {

38 - /* Packfile to which this bitmap index belongs to */

39 + /*

40 + * The pack or multi-pack index (MIDX) that this bitmap index belongs

41 + * to.

42 + *

43 + * Exactly one of these must be non-NULL; this specifies the object

44 + * order used to interpret this bitmap.

45 + */

46 struct packed_git *pack;

47 + struct multi_pack_index *midx;

49 /*

50 * Mark the first `reuse_objects` in the packfile as reused:

@@ -71,6 +79,9 @@ struct bitmap_index {

79 /* If not NULL, this is a name-hash cache pointing into map. */

80 uint32_t *hashes;

82 + /* The checksum of the packfile or MIDX; points into map. */

83 + const unsigned char *checksum;

84 +

85 /*

86 * Extended index.

87 *

@@ -138,6 +149,8 @@ static struct ewah_bitmap *read_bitmap_1(struct bitmap_index *index)

149

150 static uint32_t bitmap_num_objects(struct bitmap_index *index)

151 {

152 + if (index->midx)

153 + return index->midx->num_objects;

154 return index->pack->num_objects;

155 }

156

@@ -175,6 +188,7 @@ static int load_bitmap_header(struct bitmap_index *index)

188 }

189

190 index->entry_count = ntohl(header->entry_count);

191 + index->checksum = header->checksum;

192 index->map_pos += header_size;

193 return 0;

194 }

@@ -227,6 +241,8 @@ static int nth_bitmap_object_oid(struct bitmap_index *index,

241 struct object_id *oid,

242 uint32_t n)

243 {

244 + if (index->midx)

245 + return nth_midxed_object_oid(oid, index->midx, n) ? 0 : -1;

246 return nth_packed_object_id(oid, index->pack, n);

247 }

248

@@ -274,7 +290,14 @@ static int load_bitmap_entries_v1(struct bitmap_index *index)

290 return 0;

291 }

292

277 -static char *pack_bitmap_filename(struct packed_git *p)

293 +char *midx_bitmap_filename(struct multi_pack_index *midx)

294 +{

295 + return xstrfmt("%s-%s.bitmap",

296 + get_midx_filename(midx->object_dir),

297 + hash_to_hex(get_midx_checksum(midx)));

298 +}

299 +

300 +char *pack_bitmap_filename(struct packed_git *p)

301 {

302 size_t len;

303

@@ -283,6 +306,57 @@ static char *pack_bitmap_filename(struct packed_git *p)

306 return xstrfmt("%.*s.bitmap", (int)len, p->pack_name);

307 }

308

309 +static int open_midx_bitmap_1(struct bitmap_index *bitmap_git,

310 + struct multi_pack_index *midx)

311 +{

312 + struct stat st;

313 + char *idx_name = midx_bitmap_filename(midx);

314 + int fd = git_open(idx_name);

315 +

316 + free(idx_name);

317 +

318 + if (fd < 0)

319 + return -1;

320 +

321 + if (fstat(fd, &st)) {

322 + close(fd);

323 + return -1;

324 + }

325 +

326 + if (bitmap_git->pack || bitmap_git->midx) {

327 + /* ignore extra bitmap file; we can only handle one */

328 + warning("ignoring extra bitmap file: %s",

329 + get_midx_filename(midx->object_dir));

330 + close(fd);

331 + return -1;

332 + }

333 +

334 + bitmap_git->midx = midx;

335 + bitmap_git->map_size = xsize_t(st.st_size);

336 + bitmap_git->map_pos = 0;

337 + bitmap_git->map = xmmap(NULL, bitmap_git->map_size, PROT_READ,

338 + MAP_PRIVATE, fd, 0);

339 + close(fd);

340 +

341 + if (load_bitmap_header(bitmap_git) < 0)

342 + goto cleanup;

343 +

344 + if (!hasheq(get_midx_checksum(bitmap_git->midx), bitmap_git->checksum))

345 + goto cleanup;

346 +

347 + if (load_midx_revindex(bitmap_git->midx) < 0) {

348 + warning(_("multi-pack bitmap is missing required reverse index"));

349 + goto cleanup;

350 + }

351 + return 0;

352 +

353 +cleanup:

354 + munmap(bitmap_git->map, bitmap_git->map_size);

355 + bitmap_git->map_size = 0;

356 + bitmap_git->map = NULL;

357 + return -1;

358 +}

359 +

360 static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git *packfile)

361 {

362 int fd;

@@ -304,7 +378,8 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git

378 return -1;

379 }

380

307 - if (bitmap_git->pack) {

381 + if (bitmap_git->pack || bitmap_git->midx) {

382 + /* ignore extra bitmap file; we can only handle one */

383 warning("ignoring extra bitmap file: %s", packfile->pack_name);

384 close(fd);

385 return -1;

@@ -331,13 +406,39 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git

406 return 0;

407 }

408

334 -static int load_pack_bitmap(struct bitmap_index *bitmap_git)

409 +static int load_reverse_index(struct bitmap_index *bitmap_git)

410 +{

411 + if (bitmap_is_midx(bitmap_git)) {

412 + uint32_t i;

413 + int ret;

414 +

415 + /*

416 + * The multi-pack-index's .rev file is already loaded via

417 + * open_pack_bitmap_1().

418 + *

419 + * But we still need to open the individual pack .rev files,

420 + * since we will need to make use of them in pack-objects.

421 + */

422 + for (i = 0; i < bitmap_git->midx->num_packs; i++) {

423 + if (prepare_midx_pack(the_repository, bitmap_git->midx, i))

424 + die(_("load_reverse_index: could not open pack"));

425 + ret = load_pack_revindex(bitmap_git->midx->packs[i]);

426 + if (ret)

427 + return ret;

428 + }

429 + return 0;

430 + }

431 + return load_pack_revindex(bitmap_git->pack);

432 +}

433 +

434 +static int load_bitmap(struct bitmap_index *bitmap_git)

435 {

436 assert(bitmap_git->map);

437

438 bitmap_git->bitmaps = kh_init_oid_map();

439 bitmap_git->ext_index.positions = kh_init_oid_pos();

340 - if (load_pack_revindex(bitmap_git->pack))

440 +

441 + if (load_reverse_index(bitmap_git))

442 goto failed;

443

444 if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) ||

@@ -381,11 +482,47 @@ static int open_pack_bitmap(struct repository *r,

482 return ret;

483 }

484

485 +static int open_midx_bitmap(struct repository *r,

486 + struct bitmap_index *bitmap_git)

487 +{

488 + struct multi_pack_index *midx;

489 +

490 + assert(!bitmap_git->map);

491 +

492 + for (midx = get_multi_pack_index(r); midx; midx = midx->next) {

493 + if (!open_midx_bitmap_1(bitmap_git, midx))

494 + return 0;

495 + }

496 + return -1;

497 +}

498 +

499 +static int open_bitmap(struct repository *r,

500 + struct bitmap_index *bitmap_git)

501 +{

502 + assert(!bitmap_git->map);

503 +

504 + if (!open_midx_bitmap(r, bitmap_git))

505 + return 0;

506 + return open_pack_bitmap(r, bitmap_git);

507 +}

508 +

509 struct bitmap_index *prepare_bitmap_git(struct repository *r)

510 {

511 struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git));

512

388 - if (!open_pack_bitmap(r, bitmap_git) && !load_pack_bitmap(bitmap_git))

513 + if (!open_bitmap(r, bitmap_git) && !load_bitmap(bitmap_git))

514 + return bitmap_git;

515 +

516 + free_bitmap_index(bitmap_git);

517 + return NULL;

518 +}

519 +

520 +struct bitmap_index *prepare_midx_bitmap_git(struct repository *r,

521 + struct multi_pack_index *midx)

522 +{

523 + struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git));

524 +

525 + if (!open_midx_bitmap_1(bitmap_git, midx) && !load_bitmap(bitmap_git))

526 return bitmap_git;

527

528 free_bitmap_index(bitmap_git);

@@ -435,10 +572,26 @@ static inline int bitmap_position_packfile(struct bitmap_index *bitmap_git,

572 return pos;

573 }

574

575 +static int bitmap_position_midx(struct bitmap_index *bitmap_git,

576 + const struct object_id *oid)

577 +{

578 + uint32_t want, got;

579 + if (!bsearch_midx(oid, bitmap_git->midx, &want))

580 + return -1;

581 +

582 + if (midx_to_pack_pos(bitmap_git->midx, want, &got) < 0)

583 + return -1;

584 + return got;

585 +}

586 +

587 static int bitmap_position(struct bitmap_index *bitmap_git,

588 const struct object_id *oid)

589 {

441 - int pos = bitmap_position_packfile(bitmap_git, oid);

590 + int pos;

591 + if (bitmap_is_midx(bitmap_git))

592 + pos = bitmap_position_midx(bitmap_git, oid);

593 + else

594 + pos = bitmap_position_packfile(bitmap_git, oid);

595 return (pos >= 0) ? pos : bitmap_position_extended(bitmap_git, oid);

596 }

597

@@ -749,6 +902,7 @@ static void show_objects_for_type(

902 continue;

903

904 for (offset = 0; offset < BITS_IN_EWORD; ++offset) {

905 + struct packed_git *pack;

906 struct object_id oid;

907 uint32_t hash = 0, index_pos;

908 off_t ofs;

@@ -758,14 +912,28 @@ static void show_objects_for_type(

912

913 offset += ewah_bit_ctz64(word >> offset);

914

761 - index_pos = pack_pos_to_index(bitmap_git->pack, pos + offset);

762 - ofs = pack_pos_to_offset(bitmap_git->pack, pos + offset);

763 - nth_packed_object_id(&oid, bitmap_git->pack, index_pos);

915 + if (bitmap_is_midx(bitmap_git)) {

916 + struct multi_pack_index *m = bitmap_git->midx;

917 + uint32_t pack_id;

918 +

919 + index_pos = pack_pos_to_midx(m, pos + offset);

920 + ofs = nth_midxed_offset(m, index_pos);

921 + nth_midxed_object_oid(&oid, m, index_pos);

922 +

923 + pack_id = nth_midxed_pack_int_id(m, index_pos);

924 + pack = bitmap_git->midx->packs[pack_id];

925 + } else {

926 + index_pos = pack_pos_to_index(bitmap_git->pack, pos + offset);

927 + ofs = pack_pos_to_offset(bitmap_git->pack, pos + offset);

928 + nth_bitmap_object_oid(bitmap_git, &oid, index_pos);

929 +

930 + pack = bitmap_git->pack;

931 + }

932

933 if (bitmap_git->hashes)

934 hash = get_be32(bitmap_git->hashes + index_pos);

935

768 - show_reach(&oid, object_type, 0, hash, bitmap_git->pack, ofs);

936 + show_reach(&oid, object_type, 0, hash, pack, ofs);

937 }

938 }

939 }

@@ -777,8 +945,13 @@ static int in_bitmapped_pack(struct bitmap_index *bitmap_git,

945 struct object *object = roots->item;

946 roots = roots->next;

947

780 - if (find_pack_entry_one(object->oid.hash, bitmap_git->pack) > 0)

781 - return 1;

948 + if (bitmap_is_midx(bitmap_git)) {

949 + if (bsearch_midx(&object->oid, bitmap_git->midx, NULL))

950 + return 1;

951 + } else {

952 + if (find_pack_entry_one(object->oid.hash, bitmap_git->pack) > 0)

953 + return 1;

954 + }

955 }

956

957 return 0;

@@ -865,14 +1038,26 @@ static void filter_bitmap_blob_none(struct bitmap_index *bitmap_git,

1038 static unsigned long get_size_by_pos(struct bitmap_index *bitmap_git,

1039 uint32_t pos)

1040 {

868 - struct packed_git *pack = bitmap_git->pack;

1041 unsigned long size;

1042 struct object_info oi = OBJECT_INFO_INIT;

1043

1044 oi.sizep = &size;

1045

1046 if (pos < bitmap_num_objects(bitmap_git)) {

875 - off_t ofs = pack_pos_to_offset(pack, pos);

1047 + struct packed_git *pack;

1048 + off_t ofs;

1049 +

1050 + if (bitmap_is_midx(bitmap_git)) {

1051 + uint32_t midx_pos = pack_pos_to_midx(bitmap_git->midx, pos);

1052 + uint32_t pack_id = nth_midxed_pack_int_id(bitmap_git->midx, midx_pos);

1053 +

1054 + pack = bitmap_git->midx->packs[pack_id];

1055 + ofs = nth_midxed_offset(bitmap_git->midx, midx_pos);

1056 + } else {

1057 + pack = bitmap_git->pack;

1058 + ofs = pack_pos_to_offset(pack, pos);

1059 + }

1060 +

1061 if (packed_object_info(the_repository, pack, ofs, &oi) < 0) {

1062 struct object_id oid;

1063 nth_bitmap_object_oid(bitmap_git, &oid,

@@ -1053,7 +1238,7 @@ struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs,

1238 /* try to open a bitmapped pack, but don't parse it yet

1239 * because we may not need to use it */

1240 CALLOC_ARRAY(bitmap_git, 1);

1056 - if (open_pack_bitmap(revs->repo, bitmap_git) < 0)

1241 + if (open_bitmap(revs->repo, bitmap_git) < 0)

1242 goto cleanup;

1243

1244 for (i = 0; i < revs->pending.nr; ++i) {

@@ -1097,7 +1282,7 @@ struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs,

1282 * from disk. this is the point of no return; after this the rev_list

1283 * becomes invalidated and we must perform the revwalk through bitmaps

1284 */

1100 - if (load_pack_bitmap(bitmap_git) < 0)

1285 + if (load_bitmap(bitmap_git) < 0)

1286 goto cleanup;

1287

1288 object_array_clear(&revs->pending);

@@ -1145,19 +1330,43 @@ cleanup:

1330 * reused, but you can keep feeding bits.

1331 */

1332 static int try_partial_reuse(struct bitmap_index *bitmap_git,

1333 + struct packed_git *pack,

1334 size_t pos,

1335 struct bitmap *reuse,

1336 struct pack_window **w_curs)

1337 {

1152 - off_t offset, header;

1338 + off_t offset, delta_obj_offset;

1339 enum object_type type;

1340 unsigned long size;

1341

1156 - if (pos >= bitmap_num_objects(bitmap_git))

1157 - return -1; /* not actually in the pack or MIDX */

1342 + /*

1343 + * try_partial_reuse() is called either on (a) objects in the

1344 + * bitmapped pack (in the case of a single-pack bitmap) or (b)

1345 + * objects in the preferred pack of a multi-pack bitmap.

1346 + * Importantly, the latter can pretend as if only a single pack

1347 + * exists because:

1348 + *

1349 + * - The first pack->num_objects bits of a MIDX bitmap are

1350 + * reserved for the preferred pack, and

1351 + *

1352 + * - Ties due to duplicate objects are always resolved in

1353 + * favor of the preferred pack.

1354 + *

1355 + * Therefore we do not need to ever ask the MIDX for its copy of

1356 + * an object by OID, since it will always select it from the

1357 + * preferred pack. Likewise, the selected copy of the base

1358 + * object for any deltas will reside in the same pack.

1359 + *

1360 + * This means that we can reuse pos when looking up the bit in

1361 + * the reuse bitmap, too, since bits corresponding to the

1362 + * preferred pack precede all bits from other packs.

1363 + */

1364 +

1365 + if (pos >= pack->num_objects)

1366 + return -1; /* not actually in the pack or MIDX preferred pack */

1367

1159 - offset = header = pack_pos_to_offset(bitmap_git->pack, pos);

1160 - type = unpack_object_header(bitmap_git->pack, w_curs, &offset, &size);

1368 + offset = delta_obj_offset = pack_pos_to_offset(pack, pos);

1369 + type = unpack_object_header(pack, w_curs, &offset, &size);

1370 if (type < 0)

1371 return -1; /* broken packfile, punt */

1372

@@ -1173,11 +1382,11 @@ static int try_partial_reuse(struct bitmap_index *bitmap_git,

1382 * and the normal slow path will complain about it in

1383 * more detail.

1384 */

1176 - base_offset = get_delta_base(bitmap_git->pack, w_curs,

1177 - &offset, type, header);

1385 + base_offset = get_delta_base(pack, w_curs, &offset, type,

1386 + delta_obj_offset);

1387 if (!base_offset)

1388 return 0;

1180 - if (offset_to_pack_pos(bitmap_git->pack, base_offset, &base_pos) < 0)

1389 + if (offset_to_pack_pos(pack, base_offset, &base_pos) < 0)

1390 return 0;

1391

1392 /*

@@ -1211,24 +1420,48 @@ static int try_partial_reuse(struct bitmap_index *bitmap_git,

1420 return 0;

1421 }

1422

1423 +static uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git)

1424 +{

1425 + struct multi_pack_index *m = bitmap_git->midx;

1426 + if (!m)

1427 + BUG("midx_preferred_pack: requires non-empty MIDX");

1428 + return nth_midxed_pack_int_id(m, pack_pos_to_midx(bitmap_git->midx, 0));

1429 +}

1430 +

1431 int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git,

1432 struct packed_git **packfile_out,

1433 uint32_t *entries,

1434 struct bitmap **reuse_out)

1435 {

1436 + struct packed_git *pack;

1437 struct bitmap *result = bitmap_git->result;

1438 struct bitmap *reuse;

1439 struct pack_window *w_curs = NULL;

1440 size_t i = 0;

1441 uint32_t offset;

1224 - uint32_t objects_nr = bitmap_num_objects(bitmap_git);

1442 + uint32_t objects_nr;

1443

1444 assert(result);

1445

1446 + load_reverse_index(bitmap_git);

1447 +

1448 + if (bitmap_is_midx(bitmap_git))

1449 + pack = bitmap_git->midx->packs[midx_preferred_pack(bitmap_git)];

1450 + else

1451 + pack = bitmap_git->pack;

1452 + objects_nr = pack->num_objects;

1453 +

1454 while (i < result->word_alloc && result->words[i] == (eword_t)~0)

1455 i++;

1456

1231 - /* Don't mark objects not in the packfile */

1457 + /*

1458 + * Don't mark objects not in the packfile or preferred pack. This bitmap

1459 + * marks objects eligible for reuse, but the pack-reuse code only

1460 + * understands how to reuse a single pack. Since the preferred pack is

1461 + * guaranteed to have all bases for its deltas (in a multi-pack bitmap),

1462 + * we use it instead of another pack. In single-pack bitmaps, the choice

1463 + * is made for us.

1464 + */

1465 if (i > objects_nr / BITS_IN_EWORD)

1466 i = objects_nr / BITS_IN_EWORD;

1467

@@ -1244,8 +1477,8 @@ int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git,

1477 break;

1478

1479 offset += ewah_bit_ctz64(word >> offset);

1247 - if (try_partial_reuse(bitmap_git, pos + offset, reuse,

1248 - &w_curs) < 0) {

1480 + if (try_partial_reuse(bitmap_git, pack, pos + offset,

1481 + reuse, &w_curs) < 0) {

1482 /*

1483 * try_partial_reuse indicated we couldn't reuse

1484 * any bits, so there is no point in trying more

@@ -1274,7 +1507,7 @@ done:

1507 * need to be handled separately.

1508 */

1509 bitmap_and_not(result, reuse);

1277 - *packfile_out = bitmap_git->pack;

1510 + *packfile_out = pack;

1511 *reuse_out = reuse;

1512 return 0;

1513 }

@@ -1548,6 +1781,12 @@ uint32_t *create_bitmap_mapping(struct bitmap_index *bitmap_git,

1781 uint32_t i, num_objects;

1782 uint32_t *reposition;

1783

1784 + if (!bitmap_is_midx(bitmap_git))

1785 + load_reverse_index(bitmap_git);

1786 + else if (load_midx_revindex(bitmap_git->midx) < 0)

1787 + BUG("rebuild_existing_bitmaps: missing required rev-cache "

1788 + "extension");

1789 +

1790 num_objects = bitmap_num_objects(bitmap_git);

1791 CALLOC_ARRAY(reposition, num_objects);

1792

@@ -1555,8 +1794,13 @@ uint32_t *create_bitmap_mapping(struct bitmap_index *bitmap_git,

1794 struct object_id oid;

1795 struct object_entry *oe;

1796

1558 - nth_packed_object_id(&oid, bitmap_git->pack,

1559 - pack_pos_to_index(bitmap_git->pack, i));

1797 + if (bitmap_is_midx(bitmap_git))

1798 + nth_midxed_object_oid(&oid,

1799 + bitmap_git->midx,

1800 + pack_pos_to_midx(bitmap_git->midx, i));

1801 + else

1802 + nth_packed_object_id(&oid, bitmap_git->pack,

1803 + pack_pos_to_index(bitmap_git->pack, i));

1804 oe = packlist_find(mapping, &oid);

1805

1806 if (oe)

@@ -1582,6 +1826,19 @@ void free_bitmap_index(struct bitmap_index *b)

1826 free(b->ext_index.hashes);

1827 bitmap_free(b->result);

1828 bitmap_free(b->haves);

1829 + if (bitmap_is_midx(b)) {

1830 + /*

1831 + * Multi-pack bitmaps need to have resources associated with

1832 + * their on-disk reverse indexes unmapped so that stale .rev and

1833 + * .bitmap files can be removed.

1834 + *

1835 + * Unlike pack-based bitmaps, multi-pack bitmaps can be read and

1836 + * written in the same 'git multi-pack-index write --bitmap'

1837 + * process. Close resources so they can be removed safely on

1838 + * platforms like Windows.

1839 + */

1840 + close_midx_revindex(b->midx);

1841 + }

1842 free(b);

1843 }

1844

@@ -1596,7 +1853,6 @@ static off_t get_disk_usage_for_type(struct bitmap_index *bitmap_git,

1853 enum object_type object_type)

1854 {

1855 struct bitmap *result = bitmap_git->result;

1599 - struct packed_git *pack = bitmap_git->pack;

1856 off_t total = 0;

1857 struct ewah_iterator it;

1858 eword_t filter;

@@ -1613,15 +1869,35 @@ static off_t get_disk_usage_for_type(struct bitmap_index *bitmap_git,

1869 continue;

1870

1871 for (offset = 0; offset < BITS_IN_EWORD; offset++) {

1616 - size_t pos;

1617 -

1872 if ((word >> offset) == 0)

1873 break;

1874

1875 offset += ewah_bit_ctz64(word >> offset);

1622 - pos = base + offset;

1623 - total += pack_pos_to_offset(pack, pos + 1) -

1624 - pack_pos_to_offset(pack, pos);

1876 +

1877 + if (bitmap_is_midx(bitmap_git)) {

1878 + uint32_t pack_pos;

1879 + uint32_t midx_pos = pack_pos_to_midx(bitmap_git->midx, base + offset);

1880 + off_t offset = nth_midxed_offset(bitmap_git->midx, midx_pos);

1881 +

1882 + uint32_t pack_id = nth_midxed_pack_int_id(bitmap_git->midx, midx_pos);

1883 + struct packed_git *pack = bitmap_git->midx->packs[pack_id];

1884 +

1885 + if (offset_to_pack_pos(pack, offset, &pack_pos) < 0) {

1886 + struct object_id oid;

1887 + nth_midxed_object_oid(&oid, bitmap_git->midx, midx_pos);

1888 +

1889 + die(_("could not find %s in pack %s at offset %"PRIuMAX),

1890 + oid_to_hex(&oid),

1891 + pack->pack_name,

1892 + (uintmax_t)offset);

1893 + }

1894 +

1895 + total += pack_pos_to_offset(pack, pack_pos + 1) - offset;

1896 + } else {

1897 + size_t pos = base + offset;

1898 + total += pack_pos_to_offset(bitmap_git->pack, pos + 1) -

1899 + pack_pos_to_offset(bitmap_git->pack, pos);

1900 + }

1901 }

1902 }

1903

@@ -1672,6 +1948,11 @@ off_t get_disk_usage_from_bitmap(struct bitmap_index *bitmap_git,

1948 return total;

1949 }

1950

1951 +int bitmap_is_midx(struct bitmap_index *bitmap_git)

1952 +{

1953 + return !!bitmap_git->midx;

1954 +}

1955 +

1956 const struct string_list *bitmap_preferred_tips(struct repository *r)

1957 {

1958 return repo_config_get_value_multi(r, "pack.preferbitmaptips");

pack-bitmap.h

@@ -44,6 +44,8 @@ typedef int (*show_reachable_fn)(

44 struct bitmap_index;

46 struct bitmap_index *prepare_bitmap_git(struct repository *r);

47 +struct bitmap_index *prepare_midx_bitmap_git(struct repository *r,

48 + struct multi_pack_index *midx);

49 void count_bitmap_commit_list(struct bitmap_index *, uint32_t *commits,

50 uint32_t *trees, uint32_t *blobs, uint32_t *tags);

51 void traverse_bitmap_commit_list(struct bitmap_index *,

@@ -92,6 +94,10 @@ void bitmap_writer_finish(struct pack_idx_entry **index,

94 uint32_t index_nr,

95 const char *filename,

96 uint16_t options);

97 +char *midx_bitmap_filename(struct multi_pack_index *midx);

98 +char *pack_bitmap_filename(struct packed_git *p);

99 +

100 +int bitmap_is_midx(struct bitmap_index *bitmap_git);

101

102 const struct string_list *bitmap_preferred_tips(struct repository *r);

103 int bitmap_is_preferred_refname(struct repository *r, const char *refname);

packfile.c

+1 -1

@@ -860,7 +860,7 @@ static void prepare_pack(const char *full_name, size_t full_name_len,

860 if (!strcmp(file_name, "multi-pack-index"))

861 return;

862 if (starts_with(file_name, "multi-pack-index") &&

863 - ends_with(file_name, ".rev"))

863 + (ends_with(file_name, ".bitmap") || ends_with(file_name, ".rev")))

864 return;

865 if (ends_with(file_name, ".idx") ||

866 ends_with(file_name, ".rev") ||