@matheusbsbs / matheus-bagliano / commits / 341fa34887

pack-bitmap-write: use existing bitmaps

When constructing new bitmaps, we perform a commit and tree walk in fill_bitmap_commit() and fill_bitmap_tree(). This walk would benefit from using existing bitmaps when available. We must track the existing bitmaps and translate them into the new object order, but this is generally faster than parsing trees. In fill_bitmap_commit(), we must reorder thing somewhat. The priority queue walks commits from newest-to-oldest, which means we correctly stop walking when reaching a commit with a bitmap. However, if we walk trees interleaved with the commits, then we might be parsing trees that are actually part of a re-used bitmap. To avoid over-walking trees, add them to a LIFO queue and walk them after exploring commits completely. On git.git, this reduces a second immediate bitmap computation from 2.0s to 1.0s. On linux.git, we go from 32s to 22s. On chromium's fork network, we go from 227s to 198s. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Dec 8, 2020 at 17:05 UTC 341fa34887630a7adf6b3a771ae866ce66e7967e

1 file changed +36 -4

pack-bitmap-write.c

+36 -4

@@ -340,20 +340,37 @@ static void fill_bitmap_tree(struct bitmap *bitmap,

340

341 static void fill_bitmap_commit(struct bb_commit *ent,

342 struct commit *commit,

343 - struct prio_queue *queue)

343 + struct prio_queue *queue,

344 + struct prio_queue *tree_queue,

345 + struct bitmap_index *old_bitmap,

346 + const uint32_t *mapping)

347 {

348 if (!ent->bitmap)

349 ent->bitmap = bitmap_new();

350

348 - bitmap_set(ent->bitmap, find_object_pos(&commit->object.oid));

351 prio_queue_put(queue, commit);

352

353 while (queue->nr) {

354 struct commit_list *p;

355 struct commit *c = prio_queue_get(queue);

356

357 + if (old_bitmap && mapping) {

358 + struct ewah_bitmap *old = bitmap_for_commit(old_bitmap, c);

359 + /*

360 + * If this commit has an old bitmap, then translate that

361 + * bitmap and add its bits to this one. No need to walk

362 + * parents or the tree for this commit.

363 + */

364 + if (old && !rebuild_bitmap(mapping, old, ent->bitmap))

365 + continue;

366 + }

367 +

368 + /*

369 + * Mark ourselves and queue our tree. The commit

370 + * walk ensures we cover all parents.

371 + */

372 bitmap_set(ent->bitmap, find_object_pos(&c->object.oid));

356 - fill_bitmap_tree(ent->bitmap, get_commit_tree(c));

373 + prio_queue_put(tree_queue, get_commit_tree(c));

374

375 for (p = c->parents; p; p = p->next) {

376 int pos = find_object_pos(&p->item->object.oid);

@@ -363,6 +380,9 @@ static void fill_bitmap_commit(struct bb_commit *ent,

380 }

381 }

382 }

383 +

384 + while (tree_queue->nr)

385 + fill_bitmap_tree(ent->bitmap, prio_queue_get(tree_queue));

386 }

387

388 static void store_selected(struct bb_commit *ent, struct commit *commit)

@@ -386,6 +406,9 @@ void bitmap_writer_build(struct packing_data *to_pack)

406 size_t i;

407 int nr_stored = 0; /* for progress */

408 struct prio_queue queue = { compare_commits_by_gen_then_commit_date };

409 + struct prio_queue tree_queue = { NULL };

410 + struct bitmap_index *old_bitmap;

411 + uint32_t *mapping;

412

413 writer.bitmaps = kh_init_oid_map();

414 writer.to_pack = to_pack;

@@ -395,6 +418,12 @@ void bitmap_writer_build(struct packing_data *to_pack)

418 trace2_region_enter("pack-bitmap-write", "building_bitmaps_total",

419 the_repository);

420

421 + old_bitmap = prepare_bitmap_git(to_pack->repo);

422 + if (old_bitmap)

423 + mapping = create_bitmap_mapping(old_bitmap, to_pack);

424 + else

425 + mapping = NULL;

426 +

427 bitmap_builder_init(&bb, &writer);

428 for (i = bb.commits_nr; i > 0; i--) {

429 struct commit *commit = bb.commits[i-1];

@@ -402,7 +431,8 @@ void bitmap_writer_build(struct packing_data *to_pack)

431 struct commit *child;

432 int reused = 0;

433

405 - fill_bitmap_commit(ent, commit, &queue);

434 + fill_bitmap_commit(ent, commit, &queue, &tree_queue,

435 + old_bitmap, mapping);

436

437 if (ent->selected) {

438 store_selected(ent, commit);

@@ -428,7 +458,9 @@ void bitmap_writer_build(struct packing_data *to_pack)

458 ent->bitmap = NULL;

459 }

460 clear_prio_queue(&queue);

461 + clear_prio_queue(&tree_queue);

462 bitmap_builder_clear(&bb);

463 + free(mapping);

464

465 trace2_region_leave("pack-bitmap-write", "building_bitmaps_total",

466 the_repository);