@matheusbsbs / matheus-bagliano / commits / f077b0a986

pack-bitmap-write: better reuse bitmaps

If the old bitmap file contains a bitmap for a given commit, then that commit does not need help from intermediate commits in its history to compute its final bitmap. Eject that commit from the walk and insert it into a separate list of reusable commits that are eventually stored in the list of commits for computing bitmaps. This helps the repeat bitmap computation task, even if the selected commits shift drastically. This helps when a previously-bitmapped commit exists in the first-parent history of a newly-selected commit. Since we stop the walk at these commits and we use a first-parent walk, it is harder to walk "around" these bitmapped commits. It's not impossible, but we can greatly reduce the computation time for many selected commits. | runtime (sec) | peak heap (GB) | | | | | from | with | from | with | | scratch | existing | scratch | existing | -----------+---------+----------+---------+----------- last patch | 88.478 | 53.218 | 2.157 | 2.224 | this patch | 86.681 | 16.164 | 2.157 | 2.222 | Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Dec 8, 2020 at 17:05 UTC f077b0a9860f383a3cd0ce3a0e2a11ff2f27fc65
1 file changed +38 -2
pack-bitmap-write.c
+38 -2
@@ -195,10 +195,13 @@ struct bitmap_builder {
195 };
196
197 static void bitmap_builder_init(struct bitmap_builder *bb,
198 - struct bitmap_writer *writer)
198 + struct bitmap_writer *writer,
199 + struct bitmap_index *old_bitmap)
200 {
201 struct rev_info revs;
202 struct commit *commit;
203 + struct commit_list *reusable = NULL;
204 + struct commit_list *r;
205 unsigned int i, num_maximal = 0;
206
207 memset(bb, 0, sizeof(*bb));
@@ -234,6 +237,31 @@ static void bitmap_builder_init(struct bitmap_builder *bb,
237
238 c_ent = bb_data_at(&bb->data, commit);
239
240 + /*
241 + * If there is no commit_mask, there is no reason to iterate
242 + * over this commit; it is not selected (if it were, it would
243 + * not have a blank commit mask) and all its children have
244 + * existing bitmaps (see the comment starting with "This commit
245 + * has an existing bitmap" below), so it does not contribute
246 + * anything to the final bitmap file or its descendants.
247 + */
248 + if (!c_ent->commit_mask)
249 + continue;
250 +
251 + if (old_bitmap && bitmap_for_commit(old_bitmap, commit)) {
252 + /*
253 + * This commit has an existing bitmap, so we can
254 + * get its bits immediately without an object
255 + * walk. That is, it is reusable as-is and there is no
256 + * need to continue walking beyond it.
257 + *
258 + * Mark it as such and add it to bb->commits separately
259 + * to avoid allocating a position in the commit mask.
260 + */
261 + commit_list_insert(commit, &reusable);
262 + goto next;
263 + }
264 +
265 if (c_ent->maximal) {
266 num_maximal++;
267 ALLOC_GROW(bb->commits, bb->commits_nr + 1, bb->commits_alloc);
@@ -278,14 +306,22 @@ static void bitmap_builder_init(struct bitmap_builder *bb,
306 }
307 }
308
309 +next:
310 bitmap_free(c_ent->commit_mask);
311 c_ent->commit_mask = NULL;
312 }
313
314 + for (r = reusable; r; r = r->next) {
315 + ALLOC_GROW(bb->commits, bb->commits_nr + 1, bb->commits_alloc);
316 + bb->commits[bb->commits_nr++] = r->item;
317 + }
318 +
319 trace2_data_intmax("pack-bitmap-write", the_repository,
320 "num_selected_commits", writer->selected_nr);
321 trace2_data_intmax("pack-bitmap-write", the_repository,
322 "num_maximal_commits", num_maximal);
323 +
324 + free_commit_list(reusable);
325 }
326
327 static void bitmap_builder_clear(struct bitmap_builder *bb)
@@ -420,7 +456,7 @@ void bitmap_writer_build(struct packing_data *to_pack)
456 else
457 mapping = NULL;
458
423 - bitmap_builder_init(&bb, &writer);
459 + bitmap_builder_init(&bb, &writer, old_bitmap);
460 for (i = bb.commits_nr; i > 0; i--) {
461 struct commit *commit = bb.commits[i-1];
462 struct bb_commit *ent = bb_data_at(&bb.data, commit);