pack-bitmap-write: support storing pseudo-merge commits
Prepare to write pseudo-merge bitmaps by annotating individual bitmapped commits (which are represented by the `bitmapped_commit` structure) with an extra bit indicating whether or not they are a pseudo-merge. In subsequent commits, pseudo-merge bitmaps will be generated by allocating a fake commit node with parents covering the full set of commits represented by the pseudo-merge bitmap. These commits will be added to the set of "selected" commits as usual, but will be written specially instead of being included with the rest of the selected commits. Mechanically speaking, there are two parts of this change: - The bitmapped_commit struct gets a new bit indicating whether it is a pseudo-merge, or an ordinary commit selected for bitmaps. - A handful of changes to only write out the non-pseudo-merge commits when enumerating through the selected array (see the new `bitmap_writer_selected_nr()` function). Pseudo-merge commits appear after all non-pseudo-merge commits, so it is safe to enumerate through the selected array like so: for (i = 0; i < bitmap_writer_selected_nr(); i++) if (writer.selected[i].pseudo_merge) BUG("unexpected pseudo-merge"); without encountering the BUG(). Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 23, 2024 at 17:26 UTC
0d41b18317ece0d46e2cfdf1d99ba2b2166c0248
3 files changed
+67
-34
object.h
+1
-1
@@ -81,7 +81,7 @@ void object_array_init(struct object_array *array);
81
* reflog.c: 10--12
82
* builtin/show-branch.c: 0-------------------------------------------26
83
* builtin/unpack-objects.c: 2021
84
- * pack-bitmap.h: 22
84
+ * pack-bitmap.h: 2122
85
*/
86
#define FLAG_BITS 28
87
pack-bitmap-write.c
+63
-33
@@ -25,8 +25,14 @@ struct bitmapped_commit {
25
int flags;
26
int xor_offset;
27
uint32_t commit_pos;
28
+ unsigned pseudo_merge : 1;
29
};
30
31
+static inline int bitmap_writer_nr_selected_commits(struct bitmap_writer *writer)
32
+{
33
+ return writer->selected_nr - writer->pseudo_merges_nr;
34
+}
35
+
36
void bitmap_writer_init(struct bitmap_writer *writer, struct repository *r)
37
{
38
memset(writer, 0, sizeof(struct bitmap_writer));
@@ -129,27 +135,31 @@ void bitmap_writer_build_type_index(struct bitmap_writer *writer,
135
*/
136
137
static inline void push_bitmapped_commit(struct bitmap_writer *writer,
132
- struct commit *commit)
138
+ struct commit *commit,
139
+ unsigned pseudo_merge)
140
{
134
- int hash_ret;
135
- khiter_t hash_pos;
136
-
141
if (writer->selected_nr >= writer->selected_alloc) {
142
writer->selected_alloc = (writer->selected_alloc + 32) * 2;
143
REALLOC_ARRAY(writer->selected, writer->selected_alloc);
144
}
145
142
- hash_pos = kh_put_oid_map(writer->bitmaps, commit->object.oid,
143
- &hash_ret);
144
- if (!hash_ret)
145
- die(_("duplicate entry when writing bitmap index: %s"),
146
- oid_to_hex(&commit->object.oid));
147
- kh_value(writer->bitmaps, hash_pos) = NULL;
146
+ if (!pseudo_merge) {
147
+ int hash_ret;
148
+ khiter_t hash_pos = kh_put_oid_map(writer->bitmaps,
149
+ commit->object.oid,
150
+ &hash_ret);
151
+
152
+ if (!hash_ret)
153
+ die(_("duplicate entry when writing bitmap index: %s"),
154
+ oid_to_hex(&commit->object.oid));
155
+ kh_value(writer->bitmaps, hash_pos) = NULL;
156
+ }
157
158
writer->selected[writer->selected_nr].commit = commit;
159
writer->selected[writer->selected_nr].bitmap = NULL;
160
writer->selected[writer->selected_nr].write_as = NULL;
161
writer->selected[writer->selected_nr].flags = 0;
162
+ writer->selected[writer->selected_nr].pseudo_merge = pseudo_merge;
163
164
writer->selected_nr++;
165
}
@@ -180,16 +190,20 @@ static void compute_xor_offsets(struct bitmap_writer *writer)
190
191
while (next < writer->selected_nr) {
192
struct bitmapped_commit *stored = &writer->selected[next];
183
-
193
int best_offset = 0;
194
struct ewah_bitmap *best_bitmap = stored->bitmap;
195
struct ewah_bitmap *test_xor;
196
197
+ if (stored->pseudo_merge)
198
+ goto next;
199
+
200
for (i = 1; i <= MAX_XOR_OFFSET_SEARCH; ++i) {
201
int curr = next - i;
202
203
if (curr < 0)
204
break;
205
+ if (writer->selected[curr].pseudo_merge)
206
+ continue;
207
208
test_xor = ewah_pool_new();
209
ewah_xor(writer->selected[curr].bitmap, stored->bitmap, test_xor);
@@ -205,6 +219,7 @@ static void compute_xor_offsets(struct bitmap_writer *writer)
219
}
220
}
221
222
+next:
223
stored->xor_offset = best_offset;
224
stored->write_as = best_bitmap;
225
@@ -217,7 +232,8 @@ struct bb_commit {
232
struct bitmap *commit_mask;
233
struct bitmap *bitmap;
234
unsigned selected:1,
220
- maximal:1;
235
+ maximal:1,
236
+ pseudo_merge:1;
237
unsigned idx; /* within selected array */
238
};
239
@@ -255,17 +271,18 @@ static void bitmap_builder_init(struct bitmap_builder *bb,
271
revs.first_parent_only = 1;
272
273
for (i = 0; i < writer->selected_nr; i++) {
258
- struct commit *c = writer->selected[i].commit;
259
- struct bb_commit *ent = bb_data_at(&bb->data, c);
274
+ struct bitmapped_commit *bc = &writer->selected[i];
275
+ struct bb_commit *ent = bb_data_at(&bb->data, bc->commit);
276
277
ent->selected = 1;
278
ent->maximal = 1;
279
+ ent->pseudo_merge = bc->pseudo_merge;
280
ent->idx = i;
281
282
ent->commit_mask = bitmap_new();
283
bitmap_set(ent->commit_mask, i);
284
268
- add_pending_object(&revs, &c->object, "");
285
+ add_pending_object(&revs, &bc->commit->object, "");
286
}
287
288
if (prepare_revision_walk(&revs))
@@ -444,8 +461,13 @@ static int fill_bitmap_commit(struct bitmap_writer *writer,
461
struct commit *c = prio_queue_get(queue);
462
463
if (old_bitmap && mapping) {
447
- struct ewah_bitmap *old = bitmap_for_commit(old_bitmap, c);
464
+ struct ewah_bitmap *old;
465
struct bitmap *remapped = bitmap_new();
466
+
467
+ if (commit->object.flags & BITMAP_PSEUDO_MERGE)
468
+ old = NULL;
469
+ else
470
+ old = bitmap_for_commit(old_bitmap, c);
471
/*
472
* If this commit has an old bitmap, then translate that
473
* bitmap and add its bits to this one. No need to walk
@@ -464,12 +486,14 @@ static int fill_bitmap_commit(struct bitmap_writer *writer,
486
* Mark ourselves and queue our tree. The commit
487
* walk ensures we cover all parents.
488
*/
467
- pos = find_object_pos(writer, &c->object.oid, &found);
468
- if (!found)
469
- return -1;
470
- bitmap_set(ent->bitmap, pos);
471
- prio_queue_put(tree_queue,
472
- repo_get_commit_tree(the_repository, c));
489
+ if (!(c->object.flags & BITMAP_PSEUDO_MERGE)) {
490
+ pos = find_object_pos(writer, &c->object.oid, &found);
491
+ if (!found)
492
+ return -1;
493
+ bitmap_set(ent->bitmap, pos);
494
+ prio_queue_put(tree_queue,
495
+ repo_get_commit_tree(the_repository, c));
496
+ }
497
498
for (p = c->parents; p; p = p->next) {
499
pos = find_object_pos(writer, &p->item->object.oid,
@@ -499,6 +523,9 @@ static void store_selected(struct bitmap_writer *writer,
523
524
stored->bitmap = bitmap_to_ewah(ent->bitmap);
525
526
+ if (ent->pseudo_merge)
527
+ return;
528
+
529
hash_pos = kh_get_oid_map(writer->bitmaps, commit->object.oid);
530
if (hash_pos == kh_end(writer->bitmaps))
531
die(_("attempted to store non-selected commit: '%s'"),
@@ -631,7 +658,7 @@ void bitmap_writer_select_commits(struct bitmap_writer *writer,
658
659
if (indexed_commits_nr < 100) {
660
for (i = 0; i < indexed_commits_nr; ++i)
634
- push_bitmapped_commit(writer, indexed_commits[i]);
661
+ push_bitmapped_commit(writer, indexed_commits[i], 0);
662
return;
663
}
664
@@ -664,7 +691,7 @@ void bitmap_writer_select_commits(struct bitmap_writer *writer,
691
}
692
}
693
667
- push_bitmapped_commit(writer, chosen);
694
+ push_bitmapped_commit(writer, chosen, 0);
695
696
i += next + 1;
697
display_progress(writer->progress, i);
@@ -701,8 +728,11 @@ static void write_selected_commits_v1(struct bitmap_writer *writer,
728
{
729
int i;
730
704
- for (i = 0; i < writer->selected_nr; ++i) {
731
+ for (i = 0; i < bitmap_writer_nr_selected_commits(writer); ++i) {
732
struct bitmapped_commit *stored = &writer->selected[i];
733
+ if (stored->pseudo_merge)
734
+ BUG("unexpected pseudo-merge among selected: %s",
735
+ oid_to_hex(&stored->commit->object.oid));
736
737
if (offsets)
738
offsets[i] = hashfile_total(f);
@@ -735,10 +765,10 @@ static void write_lookup_table(struct bitmap_writer *writer, struct hashfile *f,
765
uint32_t i;
766
uint32_t *table, *table_inv;
767
738
- ALLOC_ARRAY(table, writer->selected_nr);
739
- ALLOC_ARRAY(table_inv, writer->selected_nr);
768
+ ALLOC_ARRAY(table, bitmap_writer_nr_selected_commits(writer));
769
+ ALLOC_ARRAY(table_inv, bitmap_writer_nr_selected_commits(writer));
770
741
- for (i = 0; i < writer->selected_nr; i++)
771
+ for (i = 0; i < bitmap_writer_nr_selected_commits(writer); i++)
772
table[i] = i;
773
774
/*
@@ -746,16 +776,16 @@ static void write_lookup_table(struct bitmap_writer *writer, struct hashfile *f,
776
* bitmap corresponds to j'th bitmapped commit (among the selected
777
* commits) in lex order of OIDs.
778
*/
749
- QSORT_S(table, writer->selected_nr, table_cmp, writer);
779
+ QSORT_S(table, bitmap_writer_nr_selected_commits(writer), table_cmp, writer);
780
781
/* table_inv helps us discover that relationship (i'th bitmap
782
* to j'th commit by j = table_inv[i])
783
*/
754
- for (i = 0; i < writer->selected_nr; i++)
784
+ for (i = 0; i < bitmap_writer_nr_selected_commits(writer); i++)
785
table_inv[table[i]] = i;
786
787
trace2_region_enter("pack-bitmap-write", "writing_lookup_table", the_repository);
758
- for (i = 0; i < writer->selected_nr; i++) {
788
+ for (i = 0; i < bitmap_writer_nr_selected_commits(writer); i++) {
789
struct bitmapped_commit *selected = &writer->selected[table[i]];
790
uint32_t xor_offset = selected->xor_offset;
791
uint32_t xor_row;
@@ -827,7 +857,7 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
857
memcpy(header.magic, BITMAP_IDX_SIGNATURE, sizeof(BITMAP_IDX_SIGNATURE));
858
header.version = htons(default_version);
859
header.options = htons(flags | options);
830
- header.entry_count = htonl(writer->selected_nr);
860
+ header.entry_count = htonl(bitmap_writer_nr_selected_commits(writer));
861
hashcpy(header.checksum, writer->pack_checksum);
862
863
hashwrite(f, &header, sizeof(header) - GIT_MAX_RAWSZ + the_hash_algo->rawsz);
@@ -839,7 +869,7 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
869
if (options & BITMAP_OPT_LOOKUP_TABLE)
870
CALLOC_ARRAY(offsets, index_nr);
871
842
- for (i = 0; i < writer->selected_nr; i++) {
872
+ for (i = 0; i < bitmap_writer_nr_selected_commits(writer); i++) {
873
struct bitmapped_commit *stored = &writer->selected[i];
874
int commit_pos = oid_pos(&stored->commit->object.oid, index,
875
index_nr, oid_access);
pack-bitmap.h
+3
@@ -21,6 +21,7 @@ struct bitmap_disk_header {
21
unsigned char checksum[GIT_MAX_RAWSZ];
22
};
23
24
+#define BITMAP_PSEUDO_MERGE (1u<<21)
25
#define NEEDS_BITMAP (1u<<22)
26
27
/*
@@ -109,6 +110,8 @@ struct bitmap_writer {
110
struct bitmapped_commit *selected;
111
unsigned int selected_nr, selected_alloc;
112
113
+ uint32_t pseudo_merges_nr;
114
+
115
struct progress *progress;
116
int show_progress;
117
unsigned char pack_checksum[GIT_MAX_RAWSZ];