pack-bitmap-write.c: write pseudo-merge table
Now that the pack-bitmap writer machinery understands how to select and store pseudo-merge commits, teach it how to write the new optional pseudo-merge .bitmap extension. No readers yet exist for this new extension to the .bitmap format. The following commits will take any preparatory step(s) necessary before then implementing the routines necessary to read this new table. In the meantime, the new `write_pseudo_merges()` function implements writing this new format as described by a previous commit in Documentation/technical/bitmap-format.txt. Writing this table is fairly straightforward and consists of a few sub-components: - a pair of bitmaps for each pseudo-merge (one for the pseudo-merge "parents", and another for the objects reachable from those parents) - for each commit, the offset of either (a) the pseudo-merge it belongs to, or (b) an extended lookup table if it belongs to >1 pseudo-merge groups - if there are any commits belonging to >1 pseudo-merge group, the extended lookup tables (which each consist of the number of pseudo-merge groups a commit appears in, and then that many 4-byte unsigned ) Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 23, 2024 at 17:26 UTC
53ea3ec47990e9ad00a238fd962f36e5d87a6386
2 files changed
+132
pack-bitmap-write.c
+131
@@ -18,6 +18,7 @@
18
#include "tree.h"
19
#include "tree-walk.h"
20
#include "pseudo-merge.h"
21
+#include "oid-array.h"
22
23
struct bitmapped_commit {
24
struct commit *commit;
@@ -771,6 +772,130 @@ static void write_selected_commits_v1(struct bitmap_writer *writer,
772
}
773
}
774
775
+static void write_pseudo_merges(struct bitmap_writer *writer,
776
+ struct hashfile *f)
777
+{
778
+ struct oid_array commits = OID_ARRAY_INIT;
779
+ struct bitmap **commits_bitmap = NULL;
780
+ off_t *pseudo_merge_ofs = NULL;
781
+ off_t start, table_start, next_ext;
782
+
783
+ uint32_t base = bitmap_writer_nr_selected_commits(writer);
784
+ size_t i, j = 0;
785
+
786
+ CALLOC_ARRAY(commits_bitmap, writer->pseudo_merges_nr);
787
+ CALLOC_ARRAY(pseudo_merge_ofs, writer->pseudo_merges_nr);
788
+
789
+ for (i = 0; i < writer->pseudo_merges_nr; i++) {
790
+ struct bitmapped_commit *merge = &writer->selected[base + i];
791
+ struct commit_list *p;
792
+
793
+ if (!merge->pseudo_merge)
794
+ BUG("found non-pseudo merge commit at %"PRIuMAX, (uintmax_t)i);
795
+
796
+ commits_bitmap[i] = bitmap_new();
797
+
798
+ for (p = merge->commit->parents; p; p = p->next)
799
+ bitmap_set(commits_bitmap[i],
800
+ find_object_pos(writer, &p->item->object.oid,
801
+ NULL));
802
+ }
803
+
804
+ start = hashfile_total(f);
805
+
806
+ for (i = 0; i < writer->pseudo_merges_nr; i++) {
807
+ struct ewah_bitmap *commits_ewah = bitmap_to_ewah(commits_bitmap[i]);
808
+
809
+ pseudo_merge_ofs[i] = hashfile_total(f);
810
+
811
+ dump_bitmap(f, commits_ewah);
812
+ dump_bitmap(f, writer->selected[base+i].write_as);
813
+
814
+ ewah_free(commits_ewah);
815
+ }
816
+
817
+ next_ext = st_add(hashfile_total(f),
818
+ st_mult(kh_size(writer->pseudo_merge_commits),
819
+ sizeof(uint64_t)));
820
+
821
+ table_start = hashfile_total(f);
822
+
823
+ commits.alloc = kh_size(writer->pseudo_merge_commits);
824
+ CALLOC_ARRAY(commits.oid, commits.alloc);
825
+
826
+ for (i = kh_begin(writer->pseudo_merge_commits); i != kh_end(writer->pseudo_merge_commits); i++) {
827
+ if (!kh_exist(writer->pseudo_merge_commits, i))
828
+ continue;
829
+ oid_array_append(&commits, &kh_key(writer->pseudo_merge_commits, i));
830
+ }
831
+
832
+ oid_array_sort(&commits);
833
+
834
+ /* write lookup table (non-extended) */
835
+ for (i = 0; i < commits.nr; i++) {
836
+ int hash_pos;
837
+ struct pseudo_merge_commit_idx *c;
838
+
839
+ hash_pos = kh_get_oid_map(writer->pseudo_merge_commits,
840
+ commits.oid[i]);
841
+ if (hash_pos == kh_end(writer->pseudo_merge_commits))
842
+ BUG("could not find pseudo-merge commit %s",
843
+ oid_to_hex(&commits.oid[i]));
844
+
845
+ c = kh_value(writer->pseudo_merge_commits, hash_pos);
846
+
847
+ hashwrite_be32(f, find_object_pos(writer, &commits.oid[i],
848
+ NULL));
849
+ if (c->nr == 1)
850
+ hashwrite_be64(f, pseudo_merge_ofs[c->pseudo_merge[0]]);
851
+ else if (c->nr > 1) {
852
+ if (next_ext & ((uint64_t)1<<63))
853
+ die(_("too many pseudo-merges"));
854
+ hashwrite_be64(f, next_ext | ((uint64_t)1<<63));
855
+ next_ext = st_add3(next_ext,
856
+ sizeof(uint32_t),
857
+ st_mult(c->nr, sizeof(uint64_t)));
858
+ } else
859
+ BUG("expected commit '%s' to have at least one "
860
+ "pseudo-merge", oid_to_hex(&commits.oid[i]));
861
+ }
862
+
863
+ /* write lookup table (extended) */
864
+ for (i = 0; i < commits.nr; i++) {
865
+ int hash_pos;
866
+ struct pseudo_merge_commit_idx *c;
867
+
868
+ hash_pos = kh_get_oid_map(writer->pseudo_merge_commits,
869
+ commits.oid[i]);
870
+ if (hash_pos == kh_end(writer->pseudo_merge_commits))
871
+ BUG("could not find pseudo-merge commit %s",
872
+ oid_to_hex(&commits.oid[i]));
873
+
874
+ c = kh_value(writer->pseudo_merge_commits, hash_pos);
875
+ if (c->nr == 1)
876
+ continue;
877
+
878
+ hashwrite_be32(f, c->nr);
879
+ for (j = 0; j < c->nr; j++)
880
+ hashwrite_be64(f, pseudo_merge_ofs[c->pseudo_merge[j]]);
881
+ }
882
+
883
+ /* write positions for all pseudo merges */
884
+ for (i = 0; i < writer->pseudo_merges_nr; i++)
885
+ hashwrite_be64(f, pseudo_merge_ofs[i]);
886
+
887
+ hashwrite_be32(f, writer->pseudo_merges_nr);
888
+ hashwrite_be32(f, kh_size(writer->pseudo_merge_commits));
889
+ hashwrite_be64(f, table_start - start);
890
+ hashwrite_be64(f, hashfile_total(f) - start + sizeof(uint64_t));
891
+
892
+ for (i = 0; i < writer->pseudo_merges_nr; i++)
893
+ bitmap_free(commits_bitmap[i]);
894
+
895
+ free(pseudo_merge_ofs);
896
+ free(commits_bitmap);
897
+}
898
+
899
static int table_cmp(const void *_va, const void *_vb, void *_data)
900
{
901
struct bitmap_writer *writer = _data;
@@ -878,6 +1003,9 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
1003
1004
int fd = odb_mkstemp(&tmp_file, "pack/tmp_bitmap_XXXXXX");
1005
1006
+ if (writer->pseudo_merges_nr)
1007
+ options |= BITMAP_OPT_PSEUDO_MERGES;
1008
+
1009
f = hashfd(fd, tmp_file.buf);
1010
1011
memcpy(header.magic, BITMAP_IDX_SIGNATURE, sizeof(BITMAP_IDX_SIGNATURE));
@@ -907,6 +1035,9 @@ void bitmap_writer_finish(struct bitmap_writer *writer,
1035
1036
write_selected_commits_v1(writer, f, offsets);
1037
1038
+ if (options & BITMAP_OPT_PSEUDO_MERGES)
1039
+ write_pseudo_merges(writer, f);
1040
+
1041
if (options & BITMAP_OPT_LOOKUP_TABLE)
1042
write_lookup_table(writer, f, offsets);
1043
pack-bitmap.h
+1
@@ -37,6 +37,7 @@ enum pack_bitmap_opts {
37
BITMAP_OPT_FULL_DAG = 0x1,
38
BITMAP_OPT_HASH_CACHE = 0x4,
39
BITMAP_OPT_LOOKUP_TABLE = 0x10,
40
+ BITMAP_OPT_PSEUDO_MERGES = 0x20,
41
};
42
43
enum pack_bitmap_flags {