pack-bitmap-write.c: write lookup table extension
The bitmap lookup table extension was documented by an earlier change, but Git does not yet know how to write that extension. Teach Git to write bitmap lookup table extension. The table contains the list of `N` <commit_pos, offset, xor_row>` triplets. These triplets are sorted according to their commit pos (ascending order). The meaning of each data in the i'th triplet is given below: - commit_pos stores commit position (in the pack-index or midx). It is a 4 byte network byte order unsigned integer. - offset is the position (in the bitmap file) from which that commit's bitmap can be read. - xor_row is the position of the triplet in the lookup table whose bitmap is used to compress this bitmap, or `0xffffffff` if no such bitmap exists. Mentored-by: Taylor Blau <me@ttaylorr.com> Co-mentored-by: Kaartic Sivaraam <kaartic.sivaraam@gmail.com> Co-authored-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Abhradeep Chakraborty <chakrabortyabhradeep79@gmail.com> Reviewed-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Abhradeep Chakraborty committed
Aug 14, 2022 at 16:55 UTC
93eb41e2403788fa9105211956e87b6b2c22c68c
2 files changed
+92
-4
pack-bitmap-write.c
+89
-2
@@ -651,13 +651,17 @@ static const struct object_id *oid_access(size_t pos, const void *table)
651
static void write_selected_commits_v1(struct hashfile *f,
652
struct pack_idx_entry **index,
653
uint32_t index_nr,
654
- uint32_t *commit_positions)
654
+ uint32_t *commit_positions,
655
+ off_t *offsets)
656
{
657
int i;
658
659
for (i = 0; i < writer.selected_nr; ++i) {
660
struct bitmapped_commit *stored = &writer.selected[i];
661
662
+ if (offsets)
663
+ offsets[i] = hashfile_total(f);
664
+
665
hashwrite_be32(f, commit_positions[i]);
666
hashwrite_u8(f, stored->xor_offset);
667
hashwrite_u8(f, stored->flags);
@@ -666,6 +670,81 @@ static void write_selected_commits_v1(struct hashfile *f,
670
}
671
}
672
673
+static int table_cmp(const void *_va, const void *_vb, void *_data)
674
+{
675
+ uint32_t *commit_positions = _data;
676
+ uint32_t a = commit_positions[*(uint32_t *)_va];
677
+ uint32_t b = commit_positions[*(uint32_t *)_vb];
678
+
679
+ if (a > b)
680
+ return 1;
681
+ else if (a < b)
682
+ return -1;
683
+
684
+ return 0;
685
+}
686
+
687
+static void write_lookup_table(struct hashfile *f,
688
+ struct pack_idx_entry **index,
689
+ uint32_t index_nr,
690
+ uint32_t *commit_positions,
691
+ off_t *offsets)
692
+{
693
+ uint32_t i;
694
+ uint32_t *table, *table_inv;
695
+
696
+ ALLOC_ARRAY(table, writer.selected_nr);
697
+ ALLOC_ARRAY(table_inv, writer.selected_nr);
698
+
699
+ for (i = 0; i < writer.selected_nr; i++)
700
+ table[i] = i;
701
+
702
+ /*
703
+ * At the end of this sort table[j] = i means that the i'th
704
+ * bitmap corresponds to j'th bitmapped commit (among the selected
705
+ * commits) in lex order of OIDs.
706
+ */
707
+ QSORT_S(table, writer.selected_nr, table_cmp, commit_positions);
708
+
709
+ /* table_inv helps us discover that relationship (i'th bitmap
710
+ * to j'th commit by j = table_inv[i])
711
+ */
712
+ for (i = 0; i < writer.selected_nr; i++)
713
+ table_inv[table[i]] = i;
714
+
715
+ trace2_region_enter("pack-bitmap-write", "writing_lookup_table", the_repository);
716
+ for (i = 0; i < writer.selected_nr; i++) {
717
+ struct bitmapped_commit *selected = &writer.selected[table[i]];
718
+ uint32_t xor_offset = selected->xor_offset;
719
+ uint32_t xor_row;
720
+
721
+ if (xor_offset) {
722
+ /*
723
+ * xor_index stores the index (in the bitmap entries)
724
+ * of the corresponding xor bitmap. But we need to convert
725
+ * this index into lookup table's index. So, table_inv[xor_index]
726
+ * gives us the index position w.r.t. the lookup table.
727
+ *
728
+ * If "k = table[i] - xor_offset" then the xor base is the k'th
729
+ * bitmap. `table_inv[k]` gives us the position of that bitmap
730
+ * in the lookup table.
731
+ */
732
+ uint32_t xor_index = table[i] - xor_offset;
733
+ xor_row = table_inv[xor_index];
734
+ } else {
735
+ xor_row = 0xffffffff;
736
+ }
737
+
738
+ hashwrite_be32(f, commit_positions[table[i]]);
739
+ hashwrite_be64(f, (uint64_t)offsets[table[i]]);
740
+ hashwrite_be32(f, xor_row);
741
+ }
742
+ trace2_region_leave("pack-bitmap-write", "writing_lookup_table", the_repository);
743
+
744
+ free(table);
745
+ free(table_inv);
746
+}
747
+
748
static void write_hash_cache(struct hashfile *f,
749
struct pack_idx_entry **index,
750
uint32_t index_nr)
@@ -693,6 +772,7 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
772
struct strbuf tmp_file = STRBUF_INIT;
773
struct hashfile *f;
774
uint32_t *commit_positions = NULL;
775
+ off_t *offsets = NULL;
776
uint32_t i;
777
778
struct bitmap_disk_header header;
@@ -713,6 +793,9 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
793
dump_bitmap(f, writer.blobs);
794
dump_bitmap(f, writer.tags);
795
796
+ if (options & BITMAP_OPT_LOOKUP_TABLE)
797
+ CALLOC_ARRAY(offsets, index_nr);
798
+
799
ALLOC_ARRAY(commit_positions, writer.selected_nr);
800
801
for (i = 0; i < writer.selected_nr; i++) {
@@ -725,7 +808,10 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
808
commit_positions[i] = commit_pos;
809
}
810
728
- write_selected_commits_v1(f, index, index_nr, commit_positions);
811
+ write_selected_commits_v1(f, index, index_nr, commit_positions, offsets);
812
+
813
+ if (options & BITMAP_OPT_LOOKUP_TABLE)
814
+ write_lookup_table(f, index, index_nr, commit_positions, offsets);
815
816
if (options & BITMAP_OPT_HASH_CACHE)
817
write_hash_cache(f, index, index_nr);
@@ -741,4 +827,5 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
827
828
strbuf_release(&tmp_file);
829
free(commit_positions);
830
+ free(offsets);
831
}
pack-bitmap.h
+3
-2
@@ -24,8 +24,9 @@ struct bitmap_disk_header {
24
#define NEEDS_BITMAP (1u<<22)
25
26
enum pack_bitmap_opts {
27
- BITMAP_OPT_FULL_DAG = 1,
28
- BITMAP_OPT_HASH_CACHE = 4,
27
+ BITMAP_OPT_FULL_DAG = 0x1,
28
+ BITMAP_OPT_HASH_CACHE = 0x4,
29
+ BITMAP_OPT_LOOKUP_TABLE = 0x10,
30
};
31
32
enum pack_bitmap_flags {