commit-graph: write Bloom filters to commit graph file
Update the technical documentation for commit-graph-format with the formats for the Bloom filter index (BIDX) and Bloom filter data (BDAT) chunks. Write the computed Bloom filters information to the commit graph file using this format. Helped-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Garima Singh <garima.singh@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Garima Singh committed
Apr 6, 2020 at 16:59 UTC
76ffbca71a9c89d1e530f734e16a70b3924f4bea
3 files changed
+147
-1
Documentation/technical/commit-graph-format.txt
+30
@@ -17,6 +17,9 @@ metadata, including:
17
- The parents of the commit, stored using positional references within
18
the graph file.
19
20
+- The Bloom filter of the commit carrying the paths that were changed between
21
+ the commit and its first parent, if requested.
22
+
23
These positional references are stored as unsigned 32-bit integers
24
corresponding to the array position within the list of commit OIDs. Due
25
to some special constants we use to track parents, we can store at most
@@ -93,6 +96,33 @@ CHUNK DATA:
96
positions for the parents until reaching a value with the most-significant
97
bit on. The other bits correspond to the position of the last parent.
98
99
+ Bloom Filter Index (ID: {'B', 'I', 'D', 'X'}) (N * 4 bytes) [Optional]
100
+ * The ith entry, BIDX[i], stores the number of 8-byte word blocks in all
101
+ Bloom filters from commit 0 to commit i (inclusive) in lexicographic
102
+ order. The Bloom filter for the i-th commit spans from BIDX[i-1] to
103
+ BIDX[i] (plus header length), where BIDX[-1] is 0.
104
+ * The BIDX chunk is ignored if the BDAT chunk is not present.
105
+
106
+ Bloom Filter Data (ID: {'B', 'D', 'A', 'T'}) [Optional]
107
+ * It starts with header consisting of three unsigned 32-bit integers:
108
+ - Version of the hash algorithm being used. We currently only support
109
+ value 1 which corresponds to the 32-bit version of the murmur3 hash
110
+ implemented exactly as described in
111
+ https://en.wikipedia.org/wiki/MurmurHash#Algorithm and the double
112
+ hashing technique using seed values 0x293ae76f and 0x7e646e2 as
113
+ described in https://doi.org/10.1007/978-3-540-30494-4_26 "Bloom Filters
114
+ in Probabilistic Verification"
115
+ - The number of times a path is hashed and hence the number of bit positions
116
+ that cumulatively determine whether a file is present in the commit.
117
+ - The minimum number of bits 'b' per entry in the Bloom filter. If the filter
118
+ contains 'n' entries, then the filter size is the minimum number of 64-bit
119
+ words that contain n*b bits.
120
+ * The rest of the chunk is the concatenation of all the computed Bloom
121
+ filters for the commits in lexicographic order.
122
+ * Note: Commits with no changes or more than 512 changes have Bloom filters
123
+ of length zero.
124
+ * The BDAT chunk is present if and only if BIDX is present.
125
+
126
Base Graphs List (ID: {'B', 'A', 'S', 'E'}) [Optional]
127
This list of H-byte hashes describe a set of B commit-graph files that
128
form a commit-graph chain. The graph position for the ith commit in this
commit-graph.c
+112
-1
@@ -24,8 +24,10 @@
24
#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
25
#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
26
#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
27
+#define GRAPH_CHUNKID_BLOOMINDEXES 0x42494458 /* "BIDX" */
28
+#define GRAPH_CHUNKID_BLOOMDATA 0x42444154 /* "BDAT" */
29
#define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
28
-#define MAX_NUM_CHUNKS 5
30
+#define MAX_NUM_CHUNKS 7
31
32
#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
33
@@ -319,6 +321,32 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
321
chunk_repeated = 1;
322
else
323
graph->chunk_base_graphs = data + chunk_offset;
324
+ break;
325
+
326
+ case GRAPH_CHUNKID_BLOOMINDEXES:
327
+ if (graph->chunk_bloom_indexes)
328
+ chunk_repeated = 1;
329
+ else
330
+ graph->chunk_bloom_indexes = data + chunk_offset;
331
+ break;
332
+
333
+ case GRAPH_CHUNKID_BLOOMDATA:
334
+ if (graph->chunk_bloom_data)
335
+ chunk_repeated = 1;
336
+ else {
337
+ uint32_t hash_version;
338
+ graph->chunk_bloom_data = data + chunk_offset;
339
+ hash_version = get_be32(data + chunk_offset);
340
+
341
+ if (hash_version != 1)
342
+ break;
343
+
344
+ graph->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings));
345
+ graph->bloom_filter_settings->hash_version = hash_version;
346
+ graph->bloom_filter_settings->num_hashes = get_be32(data + chunk_offset + 4);
347
+ graph->bloom_filter_settings->bits_per_entry = get_be32(data + chunk_offset + 8);
348
+ }
349
+ break;
350
}
351
352
if (chunk_repeated) {
@@ -337,6 +365,15 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
365
last_chunk_offset = chunk_offset;
366
}
367
368
+ if (graph->chunk_bloom_indexes && graph->chunk_bloom_data) {
369
+ init_bloom_filters();
370
+ } else {
371
+ /* We need both the bloom chunks to exist together. Else ignore the data */
372
+ graph->chunk_bloom_indexes = NULL;
373
+ graph->chunk_bloom_data = NULL;
374
+ graph->bloom_filter_settings = NULL;
375
+ }
376
+
377
hashcpy(graph->oid.hash, graph->data + graph->data_len - graph->hash_len);
378
379
if (verify_commit_graph_lite(graph)) {
@@ -1034,6 +1071,59 @@ static void write_graph_chunk_extra_edges(struct hashfile *f,
1071
}
1072
}
1073
1074
+static void write_graph_chunk_bloom_indexes(struct hashfile *f,
1075
+ struct write_commit_graph_context *ctx)
1076
+{
1077
+ struct commit **list = ctx->commits.list;
1078
+ struct commit **last = ctx->commits.list + ctx->commits.nr;
1079
+ uint32_t cur_pos = 0;
1080
+ struct progress *progress = NULL;
1081
+ int i = 0;
1082
+
1083
+ if (ctx->report_progress)
1084
+ progress = start_delayed_progress(
1085
+ _("Writing changed paths Bloom filters index"),
1086
+ ctx->commits.nr);
1087
+
1088
+ while (list < last) {
1089
+ struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);
1090
+ cur_pos += filter->len;
1091
+ display_progress(progress, ++i);
1092
+ hashwrite_be32(f, cur_pos);
1093
+ list++;
1094
+ }
1095
+
1096
+ stop_progress(&progress);
1097
+}
1098
+
1099
+static void write_graph_chunk_bloom_data(struct hashfile *f,
1100
+ struct write_commit_graph_context *ctx,
1101
+ const struct bloom_filter_settings *settings)
1102
+{
1103
+ struct commit **list = ctx->commits.list;
1104
+ struct commit **last = ctx->commits.list + ctx->commits.nr;
1105
+ struct progress *progress = NULL;
1106
+ int i = 0;
1107
+
1108
+ if (ctx->report_progress)
1109
+ progress = start_delayed_progress(
1110
+ _("Writing changed paths Bloom filters data"),
1111
+ ctx->commits.nr);
1112
+
1113
+ hashwrite_be32(f, settings->hash_version);
1114
+ hashwrite_be32(f, settings->num_hashes);
1115
+ hashwrite_be32(f, settings->bits_per_entry);
1116
+
1117
+ while (list < last) {
1118
+ struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);
1119
+ display_progress(progress, ++i);
1120
+ hashwrite(f, filter->data, filter->len * sizeof(unsigned char));
1121
+ list++;
1122
+ }
1123
+
1124
+ stop_progress(&progress);
1125
+}
1126
+
1127
static int oid_compare(const void *_a, const void *_b)
1128
{
1129
const struct object_id *a = (const struct object_id *)_a;
@@ -1438,6 +1528,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
1528
struct strbuf progress_title = STRBUF_INIT;
1529
int num_chunks = 3;
1530
struct object_id file_hash;
1531
+ const struct bloom_filter_settings bloom_settings = DEFAULT_BLOOM_FILTER_SETTINGS;
1532
1533
if (ctx->split) {
1534
struct strbuf tmp_file = STRBUF_INIT;
@@ -1482,6 +1573,12 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
1573
chunk_ids[num_chunks] = GRAPH_CHUNKID_EXTRAEDGES;
1574
num_chunks++;
1575
}
1576
+ if (ctx->changed_paths) {
1577
+ chunk_ids[num_chunks] = GRAPH_CHUNKID_BLOOMINDEXES;
1578
+ num_chunks++;
1579
+ chunk_ids[num_chunks] = GRAPH_CHUNKID_BLOOMDATA;
1580
+ num_chunks++;
1581
+ }
1582
if (ctx->num_commit_graphs_after > 1) {
1583
chunk_ids[num_chunks] = GRAPH_CHUNKID_BASE;
1584
num_chunks++;
@@ -1500,6 +1597,15 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
1597
4 * ctx->num_extra_edges;
1598
num_chunks++;
1599
}
1600
+ if (ctx->changed_paths) {
1601
+ chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +
1602
+ sizeof(uint32_t) * ctx->commits.nr;
1603
+ num_chunks++;
1604
+
1605
+ chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +
1606
+ sizeof(uint32_t) * 3 + ctx->total_bloom_filter_data_size;
1607
+ num_chunks++;
1608
+ }
1609
if (ctx->num_commit_graphs_after > 1) {
1610
chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +
1611
hashsz * (ctx->num_commit_graphs_after - 1);
@@ -1537,6 +1643,10 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
1643
write_graph_chunk_data(f, hashsz, ctx);
1644
if (ctx->num_extra_edges)
1645
write_graph_chunk_extra_edges(f, ctx);
1646
+ if (ctx->changed_paths) {
1647
+ write_graph_chunk_bloom_indexes(f, ctx);
1648
+ write_graph_chunk_bloom_data(f, ctx, &bloom_settings);
1649
+ }
1650
if (ctx->num_commit_graphs_after > 1 &&
1651
write_graph_chunk_base(f, ctx)) {
1652
return -1;
@@ -2184,6 +2294,7 @@ void free_commit_graph(struct commit_graph *g)
2294
close(g->graph_fd);
2295
}
2296
free(g->filename);
2297
+ free(g->bloom_filter_settings);
2298
free(g);
2299
}
2300
commit-graph.h
+5
@@ -11,6 +11,7 @@
11
#define GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD "GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD"
12
13
struct commit;
14
+struct bloom_filter_settings;
15
16
char *get_commit_graph_filename(struct object_directory *odb);
17
int open_commit_graph(const char *graph_file, int *fd, struct stat *st);
@@ -59,6 +60,10 @@ struct commit_graph {
60
const unsigned char *chunk_commit_data;
61
const unsigned char *chunk_extra_edges;
62
const unsigned char *chunk_base_graphs;
63
+ const unsigned char *chunk_bloom_indexes;
64
+ const unsigned char *chunk_bloom_data;
65
+
66
+ struct bloom_filter_settings *bloom_filter_settings;
67
};
68
69
struct commit_graph *load_commit_graph_one_fd_st(int fd, struct stat *st,