@matheusbsbs / matheus-bagliano / commits / 1217c03e7b

Fork 0

commit-graph: reuse existing Bloom filters during write

Add logic to a) parse Bloom filter information from the commit graph file and, b) re-use existing Bloom filters. See Documentation/technical/commit-graph-format for the format in which the Bloom filter information is written to the commit graph file. To read Bloom filter for a given commit with lexicographic position 'i' we need to: 1. Read BIDX[i] which essentially gives us the starting index in BDAT for filter of commit i+1. It is essentially the index past the end of the filter of commit i. It is called end_index in the code. 2. For i>0, read BIDX[i-1] which will give us the starting index in BDAT for filter of commit i. It is called the start_index in the code. For the first commit, where i = 0, Bloom filter data starts at the beginning, just past the header in the BDAT chunk. Hence, start_index will be 0. 3. The length of the filter will be end_index - start_index, because BIDX[i] gives the cumulative 8-byte words including the ith commit's filter. We toggle whether Bloom filters should be recomputed based on the compute_if_not_present flag. Helped-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Garima Singh <garima.singh@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Garima Singh committed Apr 6, 2020 at 16:59 UTC 1217c03e7b87b15f2c78af5b1e1915a675050454

4 files changed +55 -6

bloom.c

+48 -1

@@ -4,6 +4,8 @@

4 #include "diffcore.h"

5 #include "revision.h"

6 #include "hashmap.h"

7 +#include "commit-graph.h"

8 +#include "commit.h"

10 define_commit_slab(bloom_filter_slab, struct bloom_filter);

@@ -26,6 +28,36 @@ static inline unsigned char get_bitmask(uint32_t pos)

28 return ((unsigned char)1) << (pos & (BITS_PER_WORD - 1));

29 }

31 +static int load_bloom_filter_from_graph(struct commit_graph *g,

32 + struct bloom_filter *filter,

33 + struct commit *c)

34 +{

35 + uint32_t lex_pos, start_index, end_index;

36 +

37 + while (c->graph_pos < g->num_commits_in_base)

38 + g = g->base_graph;

39 +

40 + /* The commit graph commit 'c' lives in doesn't carry bloom filters. */

41 + if (!g->chunk_bloom_indexes)

42 + return 0;

43 +

44 + lex_pos = c->graph_pos - g->num_commits_in_base;

45 +

46 + end_index = get_be32(g->chunk_bloom_indexes + 4 * lex_pos);

47 +

48 + if (lex_pos > 0)

49 + start_index = get_be32(g->chunk_bloom_indexes + 4 * (lex_pos - 1));

50 + else

51 + start_index = 0;

52 +

53 + filter->len = end_index - start_index;

54 + filter->data = (unsigned char *)(g->chunk_bloom_data +

55 + sizeof(unsigned char) * start_index +

56 + BLOOMDATA_CHUNK_HEADER_SIZE);

57 +

58 + return 1;

59 +}

60 +

61 /*

62 * Calculate the murmur3 32-bit hash value for the given data

63 * using the given seed.

@@ -127,7 +159,8 @@ void init_bloom_filters(void)

159 }

160

161 struct bloom_filter *get_bloom_filter(struct repository *r,

130 - struct commit *c)

162 + struct commit *c,

163 + int compute_if_not_present)

164 {

165 struct bloom_filter *filter;

166 struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;

@@ -140,6 +173,20 @@ struct bloom_filter *get_bloom_filter(struct repository *r,

173

174 filter = bloom_filter_slab_at(&bloom_filters, c);

175

176 + if (!filter->data) {

177 + load_commit_graph_info(r, c);

178 + if (c->graph_pos != COMMIT_NOT_FROM_GRAPH &&

179 + r->objects->commit_graph->chunk_bloom_indexes) {

180 + if (load_bloom_filter_from_graph(r->objects->commit_graph, filter, c))

181 + return filter;

182 + else

183 + return NULL;

184 + }

185 + }

186 +

187 + if (filter->data || !compute_if_not_present)

188 + return filter;

189 +

190 repo_diff_setup(r, &diffopt);

191 diffopt.flags.recursive = 1;

192 diffopt.max_changes = max_changes;

bloom.h

+3 -1

@@ -32,6 +32,7 @@ struct bloom_filter_settings {

33 #define DEFAULT_BLOOM_FILTER_SETTINGS { 1, 7, 10 }

34 #define BITS_PER_WORD 8

35 +#define BLOOMDATA_CHUNK_HEADER_SIZE 3 * sizeof(uint32_t)

37 /*

38 * A bloom_filter struct represents a data segment to

@@ -79,6 +80,7 @@ void add_key_to_filter(const struct bloom_key *key,

80 void init_bloom_filters(void);

82 struct bloom_filter *get_bloom_filter(struct repository *r,

82 - struct commit *c);

83 + struct commit *c,

84 + int compute_if_not_present);

86 #endif

\ No newline at end of file

commit-graph.c

+3 -3

@@ -1086,7 +1086,7 @@ static void write_graph_chunk_bloom_indexes(struct hashfile *f,

1086 ctx->commits.nr);

1087

1088 while (list < last) {

1089 - struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);

1089 + struct bloom_filter *filter = get_bloom_filter(ctx->r, *list, 0);

1090 cur_pos += filter->len;

1091 display_progress(progress, ++i);

1092 hashwrite_be32(f, cur_pos);

@@ -1115,7 +1115,7 @@ static void write_graph_chunk_bloom_data(struct hashfile *f,

1115 hashwrite_be32(f, settings->bits_per_entry);

1116

1117 while (list < last) {

1118 - struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);

1118 + struct bloom_filter *filter = get_bloom_filter(ctx->r, *list, 0);

1119 display_progress(progress, ++i);

1120 hashwrite(f, filter->data, filter->len * sizeof(unsigned char));

1121 list++;

@@ -1296,7 +1296,7 @@ static void compute_bloom_filters(struct write_commit_graph_context *ctx)

1296

1297 for (i = 0; i < ctx->commits.nr; i++) {

1298 struct commit *c = sorted_commits[i];

1299 - struct bloom_filter *filter = get_bloom_filter(ctx->r, c);

1299 + struct bloom_filter *filter = get_bloom_filter(ctx->r, c, 1);

1300 ctx->total_bloom_filter_data_size += sizeof(unsigned char) * filter->len;

1301 display_progress(progress, i + 1);

1302 }

t/helper/test-bloom.c

+1 -1

@@ -39,7 +39,7 @@ static void get_bloom_filter_for_commit(const struct object_id *commit_oid)

39 struct bloom_filter *filter;

40 setup_git_directory();

41 c = lookup_commit(the_repository, commit_oid);

42 - filter = get_bloom_filter(the_repository, c);

42 + filter = get_bloom_filter(the_repository, c, 1);

43 print_bloom_filter(filter);

44 }