bloom.c: core Bloom filter implementation for changed paths.
Add the core implementation for computing Bloom filters for the paths changed between a commit and it's first parent. We fill the Bloom filters as (const char *data, int len) pairs as `struct bloom_filters" within a commit slab. Filters for commits with no changes and more than 512 changes, is represented with a filter of length zero. There is no gain in distinguishing between a computed filter of length zero for a commit with no changes, and an uncomputed filter for new commits or for commits with more than 512 changes. The effect on `git log -- path` is the same in both cases. We will fall back to the normal diffing algorithm when we can't benefit from the existence of Bloom filters. Helped-by: Jeff King <peff@peff.net> Helped-by: Derrick Stolee <dstolee@microsoft.com> Reviewed-by: Jakub Narębski <jnareb@gmail.com> Signed-off-by: Garima Singh <garima.singh@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Garima Singh committed
Mar 30, 2020 at 00:31 UTC
ed591febb4a201ce48b34a4e90027414cd0d7966
4 files changed
+172
bloom.c
+97
@@ -1,5 +1,18 @@
1
#include "git-compat-util.h"
2
#include "bloom.h"
3
+#include "diff.h"
4
+#include "diffcore.h"
5
+#include "revision.h"
6
+#include "hashmap.h"
7
+
8
+define_commit_slab(bloom_filter_slab, struct bloom_filter);
9
+
10
+struct bloom_filter_slab bloom_filters;
11
+
12
+struct pathmap_hash_entry {
13
+ struct hashmap_entry entry;
14
+ const char path[FLEX_ARRAY];
15
+};
16
17
static uint32_t rotate_left(uint32_t value, int32_t count)
18
{
@@ -107,3 +120,87 @@ void add_key_to_filter(const struct bloom_key *key,
120
filter->data[block_pos] |= get_bitmask(hash_mod);
121
}
122
}
123
+
124
+void init_bloom_filters(void)
125
+{
126
+ init_bloom_filter_slab(&bloom_filters);
127
+}
128
+
129
+struct bloom_filter *get_bloom_filter(struct repository *r,
130
+ struct commit *c)
131
+{
132
+ struct bloom_filter *filter;
133
+ struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;
134
+ int i;
135
+ struct diff_options diffopt;
136
+
137
+ if (bloom_filters.slab_size == 0)
138
+ return NULL;
139
+
140
+ filter = bloom_filter_slab_at(&bloom_filters, c);
141
+
142
+ repo_diff_setup(r, &diffopt);
143
+ diffopt.flags.recursive = 1;
144
+ diff_setup_done(&diffopt);
145
+
146
+ if (c->parents)
147
+ diff_tree_oid(&c->parents->item->object.oid, &c->object.oid, "", &diffopt);
148
+ else
149
+ diff_tree_oid(NULL, &c->object.oid, "", &diffopt);
150
+ diffcore_std(&diffopt);
151
+
152
+ if (diff_queued_diff.nr <= 512) {
153
+ struct hashmap pathmap;
154
+ struct pathmap_hash_entry *e;
155
+ struct hashmap_iter iter;
156
+ hashmap_init(&pathmap, NULL, NULL, 0);
157
+
158
+ for (i = 0; i < diff_queued_diff.nr; i++) {
159
+ const char *path = diff_queued_diff.queue[i]->two->path;
160
+
161
+ /*
162
+ * Add each leading directory of the changed file, i.e. for
163
+ * 'dir/subdir/file' add 'dir' and 'dir/subdir' as well, so
164
+ * the Bloom filter could be used to speed up commands like
165
+ * 'git log dir/subdir', too.
166
+ *
167
+ * Note that directories are added without the trailing '/'.
168
+ */
169
+ do {
170
+ char *last_slash = strrchr(path, '/');
171
+
172
+ FLEX_ALLOC_STR(e, path, path);
173
+ hashmap_entry_init(&e->entry, strhash(path));
174
+ hashmap_add(&pathmap, &e->entry);
175
+
176
+ if (!last_slash)
177
+ last_slash = (char*)path;
178
+ *last_slash = '\0';
179
+
180
+ } while (*path);
181
+
182
+ diff_free_filepair(diff_queued_diff.queue[i]);
183
+ }
184
+
185
+ filter->len = (hashmap_get_size(&pathmap) * settings.bits_per_entry + BITS_PER_WORD - 1) / BITS_PER_WORD;
186
+ filter->data = xcalloc(filter->len, sizeof(unsigned char));
187
+
188
+ hashmap_for_each_entry(&pathmap, &iter, e, entry) {
189
+ struct bloom_key key;
190
+ fill_bloom_key(e->path, strlen(e->path), &key, &settings);
191
+ add_key_to_filter(&key, filter, &settings);
192
+ }
193
+
194
+ hashmap_free_entries(&pathmap, struct pathmap_hash_entry, entry);
195
+ } else {
196
+ for (i = 0; i < diff_queued_diff.nr; i++)
197
+ diff_free_filepair(diff_queued_diff.queue[i]);
198
+ filter->data = NULL;
199
+ filter->len = 0;
200
+ }
201
+
202
+ free(diff_queued_diff.queue);
203
+ DIFF_QUEUE_CLEAR(&diff_queued_diff);
204
+
205
+ return filter;
206
+}
bloom.h
+8
@@ -1,6 +1,9 @@
1
#ifndef BLOOM_H
2
#define BLOOM_H
3
4
+struct commit;
5
+struct repository;
6
+
7
struct bloom_filter_settings {
8
/*
9
* The version of the hashing technique being used.
@@ -73,4 +76,9 @@ void add_key_to_filter(const struct bloom_key *key,
76
struct bloom_filter *filter,
77
const struct bloom_filter_settings *settings);
78
79
+void init_bloom_filters(void);
80
+
81
+struct bloom_filter *get_bloom_filter(struct repository *r,
82
+ struct commit *c);
83
+
84
#endif
\ No newline at end of file
t/helper/test-bloom.c
+20
@@ -1,6 +1,7 @@
1
#include "git-compat-util.h"
2
#include "bloom.h"
3
#include "test-tool.h"
4
+#include "commit.h"
5
6
struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;
7
@@ -32,6 +33,16 @@ static void print_bloom_filter(struct bloom_filter *filter) {
33
printf("\n");
34
}
35
36
+static void get_bloom_filter_for_commit(const struct object_id *commit_oid)
37
+{
38
+ struct commit *c;
39
+ struct bloom_filter *filter;
40
+ setup_git_directory();
41
+ c = lookup_commit(the_repository, commit_oid);
42
+ filter = get_bloom_filter(the_repository, c);
43
+ print_bloom_filter(filter);
44
+}
45
+
46
int cmd__bloom(int argc, const char **argv)
47
{
48
if (!strcmp(argv[1], "get_murmur3")) {
@@ -57,5 +68,14 @@ int cmd__bloom(int argc, const char **argv)
68
print_bloom_filter(&filter);
69
}
70
71
+ if (!strcmp(argv[1], "get_filter_for_commit")) {
72
+ struct object_id oid;
73
+ const char *end;
74
+ if (parse_oid_hex(argv[2], &oid, &end))
75
+ die("cannot parse oid '%s'", argv[2]);
76
+ init_bloom_filters();
77
+ get_bloom_filter_for_commit(&oid);
78
+ }
79
+
80
return 0;
81
}
\ No newline at end of file
t/t0095-bloom.sh
+47
@@ -67,4 +67,51 @@ test_expect_success 'compute bloom key for test string 2' '
67
test_cmp expect actual
68
'
69
70
+test_expect_success 'get bloom filters for commit with no changes' '
71
+ git init &&
72
+ git commit --allow-empty -m "c0" &&
73
+ cat >expect <<-\EOF &&
74
+ Filter_Length:0
75
+ Filter_Data:
76
+ EOF
77
+ test-tool bloom get_filter_for_commit "$(git rev-parse HEAD)" >actual &&
78
+ test_cmp expect actual
79
+'
80
+
81
+test_expect_success 'get bloom filter for commit with 10 changes' '
82
+ rm actual &&
83
+ rm expect &&
84
+ mkdir smallDir &&
85
+ for i in $(test_seq 0 9)
86
+ do
87
+ echo $i >smallDir/$i
88
+ done &&
89
+ git add smallDir &&
90
+ git commit -m "commit with 10 changes" &&
91
+ cat >expect <<-\EOF &&
92
+ Filter_Length:25
93
+ Filter_Data:82|a0|65|47|0c|92|90|c0|a1|40|02|a0|e2|40|e0|04|0a|9a|66|cf|80|19|85|42|23|
94
+ EOF
95
+ test-tool bloom get_filter_for_commit "$(git rev-parse HEAD)" >actual &&
96
+ test_cmp expect actual
97
+'
98
+
99
+test_expect_success EXPENSIVE 'get bloom filter for commit with 513 changes' '
100
+ rm actual &&
101
+ rm expect &&
102
+ mkdir bigDir &&
103
+ for i in $(test_seq 0 512)
104
+ do
105
+ echo $i >bigDir/$i
106
+ done &&
107
+ git add bigDir &&
108
+ git commit -m "commit with 513 changes" &&
109
+ cat >expect <<-\EOF &&
110
+ Filter_Length:0
111
+ Filter_Data:
112
+ EOF
113
+ test-tool bloom get_filter_for_commit "$(git rev-parse HEAD)" >actual &&
114
+ test_cmp expect actual
115
+'
116
+
117
test_done
\ No newline at end of file