@matheusbsbs / matheus-bagliano / commits / 6d192462eb

bloom: enable bloom filter with wildcard pathspec in revision traversal

When traversing commits, a pathspec item can be used to limit the traversal to commits that modify the specified paths. And the commit-graph includes a Bloom filter to exclude commits that definitely did not modify a given pathspec item. During commit traversal, the Bloom filter can significantly improve performance. However, it is disabled if the specified pathspec item contains wildcard characters or magic signatures. For performance reason, enable Bloom filter even if a pathspec item contains wildcard characters by filtering only the non-wildcard part of the pathspec item. The function of pathspec magic signature is generally to narrow down the path specified by the pathspecs. So, enable Bloom filter when the magic signature is "top", "glob", "attr", "--depth" or "literal". "exclude" is used to select paths other than the specified path, rather than serving as a filtering function, so it cannot be used together with the Bloom filter. Since Bloom filter is not case insensitive even in case insensitive system (e.g. MacOS), it cannot be used together with "icase" magic. With this optimization, we get some improvements for pathspecs with wildcards or magic signatures. First, in the Git repository we see these modest results: git log -100 -- "t/*" Benchmark 1: new Time (mean ± σ): 20.4 ms ± 0.6 ms Range (min … max): 19.3 ms … 24.4 ms Benchmark 2: old Time (mean ± σ): 23.4 ms ± 0.5 ms Range (min … max): 22.5 ms … 24.7 ms git log -100 -- ":(top)t" Benchmark 1: new Time (mean ± σ): 16.2 ms ± 0.4 ms Range (min … max): 15.3 ms … 17.2 ms Benchmark 2: old Time (mean ± σ): 18.6 ms ± 0.5 ms Range (min … max): 17.6 ms … 20.4 ms But in a larger repo, such as the LLVM project repo below, we get even better results: git log -100 -- "libc/*" Benchmark 1: new Time (mean ± σ): 16.0 ms ± 0.6 ms Range (min … max): 14.7 ms … 17.8 ms Benchmark 2: old Time (mean ± σ): 26.7 ms ± 0.5 ms Range (min … max): 25.4 ms … 27.8 ms git log -100 -- ":(top)libc" Benchmark 1: new Time (mean ± σ): 15.6 ms ± 0.6 ms Range (min … max): 14.4 ms … 17.7 ms Benchmark 2: old Time (mean ± σ): 19.6 ms ± 0.5 ms Range (min … max): 18.6 ms … 20.6 ms Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: Lidong Yan <yldhome2d2@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Lidong Yan committed Aug 11, 2025 at 14:01 UTC 6d192462eb3ca605600731a9717fd0d9aa72eff0

2 files changed +56 -17

revision.c

+29 -13

@@ -671,12 +671,17 @@ static void trace2_bloom_filter_statistics_atexit(void)

671

672 static int forbid_bloom_filters(struct pathspec *spec)

673 {

674 - if (spec->has_wildcard)

675 - return 1;

676 - if (spec->magic & ~PATHSPEC_LITERAL)

674 + unsigned int allowed_magic =

675 + PATHSPEC_FROMTOP |

676 + PATHSPEC_MAXDEPTH |

677 + PATHSPEC_LITERAL |

678 + PATHSPEC_GLOB |

679 + PATHSPEC_ATTR;

680 +

681 + if (spec->magic & ~allowed_magic)

682 return 1;

683 for (size_t nr = 0; nr < spec->nr; nr++)

679 - if (spec->items[nr].magic & ~PATHSPEC_LITERAL)

684 + if (spec->items[nr].magic & ~allowed_magic)

685 return 1;

686

687 return 0;

@@ -691,23 +696,34 @@ static int convert_pathspec_to_bloom_keyvec(struct bloom_keyvec **out,

696 char *path_alloc = NULL;

697 const char *path;

698 size_t len;

694 - int res = 0;

699 + int res = -1;

700

701 + len = pi->nowildcard_len;

702 + if (len != pi->len) {

703 + /*

704 + * for path like "dir/file*", nowildcard part would be

705 + * "dir/file", but only "dir" should be used for the

706 + * bloom filter.

707 + */

708 + while (len > 0 && pi->match[len - 1] != '/')

709 + len--;

710 + }

711 /* remove single trailing slash from path, if needed */

697 - if (pi->len > 0 && pi->match[pi->len - 1] == '/') {

698 - path_alloc = xmemdupz(pi->match, pi->len - 1);

712 + if (len > 0 && pi->match[len - 1] == '/')

713 + len--;

714 +

715 + if (!len)

716 + goto cleanup;

717 +

718 + if (len != pi->len) {

719 + path_alloc = xmemdupz(pi->match, len);

720 path = path_alloc;

721 } else

722 path = pi->match;

723

703 - len = strlen(path);

704 - if (!len) {

705 - res = -1;

706 - goto cleanup;

707 - }

708 -

724 *out = bloom_keyvec_new(path, len, settings);

725

726 + res = 0;

727 cleanup:

728 free(path_alloc);

729 return res;

t/t4216-log-bloom.sh

+27 -4

@@ -154,11 +154,34 @@ test_expect_success 'git log with multiple literal paths uses Bloom filter' '

154 test_bloom_filters_used "-- file*"

155 '

156

157 -test_expect_success 'git log with path contains a wildcard does not use Bloom filter' '

157 +test_expect_success 'git log with paths all contain non-wildcard part uses Bloom filter' '

158 + test_bloom_filters_used "-- A/\* file4" &&

159 + test_bloom_filters_used "-- A/file\*" &&

160 + test_bloom_filters_used "-- * A/\*"

161 +'

162 +

163 +test_expect_success 'git log with path only contains wildcard part does not use Bloom filter' '

164 test_bloom_filters_not_used "-- file\*" &&

159 - test_bloom_filters_not_used "-- A/\* file4" &&

160 - test_bloom_filters_not_used "-- file4 A/\*" &&

161 - test_bloom_filters_not_used "-- * A/\*"

165 + test_bloom_filters_not_used "-- file\* A/\*" &&

166 + test_bloom_filters_not_used "-- file\* *" &&

167 + test_bloom_filters_not_used "-- \*"

168 +'

169 +

170 +test_expect_success 'git log with path contains various magic signatures' '

171 + cd A &&

172 + test_bloom_filters_used "-- \:\(top\)B" &&

173 + cd .. &&

174 +

175 + test_bloom_filters_used "-- \:\(glob\)A/\*\*/C" &&

176 + test_bloom_filters_not_used "-- \:\(icase\)FILE4" &&

177 + test_bloom_filters_not_used "-- \:\(exclude\)A/B/C" &&

178 +

179 + test_when_finished "rm -f .gitattributes" &&

180 + cat >.gitattributes <<-EOF &&

181 + A/file1 text

182 + A/B/file2 -text

183 + EOF

184 + test_bloom_filters_used "-- \:\(attr\:text\)A"

185 '

186

187 test_expect_success 'setup - add commit-graph to the chain without Bloom filters' '