@matheusbsbs / matheus-bagliano / commits / b31e2680c4

ref-filter.c: find disjoint pattern prefixes

Since cfe004a5a9 (ref-filter: limit traversal to prefix, 2017-05-22), the ref-filter code has sought to limit the traversals to a prefix of the given patterns. That code stopped short of handling more than one pattern, because it means invoking 'for_each_ref_in' multiple times. If we're not careful about which patterns overlap, we will output the same refs multiple times. For instance, consider the set of patterns 'refs/heads/a/*', 'refs/heads/a/b/c', and 'refs/tags/v1.0.0'. If we naïvely ran: for_each_ref_in("refs/heads/a/*", ...); for_each_ref_in("refs/heads/a/b/c", ...); for_each_ref_in("refs/tags/v1.0.0", ...); we would see 'refs/heads/a/b/c' (and everything underneath it) twice. Instead, we want to partition the patterns into disjoint sets, where we know that no ref will be matched by any two patterns in different sets. In the above, these are: - {'refs/heads/a/*', 'refs/heads/a/b/c'}, and - {'refs/tags/v1.0.0'} Given one of these disjoint sets, what is a suitable pattern to pass to 'for_each_ref_in'? One approach is to compute the longest common prefix over all elements in that disjoint set, and let the caller cull out the refs they didn't want. Computing the longest prefix means that in most cases, we won't match too many things the caller would like to ignore. The longest common prefixes of the above are: - {'refs/heads/a/*', 'refs/heads/a/b/c'} -> refs/heads/a/* - {'refs/tags/v1.0.0'} -> refs/tags/v1.0.0 We instead invoke: for_each_ref_in("refs/heads/a/*", ...); for_each_ref_in("refs/tags/v1.0.0", ...); Which provides us with the refs we were looking for with a minimal amount of extra cruft, but never a duplicate of the ref we asked for. Implemented here is an algorithm which accomplishes the above, which works as follows: 1. Lexicographically sort the given list of patterns. 2. Initialize 'prefix' to the empty string, where our goal is to build each element in the above set of longest common prefixes. 3. Consider each pattern in the given set, and emit 'prefix' if it reaches the end of a pattern, or touches a wildcard character. The end of a string is treated as if it precedes a wildcard. (Note that there is some room for future work to detect that, e.g., 'a?b' and 'abc' are disjoint). 4. Otherwise, recurse on step (3) with the slice of the list corresponding to our current prefix (i.e., the subset of patterns that have our prefix as a literal string prefix.) This algorithm is 'O(kn + n log(n))', where 'k' is max(len(pattern)) for each pattern in the list, and 'n' is len(patterns). By discovering this set of interesting patterns, we reduce the runtime of multi-pattern 'git for-each-ref' (and other ref traversals) from O(N) to O(n log(N)), where 'N' is the total number of packed references. Running 'git for-each-ref refs/tags/a refs/tags/b' on a repository with 10,000,000 refs in 'refs/tags/huge-N', my best-of-five times drop from: real 0m5.805s user 0m5.188s sys 0m0.468s to: real 0m0.001s user 0m0.000s sys 0m0.000s On linux.git, the times to dig out two of the latest -rc tags drops from 0.002s to 0.001s, so the change on repositories with fewer tags is much less noticeable. Co-authored-by: Jeff King <peff@peff.net> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Jun 26, 2019 at 17:41 UTC b31e2680c46c43ece716080040e90b3a787b0bad
2 files changed +89 -26
ref-filter.c
+63 -26
@@ -20,6 +20,7 @@
20 #include "commit-slab.h"
21 #include "commit-graph.h"
22 #include "commit-reach.h"
23 +#include "argv-array.h"
24
25 static struct ref_msg {
26 const char *gone;
@@ -1790,21 +1791,62 @@ static int filter_pattern_match(struct ref_filter *filter, const char *refname)
1791 return match_pattern(filter, refname);
1792 }
1793
1793 -/*
1794 - * Find the longest prefix of pattern we can pass to
1795 - * `for_each_fullref_in()`, namely the part of pattern preceding the
1796 - * first glob character. (Note that `for_each_fullref_in()` is
1797 - * perfectly happy working with a prefix that doesn't end at a
1798 - * pathname component boundary.)
1799 - */
1800 -static void find_longest_prefix(struct strbuf *out, const char *pattern)
1794 +static int qsort_strcmp(const void *va, const void *vb)
1795 +{
1796 + const char *a = *(const char **)va;
1797 + const char *b = *(const char **)vb;
1798 +
1799 + return strcmp(a, b);
1800 +}
1801 +
1802 +static void find_longest_prefixes_1(struct string_list *out,
1803 + struct strbuf *prefix,
1804 + const char **patterns, size_t nr)
1805 {
1802 - const char *p;
1806 + size_t i;
1807 +
1808 + for (i = 0; i < nr; i++) {
1809 + char c = patterns[i][prefix->len];
1810 + if (!c || is_glob_special(c)) {
1811 + string_list_append(out, prefix->buf);
1812 + return;
1813 + }
1814 + }
1815 +
1816 + i = 0;
1817 + while (i < nr) {
1818 + size_t end;
1819 +
1820 + /*
1821 + * Set "end" to the index of the element _after_ the last one
1822 + * in our group.
1823 + */
1824 + for (end = i + 1; end < nr; end++) {
1825 + if (patterns[i][prefix->len] != patterns[end][prefix->len])
1826 + break;
1827 + }
1828
1804 - for (p = pattern; *p && !is_glob_special(*p); p++)
1805 - ;
1829 + strbuf_addch(prefix, patterns[i][prefix->len]);
1830 + find_longest_prefixes_1(out, prefix, patterns + i, end - i);
1831 + strbuf_setlen(prefix, prefix->len - 1);
1832
1807 - strbuf_add(out, pattern, p - pattern);
1833 + i = end;
1834 + }
1835 +}
1836 +
1837 +static void find_longest_prefixes(struct string_list *out,
1838 + const char **patterns)
1839 +{
1840 + struct argv_array sorted = ARGV_ARRAY_INIT;
1841 + struct strbuf prefix = STRBUF_INIT;
1842 +
1843 + argv_array_pushv(&sorted, patterns);
1844 + QSORT(sorted.argv, sorted.argc, qsort_strcmp);
1845 +
1846 + find_longest_prefixes_1(out, &prefix, sorted.argv, sorted.argc);
1847 +
1848 + argv_array_clear(&sorted);
1849 + strbuf_release(&prefix);
1850 }
1851
1852 /*
@@ -1817,7 +1859,8 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
1859 void *cb_data,
1860 int broken)
1861 {
1820 - struct strbuf prefix = STRBUF_INIT;
1862 + struct string_list prefixes = STRING_LIST_INIT_DUP;
1863 + struct string_list_item *prefix;
1864 int ret;
1865
1866 if (!filter->match_as_path) {
@@ -1843,21 +1886,15 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
1886 return for_each_fullref_in("", cb, cb_data, broken);
1887 }
1888
1846 - if (filter->name_patterns[1]) {
1847 - /*
1848 - * multiple patterns; in theory this could still work as long
1849 - * as the patterns are disjoint. We'd just make multiple calls
1850 - * to for_each_ref(). But if they're not disjoint, we'd end up
1851 - * reporting the same ref multiple times. So let's punt on that
1852 - * for now.
1853 - */
1854 - return for_each_fullref_in("", cb, cb_data, broken);
1855 - }
1889 + find_longest_prefixes(&prefixes, filter->name_patterns);
1890
1857 - find_longest_prefix(&prefix, filter->name_patterns[0]);
1891 + for_each_string_list_item(prefix, &prefixes) {
1892 + ret = for_each_fullref_in(prefix->string, cb, cb_data, broken);
1893 + if (ret)
1894 + break;
1895 + }
1896
1859 - ret = for_each_fullref_in(prefix.buf, cb, cb_data, broken);
1860 - strbuf_release(&prefix);
1897 + string_list_clear(&prefixes, 0);
1898 return ret;
1899 }
1900
t/t6300-for-each-ref.sh
+26
@@ -345,6 +345,32 @@ test_expect_success 'Verify descending sort' '
345 test_cmp expected actual
346 '
347
348 +cat >expected <<\EOF
349 +refs/tags/testtag
350 +refs/tags/testtag-2
351 +EOF
352 +
353 +test_expect_success 'exercise patterns with prefixes' '
354 + git tag testtag-2 &&
355 + test_when_finished "git tag -d testtag-2" &&
356 + git for-each-ref --format="%(refname)" \
357 + refs/tags/testtag refs/tags/testtag-2 >actual &&
358 + test_cmp expected actual
359 +'
360 +
361 +cat >expected <<\EOF
362 +refs/tags/testtag
363 +refs/tags/testtag-2
364 +EOF
365 +
366 +test_expect_success 'exercise glob patterns with prefixes' '
367 + git tag testtag-2 &&
368 + test_when_finished "git tag -d testtag-2" &&
369 + git for-each-ref --format="%(refname)" \
370 + refs/tags/testtag "refs/tags/testtag-*" >actual &&
371 + test_cmp expected actual
372 +'
373 +
374 cat >expected <<\EOF
375 'refs/heads/master'
376 'refs/remotes/origin/master'