fsmonitor: add test cases for fsmonitor extension
Test the ability to add/remove the fsmonitor index extension via update-index. Test that dirty files returned from the integration script are properly represented in the index extension and verify that ls-files correctly reports their state. Test that ensure status results are correct when using the new fsmonitor extension. Test untracked, modified, and new files by ensuring the results are identical to when not using the extension. Test that if the fsmonitor extension doesn't tell git about a change, it doesn't discover it on its own. This ensures git is honoring the extension and that we get the performance benefits desired. Three test integration scripts are provided: fsmonitor-all - marks all files as dirty fsmonitor-none - marks no files as dirty fsmonitor-watchman - integrates with Watchman with debug logging To run tests in the test suite while utilizing fsmonitor: First copy t/t7519/fsmonitor-all to a location in your path and then set GIT_FORCE_PRELOAD_TEST=true and GIT_FSMONITOR_TEST=fsmonitor-all and run your tests. Note: currently when using the test script fsmonitor-watchman on Windows, many tests fail due to a reported but not yet fixed bug in Watchman where it holds on to handles for directories and files which prevents the test directory from being cleaned up properly. Signed-off-by: Ben Peart <benpeart@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ben Peart committed
Sep 22, 2017 at 12:35 UTC
5c8cdcfd807311d4ec7165c71b20c5bde9566acd
4 files changed
+490
t/t7519-status-fsmonitor.sh
new
+304
@@ -0,0 +1,304 @@
1
+#!/bin/sh
2
+
3
+test_description='git status with file system watcher'
4
+
5
+. ./test-lib.sh
6
+
7
+#
8
+# To run the entire git test suite using fsmonitor:
9
+#
10
+# copy t/t7519/fsmonitor-all to a location in your path and then set
11
+# GIT_FSMONITOR_TEST=fsmonitor-all and run your tests.
12
+#
13
+
14
+# Note, after "git reset --hard HEAD" no extensions exist other than 'TREE'
15
+# "git update-index --fsmonitor" can be used to get the extension written
16
+# before testing the results.
17
+
18
+clean_repo () {
19
+ git reset --hard HEAD &&
20
+ git clean -fd
21
+}
22
+
23
+dirty_repo () {
24
+ : >untracked &&
25
+ : >dir1/untracked &&
26
+ : >dir2/untracked &&
27
+ echo 1 >modified &&
28
+ echo 2 >dir1/modified &&
29
+ echo 3 >dir2/modified &&
30
+ echo 4 >new &&
31
+ echo 5 >dir1/new &&
32
+ echo 6 >dir2/new
33
+}
34
+
35
+write_integration_script () {
36
+ write_script .git/hooks/fsmonitor-test<<-\EOF
37
+ if test "$#" -ne 2
38
+ then
39
+ echo "$0: exactly 2 arguments expected"
40
+ exit 2
41
+ fi
42
+ if test "$1" != 1
43
+ then
44
+ echo "Unsupported core.fsmonitor hook version." >&2
45
+ exit 1
46
+ fi
47
+ printf "untracked\0"
48
+ printf "dir1/untracked\0"
49
+ printf "dir2/untracked\0"
50
+ printf "modified\0"
51
+ printf "dir1/modified\0"
52
+ printf "dir2/modified\0"
53
+ printf "new\0"
54
+ printf "dir1/new\0"
55
+ printf "dir2/new\0"
56
+ EOF
57
+}
58
+
59
+test_lazy_prereq UNTRACKED_CACHE '
60
+ { git update-index --test-untracked-cache; ret=$?; } &&
61
+ test $ret -ne 1
62
+'
63
+
64
+test_expect_success 'setup' '
65
+ mkdir -p .git/hooks &&
66
+ : >tracked &&
67
+ : >modified &&
68
+ mkdir dir1 &&
69
+ : >dir1/tracked &&
70
+ : >dir1/modified &&
71
+ mkdir dir2 &&
72
+ : >dir2/tracked &&
73
+ : >dir2/modified &&
74
+ git -c core.fsmonitor= add . &&
75
+ git -c core.fsmonitor= commit -m initial &&
76
+ git config core.fsmonitor .git/hooks/fsmonitor-test &&
77
+ cat >.gitignore <<-\EOF
78
+ .gitignore
79
+ expect*
80
+ actual*
81
+ marker*
82
+ EOF
83
+'
84
+
85
+# test that the fsmonitor extension is off by default
86
+test_expect_success 'fsmonitor extension is off by default' '
87
+ test-dump-fsmonitor >actual &&
88
+ grep "^no fsmonitor" actual
89
+'
90
+
91
+# test that "update-index --fsmonitor" adds the fsmonitor extension
92
+test_expect_success 'update-index --fsmonitor" adds the fsmonitor extension' '
93
+ git update-index --fsmonitor &&
94
+ test-dump-fsmonitor >actual &&
95
+ grep "^fsmonitor last update" actual
96
+'
97
+
98
+# test that "update-index --no-fsmonitor" removes the fsmonitor extension
99
+test_expect_success 'update-index --no-fsmonitor" removes the fsmonitor extension' '
100
+ git update-index --no-fsmonitor &&
101
+ test-dump-fsmonitor >actual &&
102
+ grep "^no fsmonitor" actual
103
+'
104
+
105
+cat >expect <<EOF &&
106
+h dir1/modified
107
+H dir1/tracked
108
+h dir2/modified
109
+H dir2/tracked
110
+h modified
111
+H tracked
112
+EOF
113
+
114
+# test that "update-index --fsmonitor-valid" sets the fsmonitor valid bit
115
+test_expect_success 'update-index --fsmonitor-valid" sets the fsmonitor valid bit' '
116
+ git update-index --fsmonitor &&
117
+ git update-index --fsmonitor-valid dir1/modified &&
118
+ git update-index --fsmonitor-valid dir2/modified &&
119
+ git update-index --fsmonitor-valid modified &&
120
+ git ls-files -f >actual &&
121
+ test_cmp expect actual
122
+'
123
+
124
+cat >expect <<EOF &&
125
+H dir1/modified
126
+H dir1/tracked
127
+H dir2/modified
128
+H dir2/tracked
129
+H modified
130
+H tracked
131
+EOF
132
+
133
+# test that "update-index --no-fsmonitor-valid" clears the fsmonitor valid bit
134
+test_expect_success 'update-index --no-fsmonitor-valid" clears the fsmonitor valid bit' '
135
+ git update-index --no-fsmonitor-valid dir1/modified &&
136
+ git update-index --no-fsmonitor-valid dir2/modified &&
137
+ git update-index --no-fsmonitor-valid modified &&
138
+ git ls-files -f >actual &&
139
+ test_cmp expect actual
140
+'
141
+
142
+cat >expect <<EOF &&
143
+H dir1/modified
144
+H dir1/tracked
145
+H dir2/modified
146
+H dir2/tracked
147
+H modified
148
+H tracked
149
+EOF
150
+
151
+# test that all files returned by the script get flagged as invalid
152
+test_expect_success 'all files returned by integration script get flagged as invalid' '
153
+ write_integration_script &&
154
+ dirty_repo &&
155
+ git update-index --fsmonitor &&
156
+ git ls-files -f >actual &&
157
+ test_cmp expect actual
158
+'
159
+
160
+cat >expect <<EOF &&
161
+H dir1/modified
162
+h dir1/new
163
+H dir1/tracked
164
+H dir2/modified
165
+h dir2/new
166
+H dir2/tracked
167
+H modified
168
+h new
169
+H tracked
170
+EOF
171
+
172
+# test that newly added files are marked valid
173
+test_expect_success 'newly added files are marked valid' '
174
+ git add new &&
175
+ git add dir1/new &&
176
+ git add dir2/new &&
177
+ git ls-files -f >actual &&
178
+ test_cmp expect actual
179
+'
180
+
181
+cat >expect <<EOF &&
182
+H dir1/modified
183
+h dir1/new
184
+h dir1/tracked
185
+H dir2/modified
186
+h dir2/new
187
+h dir2/tracked
188
+H modified
189
+h new
190
+h tracked
191
+EOF
192
+
193
+# test that all unmodified files get marked valid
194
+test_expect_success 'all unmodified files get marked valid' '
195
+ # modified files result in update-index returning 1
196
+ test_must_fail git update-index --refresh --force-write-index &&
197
+ git ls-files -f >actual &&
198
+ test_cmp expect actual
199
+'
200
+
201
+cat >expect <<EOF &&
202
+H dir1/modified
203
+h dir1/tracked
204
+h dir2/modified
205
+h dir2/tracked
206
+h modified
207
+h tracked
208
+EOF
209
+
210
+# test that *only* files returned by the integration script get flagged as invalid
211
+test_expect_success '*only* files returned by the integration script get flagged as invalid' '
212
+ write_script .git/hooks/fsmonitor-test<<-\EOF &&
213
+ printf "dir1/modified\0"
214
+ EOF
215
+ clean_repo &&
216
+ git update-index --refresh --force-write-index &&
217
+ echo 1 >modified &&
218
+ echo 2 >dir1/modified &&
219
+ echo 3 >dir2/modified &&
220
+ test_must_fail git update-index --refresh --force-write-index &&
221
+ git ls-files -f >actual &&
222
+ test_cmp expect actual
223
+'
224
+
225
+# Ensure commands that call refresh_index() to move the index back in time
226
+# properly invalidate the fsmonitor cache
227
+test_expect_success 'refresh_index() invalidates fsmonitor cache' '
228
+ write_script .git/hooks/fsmonitor-test<<-\EOF &&
229
+ EOF
230
+ clean_repo &&
231
+ dirty_repo &&
232
+ git add . &&
233
+ git commit -m "to reset" &&
234
+ git reset HEAD~1 &&
235
+ git status >actual &&
236
+ git -c core.fsmonitor= status >expect &&
237
+ test_i18ncmp expect actual
238
+'
239
+
240
+# test fsmonitor with and without preloadIndex
241
+preload_values="false true"
242
+for preload_val in $preload_values
243
+do
244
+ test_expect_success "setup preloadIndex to $preload_val" '
245
+ git config core.preloadIndex $preload_val &&
246
+ if test $preload_val = true
247
+ then
248
+ GIT_FORCE_PRELOAD_TEST=$preload_val; export GIT_FORCE_PRELOAD_TEST
249
+ else
250
+ unset GIT_FORCE_PRELOAD_TEST
251
+ fi
252
+ '
253
+
254
+ # test fsmonitor with and without the untracked cache (if available)
255
+ uc_values="false"
256
+ test_have_prereq UNTRACKED_CACHE && uc_values="false true"
257
+ for uc_val in $uc_values
258
+ do
259
+ test_expect_success "setup untracked cache to $uc_val" '
260
+ git config core.untrackedcache $uc_val
261
+ '
262
+
263
+ # Status is well tested elsewhere so we'll just ensure that the results are
264
+ # the same when using core.fsmonitor.
265
+ test_expect_success 'compare status with and without fsmonitor' '
266
+ write_integration_script &&
267
+ clean_repo &&
268
+ dirty_repo &&
269
+ git add new &&
270
+ git add dir1/new &&
271
+ git add dir2/new &&
272
+ git status >actual &&
273
+ git -c core.fsmonitor= status >expect &&
274
+ test_i18ncmp expect actual
275
+ '
276
+
277
+ # Make sure it's actually skipping the check for modified and untracked
278
+ # (if enabled) files unless it is told about them.
279
+ test_expect_success "status doesn't detect unreported modifications" '
280
+ write_script .git/hooks/fsmonitor-test<<-\EOF &&
281
+ :>marker
282
+ EOF
283
+ clean_repo &&
284
+ git status &&
285
+ test_path_is_file marker &&
286
+ dirty_repo &&
287
+ rm -f marker &&
288
+ git status >actual &&
289
+ test_path_is_file marker &&
290
+ test_i18ngrep ! "Changes not staged for commit:" actual &&
291
+ if test $uc_val = true
292
+ then
293
+ test_i18ngrep ! "Untracked files:" actual
294
+ fi &&
295
+ if test $uc_val = false
296
+ then
297
+ test_i18ngrep "Untracked files:" actual
298
+ fi &&
299
+ rm -f marker
300
+ '
301
+ done
302
+done
303
+
304
+test_done
t/t7519/fsmonitor-all
new
+24
@@ -0,0 +1,24 @@
1
+#!/bin/sh
2
+#
3
+# An test hook script to integrate with git to test fsmonitor.
4
+#
5
+# The hook is passed a version (currently 1) and a time in nanoseconds
6
+# formatted as a string and outputs to stdout all files that have been
7
+# modified since the given time. Paths must be relative to the root of
8
+# the working tree and separated by a single NUL.
9
+#
10
+#echo "$0 $*" >&2
11
+
12
+if test "$#" -ne 2
13
+then
14
+ echo "$0: exactly 2 arguments expected" >&2
15
+ exit 2
16
+fi
17
+
18
+if test "$1" != 1
19
+then
20
+ echo "Unsupported core.fsmonitor hook version." >&2
21
+ exit 1
22
+fi
23
+
24
+echo "/"
t/t7519/fsmonitor-none
new
+22
@@ -0,0 +1,22 @@
1
+#!/bin/sh
2
+#
3
+# An test hook script to integrate with git to test fsmonitor.
4
+#
5
+# The hook is passed a version (currently 1) and a time in nanoseconds
6
+# formatted as a string and outputs to stdout all files that have been
7
+# modified since the given time. Paths must be relative to the root of
8
+# the working tree and separated by a single NUL.
9
+#
10
+#echo "$0 $*" >&2
11
+
12
+if test "$#" -ne 2
13
+then
14
+ echo "$0: exactly 2 arguments expected" >&2
15
+ exit 2
16
+fi
17
+
18
+if test "$1" != 1
19
+then
20
+ echo "Unsupported core.fsmonitor hook version." >&2
21
+ exit 1
22
+fi
t/t7519/fsmonitor-watchman
new
+140
@@ -0,0 +1,140 @@
1
+#!/usr/bin/perl
2
+
3
+use strict;
4
+use warnings;
5
+use IPC::Open2;
6
+
7
+# An example hook script to integrate Watchman
8
+# (https://facebook.github.io/watchman/) with git to speed up detecting
9
+# new and modified files.
10
+#
11
+# The hook is passed a version (currently 1) and a time in nanoseconds
12
+# formatted as a string and outputs to stdout all files that have been
13
+# modified since the given time. Paths must be relative to the root of
14
+# the working tree and separated by a single NUL.
15
+#
16
+# To enable this hook, rename this file to "query-watchman" and set
17
+# 'git config core.fsmonitor .git/hooks/query-watchman'
18
+#
19
+my ($version, $time) = @ARGV;
20
+#print STDERR "$0 $version $time\n";
21
+
22
+# Check the hook interface version
23
+
24
+if ($version == 1) {
25
+ # convert nanoseconds to seconds
26
+ $time = int $time / 1000000000;
27
+} else {
28
+ die "Unsupported query-fsmonitor hook version '$version'.\n" .
29
+ "Falling back to scanning...\n";
30
+}
31
+
32
+# Convert unix style paths to escaped Windows style paths when running
33
+# in Windows command prompt
34
+
35
+my $system = `uname -s`;
36
+$system =~ s/[\r\n]+//g;
37
+my $git_work_tree;
38
+
39
+if ($system =~ m/^MSYS_NT/) {
40
+ $git_work_tree = `cygpath -aw "\$PWD"`;
41
+ $git_work_tree =~ s/[\r\n]+//g;
42
+ $git_work_tree =~ s,\\,/,g;
43
+} else {
44
+ $git_work_tree = $ENV{'PWD'};
45
+}
46
+
47
+my $retry = 1;
48
+
49
+launch_watchman();
50
+
51
+sub launch_watchman {
52
+
53
+ # Set input record separator
54
+ local $/ = 0666;
55
+
56
+ my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j')
57
+ or die "open2() failed: $!\n" .
58
+ "Falling back to scanning...\n";
59
+
60
+ # In the query expression below we're asking for names of files that
61
+ # changed since $time but were not transient (ie created after
62
+ # $time but no longer exist).
63
+ #
64
+ # To accomplish this, we're using the "since" generator to use the
65
+ # recency index to select candidate nodes and "fields" to limit the
66
+ # output to file names only. Then we're using the "expression" term to
67
+ # further constrain the results.
68
+ #
69
+ # The category of transient files that we want to ignore will have a
70
+ # creation clock (cclock) newer than $time_t value and will also not
71
+ # currently exist.
72
+
73
+ my $query = <<" END";
74
+ ["query", "$git_work_tree", {
75
+ "since": $time,
76
+ "fields": ["name"],
77
+ "expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]]
78
+ }]
79
+ END
80
+
81
+ open (my $fh, ">", ".git/watchman-query.json");
82
+ print $fh $query;
83
+ close $fh;
84
+
85
+ print CHLD_IN $query;
86
+ my $response = <CHLD_OUT>;
87
+
88
+ open ($fh, ">", ".git/watchman-response.json");
89
+ print $fh $response;
90
+ close $fh;
91
+
92
+ die "Watchman: command returned no output.\n" .
93
+ "Falling back to scanning...\n" if $response eq "";
94
+ die "Watchman: command returned invalid output: $response\n" .
95
+ "Falling back to scanning...\n" unless $response =~ /^\{/;
96
+
97
+ my $json_pkg;
98
+ eval {
99
+ require JSON::XS;
100
+ $json_pkg = "JSON::XS";
101
+ 1;
102
+ } or do {
103
+ require JSON::PP;
104
+ $json_pkg = "JSON::PP";
105
+ };
106
+
107
+ my $o = $json_pkg->new->utf8->decode($response);
108
+
109
+ if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
110
+ print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
111
+ $retry--;
112
+ qx/watchman watch "$git_work_tree"/;
113
+ die "Failed to make watchman watch '$git_work_tree'.\n" .
114
+ "Falling back to scanning...\n" if $? != 0;
115
+
116
+ # Watchman will always return all files on the first query so
117
+ # return the fast "everything is dirty" flag to git and do the
118
+ # Watchman query just to get it over with now so we won't pay
119
+ # the cost in git to look up each individual file.
120
+
121
+ open ($fh, ">", ".git/watchman-output.out");
122
+ print "/\0";
123
+ close $fh;
124
+
125
+ print "/\0";
126
+ eval { launch_watchman() };
127
+ exit 0;
128
+ }
129
+
130
+ die "Watchman: $o->{error}.\n" .
131
+ "Falling back to scanning...\n" if $o->{error};
132
+
133
+ open ($fh, ">", ".git/watchman-output.out");
134
+ print $fh @{$o->{files}};
135
+ close $fh;
136
+
137
+ binmode STDOUT, ":utf8";
138
+ local $, = "\0";
139
+ print @{$o->{files}};
140
+}