fsmonitor: add a performance test
Add a test utility (test-drop-caches) that flushes all changes to disk then drops file system cache on Windows, Linux, and OSX. Add a perf test (p7519-fsmonitor.sh) for fsmonitor. By default, the performance test will utilize the Watchman file system monitor if it is installed. If Watchman is not installed, it will use a dummy integration script that does not report any new or modified files. The dummy script has very little overhead which provides optimistic results. The performance test will also use the untracked cache feature if it is available as fsmonitor uses it to speed up scanning for untracked files. There are 4 environment variables that can be used to alter the default behavior of the performance test: GIT_PERF_7519_UNTRACKED_CACHE: used to configure core.untrackedCache GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex GIT_PERF_7519_FSMONITOR: used to configure core.fsmonitor GIT_PERF_7519_DROP_CACHE: if set, the OS caches are dropped between tests The big win for using fsmonitor is the elimination of the need to scan the working directory looking for changed and untracked files. If the file information is all cached in RAM, the benefits are reduced. Signed-off-by: Ben Peart <benpeart@microsoft.com> Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ben Peart committed
Sep 22, 2017 at 12:35 UTC
14527b30022190c6e7e8cd7f7dbef4195ebeedc2
4 files changed
+350
Makefile
+1
@@ -638,6 +638,7 @@ TEST_PROGRAMS_NEED_X += test-ctype
638
TEST_PROGRAMS_NEED_X += test-config
639
TEST_PROGRAMS_NEED_X += test-date
640
TEST_PROGRAMS_NEED_X += test-delta
641
+TEST_PROGRAMS_NEED_X += test-drop-caches
642
TEST_PROGRAMS_NEED_X += test-dump-cache-tree
643
TEST_PROGRAMS_NEED_X += test-dump-fsmonitor
644
TEST_PROGRAMS_NEED_X += test-dump-split-index
t/helper/.gitignore
+1
@@ -3,6 +3,7 @@
3
/test-config
4
/test-date
5
/test-delta
6
+/test-drop-caches
7
/test-dump-cache-tree
8
/test-dump-fsmonitor
9
/test-dump-split-index
t/helper/test-drop-caches.c
new
+164
@@ -0,0 +1,164 @@
1
+#include "git-compat-util.h"
2
+
3
+#if defined(GIT_WINDOWS_NATIVE)
4
+
5
+static int cmd_sync(void)
6
+{
7
+ char Buffer[MAX_PATH];
8
+ DWORD dwRet;
9
+ char szVolumeAccessPath[] = "\\\\.\\X:";
10
+ HANDLE hVolWrite;
11
+ int success = 0;
12
+
13
+ dwRet = GetCurrentDirectory(MAX_PATH, Buffer);
14
+ if ((0 == dwRet) || (dwRet > MAX_PATH))
15
+ return error("Error getting current directory");
16
+
17
+ if ((Buffer[0] < 'A') || (Buffer[0] > 'Z'))
18
+ return error("Invalid drive letter '%c'", Buffer[0]);
19
+
20
+ szVolumeAccessPath[4] = Buffer[0];
21
+ hVolWrite = CreateFile(szVolumeAccessPath, GENERIC_READ | GENERIC_WRITE,
22
+ FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
23
+ if (INVALID_HANDLE_VALUE == hVolWrite)
24
+ return error("Unable to open volume for writing, need admin access");
25
+
26
+ success = FlushFileBuffers(hVolWrite);
27
+ if (!success)
28
+ error("Unable to flush volume");
29
+
30
+ CloseHandle(hVolWrite);
31
+
32
+ return !success;
33
+}
34
+
35
+#define STATUS_SUCCESS (0x00000000L)
36
+#define STATUS_PRIVILEGE_NOT_HELD (0xC0000061L)
37
+
38
+typedef enum _SYSTEM_INFORMATION_CLASS {
39
+ SystemMemoryListInformation = 80,
40
+} SYSTEM_INFORMATION_CLASS;
41
+
42
+typedef enum _SYSTEM_MEMORY_LIST_COMMAND {
43
+ MemoryCaptureAccessedBits,
44
+ MemoryCaptureAndResetAccessedBits,
45
+ MemoryEmptyWorkingSets,
46
+ MemoryFlushModifiedList,
47
+ MemoryPurgeStandbyList,
48
+ MemoryPurgeLowPriorityStandbyList,
49
+ MemoryCommandMax
50
+} SYSTEM_MEMORY_LIST_COMMAND;
51
+
52
+static BOOL GetPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
53
+{
54
+ BOOL bResult;
55
+ DWORD dwBufferLength;
56
+ LUID luid;
57
+ TOKEN_PRIVILEGES tpPreviousState;
58
+ TOKEN_PRIVILEGES tpNewState;
59
+
60
+ dwBufferLength = 16;
61
+ bResult = LookupPrivilegeValueA(0, lpName, &luid);
62
+ if (bResult) {
63
+ tpNewState.PrivilegeCount = 1;
64
+ tpNewState.Privileges[0].Luid = luid;
65
+ tpNewState.Privileges[0].Attributes = 0;
66
+ bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpNewState,
67
+ (DWORD)((LPBYTE)&(tpNewState.Privileges[1]) - (LPBYTE)&tpNewState),
68
+ &tpPreviousState, &dwBufferLength);
69
+ if (bResult) {
70
+ tpPreviousState.PrivilegeCount = 1;
71
+ tpPreviousState.Privileges[0].Luid = luid;
72
+ tpPreviousState.Privileges[0].Attributes = flags != 0 ? 2 : 0;
73
+ bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpPreviousState,
74
+ dwBufferLength, 0, 0);
75
+ }
76
+ }
77
+ return bResult;
78
+}
79
+
80
+static int cmd_dropcaches(void)
81
+{
82
+ HANDLE hProcess = GetCurrentProcess();
83
+ HANDLE hToken;
84
+ HMODULE ntdll;
85
+ DWORD(WINAPI *NtSetSystemInformation)(INT, PVOID, ULONG);
86
+ SYSTEM_MEMORY_LIST_COMMAND command;
87
+ int status;
88
+
89
+ if (!OpenProcessToken(hProcess, TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
90
+ return error("Can't open current process token");
91
+
92
+ if (!GetPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
93
+ return error("Can't get SeProfileSingleProcessPrivilege");
94
+
95
+ CloseHandle(hToken);
96
+
97
+ ntdll = LoadLibrary("ntdll.dll");
98
+ if (!ntdll)
99
+ return error("Can't load ntdll.dll, wrong Windows version?");
100
+
101
+ NtSetSystemInformation =
102
+ (DWORD(WINAPI *)(INT, PVOID, ULONG))GetProcAddress(ntdll, "NtSetSystemInformation");
103
+ if (!NtSetSystemInformation)
104
+ return error("Can't get function addresses, wrong Windows version?");
105
+
106
+ command = MemoryPurgeStandbyList;
107
+ status = NtSetSystemInformation(
108
+ SystemMemoryListInformation,
109
+ &command,
110
+ sizeof(SYSTEM_MEMORY_LIST_COMMAND)
111
+ );
112
+ if (status == STATUS_PRIVILEGE_NOT_HELD)
113
+ error("Insufficient privileges to purge the standby list, need admin access");
114
+ else if (status != STATUS_SUCCESS)
115
+ error("Unable to execute the memory list command %d", status);
116
+
117
+ FreeLibrary(ntdll);
118
+
119
+ return status;
120
+}
121
+
122
+#elif defined(__linux__)
123
+
124
+static int cmd_sync(void)
125
+{
126
+ return system("sync");
127
+}
128
+
129
+static int cmd_dropcaches(void)
130
+{
131
+ return system("echo 3 | sudo tee /proc/sys/vm/drop_caches");
132
+}
133
+
134
+#elif defined(__APPLE__)
135
+
136
+static int cmd_sync(void)
137
+{
138
+ return system("sync");
139
+}
140
+
141
+static int cmd_dropcaches(void)
142
+{
143
+ return system("sudo purge");
144
+}
145
+
146
+#else
147
+
148
+static int cmd_sync(void)
149
+{
150
+ return 0;
151
+}
152
+
153
+static int cmd_dropcaches(void)
154
+{
155
+ return error("drop caches not implemented on this platform");
156
+}
157
+
158
+#endif
159
+
160
+int cmd_main(int argc, const char **argv)
161
+{
162
+ cmd_sync();
163
+ return cmd_dropcaches();
164
+}
t/perf/p7519-fsmonitor.sh
new
+184
@@ -0,0 +1,184 @@
1
+#!/bin/sh
2
+
3
+test_description="Test core.fsmonitor"
4
+
5
+. ./perf-lib.sh
6
+
7
+#
8
+# Performance test for the fsmonitor feature which enables git to talk to a
9
+# file system change monitor and avoid having to scan the working directory
10
+# for new or modified files.
11
+#
12
+# By default, the performance test will utilize the Watchman file system
13
+# monitor if it is installed. If Watchman is not installed, it will use a
14
+# dummy integration script that does not report any new or modified files.
15
+# The dummy script has very little overhead which provides optimistic results.
16
+#
17
+# The performance test will also use the untracked cache feature if it is
18
+# available as fsmonitor uses it to speed up scanning for untracked files.
19
+#
20
+# There are 3 environment variables that can be used to alter the default
21
+# behavior of the performance test:
22
+#
23
+# GIT_PERF_7519_UNTRACKED_CACHE: used to configure core.untrackedCache
24
+# GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex
25
+# GIT_PERF_7519_FSMONITOR: used to configure core.fsMonitor
26
+#
27
+# The big win for using fsmonitor is the elimination of the need to scan the
28
+# working directory looking for changed and untracked files. If the file
29
+# information is all cached in RAM, the benefits are reduced.
30
+#
31
+# GIT_PERF_7519_DROP_CACHE: if set, the OS caches are dropped between tests
32
+#
33
+
34
+test_perf_large_repo
35
+test_checkout_worktree
36
+
37
+test_lazy_prereq UNTRACKED_CACHE '
38
+ { git update-index --test-untracked-cache; ret=$?; } &&
39
+ test $ret -ne 1
40
+'
41
+
42
+test_lazy_prereq WATCHMAN '
43
+ { command -v watchman >/dev/null 2>&1; ret=$?; } &&
44
+ test $ret -ne 1
45
+'
46
+
47
+if test_have_prereq WATCHMAN
48
+then
49
+ # Convert unix style paths to escaped Windows style paths for Watchman
50
+ case "$(uname -s)" in
51
+ MSYS_NT*)
52
+ GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,/,g')"
53
+ ;;
54
+ *)
55
+ GIT_WORK_TREE="$PWD"
56
+ ;;
57
+ esac
58
+fi
59
+
60
+if test -n "$GIT_PERF_7519_DROP_CACHE"
61
+then
62
+ # When using GIT_PERF_7519_DROP_CACHE, GIT_PERF_REPEAT_COUNT must be 1 to
63
+ # generate valid results. Otherwise the caching that happens for the nth
64
+ # run will negate the validity of the comparisons.
65
+ if test "$GIT_PERF_REPEAT_COUNT" -ne 1
66
+ then
67
+ echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
68
+ GIT_PERF_REPEAT_COUNT=1
69
+ fi
70
+fi
71
+
72
+test_expect_success "setup for fsmonitor" '
73
+ # set untrackedCache depending on the environment
74
+ if test -n "$GIT_PERF_7519_UNTRACKED_CACHE"
75
+ then
76
+ git config core.untrackedCache "$GIT_PERF_7519_UNTRACKED_CACHE"
77
+ else
78
+ if test_have_prereq UNTRACKED_CACHE
79
+ then
80
+ git config core.untrackedCache true
81
+ else
82
+ git config core.untrackedCache false
83
+ fi
84
+ fi &&
85
+
86
+ # set core.splitindex depending on the environment
87
+ if test -n "$GIT_PERF_7519_SPLIT_INDEX"
88
+ then
89
+ git config core.splitIndex "$GIT_PERF_7519_SPLIT_INDEX"
90
+ fi &&
91
+
92
+ # set INTEGRATION_SCRIPT depending on the environment
93
+ if test -n "$GIT_PERF_7519_FSMONITOR"
94
+ then
95
+ INTEGRATION_SCRIPT="$GIT_PERF_7519_FSMONITOR"
96
+ else
97
+ #
98
+ # Choose integration script based on existence of Watchman.
99
+ # If Watchman exists, watch the work tree and attempt a query.
100
+ # If everything succeeds, use Watchman integration script,
101
+ # else fall back to an empty integration script.
102
+ #
103
+ mkdir .git/hooks &&
104
+ if test_have_prereq WATCHMAN
105
+ then
106
+ INTEGRATION_SCRIPT=".git/hooks/fsmonitor-watchman" &&
107
+ cp "$TEST_DIRECTORY/../templates/hooks--fsmonitor-watchman.sample" "$INTEGRATION_SCRIPT" &&
108
+ watchman watch "$GIT_WORK_TREE" &&
109
+ watchman watch-list | grep -q -F "$GIT_WORK_TREE"
110
+ else
111
+ INTEGRATION_SCRIPT=".git/hooks/fsmonitor-empty" &&
112
+ write_script "$INTEGRATION_SCRIPT"<<-\EOF
113
+ EOF
114
+ fi
115
+ fi &&
116
+
117
+ git config core.fsmonitor "$INTEGRATION_SCRIPT" &&
118
+ git update-index --fsmonitor
119
+'
120
+
121
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
122
+ test-drop-caches
123
+fi
124
+
125
+test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
126
+ git status
127
+'
128
+
129
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
130
+ test-drop-caches
131
+fi
132
+
133
+test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
134
+ git status -uno
135
+'
136
+
137
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
138
+ test-drop-caches
139
+fi
140
+
141
+test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" '
142
+ git status -uall
143
+'
144
+
145
+test_expect_success "setup without fsmonitor" '
146
+ unset INTEGRATION_SCRIPT &&
147
+ git config --unset core.fsmonitor &&
148
+ git update-index --no-fsmonitor
149
+'
150
+
151
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
152
+ test-drop-caches
153
+fi
154
+
155
+test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
156
+ git status
157
+'
158
+
159
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
160
+ test-drop-caches
161
+fi
162
+
163
+test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
164
+ git status -uno
165
+'
166
+
167
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
168
+ test-drop-caches
169
+fi
170
+
171
+test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" '
172
+ git status -uall
173
+'
174
+
175
+if test_have_prereq WATCHMAN
176
+then
177
+ watchman watch-del "$GIT_WORK_TREE" >/dev/null 2>&1 &&
178
+
179
+ # Work around Watchman bug on Windows where it holds on to handles
180
+ # preventing the removal of the trash directory
181
+ watchman shutdown-server >/dev/null 2>&1
182
+fi
183
+
184
+test_done