t/perf: add iteration setup mechanism to perf-lib
Tests that affect the repo in stateful ways are easier to write if we can run setup steps outside of the measured portion of perf iteration. This change adds a "--setup 'setup-script'" parameter to test_perf. To make invocations easier to understand, I also moved the prerequisites to a new --prereq parameter. The setup facility will be used in the upcoming perf tests for batch mode, but it already helps in some existing tests, like t5302 and t7820. Signed-off-by: Neeraj Singh <neerajsi@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Neeraj Singh committed
Apr 4, 2022 at 22:20 UTC
5dccd9155f9e2774176beba3a22c14ca5e3fcdb1
6 files changed
+74
-34
t/perf/p4220-log-grep-engines.sh
+2
-1
@@ -36,7 +36,8 @@ do
36
else
37
prereq=""
38
fi
39
- test_perf $prereq "$engine log$GIT_PERF_4220_LOG_OPTS --grep='$pattern'" "
39
+ test_perf "$engine log$GIT_PERF_4220_LOG_OPTS --grep='$pattern'" \
40
+ --prereq "$prereq" "
41
git -c grep.patternType=$engine log --pretty=format:%h$GIT_PERF_4220_LOG_OPTS --grep='$pattern' >'out.$engine' || :
42
"
43
done
t/perf/p4221-log-grep-engines-fixed.sh
+2
-1
@@ -26,7 +26,8 @@ do
26
else
27
prereq=""
28
fi
29
- test_perf $prereq "$engine log$GIT_PERF_4221_LOG_OPTS --grep='$pattern'" "
29
+ test_perf "$engine log$GIT_PERF_4221_LOG_OPTS --grep='$pattern'" \
30
+ --prereq "$prereq" "
31
git -c grep.patternType=$engine log --pretty=format:%h$GIT_PERF_4221_LOG_OPTS --grep='$pattern' >'out.$engine' || :
32
"
33
done
t/perf/p5302-pack-index.sh
+6
-9
@@ -26,9 +26,8 @@ test_expect_success 'set up thread-counting tests' '
26
done
27
'
28
29
-test_perf PERF_EXTRA 'index-pack 0 threads' '
30
- rm -rf repo.git &&
31
- git init --bare repo.git &&
29
+test_perf 'index-pack 0 threads' --prereq PERF_EXTRA \
30
+ --setup 'rm -rf repo.git && git init --bare repo.git' '
31
GIT_DIR=repo.git git index-pack --threads=1 --stdin < $PACK
32
'
33
@@ -36,17 +35,15 @@ for t in $threads
35
do
36
THREADS=$t
37
export THREADS
39
- test_perf PERF_EXTRA "index-pack $t threads" '
40
- rm -rf repo.git &&
41
- git init --bare repo.git &&
38
+ test_perf "index-pack $t threads" --prereq PERF_EXTRA \
39
+ --setup 'rm -rf repo.git && git init --bare repo.git' '
40
GIT_DIR=repo.git GIT_FORCE_THREADS=1 \
41
git index-pack --threads=$THREADS --stdin <$PACK
42
'
43
done
44
47
-test_perf 'index-pack default number of threads' '
48
- rm -rf repo.git &&
49
- git init --bare repo.git &&
45
+test_perf 'index-pack default number of threads' \
46
+ --setup 'rm -rf repo.git && git init --bare repo.git' '
47
GIT_DIR=repo.git git index-pack --stdin < $PACK
48
'
49
t/perf/p7519-fsmonitor.sh
+3
-15
@@ -60,18 +60,6 @@ then
60
esac
61
fi
62
63
-if test -n "$GIT_PERF_7519_DROP_CACHE"
64
-then
65
- # When using GIT_PERF_7519_DROP_CACHE, GIT_PERF_REPEAT_COUNT must be 1 to
66
- # generate valid results. Otherwise the caching that happens for the nth
67
- # run will negate the validity of the comparisons.
68
- if test "$GIT_PERF_REPEAT_COUNT" -ne 1
69
- then
70
- echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
71
- GIT_PERF_REPEAT_COUNT=1
72
- fi
73
-fi
74
-
63
trace_start() {
64
if test -n "$GIT_PERF_7519_TRACE"
65
then
@@ -167,10 +155,10 @@ setup_for_fsmonitor() {
155
156
test_perf_w_drop_caches () {
157
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
170
- test-tool drop-caches
158
+ test_perf "$1" --setup "test-tool drop-caches" "$2"
159
+ else
160
+ test_perf "$@"
161
fi
172
-
173
- test_perf "$@"
162
}
163
164
test_fsmonitor_suite() {
t/perf/p7820-grep-engines.sh
+4
-2
@@ -49,13 +49,15 @@ do
49
fi
50
if ! test_have_prereq PERF_GREP_ENGINES_THREADS
51
then
52
- test_perf $prereq "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern'" "
52
+ test_perf "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern'" \
53
+ --prereq "$prereq" "
54
git -c grep.patternType=$engine grep$GIT_PERF_7820_GREP_OPTS -- '$pattern' >'out.$engine' || :
55
"
56
else
57
for threads in $GIT_PERF_GREP_THREADS
58
do
58
- test_perf PTHREADS,$prereq "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern' with $threads threads" "
59
+ test_perf "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern' with $threads threads"
60
+ --prereq PTHREADS,$prereq "
61
git -c grep.patternType=$engine -c grep.threads=$threads grep$GIT_PERF_7820_GREP_OPTS -- '$pattern' >'out.$engine.$threads' || :
62
"
63
done
t/perf/perf-lib.sh
+57
-6
@@ -189,19 +189,39 @@ exit $ret' >&3 2>&4
189
}
190
191
test_wrapper_ () {
192
- test_wrapper_func_=$1; shift
192
+ local test_wrapper_func_="$1"; shift
193
+ local test_title_="$1"; shift
194
test_start_
194
- test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
195
- test "$#" = 2 ||
196
- BUG "not 2 or 3 parameters to test-expect-success"
195
+ test_prereq=
196
+ test_perf_setup_=
197
+ while test $# != 0
198
+ do
199
+ case $1 in
200
+ --prereq)
201
+ test_prereq=$2
202
+ shift
203
+ ;;
204
+ --setup)
205
+ test_perf_setup_=$2
206
+ shift
207
+ ;;
208
+ *)
209
+ break
210
+ ;;
211
+ esac
212
+ shift
213
+ done
214
+ test "$#" = 1 || BUG "test_wrapper_ needs 2 positional parameters"
215
export test_prereq
198
- if ! test_skip "$@"
216
+ export test_perf_setup_
217
+
218
+ if ! test_skip "$test_title_" "$@"
219
then
220
base=$(basename "$0" .sh)
221
echo "$test_count" >>"$perf_results_dir"/$base.subtests
222
echo "$1" >"$perf_results_dir"/$base.$test_count.descr
223
base="$perf_results_dir"/"$PERF_RESULTS_PREFIX$(basename "$0" .sh)"."$test_count"
204
- "$test_wrapper_func_" "$@"
224
+ "$test_wrapper_func_" "$test_title_" "$@"
225
fi
226
227
test_finish_
@@ -214,6 +234,16 @@ test_perf_ () {
234
echo "perf $test_count - $1:"
235
fi
236
for i in $(test_seq 1 $GIT_PERF_REPEAT_COUNT); do
237
+ if test -n "$test_perf_setup_"
238
+ then
239
+ say >&3 "setup: $test_perf_setup_"
240
+ if ! test_eval_ $test_perf_setup_
241
+ then
242
+ test_failure_ "$test_perf_setup_"
243
+ break
244
+ fi
245
+
246
+ fi
247
say >&3 "running: $2"
248
if test_run_perf_ "$2"
249
then
@@ -237,11 +267,24 @@ test_perf_ () {
267
rm test_time.*
268
}
269
270
+# Usage: test_perf 'title' [options] 'perf-test'
271
+# Run the performance test script specified in perf-test with
272
+# optional prerequisite and setup steps.
273
+# Options:
274
+# --prereq prerequisites: Skip the test if prequisites aren't met
275
+# --setup "setup-steps": Run setup steps prior to each measured iteration
276
+#
277
test_perf () {
278
test_wrapper_ test_perf_ "$@"
279
}
280
281
test_size_ () {
282
+ if test -n "$test_perf_setup_"
283
+ then
284
+ say >&3 "setup: $test_perf_setup_"
285
+ test_eval_ $test_perf_setup_
286
+ fi
287
+
288
say >&3 "running: $2"
289
if test_eval_ "$2" 3>"$base".result; then
290
test_ok_ "$1"
@@ -250,6 +293,14 @@ test_size_ () {
293
fi
294
}
295
296
+# Usage: test_size 'title' [options] 'size-test'
297
+# Run the size test script specified in size-test with optional
298
+# prerequisites and setup steps. Returns the numeric value
299
+# returned by size-test.
300
+# Options:
301
+# --prereq prerequisites: Skip the test if prequisites aren't met
302
+# --setup "setup-steps": Run setup steps prior to the size measurement
303
+
304
test_size () {
305
test_wrapper_ test_size_ "$@"
306
}