t/perf: add scripts to bisect performance regressions
The new bisect_regression script can be used to automatically bisect performance regressions. It will pass the new bisect_run_script to `git bisect run`. Signed-off-by: Christian Couder <chriscool@tuxfamily.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Christian Couder committed
Apr 8, 2018 at 11:35 UTC
297e685cbaaa0d33125dc59ce2aa55f03a836598
2 files changed
+120
t/perf/bisect_regression
new
+73
@@ -0,0 +1,73 @@
1
+#!/bin/sh
2
+
3
+# Read a line coming from `./aggregate.perl --sort-by regression ...`
4
+# and automatically bisect to find the commit responsible for the
5
+# performance regression.
6
+#
7
+# Lines from `./aggregate.perl --sort-by regression ...` look like:
8
+#
9
+# +100.0% p7821-grep-engines-fixed.1 0.04(0.10+0.03) 0.08(0.11+0.08) v2.14.3 v2.15.1
10
+# +33.3% p7820-grep-engines.1 0.03(0.08+0.02) 0.04(0.08+0.02) v2.14.3 v2.15.1
11
+#
12
+
13
+die () {
14
+ echo >&2 "error: $*"
15
+ exit 1
16
+}
17
+
18
+while [ $# -gt 0 ]; do
19
+ arg="$1"
20
+ case "$arg" in
21
+ --help)
22
+ echo "usage: $0 [--config file] [--subsection subsection]"
23
+ exit 0
24
+ ;;
25
+ --config)
26
+ shift
27
+ GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
28
+ export GIT_PERF_CONFIG_FILE
29
+ shift ;;
30
+ --subsection)
31
+ shift
32
+ GIT_PERF_SUBSECTION="$1"
33
+ export GIT_PERF_SUBSECTION
34
+ shift ;;
35
+ --*)
36
+ die "unrecognised option: '$arg'" ;;
37
+ *)
38
+ die "unknown argument '$arg'"
39
+ ;;
40
+ esac
41
+done
42
+
43
+read -r regression subtest oldtime newtime oldrev newrev
44
+
45
+test_script=$(echo "$subtest" | sed -e 's/\(.*\)\.[0-9]*$/\1.sh/')
46
+test_number=$(echo "$subtest" | sed -e 's/.*\.\([0-9]*\)$/\1/')
47
+
48
+# oldtime and newtime are decimal number, not integers
49
+
50
+oldtime=$(echo "$oldtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
51
+newtime=$(echo "$newtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
52
+
53
+test $(echo "$newtime" "$oldtime" | awk '{ print ($1 > $2) }') = 1 ||
54
+ die "New time '$newtime' shoud be greater than old time '$oldtime'"
55
+
56
+tmpdir=$(mktemp -d -t bisect_regression_XXXXXX) || die "Failed to create temp directory"
57
+echo "$oldtime" >"$tmpdir/oldtime" || die "Failed to write to '$tmpdir/oldtime'"
58
+echo "$newtime" >"$tmpdir/newtime" || die "Failed to write to '$tmpdir/newtime'"
59
+
60
+# Bisecting must be performed from the top level directory (even with --no-checkout)
61
+(
62
+ toplevel_dir=$(git rev-parse --show-toplevel) || die "Failed to find top level directory"
63
+ cd "$toplevel_dir" || die "Failed to cd into top level directory '$toplevel_dir'"
64
+
65
+ git bisect start --no-checkout "$newrev" "$oldrev" || die "Failed to start bisecting"
66
+
67
+ git bisect run t/perf/bisect_run_script "$test_script" "$test_number" "$tmpdir"
68
+ res="$?"
69
+
70
+ git bisect reset
71
+
72
+ exit "$res"
73
+)
t/perf/bisect_run_script
new
+47
@@ -0,0 +1,47 @@
1
+#!/bin/sh
2
+
3
+script="$1"
4
+test_number="$2"
5
+info_dir="$3"
6
+
7
+# This aborts the bisection immediately
8
+die () {
9
+ echo >&2 "error: $*"
10
+ exit 255
11
+}
12
+
13
+bisect_head=$(git rev-parse --verify BISECT_HEAD) || die "Failed to find BISECT_HEAD ref"
14
+
15
+script_number=$(echo "$script" | sed -e "s/^p\([0-9]*\).*\$/\1/") || die "Failed to get script number for '$script'"
16
+
17
+oldtime=$(cat "$info_dir/oldtime") || die "Failed to access '$info_dir/oldtime'"
18
+newtime=$(cat "$info_dir/newtime") || die "Failed to access '$info_dir/newtime'"
19
+
20
+cd t/perf || die "Failed to cd into 't/perf'"
21
+
22
+result_file="$info_dir/perf_${script_number}_${bisect_head}_results.txt"
23
+
24
+GIT_PERF_DIRS_OR_REVS="$bisect_head"
25
+export GIT_PERF_DIRS_OR_REVS
26
+
27
+./run "$script" >"$result_file" 2>&1 || die "Failed to run perf test '$script'"
28
+
29
+rtime=$(sed -n "s/^$script_number\.$test_number:.*\([0-9]\+\.[0-9]\+\)(.*).*\$/\1/p" "$result_file")
30
+
31
+echo "newtime: $newtime"
32
+echo "rtime: $rtime"
33
+echo "oldtime: $oldtime"
34
+
35
+# Compare ($newtime - $rtime) with ($rtime - $oldtime)
36
+# Times are decimal number, not integers
37
+
38
+if test $(echo "$newtime" "$rtime" "$oldtime" | awk '{ print ($1 - $2 > $2 - $3) }') = 1
39
+then
40
+ # Current commit is considered "good/old"
41
+ echo "$rtime" >"$info_dir/oldtime"
42
+ exit 0
43
+else
44
+ # Current commit is considered "bad/new"
45
+ echo "$rtime" >"$info_dir/newtime"
46
+ exit 1
47
+fi