t/perf: add infrastructure for measuring sizes
The main objective of scripts in the perf framework is to run "test_perf", which measures the time it takes to run some operation. However, it can also be interesting to see the change in the output size of certain operations. This patch introduces test_size, which records a single numeric output from the test and shows it in the aggregated output (with pretty printing and relative size comparison). Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff King committed
Aug 17, 2018 at 16:56 UTC
22bec79d1aef8fd82c7870e62f77b7817f0575a7
3 files changed
+81
-5
t/perf/README
+25
@@ -168,3 +168,28 @@ that
168
While we have tried to make sure that it can cope with embedded
169
whitespace and other special characters, it will not work with
170
multi-line data.
171
+
172
+Rather than tracking the performance by run-time as `test_perf` does, you
173
+may also track output size by using `test_size`. The stdout of the
174
+function should be a single numeric value, which will be captured and
175
+shown in the aggregated output. For example:
176
+
177
+ test_perf 'time foo' '
178
+ ./foo >foo.out
179
+ '
180
+
181
+ test_size 'output size'
182
+ wc -c <foo.out
183
+ '
184
+
185
+might produce output like:
186
+
187
+ Test origin HEAD
188
+ -------------------------------------------------------------
189
+ 1234.1 time foo 0.37(0.79+0.02) 0.26(0.51+0.02) -29.7%
190
+ 1234.2 output size 4.3M 3.6M -14.7%
191
+
192
+The item being measured (and its units) is up to the test; the context
193
+and the test title should make it clear to the user whether bigger or
194
+smaller numbers are better. Unlike test_perf, the test code will only be
195
+run once, since output sizes tend to be more deterministic than timings.
t/perf/aggregate.perl
+43
-5
@@ -13,10 +13,16 @@ sub get_times {
13
my $line = <$fh>;
14
return undef if not defined $line;
15
close $fh or die "cannot close $name: $!";
16
- $line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/
17
- or die "bad input line: $line";
18
- my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
19
- return ($rt, $4, $5);
16
+ # times
17
+ if ($line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/) {
18
+ my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
19
+ return ($rt, $4, $5);
20
+ # size
21
+ } elsif ($line =~ /^\d+$/) {
22
+ return $&;
23
+ } else {
24
+ die "bad input line: $line";
25
+ }
26
}
27
28
sub relative_change {
@@ -32,9 +38,15 @@ sub relative_change {
38
39
sub format_times {
40
my ($r, $u, $s, $firstr) = @_;
41
+ # no value means we did not finish the test
42
if (!defined $r) {
43
return "<missing>";
44
}
45
+ # a single value means we have a size, not times
46
+ if (!defined $u) {
47
+ return format_size($r, $firstr);
48
+ }
49
+ # otherwise, we have real/user/system times
50
my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
51
$out .= ' ' . relative_change($r, $firstr) if defined $firstr;
52
return $out;
@@ -54,6 +66,25 @@ EOT
66
exit(1);
67
}
68
69
+sub human_size {
70
+ my $n = shift;
71
+ my @units = ('', qw(K M G));
72
+ while ($n > 900 && @units > 1) {
73
+ $n /= 1000;
74
+ shift @units;
75
+ }
76
+ return $n unless length $units[0];
77
+ return sprintf '%.1f%s', $n, $units[0];
78
+}
79
+
80
+sub format_size {
81
+ my ($size, $first) = @_;
82
+ # match the width of a time: 0.00(0.00+0.00)
83
+ my $out = sprintf '%15s', human_size($size);
84
+ $out .= ' ' . relative_change($size, $first) if defined $first;
85
+ return $out;
86
+}
87
+
88
my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
89
$codespeed, $sortby, $subsection, $reponame);
90
@@ -184,7 +215,14 @@ sub print_default_results {
215
my $firstr;
216
for my $i (0..$#dirs) {
217
my $d = $dirs[$i];
187
- $times{$prefixes{$d}.$t} = [get_times("$resultsdir/$prefixes{$d}$t.times")];
218
+ my $base = "$resultsdir/$prefixes{$d}$t";
219
+ $times{$prefixes{$d}.$t} = [];
220
+ foreach my $type (qw(times size)) {
221
+ if (-e "$base.$type") {
222
+ $times{$prefixes{$d}.$t} = [get_times("$base.$type")];
223
+ last;
224
+ }
225
+ }
226
my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
227
my $w = length format_times($r,$u,$s,$firstr);
228
$colwidth[$i] = $w if $w > $colwidth[$i];
t/perf/perf-lib.sh
+13
@@ -231,6 +231,19 @@ test_perf () {
231
test_wrapper_ test_perf_ "$@"
232
}
233
234
+test_size_ () {
235
+ say >&3 "running: $2"
236
+ if test_eval_ "$2" 3>"$base".size; then
237
+ test_ok_ "$1"
238
+ else
239
+ test_failure_ "$@"
240
+ fi
241
+}
242
+
243
+test_size () {
244
+ test_wrapper_ test_size_ "$@"
245
+}
246
+
247
# We extend test_done to print timings at the end (./run disables this
248
# and does it after running everything)
249
test_at_end_hook_ () {