@matheusbsbs / matheus-bagliano / commits / 342e9ef2d9

Introduce a performance testing framework

This introduces a performance testing framework under t/perf/. It tries to be as close to the test-lib.sh infrastructure as possible, and thus should be easy to get used to for git developers. The following points were considered for the implementation: 1. You usually want to compare arbitrary revisions/build trees against each other. They may not have the performance test under consideration, or even the perf-lib.sh infrastructure. To cope with this, the 'run' script lets you specify arbitrary build dirs and revisions. It even automatically builds the revisions if it doesn't have them at hand yet. 2. Usually you would not want to run all tests. It would take too long anyway. The 'run' script lets you specify which tests to run; or you can also do it manually. There is a Makefile for discoverability and 'make clean', but it is not meant for real-world use. 3. Creating test repos from scratch in every test is extremely time-consuming, and shipping or downloading such large/weird repos is out of the question. We leave this decision to the user. Two different sizes of test repos can be configured, and the scripts just copy one or more of those (using hardlinks for the object store). By default it tries to use the build tree's git.git repository. This is fairly fast and versatile. Using a copy instead of a clone preserves many properties that the user may want to test for, such as lots of loose objects, unpacked refs, etc. Signed-off-by: Thomas Rast <trast@student.ethz.ch> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Thomas Rast committed Feb 17, 2012 at 11:25 UTC 342e9ef2d9ecc802191ef62a51f4e7db2ede41b1
12 files changed +774 -5
Makefile
+21 -1
@@ -2361,6 +2361,10 @@ GIT-BUILD-OPTIONS: FORCE
2361 @echo USE_LIBPCRE=\''$(subst ','\'',$(subst ','\'',$(USE_LIBPCRE)))'\' >>$@
2362 @echo NO_PERL=\''$(subst ','\'',$(subst ','\'',$(NO_PERL)))'\' >>$@
2363 @echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@
2364 + @echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@
2365 +ifdef GIT_TEST_OPTS
2366 + @echo GIT_TEST_OPTS=\''$(subst ','\'',$(subst ','\'',$(GIT_TEST_OPTS)))'\' >>$@
2367 +endif
2368 ifdef GIT_TEST_CMP
2369 @echo GIT_TEST_CMP=\''$(subst ','\'',$(subst ','\'',$(GIT_TEST_CMP)))'\' >>$@
2370 endif
@@ -2369,7 +2373,18 @@ ifdef GIT_TEST_CMP_USE_COPIED_CONTEXT
2373 endif
2374 @echo NO_GETTEXT=\''$(subst ','\'',$(subst ','\'',$(NO_GETTEXT)))'\' >>$@
2375 @echo GETTEXT_POISON=\''$(subst ','\'',$(subst ','\'',$(GETTEXT_POISON)))'\' >>$@
2372 - @echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@
2376 +ifdef GIT_PERF_REPEAT_COUNT
2377 + @echo GIT_PERF_REPEAT_COUNT=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_REPEAT_COUNT)))'\' >>$@
2378 +endif
2379 +ifdef GIT_PERF_REPO
2380 + @echo GIT_PERF_REPO=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_REPO)))'\' >>$@
2381 +endif
2382 +ifdef GIT_PERF_LARGE_REPO
2383 + @echo GIT_PERF_LARGE_REPO=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_LARGE_REPO)))'\' >>$@
2384 +endif
2385 +ifdef GIT_PERF_MAKE_OPTS
2386 + @echo GIT_PERF_MAKE_OPTS=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_MAKE_OPTS)))'\' >>$@
2387 +endif
2388
2389 ### Detect Tck/Tk interpreter path changes
2390 ifndef NO_TCLTK
@@ -2405,6 +2420,11 @@ export NO_SVN_TESTS
2420 test: all
2421 $(MAKE) -C t/ all
2422
2423 +perf: all
2424 + $(MAKE) -C t/perf/ all
2425 +
2426 +.PHONY: test perf
2427 +
2428 test-ctype$X: ctype.o
2429
2430 test-date$X: date.o ctype.o
t/Makefile
+42 -1
@@ -73,4 +73,45 @@ gitweb-test:
73 valgrind:
74 $(MAKE) GIT_TEST_OPTS="$(GIT_TEST_OPTS) --valgrind"
75
76 -.PHONY: pre-clean $(T) aggregate-results clean valgrind
76 +perf:
77 + $(MAKE) -C perf/ all
78 +
79 +# Smoke testing targets
80 +-include ../GIT-VERSION-FILE
81 +uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo unknown')
82 +uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo unknown')
83 +
84 +test-results:
85 + mkdir -p test-results
86 +
87 +test-results/git-smoke.tar.gz: test-results
88 + $(PERL_PATH) ./harness \
89 + --archive="test-results/git-smoke.tar.gz" \
90 + $(T)
91 +
92 +smoke: test-results/git-smoke.tar.gz
93 +
94 +SMOKE_UPLOAD_FLAGS =
95 +ifdef SMOKE_USERNAME
96 + SMOKE_UPLOAD_FLAGS += -F username="$(SMOKE_USERNAME)" -F password="$(SMOKE_PASSWORD)"
97 +endif
98 +ifdef SMOKE_COMMENT
99 + SMOKE_UPLOAD_FLAGS += -F comments="$(SMOKE_COMMENT)"
100 +endif
101 +ifdef SMOKE_TAGS
102 + SMOKE_UPLOAD_FLAGS += -F tags="$(SMOKE_TAGS)"
103 +endif
104 +
105 +smoke_report: smoke
106 + curl \
107 + -H "Expect: " \
108 + -F project=Git \
109 + -F architecture="$(uname_M)" \
110 + -F platform="$(uname_S)" \
111 + -F revision="$(GIT_VERSION)" \
112 + -F report_file=@test-results/git-smoke.tar.gz \
113 + $(SMOKE_UPLOAD_FLAGS) \
114 + http://smoke.git.nix.is/app/projects/process_add_report/1 \
115 + | grep -v ^Redirecting
116 +
117 +.PHONY: pre-clean $(T) aggregate-results clean valgrind perf
t/perf/.gitignore new
+2
@@ -0,0 +1,2 @@
1 +build/
2 +test-results/
t/perf/Makefile new
+15
@@ -0,0 +1,15 @@
1 +-include ../../config.mak
2 +export GIT_TEST_OPTIONS
3 +
4 +all: perf
5 +
6 +perf: pre-clean
7 + ./run
8 +
9 +pre-clean:
10 + rm -rf test-results
11 +
12 +clean:
13 + rm -rf build "trash directory".* test-results
14 +
15 +.PHONY: all perf pre-clean clean
t/perf/README new
+146
@@ -0,0 +1,146 @@
1 +Git performance tests
2 +=====================
3 +
4 +This directory holds performance testing scripts for git tools. The
5 +first part of this document describes the various ways in which you
6 +can run them.
7 +
8 +When fixing the tools or adding enhancements, you are strongly
9 +encouraged to add tests in this directory to cover what you are
10 +trying to fix or enhance. The later part of this short document
11 +describes how your test scripts should be organized.
12 +
13 +
14 +Running Tests
15 +-------------
16 +
17 +The easiest way to run tests is to say "make". This runs all
18 +the tests on the current git repository.
19 +
20 + === Running 2 tests in this tree ===
21 + [...]
22 + Test this tree
23 + ---------------------------------------------------------
24 + 0001.1: rev-list --all 0.54(0.51+0.02)
25 + 0001.2: rev-list --all --objects 6.14(5.99+0.11)
26 + 7810.1: grep worktree, cheap regex 0.16(0.16+0.35)
27 + 7810.2: grep worktree, expensive regex 7.90(29.75+0.37)
28 + 7810.3: grep --cached, cheap regex 3.07(3.02+0.25)
29 + 7810.4: grep --cached, expensive regex 9.39(30.57+0.24)
30 +
31 +You can compare multiple repositories and even git revisions with the
32 +'run' script:
33 +
34 + $ ./run . origin/next /path/to/git-tree p0001-rev-list.sh
35 +
36 +where . stands for the current git tree. The full invocation is
37 +
38 + ./run [<revision|directory>...] [--] [<test-script>...]
39 +
40 +A '.' argument is implied if you do not pass any other
41 +revisions/directories.
42 +
43 +You can also manually test this or another git build tree, and then
44 +call the aggregation script to summarize the results:
45 +
46 + $ ./p0001-rev-list.sh
47 + [...]
48 + $ GIT_BUILD_DIR=/path/to/other/git ./p0001-rev-list.sh
49 + [...]
50 + $ ./aggregate.perl . /path/to/other/git ./p0001-rev-list.sh
51 +
52 +aggregate.perl has the same invocation as 'run', it just does not run
53 +anything beforehand.
54 +
55 +You can set the following variables (also in your config.mak):
56 +
57 + GIT_PERF_REPEAT_COUNT
58 + Number of times a test should be repeated for best-of-N
59 + measurements. Defaults to 5.
60 +
61 + GIT_PERF_MAKE_OPTS
62 + Options to use when automatically building a git tree for
63 + performance testing. E.g., -j6 would be useful.
64 +
65 + GIT_PERF_REPO
66 + GIT_PERF_LARGE_REPO
67 + Repositories to copy for the performance tests. The normal
68 + repo should be at least git.git size. The large repo should
69 + probably be about linux-2.6.git size for optimal results.
70 + Both default to the git.git you are running from.
71 +
72 +You can also pass the options taken by ordinary git tests; the most
73 +useful one is:
74 +
75 +--root=<directory>::
76 + Create "trash" directories used to store all temporary data during
77 + testing under <directory>, instead of the t/ directory.
78 + Using this option with a RAM-based filesystem (such as tmpfs)
79 + can massively speed up the test suite.
80 +
81 +
82 +Naming Tests
83 +------------
84 +
85 +The performance test files are named as:
86 +
87 + pNNNN-commandname-details.sh
88 +
89 +where N is a decimal digit. The same conventions for choosing NNNN as
90 +for normal tests apply.
91 +
92 +
93 +Writing Tests
94 +-------------
95 +
96 +The perf script starts much like a normal test script, except it
97 +sources perf-lib.sh:
98 +
99 + #!/bin/sh
100 + #
101 + # Copyright (c) 2005 Junio C Hamano
102 + #
103 +
104 + test_description='xxx performance test'
105 + . ./perf-lib.sh
106 +
107 +After that you will want to use some of the following:
108 +
109 + test_perf_default_repo # sets up a "normal" repository
110 + test_perf_large_repo # sets up a "large" repository
111 +
112 + test_perf_default_repo sub # ditto, in a subdir "sub"
113 +
114 + test_checkout_worktree # if you need the worktree too
115 +
116 +At least one of the first two is required!
117 +
118 +You can use test_expect_success as usual. For actual performance
119 +tests, use
120 +
121 + test_perf 'descriptive string' '
122 + command1 &&
123 + command2
124 + '
125 +
126 +test_perf spawns a subshell, for lack of better options. This means
127 +that
128 +
129 +* you _must_ export all variables that you need in the subshell
130 +
131 +* you _must_ flag all variables that you want to persist from the
132 + subshell with 'test_export':
133 +
134 + test_perf 'descriptive string' '
135 + foo=$(git rev-parse HEAD) &&
136 + test_export foo
137 + '
138 +
139 + The so-exported variables are automatically marked for export in the
140 + shell executing the perf test. For your convenience, test_export is
141 + the same as export in the main shell.
142 +
143 + This feature relies on a bit of magic using 'set' and 'source'.
144 + While we have tried to make sure that it can cope with embedded
145 + whitespace and other special characters, it will not work with
146 + multi-line data.
t/perf/aggregate.perl new
+166
@@ -0,0 +1,166 @@
1 +#!/usr/bin/perl
2 +
3 +use strict;
4 +use warnings;
5 +use Git;
6 +
7 +sub get_times {
8 + my $name = shift;
9 + open my $fh, "<", $name or return undef;
10 + my $line = <$fh>;
11 + return undef if not defined $line;
12 + close $fh or die "cannot close $name: $!";
13 + $line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/
14 + or die "bad input line: $line";
15 + my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
16 + return ($rt, $4, $5);
17 +}
18 +
19 +sub format_times {
20 + my ($r, $u, $s, $firstr) = @_;
21 + if (!defined $r) {
22 + return "<missing>";
23 + }
24 + my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
25 + if (defined $firstr) {
26 + if ($firstr > 0) {
27 + $out .= sprintf " %+.1f%%", 100.0*($r-$firstr)/$firstr;
28 + } elsif ($r == 0) {
29 + $out .= " =";
30 + } else {
31 + $out .= " +inf";
32 + }
33 + }
34 + return $out;
35 +}
36 +
37 +my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests);
38 +while (scalar @ARGV) {
39 + my $arg = $ARGV[0];
40 + my $dir;
41 + last if -f $arg or $arg eq "--";
42 + if (! -d $arg) {
43 + my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
44 + $dir = "build/".$rev;
45 + } else {
46 + $arg =~ s{/*$}{};
47 + $dir = $arg;
48 + $dirabbrevs{$dir} = $dir;
49 + }
50 + push @dirs, $dir;
51 + $dirnames{$dir} = $arg;
52 + my $prefix = $dir;
53 + $prefix =~ tr/^a-zA-Z0-9/_/c;
54 + $prefixes{$dir} = $prefix . '.';
55 + shift @ARGV;
56 +}
57 +
58 +if (not @dirs) {
59 + @dirs = ('.');
60 +}
61 +$dirnames{'.'} = $dirabbrevs{'.'} = "this tree";
62 +$prefixes{'.'} = '';
63 +
64 +shift @ARGV if scalar @ARGV and $ARGV[0] eq "--";
65 +
66 +@tests = @ARGV;
67 +if (not @tests) {
68 + @tests = glob "p????-*.sh";
69 +}
70 +
71 +my @subtests;
72 +my %shorttests;
73 +for my $t (@tests) {
74 + $t =~ s{(?:.*/)?(p(\d+)-[^/]+)\.sh$}{$1} or die "bad test name: $t";
75 + my $n = $2;
76 + my $fname = "test-results/$t.subtests";
77 + open my $fp, "<", $fname or die "cannot open $fname: $!";
78 + for (<$fp>) {
79 + chomp;
80 + /^(\d+)$/ or die "malformed subtest line: $_";
81 + push @subtests, "$t.$1";
82 + $shorttests{"$t.$1"} = "$n.$1";
83 + }
84 + close $fp or die "cannot close $fname: $!";
85 +}
86 +
87 +sub read_descr {
88 + my $name = shift;
89 + open my $fh, "<", $name or return "<error reading description>";
90 + my $line = <$fh>;
91 + close $fh or die "cannot close $name";
92 + chomp $line;
93 + return $line;
94 +}
95 +
96 +my %descrs;
97 +my $descrlen = 4; # "Test"
98 +for my $t (@subtests) {
99 + $descrs{$t} = $shorttests{$t}.": ".read_descr("test-results/$t.descr");
100 + $descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
101 +}
102 +
103 +sub have_duplicate {
104 + my %seen;
105 + for (@_) {
106 + return 1 if exists $seen{$_};
107 + $seen{$_} = 1;
108 + }
109 + return 0;
110 +}
111 +sub have_slash {
112 + for (@_) {
113 + return 1 if m{/};
114 + }
115 + return 0;
116 +}
117 +
118 +my %newdirabbrevs = %dirabbrevs;
119 +while (!have_duplicate(values %newdirabbrevs)) {
120 + %dirabbrevs = %newdirabbrevs;
121 + last if !have_slash(values %dirabbrevs);
122 + %newdirabbrevs = %dirabbrevs;
123 + for (values %newdirabbrevs) {
124 + s{^[^/]*/}{};
125 + }
126 +}
127 +
128 +my %times;
129 +my @colwidth = ((0)x@dirs);
130 +for my $i (0..$#dirs) {
131 + my $d = $dirs[$i];
132 + my $w = length (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
133 + $colwidth[$i] = $w if $w > $colwidth[$i];
134 +}
135 +for my $t (@subtests) {
136 + my $firstr;
137 + for my $i (0..$#dirs) {
138 + my $d = $dirs[$i];
139 + $times{$prefixes{$d}.$t} = [get_times("test-results/$prefixes{$d}$t.times")];
140 + my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
141 + my $w = length format_times($r,$u,$s,$firstr);
142 + $colwidth[$i] = $w if $w > $colwidth[$i];
143 + $firstr = $r unless defined $firstr;
144 + }
145 +}
146 +my $totalwidth = 3*@dirs+$descrlen;
147 +$totalwidth += $_ for (@colwidth);
148 +
149 +printf "%-${descrlen}s", "Test";
150 +for my $i (0..$#dirs) {
151 + my $d = $dirs[$i];
152 + printf " %-$colwidth[$i]s", (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
153 +}
154 +print "\n";
155 +print "-"x$totalwidth, "\n";
156 +for my $t (@subtests) {
157 + printf "%-${descrlen}s", $descrs{$t};
158 + my $firstr;
159 + for my $i (0..$#dirs) {
160 + my $d = $dirs[$i];
161 + my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
162 + printf " %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
163 + $firstr = $r unless defined $firstr;
164 + }
165 + print "\n";
166 +}
t/perf/min_time.perl new
+21
@@ -0,0 +1,21 @@
1 +#!/usr/bin/perl
2 +
3 +my $minrt = 1e100;
4 +my $min;
5 +
6 +while (<>) {
7 + # [h:]m:s.xx U.xx S.xx
8 + /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/
9 + or die "bad input line: $_";
10 + my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
11 + if ($rt < $minrt) {
12 + $min = $_;
13 + $minrt = $rt;
14 + }
15 +}
16 +
17 +if (!defined $min) {
18 + die "no input found";
19 +}
20 +
21 +print $min;
t/perf/p0000-perf-lib-sanity.sh new
+41
@@ -0,0 +1,41 @@
1 +#!/bin/sh
2 +
3 +test_description='Tests whether perf-lib facilities work'
4 +. ./perf-lib.sh
5 +
6 +test_perf_default_repo
7 +
8 +test_perf 'test_perf_default_repo works' '
9 + foo=$(git rev-parse HEAD) &&
10 + test_export foo
11 +'
12 +
13 +test_checkout_worktree
14 +
15 +test_perf 'test_checkout_worktree works' '
16 + wt=$(find . | wc -l) &&
17 + idx=$(git ls-files | wc -l) &&
18 + test $wt -gt $idx
19 +'
20 +
21 +baz=baz
22 +test_export baz
23 +
24 +test_expect_success 'test_export works' '
25 + echo "$foo" &&
26 + test "$foo" = "$(git rev-parse HEAD)" &&
27 + echo "$baz" &&
28 + test "$baz" = baz
29 +'
30 +
31 +test_perf 'export a weird var' '
32 + bar="weird # variable" &&
33 + test_export bar
34 +'
35 +
36 +test_expect_success 'test_export works with weird vars' '
37 + echo "$bar" &&
38 + test "$bar" = "weird # variable"
39 +'
40 +
41 +test_done
t/perf/p0001-rev-list.sh new
+17
@@ -0,0 +1,17 @@
1 +#!/bin/sh
2 +
3 +test_description="Tests history walking performance"
4 +
5 +. ./perf-lib.sh
6 +
7 +test_perf_default_repo
8 +
9 +test_perf 'rev-list --all' '
10 + git rev-list --all >/dev/null
11 +'
12 +
13 +test_perf 'rev-list --all --objects' '
14 + git rev-list --all --objects >/dev/null
15 +'
16 +
17 +test_done
t/perf/perf-lib.sh new
+198
@@ -0,0 +1,198 @@
1 +#!/bin/sh
2 +#
3 +# Copyright (c) 2011 Thomas Rast
4 +#
5 +# This program is free software: you can redistribute it and/or modify
6 +# it under the terms of the GNU General Public License as published by
7 +# the Free Software Foundation, either version 2 of the License, or
8 +# (at your option) any later version.
9 +#
10 +# This program is distributed in the hope that it will be useful,
11 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
12 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 +# GNU General Public License for more details.
14 +#
15 +# You should have received a copy of the GNU General Public License
16 +# along with this program. If not, see http://www.gnu.org/licenses/ .
17 +
18 +# do the --tee work early; it otherwise confuses our careful
19 +# GIT_BUILD_DIR mangling
20 +case "$GIT_TEST_TEE_STARTED, $* " in
21 +done,*)
22 + # do not redirect again
23 + ;;
24 +*' --tee '*|*' --va'*)
25 + mkdir -p test-results
26 + BASE=test-results/$(basename "$0" .sh)
27 + (GIT_TEST_TEE_STARTED=done ${SHELL-sh} "$0" "$@" 2>&1;
28 + echo $? > $BASE.exit) | tee $BASE.out
29 + test "$(cat $BASE.exit)" = 0
30 + exit
31 + ;;
32 +esac
33 +
34 +TEST_DIRECTORY=$(pwd)/..
35 +TEST_OUTPUT_DIRECTORY=$(pwd)
36 +if test -z "$GIT_TEST_INSTALLED"; then
37 + perf_results_prefix=
38 +else
39 + perf_results_prefix=$(printf "%s" "${GIT_TEST_INSTALLED%/bin-wrappers}" | tr -c "[a-zA-Z0-9]" "[_*]")"."
40 + # make the tested dir absolute
41 + GIT_TEST_INSTALLED=$(cd "$GIT_TEST_INSTALLED" && pwd)
42 +fi
43 +
44 +TEST_NO_CREATE_REPO=t
45 +
46 +. ../test-lib.sh
47 +
48 +perf_results_dir=$TEST_OUTPUT_DIRECTORY/test-results
49 +mkdir -p "$perf_results_dir"
50 +rm -f "$perf_results_dir"/$(basename "$0" .sh).subtests
51 +
52 +if test -z "$GIT_PERF_REPEAT_COUNT"; then
53 + GIT_PERF_REPEAT_COUNT=3
54 +fi
55 +die_if_build_dir_not_repo () {
56 + if ! ( cd "$TEST_DIRECTORY/.." &&
57 + git rev-parse --build-dir >/dev/null 2>&1 ); then
58 + error "No $1 defined, and your build directory is not a repo"
59 + fi
60 +}
61 +
62 +if test -z "$GIT_PERF_REPO"; then
63 + die_if_build_dir_not_repo '$GIT_PERF_REPO'
64 + GIT_PERF_REPO=$TEST_DIRECTORY/..
65 +fi
66 +if test -z "$GIT_PERF_LARGE_REPO"; then
67 + die_if_build_dir_not_repo '$GIT_PERF_LARGE_REPO'
68 + GIT_PERF_LARGE_REPO=$TEST_DIRECTORY/..
69 +fi
70 +
71 +test_perf_create_repo_from () {
72 + test "$#" = 2 ||
73 + error "bug in the test script: not 2 parameters to test-create-repo"
74 + repo="$1"
75 + source="$2"
76 + source_git=$source/$(cd "$source" && git rev-parse --git-dir)
77 + mkdir -p "$repo/.git"
78 + (
79 + cd "$repo/.git" &&
80 + { cp -Rl "$source_git/objects" . 2>/dev/null ||
81 + cp -R "$source_git/objects" .; } &&
82 + for stuff in "$source_git"/*; do
83 + case "$stuff" in
84 + */objects|*/hooks|*/config)
85 + ;;
86 + *)
87 + cp -R "$stuff" . || break
88 + ;;
89 + esac
90 + done &&
91 + cd .. &&
92 + git init -q &&
93 + mv .git/hooks .git/hooks-disabled 2>/dev/null
94 + ) || error "failed to copy repository '$source' to '$repo'"
95 +}
96 +
97 +# call at least one of these to establish an appropriately-sized repository
98 +test_perf_default_repo () {
99 + test_perf_create_repo_from "${1:-$TRASH_DIRECTORY}" "$GIT_PERF_REPO"
100 +}
101 +test_perf_large_repo () {
102 + if test "$GIT_PERF_LARGE_REPO" = "$GIT_BUILD_DIR"; then
103 + echo "warning: \$GIT_PERF_LARGE_REPO is \$GIT_BUILD_DIR." >&2
104 + echo "warning: This will work, but may not be a sufficiently large repo" >&2
105 + echo "warning: for representative measurements." >&2
106 + fi
107 + test_perf_create_repo_from "${1:-$TRASH_DIRECTORY}" "$GIT_PERF_LARGE_REPO"
108 +}
109 +test_checkout_worktree () {
110 + git checkout-index -u -a ||
111 + error "git checkout-index failed"
112 +}
113 +
114 +# Performance tests should never fail. If they do, stop immediately
115 +immediate=t
116 +
117 +test_run_perf_ () {
118 + test_cleanup=:
119 + test_export_="test_cleanup"
120 + export test_cleanup test_export_
121 + /usr/bin/time -f "%E %U %S" -o test_time.$i "$SHELL" -c '
122 +. '"$TEST_DIRECTORY"/../test-lib-functions.sh'
123 +test_export () {
124 + [ $# != 0 ] || return 0
125 + test_export_="$test_export_\\|$1"
126 + shift
127 + test_export "$@"
128 +}
129 +'"$1"'
130 +ret=$?
131 +set | sed -n "s'"/'/'\\\\''/g"';s/^\\($test_export_\\)/export '"'&'"'/p" >test_vars
132 +exit $ret' >&3 2>&4
133 + eval_ret=$?
134 +
135 + if test $eval_ret = 0 || test -n "$expecting_failure"
136 + then
137 + test_eval_ "$test_cleanup"
138 + . ./test_vars || error "failed to load updated environment"
139 + fi
140 + if test "$verbose" = "t" && test -n "$HARNESS_ACTIVE"; then
141 + echo ""
142 + fi
143 + return "$eval_ret"
144 +}
145 +
146 +
147 +test_perf () {
148 + test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
149 + test "$#" = 2 ||
150 + error "bug in the test script: not 2 or 3 parameters to test-expect-success"
151 + export test_prereq
152 + if ! test_skip "$@"
153 + then
154 + base=$(basename "$0" .sh)
155 + echo "$test_count" >>"$perf_results_dir"/$base.subtests
156 + echo "$1" >"$perf_results_dir"/$base.$test_count.descr
157 + if test -z "$verbose"; then
158 + echo -n "perf $test_count - $1:"
159 + else
160 + echo "perf $test_count - $1:"
161 + fi
162 + for i in $(seq 1 $GIT_PERF_REPEAT_COUNT); do
163 + say >&3 "running: $2"
164 + if test_run_perf_ "$2"
165 + then
166 + if test -z "$verbose"; then
167 + echo -n " $i"
168 + else
169 + echo "* timing run $i/$GIT_PERF_REPEAT_COUNT:"
170 + fi
171 + else
172 + test -z "$verbose" && echo
173 + test_failure_ "$@"
174 + break
175 + fi
176 + done
177 + if test -z "$verbose"; then
178 + echo " ok"
179 + else
180 + test_ok_ "$1"
181 + fi
182 + base="$perf_results_dir"/"$perf_results_prefix$(basename "$0" .sh)"."$test_count"
183 + "$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".times
184 + fi
185 + echo >&3 ""
186 +}
187 +
188 +# We extend test_done to print timings at the end (./run disables this
189 +# and does it after running everything)
190 +test_at_end_hook_ () {
191 + if test -z "$GIT_PERF_AGGREGATING_LATER"; then
192 + ( cd "$TEST_DIRECTORY"/perf && ./aggregate.perl $(basename "$0") )
193 + fi
194 +}
195 +
196 +test_export () {
197 + export "$@"
198 +}
t/perf/run new
+82
@@ -0,0 +1,82 @@
1 +#!/bin/sh
2 +
3 +case "$1" in
4 + --help)
5 + echo "usage: $0 [other_git_tree...] [--] [test_scripts]"
6 + exit 0
7 + ;;
8 +esac
9 +
10 +die () {
11 + echo >&2 "error: $*"
12 + exit 1
13 +}
14 +
15 +run_one_dir () {
16 + if test $# -eq 0; then
17 + set -- p????-*.sh
18 + fi
19 + echo "=== Running $# tests in ${GIT_TEST_INSTALLED:-this tree} ==="
20 + for t in "$@"; do
21 + ./$t $GIT_TEST_OPTS
22 + done
23 +}
24 +
25 +unpack_git_rev () {
26 + rev=$1
27 + mkdir -p build/$rev
28 + (cd "$(git rev-parse --show-cdup)" && git archive --format=tar $rev) |
29 + (cd build/$rev && tar x)
30 +}
31 +build_git_rev () {
32 + rev=$1
33 + cp ../../config.mak build/$rev/config.mak
34 + (cd build/$rev && make $GIT_PERF_MAKE_OPTS) ||
35 + die "failed to build revision '$mydir'"
36 +}
37 +
38 +run_dirs_helper () {
39 + mydir=${1%/}
40 + shift
41 + while test $# -gt 0 -a "$1" != -- -a ! -f "$1"; do
42 + shift
43 + done
44 + if test $# -gt 0 -a "$1" = --; then
45 + shift
46 + fi
47 + if [ ! -d "$mydir" ]; then
48 + rev=$(git rev-parse --verify "$mydir" 2>/dev/null) ||
49 + die "'$mydir' is neither a directory nor a valid revision"
50 + if [ ! -d build/$rev ]; then
51 + unpack_git_rev $rev
52 + fi
53 + build_git_rev $rev
54 + mydir=build/$rev
55 + fi
56 + if test "$mydir" = .; then
57 + unset GIT_TEST_INSTALLED
58 + else
59 + GIT_TEST_INSTALLED="$mydir/bin-wrappers"
60 + export GIT_TEST_INSTALLED
61 + fi
62 + run_one_dir "$@"
63 +}
64 +
65 +run_dirs () {
66 + while test $# -gt 0 -a "$1" != -- -a ! -f "$1"; do
67 + run_dirs_helper "$@"
68 + shift
69 + done
70 +}
71 +
72 +GIT_PERF_AGGREGATING_LATER=t
73 +export GIT_PERF_AGGREGATING_LATER
74 +
75 +cd "$(dirname $0)"
76 +. ../../GIT-BUILD-OPTIONS
77 +
78 +if test $# = 0 -o "$1" = -- -o -f "$1"; then
79 + set -- . "$@"
80 +fi
81 +run_dirs "$@"
82 +./aggregate.perl "$@"
t/test-lib.sh
+23 -3
@@ -55,6 +55,7 @@ unset $(perl -e '
55 .*_TEST
56 PROVE
57 VALGRIND
58 + PERF_AGGREGATING_LATER
59 ));
60 my @vars = grep(/^GIT_/ && !/^GIT_($ok)/o, @env);
61 print join("\n", @vars);
@@ -98,6 +99,8 @@ _z40=0000000000000000000000000000000000000000
99 LF='
100 '
101
102 +export _x05 _x40 _z40 LF
103 +
104 # Each test should start with something like this, after copyright notices:
105 #
106 # test_description='Description of this test...
@@ -313,11 +316,16 @@ test_skip () {
316 esac
317 }
318
319 +# stub; perf-lib overrides it
320 +test_at_end_hook_ () {
321 + :
322 +}
323 +
324 test_done () {
325 GIT_EXIT_OK=t
326
327 if test -z "$HARNESS_ACTIVE"; then
320 - test_results_dir="$TEST_DIRECTORY/test-results"
328 + test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results"
329 mkdir -p "$test_results_dir"
330 test_results_path="$test_results_dir/${0%.sh}-$$.counts"
331
@@ -356,6 +364,8 @@ test_done () {
364 cd "$(dirname "$remove_trash")" &&
365 rm -rf "$(basename "$remove_trash")"
366
367 + test_at_end_hook_
368 +
369 exit 0 ;;
370
371 *)
@@ -378,6 +388,12 @@ then
388 # itself.
389 TEST_DIRECTORY=$(pwd)
390 fi
391 +if test -z "$TEST_OUTPUT_DIRECTORY"
392 +then
393 + # Similarly, override this to store the test-results subdir
394 + # elsewhere
395 + TEST_OUTPUT_DIRECTORY=$TEST_DIRECTORY
396 +fi
397 GIT_BUILD_DIR="$TEST_DIRECTORY"/..
398
399 if test -n "$valgrind"
@@ -513,7 +529,7 @@ test="trash directory.$(basename "$0" .sh)"
529 test -n "$root" && test="$root/$test"
530 case "$test" in
531 /*) TRASH_DIRECTORY="$test" ;;
516 - *) TRASH_DIRECTORY="$TEST_DIRECTORY/$test" ;;
532 + *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$test" ;;
533 esac
534 test ! -z "$debug" || remove_trash=$TRASH_DIRECTORY
535 rm -fr "$test" || {
@@ -525,7 +541,11 @@ rm -fr "$test" || {
541 HOME="$TRASH_DIRECTORY"
542 export HOME
543
528 -test_create_repo "$test"
544 +if test -z "$TEST_NO_CREATE_REPO"; then
545 + test_create_repo "$test"
546 +else
547 + mkdir -p "$test"
548 +fi
549 # Use -P to resolve symlinks in our working directory so that the cwd
550 # in subprocesses like git equals our $PWD (for pathname comparisons).
551 cd -P "$test" || exit 1