sharness: Generate JUnit test reports
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Dec 28, 2017 at 20:11 UTC
3eb614fcb29823f3c2c5cc796b47397a8d081cb3
6 files changed
+229
-5
test/sharness/Rules.mk
+5
@@ -29,12 +29,17 @@ export MAKE_SKIP_PATH=1
29
30
$(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency
31
@echo "*** $@ ***"
32
+ifeq ($(CONTINUE_ON_S_FAILURE),1)
33
+ -@(cd $(@D) && ./$(@F)) 2>&1
34
+else
35
@(cd $(@D) && ./$(@F)) 2>&1
36
+endif
37
.PHONY: $(T_$(d))
38
39
$(d)/aggregate: $(T_$(d))
40
@echo "*** $@ ***"
41
@(cd $(@D) && ./lib/test-aggregate-results.sh)
42
+ @(cd $(@D) && ./lib/gen-junit-report.sh)
43
.PHONY: $(d)/aggregate
44
45
$(d)/clean-test-results:
test/sharness/lib/0001-Generate-partial-JUnit-reports.patch
new
+200
@@ -0,0 +1,200 @@
1
+From 0e2a489651e8e2b1aa4abf1cb2587fc3d0f78ce6 Mon Sep 17 00:00:00 2001
2
+From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
3
+Date: Thu, 28 Dec 2017 19:24:55 +0100
4
+Subject: [PATCH] Generate partial JUnit reports
5
+
6
+---
7
+ sharness.sh | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
8
+ 1 file changed, 73 insertions(+), 6 deletions(-)
9
+
10
+diff --git a/sharness.sh b/sharness.sh
11
+index 6750ff7..55e10ac 100644
12
+--- a/sharness.sh
13
++++ b/sharness.sh
14
+@@ -1,4 +1,4 @@
15
+-#!/bin/sh
16
++#!/usr/bin/env bash
17
+ #
18
+ # Copyright (c) 2011-2012 Mathias Lafeldt
19
+ # Copyright (c) 2005-2012 Git project
20
+@@ -106,6 +106,8 @@ if test -n "$color"; then
21
+ test -n "$quiet" && return;;
22
+ esac
23
+ shift
24
++
25
++ echo "$*" >> .junit/tout
26
+ printf "%s" "$*"
27
+ tput sgr0
28
+ echo
29
+@@ -115,6 +117,8 @@ else
30
+ say_color() {
31
+ test -z "$1" && test -n "$quiet" && return
32
+ shift
33
++
34
++ echo "$*" >> .junit/tout
35
+ printf "%s\n" "$*"
36
+ }
37
+ fi
38
+@@ -129,6 +133,12 @@ say() {
39
+ say_color info "$*"
40
+ }
41
+
42
++esc=$(printf '\033')
43
++
44
++esc_xml() {
45
++ sed 's/&/\&/g; s/</\</g; s/>/\>/g; s/"/\"/g; s/'"$esc"'/\'/g; s///g;'
46
++}
47
++
48
+ test -n "$test_description" || error "Test script did not set test_description."
49
+
50
+ if test "$help" = "t"; then
51
+@@ -251,30 +261,69 @@ test_have_prereq() {
52
+ test $total_prereq = $ok_prereq
53
+ }
54
+
55
++junit_testcase() {
56
++ test_name=$1
57
++ tc_file=".junit/case-$(printf "%04d" $test_count)"
58
++
59
++ shift
60
++ cat > "$tc_file" <<-EOF
61
++ <testcase name="$test_count - $(echo $test_name | esc_xml) ">
62
++ $@
63
++ EOF
64
++
65
++ if test -f .junit/tout; then
66
++ cat >> "$tc_file" <<-EOF
67
++ <system-out>
68
++ $(cat .junit/tout | esc_xml)
69
++ </system-out>
70
++ EOF
71
++ fi
72
++
73
++ if test -f .junit/terr; then
74
++ cat >> "$tc_file" <<-EOF
75
++ <system-err>
76
++ $(cat .junit/terr | esc_xml)
77
++ </system-err>
78
++ EOF
79
++ fi
80
++
81
++ echo "</testcase>" >> "$tc_file"
82
++ rm -f .junit/tout .junit/terr
83
++}
84
++
85
+ # You are not expected to call test_ok_ and test_failure_ directly, use
86
+ # the text_expect_* functions instead.
87
+
88
+ test_ok_() {
89
+ test_success=$(($test_success + 1))
90
+ say_color "" "ok $test_count - $@"
91
++
92
++ junit_testcase "$@"
93
+ }
94
+
95
+ test_failure_() {
96
+ test_failure=$(($test_failure + 1))
97
+ say_color error "not ok $test_count - $1"
98
++ test_name=$1
99
+ shift
100
+ echo "$@" | sed -e 's/^/# /'
101
++ junit_testcase "$test_name" '<failure type="">'$(echo $@ | esc_xml)'</failure>'
102
++
103
+ test "$immediate" = "" || { EXIT_OK=t; exit 1; }
104
+ }
105
+
106
+ test_known_broken_ok_() {
107
+ test_fixed=$(($test_fixed + 1))
108
+ say_color error "ok $test_count - $@ # TODO known breakage vanished"
109
++
110
++ junit_testcase "$@" '<failure type="known breakage vanished"/>'
111
+ }
112
+
113
+ test_known_broken_failure_() {
114
+ test_broken=$(($test_broken + 1))
115
+ say_color warn "not ok $test_count - $@ # TODO known breakage"
116
++
117
++ junit_testcase "$@"
118
+ }
119
+
120
+ # Public: Execute commands in debug mode.
121
+@@ -310,7 +359,7 @@ test_pause() {
122
+ test_eval_() {
123
+ # This is a separate function because some tests use
124
+ # "return" to end a test_expect_success block early.
125
+- eval </dev/null >&3 2>&4 "$*"
126
++ eval </dev/null > >(tee -a .junit/tout >&3) 2> >(tee -a .junit/terr >&4) "$*"
127
+ }
128
+
129
+ test_run_() {
130
+@@ -355,8 +404,16 @@ test_skip_() {
131
+ of_prereq=" of $test_prereq"
132
+ fi
133
+
134
+- say_color skip >&3 "skipping test: $@"
135
+- say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})"
136
++ say_color skip >&3 "skipping test: $1"
137
++ say_color skip "ok $test_count # skip $1 (missing $missing_prereqm${of_prereq})"
138
++
139
++ cat > ".junit/case-$(printf "%04d" $test_count)" <<-EOF
140
++ <testcase name="$test_count - $(echo $2 | esc_xml)">
141
++ <skipped>
142
++ skip $(echo $1 | esc_xml) (missing $missing_prereq${of_prereq})
143
++ </skipped>
144
++ </testcase>
145
++ EOF
146
+ : true
147
+ ;;
148
+ *)
149
+@@ -403,7 +460,7 @@ test_expect_success() {
150
+ test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
151
+ test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_success"
152
+ export test_prereq
153
+- if ! test_skip_ "$@"; then
154
++ if ! test_skip_ "$@" "$1"; then
155
+ say >&3 "expecting success: $2"
156
+ if test_run_ "$2"; then
157
+ test_ok_ "$1"
158
+@@ -442,7 +499,7 @@ test_expect_failure() {
159
+ test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
160
+ test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_failure"
161
+ export test_prereq
162
+- if ! test_skip_ "$@"; then
163
++ if ! test_skip_ "$@" "$1"; then
164
+ say >&3 "checking known breakage: $2"
165
+ if test_run_ "$2" expecting_failure; then
166
+ test_known_broken_ok_ "$1"
167
+@@ -675,6 +732,7 @@ test_done() {
168
+ test_results_dir="$SHARNESS_TEST_DIRECTORY/test-results"
169
+ mkdir -p "$test_results_dir"
170
+ test_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.counts"
171
++ junit_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.xml.part"
172
+
173
+ cat >>"$test_results_path" <<-EOF
174
+ total $test_count
175
+@@ -684,6 +742,12 @@ test_done() {
176
+ failed $test_failure
177
+
178
+ EOF
179
++
180
++ cat >>"$junit_results_path" <<-EOF
181
++ <testsuite errors="$test_broken" failures="$((test_failure+test_fixed))" tests="$test_count" name="$SHARNESS_TEST_FILE">
182
++ $(find .junit -name 'case-*' | sort | xargs -i cat "{}")
183
++ </testsuite>
184
++ EOF
185
+ fi
186
+
187
+ if test "$test_fixed" != 0; then
188
+@@ -771,6 +835,9 @@ mkdir -p "$test_dir" || exit 1
189
+ # in subprocesses like git equals our $PWD (for pathname comparisons).
190
+ cd -P "$test_dir" || exit 1
191
+
192
++# Prepare JUnit report dir
193
++mkdir -p .junit
194
++
195
+ this_test=${SHARNESS_TEST_FILE##*/}
196
+ this_test=${this_test%.$SHARNESS_TEST_EXTENSION}
197
+ for skp in $SKIP_TESTS; do
198
+--
199
+2.15.1
200
+
test/sharness/lib/gen-junit-report.sh
new
+8
@@ -0,0 +1,8 @@
1
+#!/usr/bin/env bash
2
+
3
+cat > test-results/sharness.xml <<-EOF
4
+<?xml version="1.0" encoding="UTF-8"?>
5
+<testsuites name="sharness">
6
+ $(find test-results -name '*.xml.part' | sort | xargs -i cat "{}")
7
+</testsuites>
8
+EOF
test/sharness/lib/install-sharness.sh
+13
-2
@@ -7,13 +7,15 @@
7
8
# settings
9
version=5eee9b51b5621cec95a64018f0cc779963b230d2
10
+patch_version=1
11
+
12
urlprefix=https://github.com/mlafeldt/sharness.git
13
if test ! -n "$clonedir" ; then
14
clonedir=lib
15
fi
16
sharnessdir=sharness
17
16
-if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_$version"
18
+if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_${version}_p${patch_version}"
19
then
20
# There is the right version file. Great, we are done!
21
exit 0
@@ -24,11 +26,20 @@ die() {
26
exit 1
27
}
28
29
+apply_patches() {
30
+ git config --local user.email "noone@nowhere"
31
+ git config --local user.name "No One"
32
+ git am ../0001-Generate-partial-JUnit-reports.patch
33
+
34
+ touch "SHARNESS_VERSION_${version}_p${patch_version}" || die "Could not create 'SHARNESS_VERSION_${version}_p${patch_version}'"
35
+}
36
+
37
checkout_version() {
38
git checkout "$version" || die "Could not checkout '$version'"
39
rm -f SHARNESS_VERSION_* || die "Could not remove 'SHARNESS_VERSION_*'"
30
- touch "SHARNESS_VERSION_$version" || die "Could not create 'SHARNESS_VERSION_$version'"
40
echo "Sharness version $version is checked out!"
41
+
42
+ apply_patches
43
}
44
45
if test -d "$clonedir/$sharnessdir/.git"
test/sharness/lib/test-lib.sh
+1
-1
@@ -214,6 +214,7 @@ test_launch_ipfs_daemon() {
214
215
test_expect_success "'ipfs daemon' succeeds" '
216
ipfs daemon $args >actual_daemon 2>daemon_err &
217
+ IPFS_PID=$!
218
'
219
220
# wait for api file to show up
@@ -225,7 +226,6 @@ test_launch_ipfs_daemon() {
226
227
# we say the daemon is ready when the API server is ready.
228
test_expect_success "'ipfs daemon' is ready" '
228
- IPFS_PID=$! &&
229
pollEndpoint -ep=/version -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
230
test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout
231
'
test/sharness/t0181-private-network.sh
+2
-2
@@ -29,7 +29,7 @@ pnet_key() {
29
random 16
30
}
31
32
-pnet_key > $IPFS_PATH/swarm.key
32
+pnet_key > "${IPFS_PATH}/swarm.key"
33
34
LIBP2P_FORCE_PNET=1 test_launch_ipfs_daemon
35
@@ -42,7 +42,7 @@ set_key() {
42
node="$1"
43
keyfile="$2"
44
45
- cp "$keyfile" "$IPTB_ROOT/$node/swarm.key"
45
+ cp "$keyfile" "${IPTB_ROOT}/${node}/swarm.key"
46
}
47
48
pnet_key > key1