tests: optionally write results as JUnit-style .xml
This will come in handy when publishing the results of Git's test suite during an automated Azure DevOps run. Note: we need to make extra sure that invalid UTF-8 encoding is turned into valid UTF-8 (using the Replacement Character, \uFFFD) because t9902's trace contains such invalid byte sequences, and the task in the Azure Pipeline that uploads the test results would refuse to do anything if it was asked to parse an .xml file with invalid UTF-8 in it. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Johannes Schindelin committed
Jan 29, 2019 at 06:19 UTC
222319081515f9f6a4c699bd3a406f181a6ddc3b
6 files changed
+175
Makefile
+1
@@ -754,6 +754,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o
754
TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o
755
TEST_BUILTINS_OBJS += test-subprocess.o
756
TEST_BUILTINS_OBJS += test-urlmatch-normalization.o
757
+TEST_BUILTINS_OBJS += test-xml-encode.o
758
TEST_BUILTINS_OBJS += test-wildmatch.o
759
TEST_BUILTINS_OBJS += test-windows-named-pipe.o
760
TEST_BUILTINS_OBJS += test-write-cache.o
t/.gitignore
+1
@@ -2,3 +2,4 @@
2
/test-results
3
/.prove
4
/chainlinttmp
5
+/out/
t/helper/test-tool.c
+1
@@ -49,6 +49,7 @@ static struct test_cmd cmds[] = {
49
{ "submodule-nested-repo-config", cmd__submodule_nested_repo_config },
50
{ "subprocess", cmd__subprocess },
51
{ "urlmatch-normalization", cmd__urlmatch_normalization },
52
+ { "xml-encode", cmd__xml_encode },
53
{ "wildmatch", cmd__wildmatch },
54
#ifdef GIT_WINDOWS_NATIVE
55
{ "windows-named-pipe", cmd__windows_named_pipe },
t/helper/test-tool.h
+1
@@ -45,6 +45,7 @@ int cmd__submodule_config(int argc, const char **argv);
45
int cmd__submodule_nested_repo_config(int argc, const char **argv);
46
int cmd__subprocess(int argc, const char **argv);
47
int cmd__urlmatch_normalization(int argc, const char **argv);
48
+int cmd__xml_encode(int argc, const char **argv);
49
int cmd__wildmatch(int argc, const char **argv);
50
#ifdef GIT_WINDOWS_NATIVE
51
int cmd__windows_named_pipe(int argc, const char **argv);
t/helper/test-xml-encode.c
new
+80
@@ -0,0 +1,80 @@
1
+#include "test-tool.h"
2
+
3
+static const char *utf8_replace_character = "�";
4
+
5
+/*
6
+ * Encodes (possibly incorrect) UTF-8 on <stdin> to <stdout>, to be embedded
7
+ * in an XML file.
8
+ */
9
+int cmd__xml_encode(int argc, const char **argv)
10
+{
11
+ unsigned char buf[1024], tmp[4], *tmp2 = NULL;
12
+ ssize_t cur = 0, len = 1, remaining = 0;
13
+ unsigned char ch;
14
+
15
+ for (;;) {
16
+ if (++cur == len) {
17
+ len = xread(0, buf, sizeof(buf));
18
+ if (!len)
19
+ return 0;
20
+ if (len < 0)
21
+ die_errno("Could not read <stdin>");
22
+ cur = 0;
23
+ }
24
+ ch = buf[cur];
25
+
26
+ if (tmp2) {
27
+ if ((ch & 0xc0) != 0x80) {
28
+ fputs(utf8_replace_character, stdout);
29
+ tmp2 = NULL;
30
+ cur--;
31
+ continue;
32
+ }
33
+ *tmp2 = ch;
34
+ tmp2++;
35
+ if (--remaining == 0) {
36
+ fwrite(tmp, tmp2 - tmp, 1, stdout);
37
+ tmp2 = NULL;
38
+ }
39
+ continue;
40
+ }
41
+
42
+ if (!(ch & 0x80)) {
43
+ /* 0xxxxxxx */
44
+ if (ch == '&')
45
+ fputs("&", stdout);
46
+ else if (ch == '\'')
47
+ fputs("'", stdout);
48
+ else if (ch == '"')
49
+ fputs(""", stdout);
50
+ else if (ch == '<')
51
+ fputs("<", stdout);
52
+ else if (ch == '>')
53
+ fputs(">", stdout);
54
+ else if (ch >= 0x20)
55
+ fputc(ch, stdout);
56
+ else if (ch == 0x09 || ch == 0x0a || ch == 0x0d)
57
+ fprintf(stdout, "&#x%02x;", ch);
58
+ else
59
+ fputs(utf8_replace_character, stdout);
60
+ } else if ((ch & 0xe0) == 0xc0) {
61
+ /* 110XXXXx 10xxxxxx */
62
+ tmp[0] = ch;
63
+ remaining = 1;
64
+ tmp2 = tmp + 1;
65
+ } else if ((ch & 0xf0) == 0xe0) {
66
+ /* 1110XXXX 10Xxxxxx 10xxxxxx */
67
+ tmp[0] = ch;
68
+ remaining = 2;
69
+ tmp2 = tmp + 1;
70
+ } else if ((ch & 0xf8) == 0xf0) {
71
+ /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
72
+ tmp[0] = ch;
73
+ remaining = 3;
74
+ tmp2 = tmp + 1;
75
+ } else
76
+ fputs(utf8_replace_character, stdout);
77
+ }
78
+
79
+ return 0;
80
+}
t/test-lib.sh
+91
@@ -139,6 +139,9 @@ do
139
verbose_log=t
140
tee=t
141
;;
142
+ --write-junit-xml)
143
+ write_junit_xml=t
144
+ ;;
145
--stress)
146
stress=t ;;
147
--stress=*)
@@ -622,11 +625,24 @@ trap 'exit $?' INT TERM HUP
625
# the test_expect_* functions instead.
626
627
test_ok_ () {
628
+ if test -n "$write_junit_xml"
629
+ then
630
+ write_junit_xml_testcase "$*"
631
+ fi
632
test_success=$(($test_success + 1))
633
say_color "" "ok $test_count - $@"
634
}
635
636
test_failure_ () {
637
+ if test -n "$write_junit_xml"
638
+ then
639
+ junit_insert="<failure message=\"not ok $test_count -"
640
+ junit_insert="$junit_insert $(xml_attr_encode "$1")\">"
641
+ junit_insert="$junit_insert $(xml_attr_encode \
642
+ "$(printf '%s\n' "$@" | sed 1d)")"
643
+ junit_insert="$junit_insert</failure>"
644
+ write_junit_xml_testcase "$1" " $junit_insert"
645
+ fi
646
test_failure=$(($test_failure + 1))
647
say_color error "not ok $test_count - $1"
648
shift
@@ -635,11 +651,19 @@ test_failure_ () {
651
}
652
653
test_known_broken_ok_ () {
654
+ if test -n "$write_junit_xml"
655
+ then
656
+ write_junit_xml_testcase "$* (breakage fixed)"
657
+ fi
658
test_fixed=$(($test_fixed+1))
659
say_color error "ok $test_count - $@ # TODO known breakage vanished"
660
}
661
662
test_known_broken_failure_ () {
663
+ if test -n "$write_junit_xml"
664
+ then
665
+ write_junit_xml_testcase "$* (known breakage)"
666
+ fi
667
test_broken=$(($test_broken+1))
668
say_color warn "not ok $test_count - $@ # TODO known breakage"
669
}
@@ -897,6 +921,10 @@ test_start_ () {
921
test_count=$(($test_count+1))
922
maybe_setup_verbose
923
maybe_setup_valgrind
924
+ if test -n "$write_junit_xml"
925
+ then
926
+ junit_start=$(test-tool date getnanos)
927
+ fi
928
}
929
930
test_finish_ () {
@@ -934,6 +962,13 @@ test_skip () {
962
963
case "$to_skip" in
964
t)
965
+ if test -n "$write_junit_xml"
966
+ then
967
+ message="$(xml_attr_encode "$skipped_reason")"
968
+ write_junit_xml_testcase "$1" \
969
+ " <skipped message=\"$message\" />"
970
+ fi
971
+
972
say_color skip >&3 "skipping test: $@"
973
say_color skip "ok $test_count # skip $1 ($skipped_reason)"
974
: true
@@ -949,9 +984,51 @@ test_at_end_hook_ () {
984
:
985
}
986
987
+write_junit_xml () {
988
+ case "$1" in
989
+ --truncate)
990
+ >"$junit_xml_path"
991
+ junit_have_testcase=
992
+ shift
993
+ ;;
994
+ esac
995
+ printf '%s\n' "$@" >>"$junit_xml_path"
996
+}
997
+
998
+xml_attr_encode () {
999
+ printf '%s\n' "$@" | test-tool xml-encode
1000
+}
1001
+
1002
+write_junit_xml_testcase () {
1003
+ junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\""
1004
+ shift
1005
+ junit_attrs="$junit_attrs classname=\"$this_test\""
1006
+ junit_attrs="$junit_attrs time=\"$(test-tool \
1007
+ date getnanos $junit_start)\""
1008
+ write_junit_xml "$(printf '%s\n' \
1009
+ " <testcase $junit_attrs>" "$@" " </testcase>")"
1010
+ junit_have_testcase=t
1011
+}
1012
+
1013
test_done () {
1014
GIT_EXIT_OK=t
1015
1016
+ if test -n "$write_junit_xml" && test -n "$junit_xml_path"
1017
+ then
1018
+ test -n "$junit_have_testcase" || {
1019
+ junit_start=$(test-tool date getnanos)
1020
+ write_junit_xml_testcase "all tests skipped"
1021
+ }
1022
+
1023
+ # adjust the overall time
1024
+ junit_time=$(test-tool date getnanos $junit_suite_start)
1025
+ sed "s/<testsuite [^>]*/& time=\"$junit_time\"/" \
1026
+ <"$junit_xml_path" >"$junit_xml_path.new"
1027
+ mv "$junit_xml_path.new" "$junit_xml_path"
1028
+
1029
+ write_junit_xml " </testsuite>" "</testsuites>"
1030
+ fi
1031
+
1032
if test -z "$HARNESS_ACTIVE"
1033
then
1034
mkdir -p "$TEST_RESULTS_DIR"
@@ -1178,6 +1255,7 @@ then
1255
else
1256
mkdir -p "$TRASH_DIRECTORY"
1257
fi
1258
+
1259
# Use -P to resolve symlinks in our working directory so that the cwd
1260
# in subprocesses like git equals our $PWD (for pathname comparisons).
1261
cd -P "$TRASH_DIRECTORY" || exit 1
@@ -1191,6 +1269,19 @@ then
1269
test_done
1270
fi
1271
1272
+if test -n "$write_junit_xml"
1273
+then
1274
+ junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
1275
+ mkdir -p "$junit_xml_dir"
1276
+ junit_xml_base=${0##*/}
1277
+ junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
1278
+ junit_attrs="name=\"${junit_xml_base%.sh}\""
1279
+ junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
1280
+ date +%Y-%m-%dT%H:%M:%S)\""
1281
+ write_junit_xml --truncate "<testsuites>" " <testsuite $junit_attrs>"
1282
+ junit_suite_start=$(test-tool date getnanos)
1283
+fi
1284
+
1285
# Provide an implementation of the 'yes' utility
1286
yes () {
1287
if test $# = 0