@matheusbsbs / matheus-bagliano / commits / af9912efaf

tests: include detailed trace logs with --write-junit-xml upon failure

The JUnit XML format lends itself to be presented in a powerful UI, where you can drill down to the information you are interested in very quickly. For test failures, this usually means that you want to see the detailed trace of the failing tests. With Travis CI, we passed the `--verbose-log` option to get those traces. However, that seems excessive, as we do not need/use the logs in almost all of those cases: only when a test fails do we have a way to include the trace. So let's do something different when using Azure DevOps: let's run all the tests with `--quiet` first, and only if a failure is encountered, try to trace the commands as they are executed. Of course, we cannot turn on `--verbose-log` after the fact. So let's just re-run the test with all the same options, adding `--verbose-log`. And then munging the output file into the JUnit XML on the fly. Note: there is an off chance that re-running the test in verbose mode "fixes" the failures (and this does happen from time to time!). That is a possibility we should be able to live with. Ideally, we would label this as "Passed upon rerun", and Azure Pipelines even know about that outcome, but it is not available when using the JUnit XML format for now: https://github.com/Microsoft/azure-pipelines-agent/blob/master/src/Agent.Worker/TestResults/JunitResultReader.cs Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Johannes Schindelin committed Jan 29, 2019 at 06:19 UTC af9912efaf9d70ef96c63dfc28043b0b856cd516
2 files changed +42 -1
t/helper/test-path-utils.c
+21
@@ -303,6 +303,27 @@ int cmd__path_utils(int argc, const char **argv)
303 return !!res;
304 }
305
306 + if (argc == 4 && !strcmp(argv[1], "skip-n-bytes")) {
307 + int fd = open(argv[2], O_RDONLY), offset = atoi(argv[3]);
308 + char buffer[65536];
309 +
310 + if (fd < 0)
311 + die_errno("could not open '%s'", argv[2]);
312 + if (lseek(fd, offset, SEEK_SET) < 0)
313 + die_errno("could not skip %d bytes", offset);
314 + for (;;) {
315 + ssize_t count = read(fd, buffer, sizeof(buffer));
316 + if (count < 0)
317 + die_errno("could not read '%s'", argv[2]);
318 + if (!count)
319 + break;
320 + if (write(1, buffer, count) < 0)
321 + die_errno("could not write to stdout");
322 + }
323 + close(fd);
324 + return 0;
325 + }
326 +
327 fprintf(stderr, "%s: unknown function name: %s\n", argv[0],
328 argv[1] ? argv[1] : "(there was none)");
329 return 1;
t/test-lib.sh
+21 -1
@@ -639,8 +639,19 @@ test_failure_ () {
639 junit_insert="<failure message=\"not ok $test_count -"
640 junit_insert="$junit_insert $(xml_attr_encode "$1")\">"
641 junit_insert="$junit_insert $(xml_attr_encode \
642 - "$(printf '%s\n' "$@" | sed 1d)")"
642 + "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
643 + then
644 + test-tool path-utils skip-n-bytes \
645 + "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
646 + else
647 + printf '%s\n' "$@" | sed 1d
648 + fi)")"
649 junit_insert="$junit_insert</failure>"
650 + if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
651 + then
652 + junit_insert="$junit_insert<system-err>$(xml_attr_encode \
653 + "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")</system-err>"
654 + fi
655 write_junit_xml_testcase "$1" " $junit_insert"
656 fi
657 test_failure=$(($test_failure + 1))
@@ -931,6 +942,11 @@ test_finish_ () {
942 echo >&3 ""
943 maybe_teardown_valgrind
944 maybe_teardown_verbose
945 + if test -n "$GIT_TEST_TEE_OFFSET"
946 + then
947 + GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \
948 + "$GIT_TEST_TEE_OUTPUT_FILE")
949 + fi
950 }
951
952 test_skip () {
@@ -1280,6 +1296,10 @@ then
1296 date +%Y-%m-%dT%H:%M:%S)\""
1297 write_junit_xml --truncate "<testsuites>" " <testsuite $junit_attrs>"
1298 junit_suite_start=$(test-tool date getnanos)
1299 + if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
1300 + then
1301 + GIT_TEST_TEE_OFFSET=0
1302 + fi
1303 fi
1304
1305 # Provide an implementation of the 'yes' utility