test-lib: add a --invert-exit-code switch
Add the ability to have those tests that fail return 0, and those tests that succeed return 1. This is useful e.g. to run "--stress" tests on tests that fail 99% of the time on some setup, i.e. to smoke out the flaky run which yielded success. In a subsequent commit a new SANITIZE=leak mode will make use of this. Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ævar Arnfjörð Bjarmason committed
Jul 28, 2022 at 01:13 UTC
46fb057aaad2cfcbf6cdf409fcf241a362ad77b3
2 files changed
+115
-2
t/t0000-basic.sh
+72
@@ -578,6 +578,78 @@ test_expect_success 'subtest: --run invalid range end' '
578
EOF_ERR
579
'
580
581
+test_expect_success 'subtest: --invert-exit-code without --immediate' '
582
+ run_sub_test_lib_test_err full-pass \
583
+ --invert-exit-code &&
584
+ check_sub_test_lib_test_err full-pass \
585
+ <<-\EOF_OUT 3<<-EOF_ERR
586
+ ok 1 - passing test #1
587
+ ok 2 - passing test #2
588
+ ok 3 - passing test #3
589
+ # passed all 3 test(s)
590
+ 1..3
591
+ # faking up non-zero exit with --invert-exit-code
592
+ EOF_OUT
593
+ EOF_ERR
594
+'
595
+
596
+test_expect_success 'subtest: --invert-exit-code with --immediate: all passed' '
597
+ run_sub_test_lib_test_err full-pass \
598
+ --invert-exit-code --immediate &&
599
+ check_sub_test_lib_test_err full-pass \
600
+ <<-\EOF_OUT 3<<-EOF_ERR
601
+ ok 1 - passing test #1
602
+ ok 2 - passing test #2
603
+ ok 3 - passing test #3
604
+ # passed all 3 test(s)
605
+ 1..3
606
+ # faking up non-zero exit with --invert-exit-code
607
+ EOF_OUT
608
+ EOF_ERR
609
+'
610
+
611
+test_expect_success 'subtest: --invert-exit-code without --immediate: partial pass' '
612
+ run_sub_test_lib_test partial-pass \
613
+ --invert-exit-code &&
614
+ check_sub_test_lib_test partial-pass <<-\EOF
615
+ ok 1 - passing test #1
616
+ not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
617
+ # false
618
+ ok 3 - passing test #3
619
+ # failed 1 among 3 test(s)
620
+ 1..3
621
+ # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
622
+ EOF
623
+'
624
+
625
+test_expect_success 'subtest: --invert-exit-code with --immediate: partial pass' '
626
+ run_sub_test_lib_test partial-pass \
627
+ --invert-exit-code --immediate &&
628
+ check_sub_test_lib_test partial-pass \
629
+ <<-\EOF_OUT 3<<-EOF_ERR
630
+ ok 1 - passing test #1
631
+ not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
632
+ # false
633
+ 1..2
634
+ # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
635
+ EOF_OUT
636
+ EOF_ERR
637
+'
638
+
639
+test_expect_success 'subtest: --invert-exit-code --immediate: got a failure' '
640
+ run_sub_test_lib_test partial-pass \
641
+ --invert-exit-code --immediate &&
642
+ check_sub_test_lib_test_err partial-pass \
643
+ <<-\EOF_OUT 3<<-EOF_ERR
644
+ ok 1 - passing test #1
645
+ not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
646
+ # false
647
+ 1..2
648
+ # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
649
+ EOF_OUT
650
+ EOF_ERR
651
+'
652
+
653
test_expect_success 'subtest: tests respect prerequisites' '
654
write_and_run_sub_test_lib_test prereqs <<-\EOF &&
655
t/test-lib.sh
+43
-2
@@ -238,6 +238,9 @@ parse_option () {
238
;;
239
esac
240
;;
241
+ --invert-exit-code)
242
+ invert_exit_code=t
243
+ ;;
244
*)
245
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
246
esac
@@ -788,15 +791,31 @@ test_ok_ () {
791
finalize_test_case_output ok "$@"
792
}
793
794
+_invert_exit_code_failure_end_blurb () {
795
+ say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code"
796
+}
797
+
798
test_failure_ () {
799
failure_label=$1
800
test_failure=$(($test_failure + 1))
794
- say_color error "not ok $test_count - $1"
801
+ local pfx=""
802
+ if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE"
803
+ then
804
+ pfx="# TODO induced breakage (--invert-exit-code):"
805
+ fi
806
+ say_color error "not ok $test_count - ${pfx:+$pfx }$1"
807
shift
808
printf '%s\n' "$*" | sed -e 's/^/# /'
809
if test -n "$immediate"
810
then
811
say_color error "1..$test_count"
812
+ if test -n "$invert_exit_code"
813
+ then
814
+ finalize_test_output
815
+ _invert_exit_code_failure_end_blurb
816
+ GIT_EXIT_OK=t
817
+ exit 0
818
+ fi
819
_error_exit
820
fi
821
finalize_test_case_output failure "$failure_label" "$@"
@@ -1229,7 +1248,14 @@ test_done () {
1248
esac
1249
fi
1250
1232
- if test -z "$debug" && test -n "$remove_trash"
1251
+ if test -n "$stress" && test -n "$invert_exit_code"
1252
+ then
1253
+ # We're about to move our "$TRASH_DIRECTORY"
1254
+ # to "$TRASH_DIRECTORY.stress-failed" if
1255
+ # --stress is combined with
1256
+ # --invert-exit-code.
1257
+ say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'"
1258
+ elif test -z "$debug" && test -n "$remove_trash"
1259
then
1260
test -d "$TRASH_DIRECTORY" ||
1261
error "Tests passed but trash directory already removed before test cleanup; aborting"
@@ -1242,6 +1268,14 @@ test_done () {
1268
} ||
1269
error "Tests passed but test cleanup failed; aborting"
1270
fi
1271
+
1272
+ if test -z "$skip_all" && test -n "$invert_exit_code"
1273
+ then
1274
+ say_color warn "# faking up non-zero exit with --invert-exit-code"
1275
+ GIT_EXIT_OK=t
1276
+ exit 1
1277
+ fi
1278
+
1279
test_at_end_hook_
1280
1281
GIT_EXIT_OK=t
@@ -1254,6 +1288,13 @@ test_done () {
1288
say "1..$test_count"
1289
fi
1290
1291
+ if test -n "$invert_exit_code"
1292
+ then
1293
+ _invert_exit_code_failure_end_blurb
1294
+ GIT_EXIT_OK=t
1295
+ exit 0
1296
+ fi
1297
+
1298
GIT_EXIT_OK=t
1299
exit 1 ;;
1300