tests: paint unexpectedly fixed known breakages in bold red
Change color of unexpectedly fixed known breakages to bold red. An unexpectedly passing test indicates that the test code is somehow broken or out of sync with the code it is testing. Either way this is an error which is potentially as bad as a failing test, and as such is no longer portrayed as a pass in the output. Signed-off-by: Adam Spiers <git@adamspiers.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Adam Spiers committed
Dec 16, 2012 at 18:28 UTC
b73d9a2363beedd59c212b5aa6b9db2977c38b2b
2 files changed
+33
-10
t/t0000-basic.sh
+24
-6
@@ -145,13 +145,31 @@ test_expect_success 'pretend we have fixed a known breakage' "
145
test_done
146
EOF
147
check_sub_test_lib_test passing-todo <<-\\EOF
148
- > ok 1 - pretend we have fixed a known breakage # TODO known breakage
149
- > # fixed 1 known breakage(s)
150
- > # passed all 1 test(s)
148
+ > ok 1 - pretend we have fixed a known breakage # TODO known breakage vanished
149
+ > # 1 known breakage(s) vanished; please update test(s)
150
> 1..1
151
EOF
152
"
153
154
+test_expect_success 'pretend we have fixed one of two known breakages (run in sub test-lib)' "
155
+ run_sub_test_lib_test partially-passing-todos \
156
+ '2 TODO tests, one passing' <<-\\EOF &&
157
+ test_expect_failure 'pretend we have a known breakage' 'false'
158
+ test_expect_success 'pretend we have a passing test' 'true'
159
+ test_expect_failure 'pretend we have fixed another known breakage' 'true'
160
+ test_done
161
+ EOF
162
+ check_sub_test_lib_test partially-passing-todos <<-\\EOF
163
+ > not ok 1 - pretend we have a known breakage # TODO known breakage
164
+ > ok 2 - pretend we have a passing test
165
+ > ok 3 - pretend we have fixed another known breakage # TODO known breakage vanished
166
+ > # 1 known breakage(s) vanished; please update test(s)
167
+ > # still have 1 known breakage(s)
168
+ > # passed all remaining 1 test(s)
169
+ > 1..3
170
+ EOF
171
+"
172
+
173
test_expect_success 'pretend we have a pass, fail, and known breakage' "
174
test_must_fail run_sub_test_lib_test \
175
mixed-results1 'mixed results #1' <<-\\EOF &&
@@ -199,10 +217,10 @@ test_expect_success 'pretend we have a mix of all possible results' "
217
> # false
218
> not ok 8 - pretend we have a known breakage # TODO known breakage
219
> not ok 9 - pretend we have a known breakage # TODO known breakage
202
- > ok 10 - pretend we have fixed a known breakage # TODO known breakage
203
- > # fixed 1 known breakage(s)
220
+ > ok 10 - pretend we have fixed a known breakage # TODO known breakage vanished
221
+ > # 1 known breakage(s) vanished; please update test(s)
222
> # still have 2 known breakage(s)
205
- > # failed 3 among remaining 8 test(s)
223
+ > # failed 3 among remaining 7 test(s)
224
> 1..10
225
EOF
226
"
t/test-lib.sh
+9
-4
@@ -278,7 +278,7 @@ test_failure_ () {
278
279
test_known_broken_ok_ () {
280
test_fixed=$(($test_fixed+1))
281
- say_color "" "ok $test_count - $@ # TODO known breakage"
281
+ say_color error "ok $test_count - $@ # TODO known breakage vanished"
282
}
283
284
test_known_broken_failure_ () {
@@ -373,13 +373,18 @@ test_done () {
373
374
if test "$test_fixed" != 0
375
then
376
- say_color pass "# fixed $test_fixed known breakage(s)"
376
+ say_color error "# $test_fixed known breakage(s) vanished; please update test(s)"
377
fi
378
if test "$test_broken" != 0
379
then
380
say_color warn "# still have $test_broken known breakage(s)"
381
- msg="remaining $(($test_count-$test_broken)) test(s)"
381
+ fi
382
+ if test "$test_broken" != 0 || test "$test_fixed" != 0
383
+ then
384
+ test_remaining=$(( $test_count - $test_broken - $test_fixed ))
385
+ msg="remaining $test_remaining test(s)"
386
else
387
+ test_remaining=$test_count
388
msg="$test_count test(s)"
389
fi
390
case "$test_failure" in
@@ -393,7 +398,7 @@ test_done () {
398
399
if test $test_external_has_tap -eq 0
400
then
396
- if test $test_count -gt 0
401
+ if test $test_remaining -gt 0
402
then
403
say_color pass "# passed all $msg"
404
fi