tests: add a special setup where prerequisites fail
As discussed in [1] there's a regression in the "pu" branch now because a new test implicitly assumed that a previous test guarded by a prerequisite had been run. Add a "GIT_TEST_FAIL_PREREQS" special test setup where we'll skip (nearly) all tests guarded by prerequisites, allowing us to easily emulate those platform where we don't run these tests. As noted in the documentation I'm adding I'm whitelisting the SYMLINKS prerequisite for now. A lot of tests started failing if we lied about not supporting symlinks. It's also unlikely that we'll have a failing test due to a hard dependency on symlinks without that being the obvious cause, so for now it's not worth the effort to make it work. 1. https://public-inbox.org/git/nycvar.QRO.7.76.6.1905131531000.44@tvgsbejvaqbjf.bet/ Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ævar Arnfjörð Bjarmason committed
May 13, 2019 at 20:32 UTC
dfe1a17df9b1bd9ab95095e248b8f376637ac395
7 files changed
+43
-10
t/README
+9
@@ -334,6 +334,15 @@ that cannot be easily covered by a few specific test cases. These
334
could be enabled by running the test suite with correct GIT_TEST_
335
environment set.
336
337
+GIT_TEST_FAIL_PREREQS<non-empty?> fails all prerequisites. This is
338
+useful for discovering issues with the tests where say a later test
339
+implicitly depends on an optional earlier test.
340
+
341
+There's a "FAIL_PREREQS" prerequisite that can be used to test for
342
+whether this mode is active, and e.g. skip some tests that are hard to
343
+refactor to deal with it. The "SYMLINKS" prerequisite is currently
344
+excluded as so much relies on it, but this might change in the future.
345
+
346
GIT_TEST_GETTEXT_POISON=<non-empty?> turns all strings marked for
347
translation into gibberish if non-empty (think "test -n"). Used for
348
spotting those tests that need to be marked with a C_LOCALE_OUTPUT
t/t0000-basic.sh
+5
-5
@@ -726,7 +726,7 @@ donthaveit=yes
726
test_expect_success DONTHAVEIT 'unmet prerequisite causes test to be skipped' '
727
donthaveit=no
728
'
729
-if test $haveit$donthaveit != yesyes
729
+if test -z "$GIT_TEST_FAIL_PREREQS" -a $haveit$donthaveit != yesyes
730
then
731
say "bug in test framework: prerequisite tags do not work reliably"
732
exit 1
@@ -747,7 +747,7 @@ donthaveiteither=yes
747
test_expect_success DONTHAVEIT,HAVEIT 'unmet prerequisites causes test to be skipped' '
748
donthaveiteither=no
749
'
750
-if test $haveit$donthaveit$donthaveiteither != yesyesyes
750
+if test -z "$GIT_TEST_FAIL_PREREQS" -a $haveit$donthaveit$donthaveiteither != yesyesyes
751
then
752
say "bug in test framework: multiple prerequisite tags do not work reliably"
753
exit 1
@@ -763,7 +763,7 @@ test_expect_success !LAZY_TRUE 'missing lazy prereqs skip tests' '
763
donthavetrue=no
764
'
765
766
-if test "$havetrue$donthavetrue" != yesyes
766
+if test -z "$GIT_TEST_FAIL_PREREQS" -a "$havetrue$donthavetrue" != yesyes
767
then
768
say 'bug in test framework: lazy prerequisites do not work'
769
exit 1
@@ -779,7 +779,7 @@ test_expect_success LAZY_FALSE 'missing negative lazy prereqs will skip' '
779
havefalse=no
780
'
781
782
-if test "$nothavefalse$havefalse" != yesyes
782
+if test -z "$GIT_TEST_FAIL_PREREQS" -a "$nothavefalse$havefalse" != yesyes
783
then
784
say 'bug in test framework: negative lazy prerequisites do not work'
785
exit 1
@@ -790,7 +790,7 @@ test_expect_success 'tests clean up after themselves' '
790
test_when_finished clean=yes
791
'
792
793
-if test $clean != yes
793
+if test -z "$GIT_TEST_FAIL_PREREQS" -a $clean != yes
794
then
795
say "bug in test framework: basic cleanup command does not work reliably"
796
exit 1
t/t4202-log.sh
+1
-1
@@ -352,7 +352,7 @@ test_expect_success 'log with grep.patternType configuration and command line' '
352
test_cmp expect actual
353
'
354
355
-test_expect_success 'log with various grep.patternType configurations & command-lines' '
355
+test_expect_success !FAIL_PREREQS 'log with various grep.patternType configurations & command-lines' '
356
git init pattern-type &&
357
(
358
cd pattern-type &&
t/t7405-submodule-merge.sh
+1
-1
@@ -417,7 +417,7 @@ test_expect_failure 'directory/submodule conflict; keep submodule clean' '
417
)
418
'
419
420
-test_expect_failure 'directory/submodule conflict; should not treat submodule files as untracked or in the way' '
420
+test_expect_failure !FAIL_PREREQS 'directory/submodule conflict; should not treat submodule files as untracked or in the way' '
421
test_when_finished "git -C directory-submodule/path reset --hard" &&
422
test_when_finished "git -C directory-submodule reset --hard" &&
423
(
t/t7810-grep.sh
+3
-3
@@ -412,7 +412,7 @@ do
412
test_cmp expected actual
413
'
414
415
- test_expect_success !PCRE "grep $L with grep.patterntype=perl errors without PCRE" '
415
+ test_expect_success !FAIL_PREREQS,!PCRE "grep $L with grep.patterntype=perl errors without PCRE" '
416
test_must_fail git -c grep.patterntype=perl grep "foo.*bar"
417
'
418
@@ -1234,7 +1234,7 @@ test_expect_success PCRE 'grep --perl-regexp pattern' '
1234
test_cmp expected actual
1235
'
1236
1237
-test_expect_success !PCRE 'grep --perl-regexp pattern errors without PCRE' '
1237
+test_expect_success !FAIL_PREREQS,!PCRE 'grep --perl-regexp pattern errors without PCRE' '
1238
test_must_fail git grep --perl-regexp "foo.*bar"
1239
'
1240
@@ -1249,7 +1249,7 @@ test_expect_success LIBPCRE2 "grep -P with (*NO_JIT) doesn't error out" '
1249
1250
'
1251
1252
-test_expect_success !PCRE 'grep -P pattern errors without PCRE' '
1252
+test_expect_success !FAIL_PREREQS,!PCRE 'grep -P pattern errors without PCRE' '
1253
test_must_fail git grep -P "foo.*bar"
1254
'
1255
t/test-lib-functions.sh
+20
@@ -309,6 +309,26 @@ test_unset_prereq () {
309
}
310
311
test_set_prereq () {
312
+ if test -n "$GIT_TEST_FAIL_PREREQS"
313
+ then
314
+ case "$1" in
315
+ # The "!" case is handled below with
316
+ # test_unset_prereq()
317
+ !*)
318
+ ;;
319
+ # (Temporary?) whitelist of things we can't easily
320
+ # pretend not to support
321
+ SYMLINKS)
322
+ ;;
323
+ # Inspecting whether GIT_TEST_FAIL_PREREQS is on
324
+ # should be unaffected.
325
+ FAIL_PREREQS)
326
+ ;;
327
+ *)
328
+ return
329
+ esac
330
+ fi
331
+
332
case "$1" in
333
!*)
334
test_unset_prereq "${1#!}"
t/test-lib.sh
+4
@@ -1607,3 +1607,7 @@ test_lazy_prereq SHA1 '
1607
test_lazy_prereq REBASE_P '
1608
test -z "$GIT_TEST_SKIP_REBASE_P"
1609
'
1610
+
1611
+test_lazy_prereq FAIL_PREREQS '
1612
+ test -n "$GIT_TEST_FAIL_PREREQS"
1613
+'