@matheusbsbs / matheus-bagliano / commits / 0445e6f0a1

test-lib: '--run' to run only specific tests

Allow better control of the set of tests that will be executed for a single test suite. Mostly useful while debugging or developing as it allows to focus on a specific test. Signed-off-by: Ilya Bobyr <ilya.bobyr@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Ilya Bobyr committed Apr 30, 2014 at 02:50 UTC 0445e6f0a1223b5d40542627607207a87a416b5b
3 files changed +537 -8
t/README
+77 -4
@@ -104,6 +104,12 @@ appropriately before running "make".
104 This causes additional long-running tests to be run (where
105 available), for more exhaustive testing.
106
107 +-r::
108 +--run=<test-selector>::
109 + Run only the subset of tests indicated by
110 + <test-selector>. See section "Skipping Tests" below for
111 + <test-selector> syntax.
112 +
113 --valgrind=<tool>::
114 Execute all Git binaries under valgrind tool <tool> and exit
115 with status 126 on errors (just like regular tests, this will
@@ -191,10 +197,77 @@ and either can match the "t[0-9]{4}" part to skip the whole
197 test, or t[0-9]{4} followed by ".$number" to say which
198 particular test to skip.
199
194 -Note that some tests in the existing test suite rely on previous
195 -test item, so you cannot arbitrarily disable one and expect the
196 -remainder of test to check what the test originally was intended
197 -to check.
200 +For an individual test suite --run could be used to specify that
201 +only some tests should be run or that some tests should be
202 +excluded from a run.
203 +
204 +The argument for --run is a list of individual test numbers or
205 +ranges with an optional negation prefix that define what tests in
206 +a test suite to include in the run. A range is two numbers
207 +separated with a dash and matches a range of tests with both ends
208 +been included. You may omit the first or the second number to
209 +mean "from the first test" or "up to the very last test"
210 +respectively.
211 +
212 +Optional prefix of '!' means that the test or a range of tests
213 +should be excluded from the run.
214 +
215 +If --run starts with an unprefixed number or range the initial
216 +set of tests to run is empty. If the first item starts with '!'
217 +all the tests are added to the initial set. After initial set is
218 +determined every test number or range is added or excluded from
219 +the set one by one, from left to right.
220 +
221 +Individual numbers or ranges could be separated either by a space
222 +or a comma.
223 +
224 +For example, to run only tests up to a specific test (21), one
225 +could do this:
226 +
227 + $ sh ./t9200-git-cvsexport-commit.sh --run='1-21'
228 +
229 +or this:
230 +
231 + $ sh ./t9200-git-cvsexport-commit.sh --run='-21'
232 +
233 +Common case is to run several setup tests (1, 2, 3) and then a
234 +specific test (21) that relies on that setup:
235 +
236 + $ sh ./t9200-git-cvsexport-commit.sh --run='1 2 3 21'
237 +
238 +or:
239 +
240 + $ sh ./t9200-git-cvsexport-commit.sh --run=1,2,3,21
241 +
242 +or:
243 +
244 + $ sh ./t9200-git-cvsexport-commit.sh --run='-3 21'
245 +
246 +As noted above, the test set is built going though items left to
247 +right, so this:
248 +
249 + $ sh ./t9200-git-cvsexport-commit.sh --run='1-4 !3'
250 +
251 +will run tests 1, 2, and 4. Items that comes later have higher
252 +precendence. It means that this:
253 +
254 + $ sh ./t9200-git-cvsexport-commit.sh --run='!3 1-4'
255 +
256 +would just run tests from 1 to 4, including 3.
257 +
258 +You may use negation with ranges. The following will run all
259 +test in the test suite except from 7 up to 11:
260 +
261 + $ sh ./t9200-git-cvsexport-commit.sh --run='!7-11'
262 +
263 +Some tests in a test suite rely on the previous tests performing
264 +certain actions, specifically some tests are designated as
265 +"setup" test, so you cannot _arbitrarily_ disable one test and
266 +expect the rest to function correctly.
267 +
268 +--run is mostly useful when you want to focus on a specific test
269 +and know what setup is needed for it. Or when you want to run
270 +everything up to a certain test.
271
272
273 Naming Tests
t/t0000-basic.sh
+352 -4
@@ -42,9 +42,9 @@ test_expect_success 'success is reported like this' '
42 :
43 '
44
45 -run_sub_test_lib_test () {
46 - name="$1" descr="$2" # stdin is the body of the test code
47 - shift 2
45 +_run_sub_test_lib_test_common () {
46 + neg="$1" name="$2" descr="$3" # stdin is the body of the test code
47 + shift 3
48 mkdir "$name" &&
49 (
50 # Pretend we're not running under a test harness, whether we
@@ -70,10 +70,23 @@ run_sub_test_lib_test () {
70 export TEST_DIRECTORY &&
71 TEST_OUTPUT_DIRECTORY=$(pwd) &&
72 export TEST_OUTPUT_DIRECTORY &&
73 - ./"$name.sh" "$@" >out 2>err
73 + if test -z "$neg"
74 + then
75 + ./"$name.sh" "$@" >out 2>err
76 + else
77 + ! ./"$name.sh" "$@" >out 2>err
78 + fi
79 )
80 }
81
82 +run_sub_test_lib_test () {
83 + _run_sub_test_lib_test_common '' "$@"
84 +}
85 +
86 +run_sub_test_lib_test_err () {
87 + _run_sub_test_lib_test_common '!' "$@"
88 +}
89 +
90 check_sub_test_lib_test () {
91 name="$1" # stdin is the expected output from the test
92 (
@@ -84,6 +97,18 @@ check_sub_test_lib_test () {
97 )
98 }
99
100 +check_sub_test_lib_test_err () {
101 + name="$1" # stdin is the expected output output from the test
102 + # expected error output is in descriptior 3
103 + (
104 + cd "$name" &&
105 + sed -e 's/^> //' -e 's/Z$//' >expect.out &&
106 + test_cmp expect.out out &&
107 + sed -e 's/^> //' -e 's/Z$//' <&3 >expect.err &&
108 + test_cmp expect.err err
109 + )
110 +}
111 +
112 test_expect_success 'pretend we have a fully passing test suite' "
113 run_sub_test_lib_test full-pass '3 passing tests' <<-\\EOF &&
114 for i in 1 2 3
@@ -333,6 +358,329 @@ test_expect_success 'GIT_SKIP_TESTS sh pattern' "
358 EOF
359 "
360
361 +test_expect_success '--run basic' "
362 + run_sub_test_lib_test run-basic \
363 + '--run basic' --run='1 3 5' <<-\\EOF &&
364 + for i in 1 2 3 4 5 6
365 + do
366 + test_expect_success \"passing test #\$i\" 'true'
367 + done
368 + test_done
369 + EOF
370 + check_sub_test_lib_test run-basic <<-\\EOF
371 + > ok 1 - passing test #1
372 + > ok 2 # skip passing test #2 (--run)
373 + > ok 3 - passing test #3
374 + > ok 4 # skip passing test #4 (--run)
375 + > ok 5 - passing test #5
376 + > ok 6 # skip passing test #6 (--run)
377 + > # passed all 6 test(s)
378 + > 1..6
379 + EOF
380 +"
381 +
382 +test_expect_success '--run with a range' "
383 + run_sub_test_lib_test run-range \
384 + '--run with a range' --run='1-3' <<-\\EOF &&
385 + for i in 1 2 3 4 5 6
386 + do
387 + test_expect_success \"passing test #\$i\" 'true'
388 + done
389 + test_done
390 + EOF
391 + check_sub_test_lib_test run-range <<-\\EOF
392 + > ok 1 - passing test #1
393 + > ok 2 - passing test #2
394 + > ok 3 - passing test #3
395 + > ok 4 # skip passing test #4 (--run)
396 + > ok 5 # skip passing test #5 (--run)
397 + > ok 6 # skip passing test #6 (--run)
398 + > # passed all 6 test(s)
399 + > 1..6
400 + EOF
401 +"
402 +
403 +test_expect_success '--run with two ranges' "
404 + run_sub_test_lib_test run-two-ranges \
405 + '--run with two ranges' --run='1-2 5-6' <<-\\EOF &&
406 + for i in 1 2 3 4 5 6
407 + do
408 + test_expect_success \"passing test #\$i\" 'true'
409 + done
410 + test_done
411 + EOF
412 + check_sub_test_lib_test run-two-ranges <<-\\EOF
413 + > ok 1 - passing test #1
414 + > ok 2 - passing test #2
415 + > ok 3 # skip passing test #3 (--run)
416 + > ok 4 # skip passing test #4 (--run)
417 + > ok 5 - passing test #5
418 + > ok 6 - passing test #6
419 + > # passed all 6 test(s)
420 + > 1..6
421 + EOF
422 +"
423 +
424 +test_expect_success '--run with a left open range' "
425 + run_sub_test_lib_test run-left-open-range \
426 + '--run with a left open range' --run='-3' <<-\\EOF &&
427 + for i in 1 2 3 4 5 6
428 + do
429 + test_expect_success \"passing test #\$i\" 'true'
430 + done
431 + test_done
432 + EOF
433 + check_sub_test_lib_test run-left-open-range <<-\\EOF
434 + > ok 1 - passing test #1
435 + > ok 2 - passing test #2
436 + > ok 3 - passing test #3
437 + > ok 4 # skip passing test #4 (--run)
438 + > ok 5 # skip passing test #5 (--run)
439 + > ok 6 # skip passing test #6 (--run)
440 + > # passed all 6 test(s)
441 + > 1..6
442 + EOF
443 +"
444 +
445 +test_expect_success '--run with a right open range' "
446 + run_sub_test_lib_test run-right-open-range \
447 + '--run with a right open range' --run='4-' <<-\\EOF &&
448 + for i in 1 2 3 4 5 6
449 + do
450 + test_expect_success \"passing test #\$i\" 'true'
451 + done
452 + test_done
453 + EOF
454 + check_sub_test_lib_test run-right-open-range <<-\\EOF
455 + > ok 1 # skip passing test #1 (--run)
456 + > ok 2 # skip passing test #2 (--run)
457 + > ok 3 # skip passing test #3 (--run)
458 + > ok 4 - passing test #4
459 + > ok 5 - passing test #5
460 + > ok 6 - passing test #6
461 + > # passed all 6 test(s)
462 + > 1..6
463 + EOF
464 +"
465 +
466 +test_expect_success '--run with basic negation' "
467 + run_sub_test_lib_test run-basic-neg \
468 + '--run with basic negation' --run='"'!3'"' <<-\\EOF &&
469 + for i in 1 2 3 4 5 6
470 + do
471 + test_expect_success \"passing test #\$i\" 'true'
472 + done
473 + test_done
474 + EOF
475 + check_sub_test_lib_test run-basic-neg <<-\\EOF
476 + > ok 1 - passing test #1
477 + > ok 2 - passing test #2
478 + > ok 3 # skip passing test #3 (--run)
479 + > ok 4 - passing test #4
480 + > ok 5 - passing test #5
481 + > ok 6 - passing test #6
482 + > # passed all 6 test(s)
483 + > 1..6
484 + EOF
485 +"
486 +
487 +test_expect_success '--run with two negations' "
488 + run_sub_test_lib_test run-two-neg \
489 + '--run with two negations' --run='"'!3 !6'"' <<-\\EOF &&
490 + for i in 1 2 3 4 5 6
491 + do
492 + test_expect_success \"passing test #\$i\" 'true'
493 + done
494 + test_done
495 + EOF
496 + check_sub_test_lib_test run-two-neg <<-\\EOF
497 + > ok 1 - passing test #1
498 + > ok 2 - passing test #2
499 + > ok 3 # skip passing test #3 (--run)
500 + > ok 4 - passing test #4
501 + > ok 5 - passing test #5
502 + > ok 6 # skip passing test #6 (--run)
503 + > # passed all 6 test(s)
504 + > 1..6
505 + EOF
506 +"
507 +
508 +test_expect_success '--run a range and negation' "
509 + run_sub_test_lib_test run-range-and-neg \
510 + '--run a range and negation' --run='"'-4 !2'"' <<-\\EOF &&
511 + for i in 1 2 3 4 5 6
512 + do
513 + test_expect_success \"passing test #\$i\" 'true'
514 + done
515 + test_done
516 + EOF
517 + check_sub_test_lib_test run-range-and-neg <<-\\EOF
518 + > ok 1 - passing test #1
519 + > ok 2 # skip passing test #2 (--run)
520 + > ok 3 - passing test #3
521 + > ok 4 - passing test #4
522 + > ok 5 # skip passing test #5 (--run)
523 + > ok 6 # skip passing test #6 (--run)
524 + > # passed all 6 test(s)
525 + > 1..6
526 + EOF
527 +"
528 +
529 +test_expect_success '--run range negation' "
530 + run_sub_test_lib_test run-range-neg \
531 + '--run range negation' --run='"'!1-3'"' <<-\\EOF &&
532 + for i in 1 2 3 4 5 6
533 + do
534 + test_expect_success \"passing test #\$i\" 'true'
535 + done
536 + test_done
537 + EOF
538 + check_sub_test_lib_test run-range-neg <<-\\EOF
539 + > ok 1 # skip passing test #1 (--run)
540 + > ok 2 # skip passing test #2 (--run)
541 + > ok 3 # skip passing test #3 (--run)
542 + > ok 4 - passing test #4
543 + > ok 5 - passing test #5
544 + > ok 6 - passing test #6
545 + > # passed all 6 test(s)
546 + > 1..6
547 + EOF
548 +"
549 +
550 +test_expect_success '--run include, exclude and include' "
551 + run_sub_test_lib_test run-inc-neg-inc \
552 + '--run include, exclude and include' \
553 + --run='"'1-5 !1-3 2'"' <<-\\EOF &&
554 + for i in 1 2 3 4 5 6
555 + do
556 + test_expect_success \"passing test #\$i\" 'true'
557 + done
558 + test_done
559 + EOF
560 + check_sub_test_lib_test run-inc-neg-inc <<-\\EOF
561 + > ok 1 # skip passing test #1 (--run)
562 + > ok 2 - passing test #2
563 + > ok 3 # skip passing test #3 (--run)
564 + > ok 4 - passing test #4
565 + > ok 5 - passing test #5
566 + > ok 6 # skip passing test #6 (--run)
567 + > # passed all 6 test(s)
568 + > 1..6
569 + EOF
570 +"
571 +
572 +test_expect_success '--run include, exclude and include, comma separated' "
573 + run_sub_test_lib_test run-inc-neg-inc-comma \
574 + '--run include, exclude and include, comma separated' \
575 + --run=1-5,\!1-3,2 <<-\\EOF &&
576 + for i in 1 2 3 4 5 6
577 + do
578 + test_expect_success \"passing test #\$i\" 'true'
579 + done
580 + test_done
581 + EOF
582 + check_sub_test_lib_test run-inc-neg-inc-comma <<-\\EOF
583 + > ok 1 # skip passing test #1 (--run)
584 + > ok 2 - passing test #2
585 + > ok 3 # skip passing test #3 (--run)
586 + > ok 4 - passing test #4
587 + > ok 5 - passing test #5
588 + > ok 6 # skip passing test #6 (--run)
589 + > # passed all 6 test(s)
590 + > 1..6
591 + EOF
592 +"
593 +
594 +test_expect_success '--run exclude and include' "
595 + run_sub_test_lib_test run-neg-inc \
596 + '--run exclude and include' \
597 + --run='"'!3- 5'"' <<-\\EOF &&
598 + for i in 1 2 3 4 5 6
599 + do
600 + test_expect_success \"passing test #\$i\" 'true'
601 + done
602 + test_done
603 + EOF
604 + check_sub_test_lib_test run-neg-inc <<-\\EOF
605 + > ok 1 - passing test #1
606 + > ok 2 - passing test #2
607 + > ok 3 # skip passing test #3 (--run)
608 + > ok 4 # skip passing test #4 (--run)
609 + > ok 5 - passing test #5
610 + > ok 6 # skip passing test #6 (--run)
611 + > # passed all 6 test(s)
612 + > 1..6
613 + EOF
614 +"
615 +
616 +test_expect_success '--run empty selectors' "
617 + run_sub_test_lib_test run-empty-sel \
618 + '--run empty selectors' \
619 + --run='1,,3,,,5' <<-\\EOF &&
620 + for i in 1 2 3 4 5 6
621 + do
622 + test_expect_success \"passing test #\$i\" 'true'
623 + done
624 + test_done
625 + EOF
626 + check_sub_test_lib_test run-empty-sel <<-\\EOF
627 + > ok 1 - passing test #1
628 + > ok 2 # skip passing test #2 (--run)
629 + > ok 3 - passing test #3
630 + > ok 4 # skip passing test #4 (--run)
631 + > ok 5 - passing test #5
632 + > ok 6 # skip passing test #6 (--run)
633 + > # passed all 6 test(s)
634 + > 1..6
635 + EOF
636 +"
637 +
638 +test_expect_success '--run invalid range start' "
639 + run_sub_test_lib_test_err run-inv-range-start \
640 + '--run invalid range start' \
641 + --run='a-5' <<-\\EOF &&
642 + test_expect_success \"passing test #1\" 'true'
643 + test_done
644 + EOF
645 + check_sub_test_lib_test_err run-inv-range-start \
646 + <<-\\EOF_OUT 3<<-\\EOF_ERR
647 + > FATAL: Unexpected exit with code 1
648 + EOF_OUT
649 + > error: --run: invalid non-numeric in range start: 'a-5'
650 + EOF_ERR
651 +"
652 +
653 +test_expect_success '--run invalid range end' "
654 + run_sub_test_lib_test_err run-inv-range-end \
655 + '--run invalid range end' \
656 + --run='1-z' <<-\\EOF &&
657 + test_expect_success \"passing test #1\" 'true'
658 + test_done
659 + EOF
660 + check_sub_test_lib_test_err run-inv-range-end \
661 + <<-\\EOF_OUT 3<<-\\EOF_ERR
662 + > FATAL: Unexpected exit with code 1
663 + EOF_OUT
664 + > error: --run: invalid non-numeric in range end: '1-z'
665 + EOF_ERR
666 +"
667 +
668 +test_expect_success '--run invalid selector' "
669 + run_sub_test_lib_test_err run-inv-selector \
670 + '--run invalid selector' \
671 + --run='1?' <<-\\EOF &&
672 + test_expect_success \"passing test #1\" 'true'
673 + test_done
674 + EOF
675 + check_sub_test_lib_test_err run-inv-selector \
676 + <<-\\EOF_OUT 3<<-\\EOF_ERR
677 + > FATAL: Unexpected exit with code 1
678 + EOF_OUT
679 + > error: --run: invalid non-numeric in test selector: '1?'
680 + EOF_ERR
681 +"
682 +
683 +
684 test_set_prereq HAVEIT
685 haveit=no
686 test_expect_success HAVEIT 'test runs if prerequisite is satisfied' '
t/test-lib.sh
+108
@@ -191,6 +191,14 @@ do
191 immediate=t; shift ;;
192 -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
193 GIT_TEST_LONG=t; export GIT_TEST_LONG; shift ;;
194 + -r)
195 + shift; test "$#" -ne 0 || {
196 + echo 'error: -r requires an argument' >&2;
197 + exit 1;
198 + }
199 + run_list=$1; shift ;;
200 + --run=*)
201 + run_list=$(expr "z$1" : 'z[^=]*=\(.*\)'); shift ;;
202 -h|--h|--he|--hel|--help)
203 help=t; shift ;;
204 -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
@@ -366,6 +374,99 @@ match_pattern_list () {
374 return 1
375 }
376
377 +match_test_selector_list () {
378 + title="$1"
379 + shift
380 + arg="$1"
381 + shift
382 + test -z "$1" && return 0
383 +
384 + # Both commas and whitespace are accepted as separators.
385 + OLDIFS=$IFS
386 + IFS=' ,'
387 + set -- $1
388 + IFS=$OLDIFS
389 +
390 + # If the first selector is negative we include by default.
391 + include=
392 + case "$1" in
393 + !*) include=t ;;
394 + esac
395 +
396 + for selector
397 + do
398 + orig_selector=$selector
399 +
400 + positive=t
401 + case "$selector" in
402 + !*)
403 + positive=
404 + selector=${selector##?}
405 + ;;
406 + esac
407 +
408 + test -z "$selector" && continue
409 +
410 + case "$selector" in
411 + *-*)
412 + if expr "z${selector%%-*}" : "z[0-9]*[^0-9]" >/dev/null
413 + then
414 + echo "error: $title: invalid non-numeric in range" \
415 + "start: '$orig_selector'" >&2
416 + exit 1
417 + fi
418 + if expr "z${selector#*-}" : "z[0-9]*[^0-9]" >/dev/null
419 + then
420 + echo "error: $title: invalid non-numeric in range" \
421 + "end: '$orig_selector'" >&2
422 + exit 1
423 + fi
424 + ;;
425 + *)
426 + if expr "z$selector" : "z[0-9]*[^0-9]" >/dev/null
427 + then
428 + echo "error: $title: invalid non-numeric in test" \
429 + "selector: '$orig_selector'" >&2
430 + exit 1
431 + fi
432 + esac
433 +
434 + # Short cut for "obvious" cases
435 + test -z "$include" && test -z "$positive" && continue
436 + test -n "$include" && test -n "$positive" && continue
437 +
438 + case "$selector" in
439 + -*)
440 + if test $arg -le ${selector#-}
441 + then
442 + include=$positive
443 + fi
444 + ;;
445 + *-)
446 + if test $arg -ge ${selector%-}
447 + then
448 + include=$positive
449 + fi
450 + ;;
451 + *-*)
452 + if test ${selector%%-*} -le $arg \
453 + && test $arg -le ${selector#*-}
454 + then
455 + include=$positive
456 + fi
457 + ;;
458 + *)
459 + if test $arg -eq $selector
460 + then
461 + include=$positive
462 + fi
463 + ;;
464 + esac
465 + done
466 +
467 + test -n "$include"
468 +}
469 +
470 maybe_teardown_verbose () {
471 test -z "$verbose_only" && return
472 exec 4>/dev/null 3>/dev/null
@@ -470,6 +571,13 @@ test_skip () {
571 fi
572 skipped_reason="missing $missing_prereq${of_prereq}"
573 fi
574 + if test -z "$to_skip" && test -n "$run_list" &&
575 + ! match_test_selector_list '--run' $test_count "$run_list"
576 + then
577 + to_skip=t
578 + skipped_reason="--run"
579 + fi
580 +
581 case "$to_skip" in
582 t)
583 say_color skip >&3 "skipping test: $@"