tests: test the test framework more thoroughly
Add 5 new full test suite runs each with a different number of passing/failing/broken/fixed tests, in order to ensure that the correct exit code and output are generated in each case. As before, these are run in a subdirectory to avoid disrupting the metrics for the parent tests. Signed-off-by: Adam Spiers <git@adamspiers.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Adam Spiers committed
Dec 16, 2012 at 18:28 UTC
5ebf89e88637fa641ced15edc10be0cf8d5dcbc8
1 file changed
+105
t/t0000-basic.sh
+105
@@ -89,6 +89,56 @@ check_sub_test_lib_test () {
89
)
90
}
91
92
+test_expect_success 'pretend we have a fully passing test suite' "
93
+ run_sub_test_lib_test full-pass '3 passing tests' <<-\\EOF &&
94
+ for i in 1 2 3
95
+ do
96
+ test_expect_success \"passing test #\$i\" 'true'
97
+ done
98
+ test_done
99
+ EOF
100
+ check_sub_test_lib_test full-pass <<-\\EOF
101
+ > ok 1 - passing test #1
102
+ > ok 2 - passing test #2
103
+ > ok 3 - passing test #3
104
+ > # passed all 3 test(s)
105
+ > 1..3
106
+ EOF
107
+"
108
+
109
+test_expect_success 'pretend we have a partially passing test suite' "
110
+ test_must_fail run_sub_test_lib_test \
111
+ partial-pass '2/3 tests passing' <<-\\EOF &&
112
+ test_expect_success 'passing test #1' 'true'
113
+ test_expect_success 'failing test #2' 'false'
114
+ test_expect_success 'passing test #3' 'true'
115
+ test_done
116
+ EOF
117
+ check_sub_test_lib_test partial-pass <<-\\EOF
118
+ > ok 1 - passing test #1
119
+ > not ok 2 - failing test #2
120
+ # false
121
+ > ok 3 - passing test #3
122
+ > # failed 1 among 3 test(s)
123
+ > 1..3
124
+ EOF
125
+"
126
+
127
+test_expect_success 'pretend we have a known breakage' "
128
+ run_sub_test_lib_test failing-todo 'A failing TODO test' <<-\\EOF &&
129
+ test_expect_success 'passing test' 'true'
130
+ test_expect_failure 'pretend we have a known breakage' 'false'
131
+ test_done
132
+ EOF
133
+ check_sub_test_lib_test failing-todo <<-\\EOF
134
+ > ok 1 - passing test
135
+ > not ok 2 - pretend we have a known breakage # TODO known breakage
136
+ > # still have 1 known breakage(s)
137
+ > # passed all remaining 1 test(s)
138
+ > 1..2
139
+ EOF
140
+"
141
+
142
test_expect_success 'pretend we have fixed a known breakage' "
143
run_sub_test_lib_test passing-todo 'A passing TODO test' <<-\\EOF &&
144
test_expect_failure 'pretend we have fixed a known breakage' 'true'
@@ -102,6 +152,61 @@ test_expect_success 'pretend we have fixed a known breakage' "
152
EOF
153
"
154
155
+test_expect_success 'pretend we have a pass, fail, and known breakage' "
156
+ test_must_fail run_sub_test_lib_test \
157
+ mixed-results1 'mixed results #1' <<-\\EOF &&
158
+ test_expect_success 'passing test' 'true'
159
+ test_expect_success 'failing test' 'false'
160
+ test_expect_failure 'pretend we have a known breakage' 'false'
161
+ test_done
162
+ EOF
163
+ check_sub_test_lib_test mixed-results1 <<-\\EOF
164
+ > ok 1 - passing test
165
+ > not ok 2 - failing test
166
+ > # false
167
+ > not ok 3 - pretend we have a known breakage # TODO known breakage
168
+ > # still have 1 known breakage(s)
169
+ > # failed 1 among remaining 2 test(s)
170
+ > 1..3
171
+ EOF
172
+"
173
+
174
+test_expect_success 'pretend we have a mix of all possible results' "
175
+ test_must_fail run_sub_test_lib_test \
176
+ mixed-results2 'mixed results #2' <<-\\EOF &&
177
+ test_expect_success 'passing test' 'true'
178
+ test_expect_success 'passing test' 'true'
179
+ test_expect_success 'passing test' 'true'
180
+ test_expect_success 'passing test' 'true'
181
+ test_expect_success 'failing test' 'false'
182
+ test_expect_success 'failing test' 'false'
183
+ test_expect_success 'failing test' 'false'
184
+ test_expect_failure 'pretend we have a known breakage' 'false'
185
+ test_expect_failure 'pretend we have a known breakage' 'false'
186
+ test_expect_failure 'pretend we have fixed a known breakage' 'true'
187
+ test_done
188
+ EOF
189
+ check_sub_test_lib_test mixed-results2 <<-\\EOF
190
+ > ok 1 - passing test
191
+ > ok 2 - passing test
192
+ > ok 3 - passing test
193
+ > ok 4 - passing test
194
+ > not ok 5 - failing test
195
+ > # false
196
+ > not ok 6 - failing test
197
+ > # false
198
+ > not ok 7 - failing test
199
+ > # false
200
+ > not ok 8 - pretend we have a known breakage # TODO known breakage
201
+ > not ok 9 - pretend we have a known breakage # TODO known breakage
202
+ > ok 10 - pretend we have fixed a known breakage # TODO known breakage
203
+ > # fixed 1 known breakage(s)
204
+ > # still have 2 known breakage(s)
205
+ > # failed 3 among remaining 8 test(s)
206
+ > 1..10
207
+ EOF
208
+"
209
+
210
test_set_prereq HAVEIT
211
haveit=no
212
test_expect_success HAVEIT 'test runs if prerequisite is satisfied' '