[snap tester] Handle unexpected errors by failing fixture
Mofei Zhang committed
May 3, 2023 at 14:50 UTC
7e990419c1314252ba78d2b0fa14c8dddaab72d4
2 files changed
+51
-12
compiler/forget/packages/snap/src/compiler-worker.ts
+30
-8
@@ -23,6 +23,7 @@ export type TestResult = {
23
outputPath: string;
24
actual: string | null; // null == input did not exist
25
expected: string | null; // null == output did not exist
26
+ unexpectedError: string | null;
27
};
28
29
export async function compile(
@@ -56,6 +57,7 @@ export async function compile(
57
outputPath,
58
actual: null,
59
expected,
60
+ unexpectedError: null,
61
};
62
}
63
@@ -127,10 +129,13 @@ export async function compile(
129
const expectError = fixture.startsWith("error.");
130
if (expectError) {
131
if (error === null) {
130
- console.log(code);
131
- throw new Error(
132
- `Expected an error to be thrown for fixture: '${fixture}', remove the 'error.' prefix if an error is not expected.`
133
- );
132
+ return {
133
+ inputPath,
134
+ outputPath,
135
+ actual: code,
136
+ expected,
137
+ unexpectedError: `Expected an error to be thrown for fixture: '${fixture}', remove the 'error.' prefix if an error is not expected.`,
138
+ };
139
} else if (code != null) {
140
output = `${formatOutput(code)}\n${formatErrorOutput(error)}`;
141
} else {
@@ -138,11 +143,22 @@ export async function compile(
143
}
144
} else {
145
if (error !== null) {
141
- error.message = `Expected fixture '${fixture}' to succeed but it failed with error:\n\n${error.message}`;
142
- throw error;
146
+ return {
147
+ inputPath,
148
+ outputPath,
149
+ actual: code,
150
+ expected,
151
+ unexpectedError: `Expected fixture '${fixture}' to succeed but it failed with error:\n\n${error.message}`,
152
+ };
153
}
154
if (code == null || code.length === 0) {
145
- throw new Error(`Expected output for fixture '${fixture}'.`);
155
+ return {
156
+ inputPath,
157
+ outputPath,
158
+ actual: code,
159
+ expected,
160
+ unexpectedError: `Expected output for fixture '${fixture}'.`,
161
+ };
162
}
163
output = formatOutput(code);
164
}
@@ -158,7 +174,13 @@ ${output}
174
175
console.error = originalConsoleError;
176
161
- return { inputPath, outputPath, actual, expected };
177
+ return {
178
+ inputPath,
179
+ outputPath,
180
+ actual,
181
+ expected,
182
+ unexpectedError: null,
183
+ };
184
}
185
186
function formatErrorOutput(error: Error): string {
compiler/forget/packages/snap/src/runner.ts
+21
-4
@@ -181,7 +181,7 @@ async function run(
181
function report(results: Results): void {
182
const failures: Array<[string, TestResult]> = [];
183
for (const [basename, result] of results) {
184
- if (result.actual === result.expected) {
184
+ if (result.actual === result.expected && result.unexpectedError == null) {
185
console.log(
186
chalk.green.inverse.bold(" PASS ") + " " + chalk.dim(basename)
187
);
@@ -196,7 +196,13 @@ function report(results: Results): void {
196
197
for (const [basename, result] of failures) {
198
console.log(chalk.red.bold("FAIL:") + " " + basename);
199
- console.log(diff(result.actual, result.expected) + "\n");
199
+ if (result.unexpectedError != null) {
200
+ console.log(
201
+ ` >> Unexpected error during test: \n${result.unexpectedError}`
202
+ );
203
+ } else {
204
+ console.log(diff(result.actual, result.expected) + "\n");
205
+ }
206
}
207
}
208
@@ -214,8 +220,14 @@ async function update(results: Results): Promise<void> {
220
let deleted = 0;
221
let updated = 0;
222
let created = 0;
223
+ const failed = [];
224
for (const [basename, result] of results) {
218
- if (result.actual == null) {
225
+ if (result.unexpectedError != null) {
226
+ console.log(
227
+ chalk.red.inverse.bold(" FAILED ") + " " + chalk.dim(basename)
228
+ );
229
+ failed.push([basename, result.unexpectedError]);
230
+ } else if (result.actual == null) {
231
// Input was deleted but the expect file still existed, remove it
232
console.log(
233
chalk.red.inverse.bold(" REMOVE ") + " " + chalk.dim(basename)
@@ -243,7 +255,12 @@ async function update(results: Results): Promise<void> {
255
);
256
}
257
}
246
- console.log(`${deleted} Deleted, ${created} Created, ${updated} Updated`);
258
+ console.log(
259
+ `${deleted} Deleted, ${created} Created, ${updated} Updated, ${failed.length} Failed`
260
+ );
261
+ for (const [basename, errorMsg] of failed) {
262
+ console.log(`${chalk.red.bold("Fail:")} ${basename}\n${errorMsg}`);
263
+ }
264
}
265
266
function watchSrc(