Add snapshot testing on e2e test failure (#24672)
We have a currently unreproducible flaky e2e test. This PR captures snapshots on e2e test failures so we can better debug flaky e2e tests that don't fail locally.
Luna Ruan committed
Jun 6, 2022 at 10:36 UTC
254b49e5897b4e493c88ecec5c80abeacda90526
4 files changed
+9
-4
.circleci/config.yml
+2
@@ -291,6 +291,8 @@ jobs:
291
- run:
292
name: Cleanup build regression folder
293
command: rm -r ./build-regression
294
+ - store_artifacts:
295
+ path: ./tmp/screenshots
296
297
yarn_lint_build:
298
docker: *docker
packages/react-devtools-inline/__tests__/__e2e__/components.test.js
-1
@@ -210,7 +210,6 @@ test.describe('Components', () => {
210
let count = await getComponentSearchResultsCount();
211
expect(count).toBe('1 | 4');
212
213
- await focusComponentSearch();
213
page.keyboard.insertText('Item');
214
count = await getComponentSearchResultsCount();
215
expect(count).toBe('1 | 3');
packages/react-devtools-inline/playwright.config.js
+2
-1
@@ -18,9 +18,10 @@ const config = {
18
react_version: process.env.REACT_VERSION
19
? semver.coerce(process.env.REACT_VERSION).version
20
: reactVersion,
21
+ trace: 'retain-on-failure',
22
},
23
// Some of our e2e tests can be flaky. Retry tests to make sure the error isn't transient
23
- retries: 2,
24
+ retries: 3,
25
};
26
27
module.exports = config;
scripts/circleci/run_devtools_e2e_tests.js
+5
-2
@@ -9,6 +9,7 @@ const ROOT_PATH = join(__dirname, '..', '..');
9
const reactVersion = process.argv[2];
10
const inlinePackagePath = join(ROOT_PATH, 'packages', 'react-devtools-inline');
11
const shellPackagePath = join(ROOT_PATH, 'packages', 'react-devtools-shell');
12
+const screenshotPath = join(ROOT_PATH, 'tmp', 'screenshots');
13
14
let buildProcess = null;
15
let serverProcess = null;
@@ -115,9 +116,11 @@ function runTestShell() {
116
async function runEndToEndTests() {
117
logBright('Running e2e tests');
118
if (!reactVersion) {
118
- testProcess = spawn('yarn', ['test:e2e'], {cwd: inlinePackagePath});
119
+ testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
120
+ cwd: inlinePackagePath,
121
+ });
122
} else {
120
- testProcess = spawn('yarn', ['test:e2e'], {
123
+ testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
124
cwd: inlinePackagePath,
125
env: {...process.env, REACT_VERSION: reactVersion},
126
});