CI: try to make caching more reliable (#25259)
- `~/.yarn/cache` is now restored from an hierarchical cache key, if no precise match is found, we fallback to less precise ones. - The yarn install in `fixtures/dom` is also cached. Notably, is utilizes the cache from root, but stores into its more precise key. - Steps running in root no longer have a `yarn install` and rely on the cache from the setup step. - Retry `yarn install` once on failure.
Jan Kassens committed
Sep 14, 2022 at 13:22 UTC
c327b91d22d95de2d43c899c3d8eb87a3ffca048
1 file changed
+90
-64
.circleci/config.yml
+90
-64
@@ -10,18 +10,93 @@ aliases:
10
- &restore_yarn_cache
11
restore_cache:
12
name: Restore yarn cache
13
- key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
13
+ keys:
14
+ - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
15
+ - v1-yarn_cache-{{ arch }}-
16
+ - v1-yarn_cache-
17
+
18
+ - &yarn_install
19
+ run:
20
+ name: Install dependencies
21
+ command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
22
+
23
+ - &yarn_install_retry
24
+ run:
25
+ name: Install dependencies (retry)
26
+ when: on_fail
27
+ command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
28
+
29
+ - &save_yarn_cache
30
+ save_cache:
31
+ name: Save yarn cache
32
+ key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
33
+ paths:
34
+ - ~/.cache/yarn
35
+
36
+ - &restore_yarn_cache_fixtures_dom
37
+ restore_cache:
38
+ name: Restore yarn cache for fixtures/dom
39
+ keys:
40
+ - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
41
+ - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
42
+ - v1-yarn_cache-{{ arch }}-
43
+ - v1-yarn_cache-
44
+
45
+ - &yarn_install_fixtures_dom
46
+ run:
47
+ name: Install dependencies in fixtures/dom
48
+ working_directory: fixtures/dom
49
+ command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
50
15
- - &prepare_node_modules_cache_key
51
+ - &yarn_install_fixtures_dom_retry
52
run:
17
- name: Preparing node_modules cache key
18
- command: yarn workspaces info | head -n -1 > workspace_info.txt
53
+ name: Install dependencies in fixtures/dom (retry)
54
+ when: on_fail
55
+ working_directory: fixtures/dom
56
+ command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
57
+
58
+ - &save_yarn_cache_fixtures_dom
59
+ save_cache:
60
+ name: Save yarn cache for fixtures/dom
61
+ key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
62
+ paths:
63
+ - ~/.cache/yarn
64
+
65
+ - &save_node_modules
66
+ save_cache:
67
+ name: Save node_modules cache
68
+ # Cache only for the current revision to prevent cache injections from
69
+ # malicious PRs.
70
+ key: v1-node_modules-{{ arch }}-{{ .Revision }}
71
+ paths:
72
+ - node_modules
73
+ - packages/eslint-plugin-react-hooks/node_modules
74
+ - packages/react-art/node_modules
75
+ - packages/react-client/node_modules
76
+ - packages/react-devtools-core/node_modules
77
+ - packages/react-devtools-extensions/node_modules
78
+ - packages/react-devtools-inline/node_modules
79
+ - packages/react-devtools-shared/node_modules
80
+ - packages/react-devtools-shell/node_modules
81
+ - packages/react-devtools-timeline/node_modules
82
+ - packages/react-devtools/node_modules
83
+ - packages/react-dom/node_modules
84
+ - packages/react-interactions/node_modules
85
+ - packages/react-native-renderer/node_modules
86
+ - packages/react-reconciler/node_modules
87
+ - packages/react-server-dom-relay/node_modules
88
+ - packages/react-server-dom-webpack/node_modules
89
+ - packages/react-server-native-relay/node_modules
90
+ - packages/react-server/node_modules
91
+ - packages/react-test-renderer/node_modules
92
+ - packages/react/node_modules
93
+ - packages/scheduler/node_modules
94
95
- &restore_node_modules
96
restore_cache:
97
name: Restore node_modules cache
98
keys:
24
- - v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
99
+ - v1-node_modules-{{ arch }}-{{ .Revision }}
100
101
- &TEST_PARALLELISM 20
102
@@ -49,27 +124,11 @@ jobs:
124
name: Nodejs Version
125
command: node --version
126
- *restore_yarn_cache
52
- - run:
53
- name: Install Packages
54
- command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
55
- - run: yarn workspaces info | head -n -1 > workspace_info.txt
56
- - save_cache:
57
- # Store the yarn cache globally for all lock files with this same
58
- # checksum. This will speed up the setup job for all PRs where the
59
- # lockfile is the same.
60
- name: Save yarn cache for future installs
61
- key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
62
- paths:
63
- - ~/.cache/yarn
64
- - save_cache:
65
- # Store node_modules for all jobs in this workflow so that they don't
66
- # need to each run a yarn install for each job. This will speed up
67
- # all jobs run on this branch with the same lockfile.
68
- name: Save node_modules cache
69
- # This cache key is per branch, a yarn install in setup is required.
70
- key: v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
71
- paths:
72
- - node_modules
127
+ - *restore_node_modules
128
+ - *yarn_install
129
+ - *yarn_install_retry
130
+ - *save_yarn_cache
131
+ - *save_node_modules
132
133
yarn_lint:
134
docker: *docker
@@ -77,7 +136,6 @@ jobs:
136
137
steps:
138
- checkout
80
- - *prepare_node_modules_cache_key
139
- *restore_node_modules
140
- run: node ./scripts/prettier/index
141
- run: node ./scripts/tasks/eslint
@@ -92,7 +150,6 @@ jobs:
150
151
steps:
152
- checkout
95
- - *prepare_node_modules_cache_key
153
- *restore_node_modules
154
- run: node ./scripts/tasks/flow-ci
155
@@ -102,7 +159,6 @@ jobs:
159
160
steps:
161
- checkout
105
- - *prepare_node_modules_cache_key
162
- *restore_node_modules
163
- run:
164
command: |
@@ -119,7 +175,6 @@ jobs:
175
parallelism: 40
176
steps:
177
- checkout
122
- - *prepare_node_modules_cache_key
178
- *restore_node_modules
179
- run: yarn build-combined
180
- persist_to_workspace:
@@ -135,7 +190,6 @@ jobs:
190
type: string
191
steps:
192
- checkout
138
- - *prepare_node_modules_cache_key
193
- *restore_node_modules
194
- run:
195
name: Download artifacts for revision
@@ -153,7 +207,6 @@ jobs:
207
environment: *environment
208
steps:
209
- checkout
156
- - *prepare_node_modules_cache_key
210
- *restore_node_modules
211
- run:
212
name: Download artifacts for base revision
@@ -182,7 +235,6 @@ jobs:
235
- checkout
236
- attach_workspace:
237
at: .
185
- - *prepare_node_modules_cache_key
238
- *restore_node_modules
239
- run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
240
# Compress build directory into a single tarball for easy download
@@ -202,7 +254,6 @@ jobs:
254
- attach_workspace:
255
at: .
256
- run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
205
- - *prepare_node_modules_cache_key
257
- *restore_node_modules
258
- run:
259
command: node ./scripts/tasks/danger
@@ -214,11 +265,7 @@ jobs:
265
- checkout
266
- attach_workspace:
267
at: .
217
- - *prepare_node_modules_cache_key
268
- *restore_node_modules
219
- - run:
220
- name: Install Packages
221
- command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
269
- run:
270
environment:
271
RELEASE_CHANNEL: experimental
@@ -233,11 +280,7 @@ jobs:
280
- checkout
281
- attach_workspace:
282
at: .
236
- - *prepare_node_modules_cache_key
283
- *restore_node_modules
238
- - run:
239
- name: Install Packages
240
- command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
284
- run:
285
name: Playwright install deps
286
command: |
@@ -259,11 +302,7 @@ jobs:
302
- checkout
303
- attach_workspace:
304
at: .
262
- - *prepare_node_modules_cache_key
305
- *restore_node_modules
264
- - run:
265
- name: Install nested packages from Yarn cache
266
- command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
306
- run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild
307
- run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci
308
@@ -278,11 +317,7 @@ jobs:
317
- checkout
318
- attach_workspace:
319
at: .
281
- - *prepare_node_modules_cache_key
320
- *restore_node_modules
283
- - run:
284
- name: Install nested packages from Yarn cache
285
- command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
321
- run:
322
name: Playwright install deps
323
command: |
@@ -306,7 +341,6 @@ jobs:
341
- checkout
342
- attach_workspace:
343
at: .
309
- - *prepare_node_modules_cache_key
344
- *restore_node_modules
345
- run: yarn lint-build
346
@@ -317,7 +351,6 @@ jobs:
351
- checkout
352
- attach_workspace:
353
at: .
320
- - *prepare_node_modules_cache_key
354
- *restore_node_modules
355
- run: yarn check-release-dependencies
356
@@ -328,7 +361,6 @@ jobs:
361
steps:
362
- checkout
363
- attach_workspace: *attach_workspace
331
- - *prepare_node_modules_cache_key
364
- *restore_node_modules
365
- run:
366
name: Search build artifacts for unminified errors
@@ -345,7 +377,6 @@ jobs:
377
type: string
378
steps:
379
- checkout
348
- - *prepare_node_modules_cache_key
380
- *restore_node_modules
381
- run: yarn test <<parameters.args>> --ci
382
@@ -360,11 +391,7 @@ jobs:
391
- checkout
392
- attach_workspace:
393
at: .
363
- - *prepare_node_modules_cache_key
394
- *restore_node_modules
365
- - run:
366
- name: Install nested packages from Yarn cache
367
- command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
395
- run: yarn test --build <<parameters.args>> --ci
396
397
RELEASE_CHANNEL_stable_yarn_test_dom_fixtures:
@@ -374,15 +401,17 @@ jobs:
401
- checkout
402
- attach_workspace:
403
at: .
377
- - *prepare_node_modules_cache_key
404
- *restore_node_modules
405
+ - *restore_yarn_cache_fixtures_dom
406
+ - *yarn_install_fixtures_dom
407
+ - *yarn_install_fixtures_dom_retry
408
+ - *save_yarn_cache_fixtures_dom
409
- run:
410
name: Run DOM fixture tests
411
environment:
412
RELEASE_CHANNEL: stable
413
+ working_directory: fixtures/dom
414
command: |
384
- cd fixtures/dom
385
- yarn --frozen-lockfile
415
yarn prestart
416
yarn test --maxWorkers=2
417
@@ -391,7 +420,6 @@ jobs:
420
environment: *environment
421
steps:
422
- checkout
394
- - *prepare_node_modules_cache_key
423
- *restore_node_modules
424
- run:
425
name: Run fuzz tests
@@ -411,7 +439,6 @@ jobs:
439
environment: *environment
440
steps:
441
- checkout
414
- - *prepare_node_modules_cache_key
442
- *restore_node_modules
443
- run:
444
name: Run publish script
@@ -430,7 +457,6 @@ jobs:
457
environment: *environment
458
steps:
459
- checkout
433
- - *prepare_node_modules_cache_key
460
- *restore_node_modules
461
- run:
462
name: Fetch revisions that contain an intentional fork