@samitouri / QOS-React-2 / commits / 1445acf777

[CI] cache yarn instead of node_modules (#25834)

The current caching of steps of `node_modules` doesn't work reliable as is because it includes `arch` in the cache key. `arch` might be different across workers in the same commit. I couldn't find a way to optionally restore caches, so what this PR does is: - remove the setup step that ran before all other steps and essentially just populates a circle CI cache key - all other steps now do: restore yarn cache, `yarn install`, save yarn cache (fast if already exists) With this change the initial batch of jobs all race to populate the cache, but any subsequent jobs should find the existing cache. The expected downside would be slightly more worker CPU time with all the parallel jobs, but wall time might be shorter (1 step less in the critical path) and we should be more reliable as we no longer have the failure with multiple archs. ## Alternative 1 Remove the `{arch}` from the cache key. Downside: this might run into weird issues with native dependencies. ## Alternative 2 Somehow check if the cache was restored and only then run a yarn install. Downside: couldn't figure out if it's possible to only restore the yarn cache if restoring the node_modules cache failed. Without that we'd either always restore both the yarn and node_modules cache or do yarn installs w/o cache which are prone to failure in the past.

Jan Kassens committed Feb 6, 2023 at 16:11 UTC 1445acf777a6e503e6c15e78dd4257abbd7a2264
1 file changed +67 -128
.circleci/config.yml
+67 -128
@@ -7,32 +7,6 @@ aliases:
7 - &environment
8 TZ: /usr/share/zoneinfo/America/Los_Angeles
9
10 - - &restore_yarn_cache
11 - restore_cache:
12 - name: Restore yarn cache
13 - keys:
14 - - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
15 - - v1-yarn_cache-{{ arch }}-
16 - - v1-yarn_cache-
17 -
18 - - &yarn_install
19 - run:
20 - name: Install dependencies
21 - command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
22 -
23 - - &yarn_install_retry
24 - run:
25 - name: Install dependencies (retry)
26 - when: on_fail
27 - command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
28 -
29 - - &save_yarn_cache
30 - save_cache:
31 - name: Save yarn cache
32 - key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
33 - paths:
34 - - ~/.cache/yarn
35 -
10 - &restore_yarn_cache_fixtures_dom
11 restore_cache:
12 name: Restore yarn cache for fixtures/dom
@@ -62,47 +36,34 @@ aliases:
36 paths:
37 - ~/.cache/yarn
38
65 - - &save_node_modules
66 - save_cache:
67 - name: Save node_modules cache
68 - # Cache only for the current revision to prevent cache injections from
69 - # malicious PRs.
70 - key: v1-node_modules-{{ arch }}-{{ .Revision }}
71 - paths:
72 - - node_modules
73 - - packages/eslint-plugin-react-hooks/node_modules
74 - - packages/react-art/node_modules
75 - - packages/react-client/node_modules
76 - - packages/react-devtools-core/node_modules
77 - - packages/react-devtools-extensions/node_modules
78 - - packages/react-devtools-inline/node_modules
79 - - packages/react-devtools-shared/node_modules
80 - - packages/react-devtools-shell/node_modules
81 - - packages/react-devtools-timeline/node_modules
82 - - packages/react-devtools/node_modules
83 - - packages/react-dom/node_modules
84 - - packages/react-interactions/node_modules
85 - - packages/react-native-renderer/node_modules
86 - - packages/react-reconciler/node_modules
87 - - packages/react-server-dom-relay/node_modules
88 - - packages/react-server-dom-webpack/node_modules
89 - - packages/react-server-native-relay/node_modules
90 - - packages/react-server/node_modules
91 - - packages/react-test-renderer/node_modules
92 - - packages/react/node_modules
93 - - packages/scheduler/node_modules
94 -
95 - - &restore_node_modules
96 - restore_cache:
97 - name: Restore node_modules cache
98 - keys:
99 - - v1-node_modules-{{ arch }}-{{ .Revision }}
100 -
39 - &TEST_PARALLELISM 20
40
41 - &attach_workspace
42 at: build
43
44 +commands:
45 + setup_node_modules:
46 + description: "Restore node_modules"
47 + steps:
48 + - restore_cache:
49 + name: Restore yarn cache
50 + keys:
51 + - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
52 + - v1-yarn_cache-{{ arch }}-
53 + - v1-yarn_cache-
54 + - run:
55 + name: Install dependencies
56 + command: |
57 + yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
58 + if [ $? -ne 0 ]; then
59 + yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
60 + fi
61 + - save_cache:
62 + name: Save yarn cache
63 + key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
64 + paths:
65 + - ~/.cache/yarn
66 +
67 # The CircleCI API doesn't yet support triggering a specific workflow, but it
68 # does support triggering a pipeline. So as a workaround you can triggger the
69 # entire pipeline and use parameters to disable everything except the workflow
@@ -115,28 +76,13 @@ parameters:
76 default: ''
77
78 jobs:
118 - setup:
119 - docker: *docker
120 - environment: *environment
121 - steps:
122 - - checkout
123 - - run:
124 - name: NodeJS Version
125 - command: node --version
126 - - *restore_yarn_cache
127 - - *restore_node_modules
128 - - *yarn_install
129 - - *yarn_install_retry
130 - - *save_yarn_cache
131 - - *save_node_modules
132 -
79 yarn_lint:
80 docker: *docker
81 environment: *environment
82
83 steps:
84 - checkout
139 - - *restore_node_modules
85 + - setup_node_modules
86 - run: node ./scripts/prettier/index
87 - run: node ./scripts/tasks/eslint
88 - run: ./scripts/circleci/check_license.sh
@@ -150,7 +96,7 @@ jobs:
96
97 steps:
98 - checkout
153 - - *restore_node_modules
99 + - setup_node_modules
100 - run: node ./scripts/tasks/flow-ci
101
102 scrape_warning_messages:
@@ -159,7 +105,7 @@ jobs:
105
106 steps:
107 - checkout
162 - - *restore_node_modules
108 + - setup_node_modules
109 - run:
110 command: |
111 mkdir -p ./build
@@ -175,7 +121,7 @@ jobs:
121 parallelism: 40
122 steps:
123 - checkout
178 - - *restore_node_modules
124 + - setup_node_modules
125 - run: yarn build-combined
126 - persist_to_workspace:
127 root: .
@@ -190,7 +136,7 @@ jobs:
136 type: string
137 steps:
138 - checkout
193 - - *restore_node_modules
139 + - setup_node_modules
140 - run:
141 name: Download artifacts for revision
142 command: |
@@ -207,7 +153,7 @@ jobs:
153 environment: *environment
154 steps:
155 - checkout
210 - - *restore_node_modules
156 + - setup_node_modules
157 - run:
158 name: Download artifacts for base revision
159 command: |
@@ -235,7 +181,7 @@ jobs:
181 - checkout
182 - attach_workspace:
183 at: .
238 - - *restore_node_modules
184 + - setup_node_modules
185 - run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
186 # Compress build directory into a single tarball for easy download
187 - run: tar -zcvf ./build.tgz ./build
@@ -254,7 +200,7 @@ jobs:
200 - attach_workspace:
201 at: .
202 - run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
257 - - *restore_node_modules
203 + - setup_node_modules
204 - run:
205 command: node ./scripts/tasks/danger
206
@@ -265,7 +211,7 @@ jobs:
211 - checkout
212 - attach_workspace:
213 at: .
268 - - *restore_node_modules
214 + - setup_node_modules
215 - run:
216 environment:
217 RELEASE_CHANNEL: experimental
@@ -280,7 +226,7 @@ jobs:
226 - checkout
227 - attach_workspace:
228 at: .
283 - - *restore_node_modules
229 + - setup_node_modules
230 - run:
231 name: Playwright install deps
232 command: |
@@ -302,7 +248,7 @@ jobs:
248 - checkout
249 - attach_workspace:
250 at: .
305 - - *restore_node_modules
251 + - setup_node_modules
252 - run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild
253 - run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci
254
@@ -317,7 +263,7 @@ jobs:
263 - checkout
264 - attach_workspace:
265 at: .
320 - - *restore_node_modules
266 + - setup_node_modules
267 - run:
268 name: Playwright install deps
269 command: |
@@ -341,7 +287,7 @@ jobs:
287 - checkout
288 - attach_workspace:
289 at: .
344 - - *restore_node_modules
290 + - setup_node_modules
291 - run: yarn lint-build
292
293 yarn_check_release_dependencies:
@@ -351,7 +297,7 @@ jobs:
297 - checkout
298 - attach_workspace:
299 at: .
354 - - *restore_node_modules
300 + - setup_node_modules
301 - run: yarn check-release-dependencies
302
303
@@ -361,7 +307,7 @@ jobs:
307 steps:
308 - checkout
309 - attach_workspace: *attach_workspace
364 - - *restore_node_modules
310 + - setup_node_modules
311 - run:
312 name: Search build artifacts for unminified errors
313 command: |
@@ -374,7 +320,7 @@ jobs:
320 steps:
321 - checkout
322 - attach_workspace: *attach_workspace
377 - - *restore_node_modules
323 + - setup_node_modules
324 - run:
325 name: Confirm generated inline Fizz runtime is up to date
326 command: |
@@ -390,7 +336,7 @@ jobs:
336 type: string
337 steps:
338 - checkout
393 - - *restore_node_modules
339 + - setup_node_modules
340 - run: yarn test <<parameters.args>> --ci
341
342 yarn_test_build:
@@ -404,7 +350,7 @@ jobs:
350 - checkout
351 - attach_workspace:
352 at: .
407 - - *restore_node_modules
353 + - setup_node_modules
354 - run: yarn test --build <<parameters.args>> --ci
355
356 RELEASE_CHANNEL_stable_yarn_test_dom_fixtures:
@@ -414,7 +360,7 @@ jobs:
360 - checkout
361 - attach_workspace:
362 at: .
417 - - *restore_node_modules
363 + - setup_node_modules
364 - *restore_yarn_cache_fixtures_dom
365 - *yarn_install_fixtures_dom
366 - *yarn_install_fixtures_dom_retry
@@ -433,7 +379,7 @@ jobs:
379 environment: *environment
380 steps:
381 - checkout
436 - - *restore_node_modules
382 + - setup_node_modules
383 - run:
384 name: Run fuzz tests
385 command: |
@@ -452,7 +398,7 @@ jobs:
398 environment: *environment
399 steps:
400 - checkout
455 - - *restore_node_modules
401 + - setup_node_modules
402 - run:
403 name: Run publish script
404 command: |
@@ -465,27 +411,29 @@ jobs:
411 workflows:
412 version: 2
413
468 - # New workflow that will replace "stable" and "experimental"
414 build_and_test:
415 unless: << pipeline.parameters.prerelease_commit_sha >>
416 jobs:
472 - - setup:
417 + - yarn_flow:
418 filters:
419 branches:
420 ignore:
421 - builds/facebook-www
477 - - yarn_flow:
478 - requires:
479 - - setup
422 - check_generated_fizz_runtime:
481 - requires:
482 - - setup
423 + filters:
424 + branches:
425 + ignore:
426 + - builds/facebook-www
427 - yarn_lint:
484 - requires:
485 - - setup
428 + filters:
429 + branches:
430 + ignore:
431 + - builds/facebook-www
432 - yarn_test:
487 - requires:
488 - - setup
433 + filters:
434 + branches:
435 + ignore:
436 + - builds/facebook-www
437 matrix:
438 parameters:
439 args:
@@ -509,11 +457,15 @@ workflows:
457 - '-r=stable --env=development --persistent'
458 - '-r=experimental --env=development --persistent'
459 - yarn_build_combined:
512 - requires:
513 - - setup
460 + filters:
461 + branches:
462 + ignore:
463 + - builds/facebook-www
464 - scrape_warning_messages:
515 - requires:
516 - - setup
465 + filters:
466 + branches:
467 + ignore:
468 + - builds/facebook-www
469 - process_artifacts_combined:
470 requires:
471 - scrape_warning_messages
@@ -551,8 +503,7 @@ workflows:
503 branches:
504 ignore:
505 - main
554 - requires:
555 - - setup
506 + - builds/facebook-www
507 - sizebot:
508 filters:
509 branches:
@@ -591,10 +542,7 @@ workflows:
542 only:
543 - main
544 jobs:
594 - - setup
595 - - test_fuzz:
596 - requires:
597 - - setup
545 + - test_fuzz
546
547 devtools_regression_tests:
548 unless: << pipeline.parameters.prerelease_commit_sha >>
@@ -607,10 +555,7 @@ workflows:
555 only:
556 - main
557 jobs:
610 - - setup
558 - download_build:
612 - requires:
613 - - setup
559 revision: << pipeline.git.revision >>
560 - build_devtools_and_process_artifacts:
561 requires:
@@ -642,11 +587,8 @@ workflows:
587 publish_preleases:
588 when: << pipeline.parameters.prerelease_commit_sha >>
589 jobs:
645 - - setup
590 - publish_prerelease:
591 name: Publish to Next channel
648 - requires:
649 - - setup
592 commit_sha: << pipeline.parameters.prerelease_commit_sha >>
593 release_channel: stable
594 dist_tag: "next"
@@ -674,11 +616,8 @@ workflows:
616 only:
617 - main
618 jobs:
677 - - setup
619 - publish_prerelease:
620 name: Publish to Next channel
680 - requires:
681 - - setup
621 commit_sha: << pipeline.git.revision >>
622 release_channel: stable
623 dist_tag: "next"