Parallelize CircleCI jobs using workflows (#15704)
Updates the CircleCI config to use the workflows features to run jobs in parallel, instead of the `parallelism` option. This change alone doesn't improve the overall build time much, since almost all of the total time is spent running the Rollup script, which runs entirely sequentially. But it does improve reporting, and should make it easier to add additional parallel jobs in the future.
Andrew Clark committed
May 22, 2019 at 11:14 UTC
f3109ad8a7ddf7a162e7c9a050ffe6c3efb75491
2 files changed
+162
-99
.circleci/config.yml
+162
-26
@@ -1,53 +1,189 @@
1
version: 2
2
-jobs:
3
- build:
2
5
- docker:
6
- - image: circleci/openjdk:8-jdk-node-browsers
3
+aliases:
4
+ - &docker
5
+ - image: circleci/openjdk:8-jdk-node-browsers
6
8
- environment:
9
- TZ: /usr/share/zoneinfo/America/Los_Angeles
7
+ - &environment
8
+ TZ: /usr/share/zoneinfo/America/Los_Angeles
9
11
- parallelism: 4
10
+ - &restore_node_modules
11
+ name: Restore node_modules cache
12
+ keys:
13
+ - v1-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
14
+ - v1-node-{{ arch }}-{{ .Branch }}-
15
+ - v1-node-{{ arch }}-
16
+ - &attach_workspace
17
+ at: build
18
+
19
+jobs:
20
+ setup:
21
+ docker: *docker
22
+ environment: *environment
23
24
steps:
25
- checkout
15
-
16
- - run: echo $CIRCLE_COMPARE_URL | cut -d/ -f7
17
-
18
- - restore_cache:
19
- name: Restore node_modules cache
20
- keys:
21
- - v1-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
22
- - v1-node-{{ arch }}-{{ .Branch }}-
23
- - v1-node-{{ arch }}-
24
-
26
+ - restore_cache: *restore_node_modules
27
- run:
28
name: Nodejs Version
29
command: node --version
28
-
30
- run:
31
name: Install Packages
32
command: yarn install --frozen-lockfile
32
-
33
- - run:
34
- name: Test Packages
35
- command: ./scripts/circleci/test_entry_point.sh
36
-
33
- save_cache:
34
name: Save node_modules cache
35
key: v1-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
36
paths:
37
- node_modules
38
39
+ lint:
40
+ docker: *docker
41
+ environment: *environment
42
+
43
+ steps:
44
+ - checkout
45
+ - restore_cache: *restore_node_modules
46
+ - run: node ./scripts/prettier/index
47
+ - run: node ./scripts/tasks/eslint
48
+ - run: ./scripts/circleci/check_license.sh
49
+ - run: ./scripts/circleci/check_modules.sh
50
+ - run: ./scripts/circleci/test_print_warnings.sh
51
+
52
+ flow:
53
+ docker: *docker
54
+ environment: *environment
55
+
56
+ steps:
57
+ - checkout
58
+ - restore_cache: *restore_node_modules
59
+ - run: node ./scripts/tasks/flow-ci
60
+
61
+ test-source:
62
+ docker: *docker
63
+ environment: *environment
64
+
65
+ steps:
66
+ - checkout
67
+ - restore_cache: *restore_node_modules
68
+ - run: yarn test --maxWorkers=2
69
+
70
+ test-source-persistent:
71
+ docker: *docker
72
+ environment: *environment
73
+
74
+ steps:
75
+ - checkout
76
+ - restore_cache: *restore_node_modules
77
+ - run: yarn test-persistent --maxWorkers=2
78
+
79
+ test-source-prod:
80
+ docker: *docker
81
+ environment: *environment
82
+
83
+ steps:
84
+ - checkout
85
+ - restore_cache: *restore_node_modules
86
+ - run: yarn test-prod --maxWorkers=2
87
+
88
+ test-source-fire:
89
+ docker: *docker
90
+ environment: *environment
91
+
92
+ steps:
93
+ - checkout
94
+ - restore_cache: *restore_node_modules
95
+ - run: yarn test-fire --maxWorkers=2
96
+ - run: yarn test-fire-prod --maxWorkers=2
97
+
98
+ test-coverage:
99
+ docker: *docker
100
+ environment: *environment
101
+
102
+ steps:
103
+ - checkout
104
+ - restore_cache: *restore_node_modules
105
+ - run: ./scripts/circleci/test_coverage.sh
106
+
107
+ build:
108
+ docker: *docker
109
+ environment: *environment
110
+ steps:
111
+ - checkout
112
+ - restore_cache: *restore_node_modules
113
+ - run: ./scripts/circleci/add_build_info_json.sh
114
+ - run: ./scripts/circleci/update_package_versions.sh
115
+ - run: ./scripts/circleci/build.sh
116
+ - run: yarn test-build --maxWorkers=2
117
+ - run: yarn test-build-prod --maxWorkers=2
118
+ - run: cp ./scripts/rollup/results.json ./build/bundle-sizes.json
119
+ - run: node ./scripts/tasks/danger
120
+ - run: ./scripts/circleci/upload_build.sh
121
+ - run: ./scripts/circleci/pack_and_store_artifact.sh
122
- store_artifacts:
123
path: ./node_modules.tgz
45
-
124
- store_artifacts:
125
path: ./build.tgz
48
-
126
- store_artifacts:
127
path: ./build/bundle-sizes.json
51
-
128
- store_artifacts:
129
path: ./scripts/error-codes/codes.json
130
+ - persist_to_workspace:
131
+ root: build
132
+ paths:
133
+ - facebook-www
134
+ - node_modules
135
+ - react-native
136
+
137
+ test-build:
138
+ docker: *docker
139
+ environment: *environment
140
+ steps:
141
+ - checkout
142
+ - attach_workspace: *attach_workspace
143
+ - restore_cache: *restore_node_modules
144
+ - run: yarn test-build --maxWorkers=2
145
+
146
+ test-build-prod:
147
+ docker: *docker
148
+ environment: *environment
149
+ steps:
150
+ - checkout
151
+ - attach_workspace: *attach_workspace
152
+ - restore_cache: *restore_node_modules
153
+ - run: yarn test-build-prod --maxWorkers=2
154
+
155
+workflows:
156
+ version: 2
157
+ build-and-test:
158
+ jobs:
159
+ - setup
160
+ - lint:
161
+ requires:
162
+ - setup
163
+ - flow:
164
+ requires:
165
+ - setup
166
+ - test-source:
167
+ requires:
168
+ - setup
169
+ - test-source-prod:
170
+ requires:
171
+ - setup
172
+ - test-source-persistent:
173
+ requires:
174
+ - setup
175
+ - test-source-fire:
176
+ requires:
177
+ - setup
178
+ - test-coverage:
179
+ requires:
180
+ - setup
181
+ - build:
182
+ requires:
183
+ - setup
184
+ - test-build:
185
+ requires:
186
+ - build
187
+ - test-build-prod:
188
+ requires:
189
+ - build
scripts/circleci/test_entry_point.sh
deleted
-73
@@ -1,73 +0,0 @@
1
-#!/bin/bash
2
-
3
-set -e
4
-
5
-./scripts/circleci/set_up_github_keys.sh
6
-
7
-COMMANDS_TO_RUN=()
8
-
9
-if [ $((0 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then
10
- COMMANDS_TO_RUN+=('node ./scripts/prettier/index')
11
- COMMANDS_TO_RUN+=('node ./scripts/tasks/flow-ci')
12
- COMMANDS_TO_RUN+=('node ./scripts/tasks/eslint')
13
- COMMANDS_TO_RUN+=('yarn test --maxWorkers=2')
14
- COMMANDS_TO_RUN+=('yarn test-persistent --maxWorkers=2')
15
- COMMANDS_TO_RUN+=('./scripts/circleci/check_license.sh')
16
- COMMANDS_TO_RUN+=('./scripts/circleci/check_modules.sh')
17
- COMMANDS_TO_RUN+=('./scripts/circleci/test_print_warnings.sh')
18
-fi
19
-
20
-if [ $((1 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then
21
- COMMANDS_TO_RUN+=('yarn test-prod --maxWorkers=2')
22
- # React Fire:
23
- COMMANDS_TO_RUN+=('yarn test-fire --maxWorkers=2')
24
- COMMANDS_TO_RUN+=('yarn test-fire-prod --maxWorkers=2')
25
-fi
26
-
27
-if [ $((2 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then
28
- COMMANDS_TO_RUN+=('./scripts/circleci/add_build_info_json.sh')
29
- COMMANDS_TO_RUN+=('./scripts/circleci/update_package_versions.sh')
30
- COMMANDS_TO_RUN+=('./scripts/circleci/build.sh')
31
- COMMANDS_TO_RUN+=('yarn test-build --maxWorkers=2')
32
- COMMANDS_TO_RUN+=('yarn test-build-prod --maxWorkers=2')
33
- COMMANDS_TO_RUN+=('cp ./scripts/rollup/results.json ./build/bundle-sizes.json')
34
- COMMANDS_TO_RUN+=('node ./scripts/tasks/danger')
35
- COMMANDS_TO_RUN+=('./scripts/circleci/upload_build.sh')
36
- COMMANDS_TO_RUN+=('./scripts/circleci/pack_and_store_artifact.sh')
37
-fi
38
-
39
-if [ $((3 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then
40
- COMMANDS_TO_RUN+=('./scripts/circleci/test_coverage.sh')
41
-fi
42
-
43
-RETURN_CODES=()
44
-FAILURE=0
45
-
46
-printf "Node #%s (%s total). " "$CIRCLE_NODE_INDEX" "$CIRCLE_NODE_TOTAL"
47
-if [ -n "${COMMANDS_TO_RUN[0]}" ]; then
48
- echo "Preparing to run commands:"
49
- for cmd in "${COMMANDS_TO_RUN[@]}"; do
50
- echo "- $cmd"
51
- done
52
-
53
- for cmd in "${COMMANDS_TO_RUN[@]}"; do
54
- echo
55
- echo "$ $cmd"
56
- set +e
57
- $cmd
58
- rc=$?
59
- set -e
60
- RETURN_CODES+=($rc)
61
- if [ $rc -ne 0 ]; then
62
- FAILURE=$rc
63
- fi
64
- done
65
-
66
- echo
67
- for i in "${!COMMANDS_TO_RUN[@]}"; do
68
- echo "Received return code ${RETURN_CODES[i]} from: ${COMMANDS_TO_RUN[i]}"
69
- done
70
- exit $FAILURE
71
-else
72
- echo "No commands to run."
73
-fi