Parallelizes the build script across multiple processes (#15716)
* Write size info to separate file per bundle `bundle-sizes.json` contains the combined size information for every build. This makes it easier to store and process, but it prevents us from parallelizing the build script, because each process would need to write to the same file. So I've updated the Rollup script to output individual files per build. A downstream CI job consolidates them into a single file. I have not parallelized the Rollup script yet. I'll do that next. * Parallelize the build script Uses CircleCI's `parallelism` config option to spin up multiple build processes.
Andrew Clark committed
May 29, 2019 at 14:34 UTC
1cc3bba004e96c18d8f812ed64d37d66a8bd666b
6 files changed
+97
-42
.circleci/config.yml
+20
-17
@@ -118,6 +118,7 @@ jobs:
118
build:
119
docker: *docker
120
environment: *environment
121
+ parallelism: 20
122
steps:
123
- checkout
124
- *restore_yarn_cache
@@ -125,26 +126,15 @@ jobs:
126
- run: ./scripts/circleci/add_build_info_json.sh
127
- run: ./scripts/circleci/update_package_versions.sh
128
- run: yarn build
128
- - run: cp ./scripts/rollup/results.json ./build/bundle-sizes.json
129
- - run: ./scripts/circleci/upload_build.sh
130
- - run: ./scripts/circleci/pack_and_store_artifact.sh
131
- - store_artifacts:
132
- path: ./node_modules.tgz
133
- - store_artifacts:
134
- path: ./build.tgz
135
- - store_artifacts:
136
- path: ./build/bundle-sizes.json
137
- - store_artifacts:
138
- path: ./scripts/error-codes/codes.json
129
- persist_to_workspace:
130
root: build
131
paths:
132
- facebook-www
133
- node_modules
134
- react-native
145
- - bundle-sizes.json
135
+ - sizes/*.json
136
147
- sizebot:
137
+ process_artifacts:
138
docker: *docker
139
environment: *environment
140
steps:
@@ -152,7 +142,20 @@ jobs:
142
- attach_workspace: *attach_workspace
143
- *restore_yarn_cache
144
- *run_yarn
145
+ - run: node ./scripts/rollup/consolidateBundleSizes.js
146
- run: node ./scripts/tasks/danger
147
+ - run: ./scripts/circleci/upload_build.sh
148
+ - run: ./scripts/circleci/pack_and_store_artifact.sh
149
+ - store_artifacts:
150
+ path: ./node_modules.tgz
151
+ - store_artifacts:
152
+ path: ./build.tgz
153
+ - store_artifacts:
154
+ path: ./build/bundle-sizes.json
155
+ - store_artifacts:
156
+ # TODO: Update release script to use local file instead of pulling
157
+ # from artifacts.
158
+ path: ./scripts/error-codes/codes.json
159
160
lint_build:
161
docker: *docker
@@ -223,13 +226,10 @@ workflows:
226
- test_source_fire:
227
requires:
228
- setup
226
- - test_coverage:
227
- requires:
228
- - setup
229
- build:
230
requires:
231
- setup
232
- - sizebot:
232
+ - process_artifacts:
233
requires:
234
- build
235
- lint_build:
@@ -254,3 +254,6 @@ workflows:
254
- test_fuzz:
255
requires:
256
- setup
257
+ - test_coverage:
258
+ requires:
259
+ - setup
.gitignore
-1
@@ -1,7 +1,6 @@
1
.DS_STORE
2
node_modules
3
scripts/flow/*/.flowconfig
4
-scripts/rollup/results.json
4
*~
5
*.pyc
6
.grunt
scripts/circleci/upload_build.sh
+1
-1
@@ -10,7 +10,7 @@ if [ -z "$CI_PULL_REQUEST" ] && [ -n "$BUILD_SERVER_ENDPOINT" ]; then
10
-F "react-dom.production.min=@build/dist/react-dom.production.min.js" \
11
-F "react-dom-server.browser.development=@build/dist/react-dom-server.browser.development.js" \
12
-F "react-dom-server.browser.production.min=@build/dist/react-dom-server.browser.production.min.js" \
13
- -F "results.json=@build/../scripts/rollup/results.json" \
13
+ -F "results.json=@build/../build/bundle-sizes.json" \
14
-F "commit=$CIRCLE_SHA1" \
15
-F "date=$(git log --format='%ct' -1)" \
16
-F "pull_request=false" \
scripts/rollup/build.js
+32
-18
@@ -21,7 +21,7 @@ const useForks = require('./plugins/use-forks-plugin');
21
const stripUnusedImports = require('./plugins/strip-unused-imports');
22
const extractErrorCodes = require('../error-codes/extract-errors');
23
const Packaging = require('./packaging');
24
-const {asyncCopyTo, asyncRimRaf} = require('./utils');
24
+const {asyncCopyTo} = require('./utils');
25
const codeFrame = require('babel-code-frame');
26
const Wrappers = require('./wrappers');
27
@@ -634,27 +634,41 @@ function handleRollupError(error) {
634
}
635
636
async function buildEverything() {
637
- await asyncRimRaf('build');
638
-
637
// Run them serially for better console output
638
// and to avoid any potential race conditions.
639
+
640
+ let bundles = [];
641
// eslint-disable-next-line no-for-of-loops/no-for-of-loops
642
for (const bundle of Bundles.bundles) {
643
- await createBundle(bundle, UMD_DEV);
644
- await createBundle(bundle, UMD_PROD);
645
- await createBundle(bundle, UMD_PROFILING);
646
- await createBundle(bundle, NODE_DEV);
647
- await createBundle(bundle, NODE_PROD);
648
- await createBundle(bundle, NODE_PROFILING);
649
- await createBundle(bundle, FB_WWW_DEV);
650
- await createBundle(bundle, FB_WWW_PROD);
651
- await createBundle(bundle, FB_WWW_PROFILING);
652
- await createBundle(bundle, RN_OSS_DEV);
653
- await createBundle(bundle, RN_OSS_PROD);
654
- await createBundle(bundle, RN_OSS_PROFILING);
655
- await createBundle(bundle, RN_FB_DEV);
656
- await createBundle(bundle, RN_FB_PROD);
657
- await createBundle(bundle, RN_FB_PROFILING);
643
+ bundles.push(
644
+ [bundle, UMD_DEV],
645
+ [bundle, UMD_PROD],
646
+ [bundle, UMD_PROFILING],
647
+ [bundle, NODE_DEV],
648
+ [bundle, NODE_PROD],
649
+ [bundle, NODE_PROFILING],
650
+ [bundle, FB_WWW_DEV],
651
+ [bundle, FB_WWW_PROD],
652
+ [bundle, FB_WWW_PROFILING],
653
+ [bundle, RN_OSS_DEV],
654
+ [bundle, RN_OSS_PROD],
655
+ [bundle, RN_OSS_PROFILING],
656
+ [bundle, RN_FB_DEV],
657
+ [bundle, RN_FB_PROD],
658
+ [bundle, RN_FB_PROFILING]
659
+ );
660
+ }
661
+
662
+ if (!shouldExtractErrors && process.env.CIRCLE_NODE_TOTAL) {
663
+ // In CI, parallelize bundles across multiple tasks.
664
+ const nodeTotal = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
665
+ const nodeIndex = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
666
+ bundles = bundles.filter((_, i) => i % nodeTotal === nodeIndex);
667
+ }
668
+
669
+ // eslint-disable-next-line no-for-of-loops/no-for-of-loops
670
+ for (const [bundle, bundleType] of bundles) {
671
+ await createBundle(bundle, bundleType);
672
}
673
674
await Packaging.copyAllShims();
scripts/rollup/consolidateBundleSizes.js
new
+25
@@ -0,0 +1,25 @@
1
+'use strict';
2
+
3
+// Script that combines bundle size information for each build into a single
4
+// JSON file for easier storage and processing.
5
+
6
+const fs = require('fs');
7
+const path = require('path');
8
+
9
+const BUILD_DIR = path.join(__dirname, '../../build');
10
+
11
+const filenames = fs.readdirSync(path.join(BUILD_DIR, 'sizes'));
12
+
13
+let bundleSizes = [];
14
+for (let i = 0; i < filenames.length; i++) {
15
+ const filename = filenames[i];
16
+ if (filename.endsWith('.size.json')) {
17
+ const json = fs.readFileSync(path.join(BUILD_DIR, 'sizes', filename));
18
+ bundleSizes.push(JSON.parse(json));
19
+ }
20
+}
21
+
22
+const outputFilename = path.join(BUILD_DIR, 'bundle-sizes.json');
23
+const outputContents = JSON.stringify({bundleSizes}, null, 2);
24
+
25
+fs.writeFileSync(outputFilename, outputContents);
scripts/rollup/stats.js
+19
-5
@@ -5,21 +5,35 @@ const filesize = require('filesize');
5
const chalk = require('chalk');
6
const join = require('path').join;
7
const fs = require('fs');
8
-const prevBuildResults = fs.existsSync(__dirname + '/results.json')
9
- ? require('./results.json')
8
+const mkdirp = require('mkdirp');
9
+
10
+const BUNDLE_SIZES_FILE_NAME = join(__dirname, '../../build/bundle-sizes.json');
11
+const prevBuildResults = fs.existsSync(BUNDLE_SIZES_FILE_NAME)
12
+ ? require(BUNDLE_SIZES_FILE_NAME)
13
: {bundleSizes: []};
14
15
const currentBuildResults = {
16
// Mutated inside build.js during a build run.
14
- // We make a copy so that partial rebuilds don't erase other stats.
15
- bundleSizes: [...prevBuildResults.bundleSizes],
17
+ bundleSizes: [],
18
};
19
20
function saveResults() {
21
+ // Write all the bundle sizes to a single JSON file.
22
fs.writeFileSync(
20
- join('scripts', 'rollup', 'results.json'),
23
+ BUNDLE_SIZES_FILE_NAME,
24
JSON.stringify(currentBuildResults, null, 2)
25
);
26
+
27
+ // Also write each bundle size to a separate file. That way multiple build
28
+ // processes can run in parallel and generate separate size artifacts.
29
+ // A downstream job can combine them into a single JSON file.
30
+ mkdirp.sync('build/sizes');
31
+ currentBuildResults.bundleSizes.forEach(results => {
32
+ fs.writeFileSync(
33
+ join('build', 'sizes', `${results.filename}.size.json`),
34
+ JSON.stringify(results, null, 2)
35
+ );
36
+ });
37
}
38
39
function fractionalChange(prev, current) {