Build stable and experimental with same command (#20573)
The goal is to simplify our CI pipeline so that all configurations are built and tested in a single workflow. As a first step, this adds a new build script entry point that builds both the experimental and stable release channels into a single artifacts directory. The script works by wrapping the existing build script (which only builds a single release channel at a time), then post-processing the results to match the desired filesystem layout. A future version of the build script would output the files directly without post-processing. Because many parts of our infra depend on the existing layout of the build artifacts directory, I have left the old workflows untouched. We can incremental migrate to the new layout, then delete the old workflows after we've finished.
Andrew Clark committed
Jan 12, 2021 at 11:32 UTC
eb0fb382308f18460844d646dada2e564f82353d
4 files changed
+175
.circleci/config.yml
+50
@@ -293,6 +293,45 @@ jobs:
293
- dist
294
- sizes/*.json
295
296
+ yarn_build_combined:
297
+ docker: *docker
298
+ environment: *environment
299
+ parallelism: 40
300
+ steps:
301
+ - checkout
302
+ - run: yarn workspaces info | head -n -1 > workspace_info.txt
303
+ - *restore_node_modules
304
+ - run:
305
+ command: |
306
+ ./scripts/circleci/add_build_info_json.sh
307
+ ./scripts/circleci/update_package_versions.sh
308
+ yarn build-combined
309
+ - persist_to_workspace:
310
+ root: build2
311
+ paths:
312
+ - facebook-www
313
+ - facebook-react-native
314
+ - facebook-relay
315
+ - oss-stable
316
+ - oss-experimental
317
+ - react-native
318
+ - dist
319
+ - sizes/*.json
320
+
321
+ process_artifacts_combined:
322
+ docker: *docker
323
+ environment: *environment
324
+ steps:
325
+ - checkout
326
+ - attach_workspace:
327
+ at: build2
328
+ - run: yarn workspaces info | head -n -1 > workspace_info.txt
329
+ - *restore_node_modules
330
+ # Compress build directory into a single tarball for easy download
331
+ - run: tar -zcvf ./build2.tgz ./build2
332
+ - store_artifacts:
333
+ path: ./build2.tgz
334
+
335
build_devtools_and_process_artifacts:
336
docker: *docker
337
environment: *environment
@@ -611,6 +650,17 @@ workflows:
650
only:
651
- master
652
653
+ # New workflow that will replace "stable" and "experimental"
654
+ combined:
655
+ jobs:
656
+ - setup
657
+ - yarn_build_combined:
658
+ requires:
659
+ - setup
660
+ - process_artifacts_combined:
661
+ requires:
662
+ - yarn_build_combined
663
+
664
fuzz_tests:
665
triggers:
666
- schedule:
.gitignore
+1
@@ -8,6 +8,7 @@ scripts/flow/*/.flowconfig
8
_SpecRunner.html
9
__benchmarks__
10
build/
11
+build2/
12
remote-repo/
13
coverage/
14
.module-cache
package.json
+1
@@ -106,6 +106,7 @@
106
},
107
"scripts": {
108
"build": "node ./scripts/rollup/build.js",
109
+ "build-combined": "node ./scripts/rollup/build-all-release-channels.js",
110
"build-for-devtools": "cross-env RELEASE_CHANNEL=experimental yarn build react/index,react-dom,react-is,react-debug-tools,scheduler,react-test-renderer,react-refresh",
111
"build-for-devtools-dev": "yarn build-for-devtools --type=NODE_DEV",
112
"build-for-devtools-prod": "yarn build-for-devtools --type=NODE_PROD",
scripts/rollup/build-all-release-channels.js
new
+123
@@ -0,0 +1,123 @@
1
+'use strict';
2
+
3
+/* eslint-disable no-for-of-loops/no-for-of-loops */
4
+
5
+const fs = require('fs');
6
+const {spawnSync} = require('child_process');
7
+const tmp = require('tmp');
8
+
9
+// Runs the build script for both stable and experimental release channels,
10
+// by configuring an environment variable.
11
+
12
+if (process.env.CIRCLE_NODE_TOTAL) {
13
+ // In CI, we use multiple concurrent processes. Allocate half the processes to
14
+ // build the stable channel, and the other half for experimental. Override
15
+ // the environment variables to "trick" the underlying build script.
16
+ const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
17
+ const halfTotal = Math.floor(total / 2);
18
+ const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
19
+ if (index < halfTotal) {
20
+ const nodeTotal = halfTotal;
21
+ const nodeIndex = index;
22
+ buildForChannel('stable', nodeTotal, nodeIndex);
23
+ processStable('./build');
24
+ } else {
25
+ const nodeTotal = total - halfTotal;
26
+ const nodeIndex = index - halfTotal;
27
+ buildForChannel('experimental', nodeTotal, nodeIndex);
28
+ processExperimental('./build');
29
+ }
30
+
31
+ // TODO: Currently storing artifacts as `./build2` so that it doesn't conflict
32
+ // with old build job. Remove once we migrate rest of build/test pipeline.
33
+ fs.renameSync('./build', './build2');
34
+} else {
35
+ // Running locally, no concurrency. Move each channel's build artifacts into
36
+ // a temporary directory so that they don't conflict.
37
+ buildForChannel('stable', '', '');
38
+ const stableDir = tmp.dirSync().name;
39
+ fs.renameSync('./build', stableDir);
40
+ processStable(stableDir);
41
+
42
+ buildForChannel('experimental', '', '');
43
+ const experimentalDir = tmp.dirSync().name;
44
+ fs.renameSync('./build', experimentalDir);
45
+ processExperimental(experimentalDir);
46
+
47
+ // Then merge the experimental folder into the stable one. processExperimental
48
+ // will have already removed conflicting files.
49
+ //
50
+ // In CI, merging is handled automatically by CircleCI's workspace feature.
51
+ spawnSync('rsync', ['-ar', experimentalDir + '/', stableDir + '/']);
52
+
53
+ // Now restore the combined directory back to its original name
54
+ // TODO: Currently storing artifacts as `./build2` so that it doesn't conflict
55
+ // with old build job. Remove once we migrate rest of build/test pipeline.
56
+ fs.renameSync(stableDir, './build2');
57
+}
58
+
59
+function buildForChannel(channel, nodeTotal, nodeIndex) {
60
+ spawnSync('node', ['./scripts/rollup/build.js', ...process.argv.slice(2)], {
61
+ stdio: ['pipe', process.stdout, process.stderr],
62
+ env: {
63
+ ...process.env,
64
+ RELEASE_CHANNEL: channel,
65
+ CIRCLE_NODE_TOTAL: nodeTotal,
66
+ CIRCLE_NODE_INDEX: nodeIndex,
67
+ },
68
+ });
69
+}
70
+
71
+function processStable(buildDir) {
72
+ if (fs.existsSync(buildDir + '/node_modules')) {
73
+ fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-stable');
74
+ }
75
+
76
+ if (fs.existsSync(buildDir + '/facebook-www')) {
77
+ for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) {
78
+ const filePath = buildDir + '/facebook-www/' + fileName;
79
+ const stats = fs.statSync(filePath);
80
+ if (!stats.isDirectory()) {
81
+ fs.renameSync(filePath, filePath.replace('.js', '.classic.js'));
82
+ }
83
+ }
84
+ }
85
+
86
+ if (fs.existsSync(buildDir + '/sizes')) {
87
+ fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-stable');
88
+ }
89
+}
90
+
91
+function processExperimental(buildDir) {
92
+ if (fs.existsSync(buildDir + '/node_modules')) {
93
+ fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-experimental');
94
+ }
95
+
96
+ if (fs.existsSync(buildDir + '/facebook-www')) {
97
+ for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) {
98
+ const filePath = buildDir + '/facebook-www/' + fileName;
99
+ const stats = fs.statSync(filePath);
100
+ if (!stats.isDirectory()) {
101
+ fs.renameSync(filePath, filePath.replace('.js', '.modern.js'));
102
+ }
103
+ }
104
+ }
105
+
106
+ if (fs.existsSync(buildDir + '/sizes')) {
107
+ fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-experimental');
108
+ }
109
+
110
+ // Delete all other artifacts that weren't handled above. We assume they are
111
+ // duplicates of the corresponding artifacts in the stable channel. Ideally,
112
+ // the underlying build script should not have produced these files in the
113
+ // first place.
114
+ for (const pathName of fs.readdirSync(buildDir)) {
115
+ if (
116
+ pathName !== 'oss-experimental' &&
117
+ pathName !== 'facebook-www' &&
118
+ pathName !== 'sizes-experimental'
119
+ ) {
120
+ spawnSync('rm', ['-rm', buildDir + '/' + pathName]);
121
+ }
122
+ }
123
+}