[ci] Shard build workers by measured bundle build times (#37353)
`build_and_lint` assigns its `[bundle, bundleType]` pairs to 25 workers per channel by round-robin, which leaves the slowest worker with 56-62 seconds of rollup time while the mean worker has 40-42 seconds (measured from the timestamped `BUILDING`/`COMPLETE` lines in recent `main` run logs). This change shards by measured build time instead, and the measurement maintains itself: `yarn build` writes the timing results into `build/__shard_timings__/`, which rides along inside the existing per-worker artifacts. `process_artifacts_combined`, which already downloads all 50 artifacts and is off the critical path, combines them into `build-weights.json` and saves it to the actions cache under a per-run key. Readers restore the most recent entry via a `restore-keys` prefix. Only pushes can save to cache keys that PRs can read, so pull request runs benefit from the weights but cannot poison them. The new measurement is always written verbatim rather than merged with previous weights, so removed bundles drop out instead of accumulating. A per-bundle diff against the previous weights is logged so that we can monitor whether single-run variance is too high, in which case shards should be determined from timings across the last N runs instead. Even on this PR a perfect prediction would've only gained us ?s for the slowest shard. Co-authored-by: Claude Code (kimi-k3[1m]) <noreply@anthropic.com>
Sebastian "Sebbie" Silbermann committed
Aug 24, 2026 at 14:10 UTC
bd6ea412c6732b3b946a2827fcaac3a1c8f2e863
5 files changed
+260
-2
.github/workflows/runtime_build_and_test.yml
+36
@@ -302,11 +302,21 @@ jobs:
302
if: steps.node_modules.outputs.cache-hit != 'true'
303
- run: yarn --cwd compiler install --frozen-lockfile
304
if: steps.node_modules.outputs.cache-hit != 'true'
305
+ - name: Restore build shard weights
306
+ uses: actions/cache/restore@v4
307
+ with:
308
+ # Written by process_artifacts_combined on every push. The
309
+ # restore-keys prefix picks up the most recent entry. On a miss the
310
+ # build falls back to round-robin sharding.
311
+ path: build-weights.json
312
+ key: build-weights-v1-${{ github.run_id }}
313
+ restore-keys: build-weights-v1-
314
- run: yarn build --index=${{ matrix.worker_id }} --total=25 --r=${{ matrix.release_channel }} --ci
315
env:
316
CI: github
317
RELEASE_CHANNEL: ${{ matrix.release_channel }}
318
NODE_INDEX: ${{ matrix.worker_id }}
319
+ BUILD_SHARD_WEIGHTS: build-weights.json
320
- name: Lint build
321
run: yarn lint-build
322
- name: Display structure of build
@@ -471,6 +481,32 @@ jobs:
481
pattern: _build_*
482
path: build
483
merge-multiple: true
484
+ # Only used to log weight variance; the new measurement is what gets
485
+ # saved, so removed bundles drop out instead of accumulating.
486
+ - name: Restore previous build shard weights
487
+ uses: actions/cache/restore@v4
488
+ with:
489
+ # Must match the save step's path exactly: the cache version is a
490
+ # hash of the path, so a different path never matches the key.
491
+ path: build-weights.json
492
+ key: build-weights-v1-${{ github.run_id }}
493
+ restore-keys: build-weights-v1-
494
+ - name: Update build shard weights
495
+ run: node scripts/ci/merge-build-weights.js
496
+ - name: Save build shard weights
497
+ # Pull request saves land in the pull request's own merge-ref cache
498
+ # scope, which only that pull request can read; fork pull requests
499
+ # have a read-only cache token, so their save degrades to a warning.
500
+ # A re-run of this same workflow run reuses the cache key, and cache
501
+ # entries are immutable. Weights are an optimization, so a failed
502
+ # save must not break artifact processing.
503
+ continue-on-error: true
504
+ uses: actions/cache/save@v4
505
+ with:
506
+ path: build-weights.json
507
+ key: build-weights-v1-${{ github.run_id }}
508
+ # Keep the shard timings out of the released tarball.
509
+ - run: rm -rf build/__shard_timings__
510
- name: Display structure of build
511
run: ls -R build
512
- run: echo ${{ github.event.pull_request.head.sha || github.sha }} >> build/COMMIT_SHA
scripts/ci/merge-build-weights.js
new
+96
@@ -0,0 +1,96 @@
1
+#!/usr/bin/env node
2
+
3
+'use strict';
4
+
5
+// Combines the per-worker shard timings of the current run
6
+// (build/__shard_timings__/*.json, written by scripts/rollup/build.js) into
7
+// build-weights.json, which the workflow then saves back to the actions
8
+// cache. The previous weights (restored to build-weights.json by the
9
+// workflow) are only used to log a diff for variance monitoring; the new
10
+// measurement is always written verbatim so that removed bundles drop out
11
+// instead of accumulating. If the logged variance turns out to be too high
12
+// for stable shards, weights should be aggregated across the last N runs
13
+// instead. Weights feed scripts/rollup/sharding.js. This script never fails:
14
+// the weights are an optimization, so a broken update must not break
15
+// artifact processing.
16
+
17
+const fs = require('fs');
18
+
19
+const TIMINGS_DIR = 'build/__shard_timings__';
20
+// The workflow restores the previous weights to OUT_PATH itself, so this
21
+// script reads them from there before overwriting with the new measurement.
22
+const OUT_PATH = 'build-weights.json';
23
+
24
+function logDiff(fresh, previous) {
25
+ const deltas = [];
26
+ Object.keys(fresh).forEach(key => {
27
+ if (previous[key] !== undefined) {
28
+ deltas.push({key, delta: fresh[key] - previous[key]});
29
+ }
30
+ });
31
+ if (deltas.length === 0) {
32
+ return;
33
+ }
34
+ deltas.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));
35
+ const absDeltas = deltas
36
+ .map(entry => Math.abs(entry.delta))
37
+ .sort((a, b) => a - b);
38
+ const mean =
39
+ absDeltas.reduce((sum, delta) => sum + delta, 0) / absDeltas.length;
40
+ const p95 = absDeltas[Math.floor(absDeltas.length * 0.95)];
41
+ console.log(
42
+ `Weight changes vs the previous run: mean |delta| = ${mean.toFixed(2)}s, ` +
43
+ `p95 = ${p95.toFixed(1)}s across ${deltas.length} bundles. ` +
44
+ 'High variance here means shards should be determined from multiple runs.'
45
+ );
46
+ console.log('Largest changes:');
47
+ deltas.slice(0, 10).forEach(entry => {
48
+ console.log(
49
+ ` ${entry.delta >= 0 ? '+' : ''}${entry.delta.toFixed(1)}s ${entry.key}`
50
+ );
51
+ });
52
+}
53
+
54
+function main() {
55
+ const fresh = {};
56
+ const files = fs
57
+ .readdirSync(TIMINGS_DIR)
58
+ .filter(name => name.endsWith('.json'));
59
+ files.forEach(name => {
60
+ const timings = JSON.parse(
61
+ fs.readFileSync(TIMINGS_DIR + '/' + name, 'utf8')
62
+ );
63
+ Object.keys(timings).forEach(key => {
64
+ fresh[key] = timings[key];
65
+ });
66
+ });
67
+ const freshKeys = Object.keys(fresh);
68
+ let previous = {};
69
+ try {
70
+ previous = JSON.parse(fs.readFileSync(OUT_PATH, 'utf8')).weights;
71
+ } catch (error) {
72
+ if (error.code !== 'ENOENT') {
73
+ throw error;
74
+ }
75
+ // Expected before the first weights have ever been saved.
76
+ console.log('No previous weights found, skipping the diff.');
77
+ }
78
+ logDiff(fresh, previous);
79
+ fs.writeFileSync(
80
+ OUT_PATH,
81
+ JSON.stringify({version: 1, weights: fresh}, null, 2) + '\n'
82
+ );
83
+ console.log(`Wrote ${freshKeys.length} weights to ${OUT_PATH}.`);
84
+}
85
+
86
+try {
87
+ main();
88
+} catch (error) {
89
+ console.log(
90
+ 'Could not update build shard weights, keeping the previous ones.',
91
+ error
92
+ );
93
+ // The restored previous weights may still sit at OUT_PATH; delete them so
94
+ // the save step cannot republish data this run did not produce.
95
+ fs.rmSync(OUT_PATH, {force: true});
96
+}
scripts/rollup/build-all-release-channels.js
+4
-1
@@ -388,7 +388,10 @@ function processExperimental(buildDir, version) {
388
if (
389
pathName !== 'oss-experimental' &&
390
pathName !== 'facebook-www' &&
391
- pathName !== 'sizes-experimental'
391
+ pathName !== 'sizes-experimental' &&
392
+ // Not a duplicate: this worker's shard timings, merged into the build
393
+ // weights cache by process_artifacts_combined.
394
+ pathName !== '__shard_timings__'
395
) {
396
fs.rmSync(path.join(buildDir, pathName), {
397
recursive: true,
scripts/rollup/build.js
+20
-1
@@ -12,6 +12,7 @@ const stripBanner = require('rollup-plugin-strip-banner');
12
const chalk = require('chalk');
13
const resolve = require('@rollup/plugin-node-resolve').nodeResolve;
14
const fs = require('fs');
15
+const {performance} = require('perf_hooks');
16
const childProcess = require('child_process');
17
const argv = require('minimist')(process.argv.slice(2));
18
const Modules = require('./modules');
@@ -23,6 +24,7 @@ const useForks = require('./plugins/use-forks-plugin');
24
const dynamicImports = require('./plugins/dynamic-imports');
25
const externalRuntime = require('./plugins/external-runtime-plugin');
26
const Packaging = require('./packaging');
27
+const {selectShard, writeShardTimings} = require('./sharding');
28
const {asyncRimRaf} = require('./utils');
29
const codeFrame = require('@babel/code-frame').default;
30
const Wrappers = require('./wrappers');
@@ -866,19 +868,36 @@ async function buildEverything() {
868
return !shouldSkipBundle(bundle, bundleType);
869
});
870
871
+ // Prefixed with the channel because feature-flag forks change the cost of
872
+ // some heavy bundles.
873
+ const shardKeyOf = ([bundle, bundleType]) =>
874
+ process.env.RELEASE_CHANNEL +
875
+ '/' +
876
+ getFilename(bundle, bundleType) +
877
+ ' (' +
878
+ bundleType.toLowerCase() +
879
+ ')';
880
+
881
if (process.env.CI_TOTAL && process.env.CI_INDEX) {
882
const nodeTotal = parseInt(process.env.CI_TOTAL, 10);
883
const nodeIndex = parseInt(process.env.CI_INDEX, 10);
872
- bundles = bundles.filter((_, i) => i % nodeTotal === nodeIndex);
884
+ bundles = selectShard(bundles, shardKeyOf, nodeTotal, nodeIndex);
885
}
886
887
+ const shardTimings = [];
888
// eslint-disable-next-line no-for-of-loops/no-for-of-loops
889
for (const [bundle, bundleType] of bundles) {
890
if (bundle.prebuild) {
891
runShellCommand(bundle.prebuild);
892
}
893
+ const start = performance.now();
894
await createBundle(bundle, bundleType);
895
+ shardTimings.push({
896
+ key: shardKeyOf([bundle, bundleType]),
897
+ seconds: (performance.now() - start) / 1000,
898
+ });
899
}
900
+ writeShardTimings(shardTimings);
901
902
await Packaging.copyAllShims();
903
await Packaging.prepareNpmPackages();
scripts/rollup/sharding.js
new
+104
@@ -0,0 +1,104 @@
1
+'use strict';
2
+
3
+const fs = require('fs');
4
+
5
+function readWeights() {
6
+ const weightsPath = process.env.BUILD_SHARD_WEIGHTS;
7
+ if (!weightsPath) {
8
+ return null;
9
+ }
10
+ let weights;
11
+ try {
12
+ weights = JSON.parse(fs.readFileSync(weightsPath, 'utf8')).weights;
13
+ } catch (error) {
14
+ if (error.code === 'ENOENT') {
15
+ // Expected before the first weights have ever been saved.
16
+ console.log('No build shard weights found, using round-robin sharding.');
17
+ return null;
18
+ }
19
+ throw error;
20
+ }
21
+ if (weights === null || typeof weights !== 'object') {
22
+ return null;
23
+ }
24
+ return weights;
25
+}
26
+
27
+// Persists the durations this worker measured so that
28
+// process_artifacts_combined can merge them into the shared weights cache.
29
+// No-op outside sharded CI builds.
30
+function writeShardTimings(timings) {
31
+ const nodeIndex = process.env.CI_INDEX;
32
+ if (!process.env.CI_TOTAL || !nodeIndex) {
33
+ return;
34
+ }
35
+ const dir = 'build/__shard_timings__';
36
+ fs.mkdirSync(dir, {recursive: true});
37
+ const result = {};
38
+ timings.forEach(timing => {
39
+ result[timing.key] = Math.round(timing.seconds * 10) / 10;
40
+ });
41
+ fs.writeFileSync(
42
+ dir + '/' + nodeIndex + '-' + process.env.RELEASE_CHANNEL + '.json',
43
+ JSON.stringify(result)
44
+ );
45
+}
46
+
47
+// Assigns work items to CI workers. With measured per-item durations (see
48
+// scripts/ci/merge-build-weights.js), items are assigned longest-first to
49
+// the currently least-loaded worker so that workers finish around the same
50
+// time. Every worker computes the full assignment and then picks its own
51
+// bin, so the ordering below must stay deterministic. Without weights we
52
+// fall back to round-robin.
53
+function selectShard(items, keyFn, nodeTotal, nodeIndex) {
54
+ const weights = readWeights();
55
+ const weightedKeys = weights === null ? [] : Object.keys(weights);
56
+ if (weightedKeys.length === 0) {
57
+ return items.filter((_, i) => i % nodeTotal === nodeIndex);
58
+ }
59
+ const keys = items.map(keyFn);
60
+ const sortedWeights = weightedKeys
61
+ .map(key => weights[key])
62
+ .sort((a, b) => a - b);
63
+ const defaultWeight = sortedWeights[Math.floor(sortedWeights.length / 2)];
64
+ const weightOf = index => {
65
+ const weight = weights[keys[index]];
66
+ return weight === undefined ? defaultWeight : weight;
67
+ };
68
+ const order = items
69
+ .map((_, i) => i)
70
+ .sort((a, b) => {
71
+ const delta = weightOf(b) - weightOf(a);
72
+ if (delta !== 0) {
73
+ return delta;
74
+ }
75
+ if (keys[a] !== keys[b]) {
76
+ return keys[a] < keys[b] ? -1 : 1;
77
+ }
78
+ return a - b;
79
+ });
80
+ const bins = [];
81
+ for (let i = 0; i < nodeTotal; i++) {
82
+ bins.push({load: 0, indices: []});
83
+ }
84
+ order.forEach(i => {
85
+ // The first bin wins ties so that the assignment stays deterministic.
86
+ let target = bins[0];
87
+ for (let j = 1; j < bins.length; j++) {
88
+ if (bins[j].load < target.load) {
89
+ target = bins[j];
90
+ }
91
+ }
92
+ target.load += weightOf(i);
93
+ target.indices.push(i);
94
+ });
95
+ const shard = bins[nodeIndex].indices.sort((a, b) => a - b);
96
+ console.log(
97
+ `Sharding by measured build time: worker ${nodeIndex + 1}/${nodeTotal} ` +
98
+ `builds ${shard.length} of ${items.length} bundles ` +
99
+ `(~${Math.round(bins[nodeIndex].load)}s of rollup time).`
100
+ );
101
+ return shard.map(i => items[i]);
102
+}
103
+
104
+module.exports = {selectShard, writeShardTimings};