| 1 | 'use strict'; |
| 2 | |
| 3 | const fs = require('fs'); |
| 4 | |
| 5 | function readWeights() { |
| 6 | const weightsPath = process.env.BUILD_SHARD_WEIGHTS; |
| 7 | if (!weightsPath) { |
| 8 | return null; |
| 9 | } |
| 10 | let weights; |
| 11 | try { |
| 12 | weights = JSON.parse(fs.readFileSync(weightsPath, 'utf8')).weights; |
| 13 | } catch (error) { |
| 14 | if (error.code === 'ENOENT') { |
| 15 | // Expected before the first weights have ever been saved. |
| 16 | console.log('No build shard weights found, using round-robin sharding.'); |
| 17 | return null; |
| 18 | } |
| 19 | throw error; |
| 20 | } |
| 21 | if (weights === null || typeof weights !== 'object') { |
| 22 | return null; |
| 23 | } |
| 24 | return weights; |
| 25 | } |
| 26 | |
| 27 | // Persists the durations this worker measured so that |
| 28 | // process_artifacts_combined can merge them into the shared weights cache. |
| 29 | // No-op outside sharded CI builds. |
| 30 | function writeShardTimings(timings) { |
| 31 | const nodeIndex = process.env.CI_INDEX; |
| 32 | if (!process.env.CI_TOTAL || !nodeIndex) { |
| 33 | return; |
| 34 | } |
| 35 | const dir = 'build/__shard_timings__'; |
| 36 | fs.mkdirSync(dir, {recursive: true}); |
| 37 | const result = {}; |
| 38 | timings.forEach(timing => { |
| 39 | result[timing.key] = Math.round(timing.seconds * 10) / 10; |
| 40 | }); |
| 41 | fs.writeFileSync( |
| 42 | dir + '/' + nodeIndex + '-' + process.env.RELEASE_CHANNEL + '.json', |
| 43 | JSON.stringify(result) |
| 44 | ); |
| 45 | } |
| 46 | |
| 47 | // Assigns work items to CI workers. With measured per-item durations (see |
| 48 | // scripts/ci/merge-build-weights.js), items are assigned longest-first to |
| 49 | // the currently least-loaded worker so that workers finish around the same |
| 50 | // time. Every worker computes the full assignment and then picks its own |
| 51 | // bin, so the ordering below must stay deterministic. Without weights we |
| 52 | // fall back to round-robin. |
| 53 | function selectShard(items, keyFn, nodeTotal, nodeIndex) { |
| 54 | const weights = readWeights(); |
| 55 | const weightedKeys = weights === null ? [] : Object.keys(weights); |
| 56 | if (weightedKeys.length === 0) { |
| 57 | return items.filter((_, i) => i % nodeTotal === nodeIndex); |
| 58 | } |
| 59 | const keys = items.map(keyFn); |
| 60 | const sortedWeights = weightedKeys |
| 61 | .map(key => weights[key]) |
| 62 | .sort((a, b) => a - b); |
| 63 | const defaultWeight = sortedWeights[Math.floor(sortedWeights.length / 2)]; |
| 64 | const weightOf = index => { |
| 65 | const weight = weights[keys[index]]; |
| 66 | return weight === undefined ? defaultWeight : weight; |
| 67 | }; |
| 68 | const order = items |
| 69 | .map((_, i) => i) |
| 70 | .sort((a, b) => { |
| 71 | const delta = weightOf(b) - weightOf(a); |
| 72 | if (delta !== 0) { |
| 73 | return delta; |
| 74 | } |
| 75 | if (keys[a] !== keys[b]) { |
| 76 | return keys[a] < keys[b] ? -1 : 1; |
| 77 | } |
| 78 | return a - b; |
| 79 | }); |
| 80 | const bins = []; |
| 81 | for (let i = 0; i < nodeTotal; i++) { |
| 82 | bins.push({load: 0, indices: []}); |
| 83 | } |
| 84 | order.forEach(i => { |
| 85 | // The first bin wins ties so that the assignment stays deterministic. |
| 86 | let target = bins[0]; |
| 87 | for (let j = 1; j < bins.length; j++) { |
| 88 | if (bins[j].load < target.load) { |
| 89 | target = bins[j]; |
| 90 | } |
| 91 | } |
| 92 | target.load += weightOf(i); |
| 93 | target.indices.push(i); |
| 94 | }); |
| 95 | const shard = bins[nodeIndex].indices.sort((a, b) => a - b); |
| 96 | console.log( |
| 97 | `Sharding by measured build time: worker ${nodeIndex + 1}/${nodeTotal} ` + |
| 98 | `builds ${shard.length} of ${items.length} bundles ` + |
| 99 | `(~${Math.round(bins[nodeIndex].load)}s of rollup time).` |
| 100 | ); |
| 101 | return shard.map(i => items[i]); |
| 102 | } |
| 103 | |
| 104 | module.exports = {selectShard, writeShardTimings}; |