| 1 | 'use strict'; |
| 2 | |
| 3 | const Lighthouse = require('lighthouse'); |
| 4 | const chromeLauncher = require('chrome-launcher'); |
| 5 | |
| 6 | const stats = require('stats-analysis'); |
| 7 | const config = require('lighthouse/lighthouse-core/config/perf-config'); |
| 8 | const spawn = require('child_process').spawn; |
| 9 | const os = require('os'); |
| 10 | |
| 11 | const timesToRun = 10; |
| 12 | |
| 13 | function wait(val) { |
| 14 | return new Promise(resolve => setTimeout(resolve, val)); |
| 15 | } |
| 16 | |
| 17 | async function runScenario(benchmark, chrome) { |
| 18 | const port = chrome.port; |
| 19 | const results = await Lighthouse( |
| 20 | `http://localhost:8080/${benchmark}/`, |
| 21 | { |
| 22 | output: 'json', |
| 23 | port, |
| 24 | }, |
| 25 | config |
| 26 | ); |
| 27 | |
| 28 | const perfMarkings = results.lhr.audits['user-timings'].details.items; |
| 29 | const entries = perfMarkings |
| 30 | .filter(({timingType}) => timingType !== 'Mark') |
| 31 | .map(({duration, name}) => ({ |
| 32 | entry: name, |
| 33 | time: duration, |
| 34 | })); |
| 35 | entries.push({ |
| 36 | entry: 'First Meaningful Paint', |
| 37 | time: results.lhr.audits['first-meaningful-paint'].rawValue, |
| 38 | }); |
| 39 | |
| 40 | return entries; |
| 41 | } |
| 42 | |
| 43 | function bootstrap(data) { |
| 44 | const len = data.length; |
| 45 | const arr = Array(len); |
| 46 | for (let j = 0; j < len; j++) { |
| 47 | arr[j] = data[(Math.random() * len) | 0]; |
| 48 | } |
| 49 | return arr; |
| 50 | } |
| 51 | |
| 52 | function calculateStandardErrorOfMean(data) { |
| 53 | const means = []; |
| 54 | for (let i = 0; i < 10000; i++) { |
| 55 | means.push(stats.mean(bootstrap(data))); |
| 56 | } |
| 57 | return stats.stdev(means); |
| 58 | } |
| 59 | |
| 60 | function calculateAverages(runs) { |
| 61 | const data = []; |
| 62 | const averages = []; |
| 63 | |
| 64 | runs.forEach((entries, x) => { |
| 65 | entries.forEach(({entry, time}, i) => { |
| 66 | if (i >= averages.length) { |
| 67 | data.push([time]); |
| 68 | averages.push({ |
| 69 | entry, |
| 70 | mean: 0, |
| 71 | sem: 0, |
| 72 | }); |
| 73 | } else { |
| 74 | data[i].push(time); |
| 75 | if (x === runs.length - 1) { |
| 76 | const dataWithoutOutliers = stats.filterMADoutliers(data[i]); |
| 77 | averages[i].mean = stats.mean(dataWithoutOutliers); |
| 78 | averages[i].sem = calculateStandardErrorOfMean(data[i]); |
| 79 | } |
| 80 | } |
| 81 | }); |
| 82 | }); |
| 83 | |
| 84 | return averages; |
| 85 | } |
| 86 | |
| 87 | async function initChrome() { |
| 88 | const platform = os.platform(); |
| 89 | |
| 90 | if (platform === 'linux') { |
| 91 | process.env.XVFBARGS = '-screen 0, 1024x768x16'; |
| 92 | process.env.LIGHTHOUSE_CHROMIUM_PATH = 'chromium-browser'; |
| 93 | const child = spawn('xvfb', ['start'], { |
| 94 | detached: true, |
| 95 | stdio: 'ignore', |
| 96 | }); |
| 97 | child.unref(); |
| 98 | // wait for chrome to load then continue |
| 99 | await wait(3000); |
| 100 | return child; |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | async function launchChrome(headless) { |
| 105 | return await chromeLauncher.launch({ |
| 106 | chromeFlags: [headless ? '--headless' : ''], |
| 107 | }); |
| 108 | } |
| 109 | |
| 110 | async function runBenchmark(benchmark, headless) { |
| 111 | const results = { |
| 112 | runs: [], |
| 113 | averages: [], |
| 114 | }; |
| 115 | |
| 116 | await initChrome(); |
| 117 | |
| 118 | for (let i = 0; i < timesToRun; i++) { |
| 119 | let chrome = await launchChrome(headless); |
| 120 | |
| 121 | results.runs.push(await runScenario(benchmark, chrome)); |
| 122 | // add a delay or sometimes it confuses lighthouse and it hangs |
| 123 | await wait(500); |
| 124 | try { |
| 125 | await chrome.kill(); |
| 126 | } catch (e) {} |
| 127 | } |
| 128 | |
| 129 | results.averages = calculateAverages(results.runs); |
| 130 | return results; |
| 131 | } |
| 132 | |
| 133 | module.exports = runBenchmark; |