| 1 | #Requires -RunAsAdministrator |
| 2 | |
| 3 | <# |
| 4 | .SYNOPSIS |
| 5 | Runs the WSL test suite repeatedly with a given filter and reports timing statistics. |
| 6 | |
| 7 | .DESCRIPTION |
| 8 | Wraps bin\<arch>\<config>\test.bat in a timing loop so a change's effect on test runtime can |
| 9 | be measured rather than estimated. Each pass is timed individually, its log is kept, and the |
| 10 | results are written incrementally to a CSV so a run in progress can be monitored. |
| 11 | |
| 12 | Building and branch selection are the caller's responsibility: this script measures whatever |
| 13 | is currently in the bin directory. To compare two revisions, build one, run this, rebuild the |
| 14 | other, and run it again with a different -Label. |
| 15 | |
| 16 | Runtime is dominated by per-invocation setup unless -f is included in the filter, which skips |
| 17 | package and distro installation (requires "wsl --set-default test_distro" to have been run). |
| 18 | |
| 19 | .PARAMETER Filter |
| 20 | Arguments passed verbatim to test.bat, typically a TAEF selection switch. |
| 21 | |
| 22 | They are written into a temporary .cmd rather than forwarded as PowerShell arguments, because |
| 23 | test.bat re-invokes powershell.exe with %*, which would otherwise re-split them. Quotes that |
| 24 | need to survive that hop must be backslash-escaped by the caller: |
| 25 | |
| 26 | /name:WSLCE2ETests::WSLCE2EPushPullTests::WSLCE2E_Image_PushPull |
| 27 | /select:\"@TestCategory='WSLC'\" |
| 28 | |
| 29 | Note that TAEF honours only the first /name switch, so several cannot be OR'd together. |
| 30 | |
| 31 | .PARAMETER Runs |
| 32 | Number of passes to execute. |
| 33 | |
| 34 | .PARAMETER Label |
| 35 | Short name used for the CSV and per-run log file names. Defaults to "benchmark". |
| 36 | |
| 37 | .PARAMETER Warmup |
| 38 | Performs one untimed run before the measured passes and discards its result. The first run of |
| 39 | a series is routinely several seconds slower than the rest because caches, the WSL session and |
| 40 | the test distro are all cold, which skews the mean and inflates the standard deviation. |
| 41 | |
| 42 | .PARAMETER OutputPath |
| 43 | Directory for the CSV and logs. Defaults to a wsl-test-benchmark folder under %TEMP%. |
| 44 | |
| 45 | .PARAMETER Arch |
| 46 | Build architecture to locate test.bat under bin. Defaults to x64. |
| 47 | |
| 48 | .PARAMETER Config |
| 49 | Build configuration to locate test.bat under bin. Defaults to debug. |
| 50 | |
| 51 | .EXAMPLE |
| 52 | .\benchmark-tests.ps1 -Runs 10 -Warmup -Filter '/name:WSLCE2ETests::WSLCE2EPushPullTests::WSLCE2E_Image_PushPull -f' |
| 53 | |
| 54 | Times one test over ten passes in fast mode, discarding a cold first run. |
| 55 | |
| 56 | .EXAMPLE |
| 57 | .\benchmark-tests.ps1 -Runs 3 -Filter '/select:\"@TestCategory=''WSLC''\"' -Label wslc-suite |
| 58 | |
| 59 | Times the whole WSLC category three times. |
| 60 | #> |
| 61 | |
| 62 | [CmdletBinding()] |
| 63 | param( |
| 64 | [Parameter(Mandatory)][string]$Filter, |
| 65 | [ValidateRange(1, 1000)][int]$Runs = 1, |
| 66 | [string]$Label = 'benchmark', |
| 67 | [switch]$Warmup, |
| 68 | [string]$OutputPath = (Join-Path $env:TEMP 'wsl-test-benchmark'), |
| 69 | [ValidateSet('x64', 'arm64')][string]$Arch = 'x64', |
| 70 | [ValidateSet('debug', 'release')][string]$Config = 'debug' |
| 71 | ) |
| 72 | |
| 73 | Set-StrictMode -Version Latest |
| 74 | $ErrorActionPreference = 'Stop' |
| 75 | |
| 76 | $repoRoot = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent |
| 77 | $testBat = Join-Path $repoRoot "bin\$Arch\$Config\test.bat" |
| 78 | if (-not (Test-Path $testBat)) |
| 79 | { |
| 80 | throw "test.bat not found at $testBat. Build the project first." |
| 81 | } |
| 82 | |
| 83 | # TAEF logs interleave UTF-16 and UTF-8, so decoding as either alone silently loses most lines. |
| 84 | # Dropping NUL bytes yields readable text for both. Shared access allows reading a live log. |
| 85 | function Read-TestLog([string]$Path) |
| 86 | { |
| 87 | if (-not (Test-Path $Path)) |
| 88 | { |
| 89 | return @() |
| 90 | } |
| 91 | |
| 92 | $stream = [IO.File]::Open($Path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) |
| 93 | try |
| 94 | { |
| 95 | $buffer = New-Object IO.MemoryStream |
| 96 | $stream.CopyTo($buffer) |
| 97 | } |
| 98 | finally |
| 99 | { |
| 100 | $stream.Close() |
| 101 | } |
| 102 | |
| 103 | $characters = foreach ($byte in $buffer.ToArray()) |
| 104 | { |
| 105 | if ($byte -ne 0) |
| 106 | { |
| 107 | [char]$byte |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | return (($characters -join '') -split "`r?`n") |
| 112 | } |
| 113 | |
| 114 | function Get-Statistics([double[]]$Values) |
| 115 | { |
| 116 | $sorted = @($Values | Sort-Object) |
| 117 | $count = $sorted.Count |
| 118 | $mean = ($sorted | Measure-Object -Average).Average |
| 119 | |
| 120 | $median = if ($count % 2) |
| 121 | { |
| 122 | $sorted[[int][math]::Floor($count / 2)] |
| 123 | } |
| 124 | else |
| 125 | { |
| 126 | ($sorted[$count / 2 - 1] + $sorted[$count / 2]) / 2 |
| 127 | } |
| 128 | |
| 129 | $standardDeviation = 0.0 |
| 130 | if ($count -gt 1) |
| 131 | { |
| 132 | $sumOfSquares = ($sorted | ForEach-Object { [math]::Pow($_ - $mean, 2) } | Measure-Object -Sum).Sum |
| 133 | $standardDeviation = [math]::Sqrt($sumOfSquares / ($count - 1)) |
| 134 | } |
| 135 | |
| 136 | return [pscustomobject]@{ |
| 137 | Runs = $count |
| 138 | Min = $sorted[0] |
| 139 | Median = $median |
| 140 | Mean = $mean |
| 141 | Max = $sorted[-1] |
| 142 | StandardDeviation = $standardDeviation |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | New-Item -ItemType Directory -Force -Path $OutputPath | Out-Null |
| 147 | $csvPath = Join-Path $OutputPath "$Label.csv" |
| 148 | $wrapperPath = Join-Path $OutputPath "$Label-wrapper.cmd" |
| 149 | |
| 150 | @" |
| 151 | @echo off |
| 152 | call "$testBat" $Filter |
| 153 | exit /b %ERRORLEVEL% |
| 154 | "@ | Set-Content -Path $wrapperPath -Encoding ASCII |
| 155 | |
| 156 | 'Run,Seconds,Total,Passed,Failed,Skipped,ExitCode,Log' | Set-Content -Path $csvPath -Encoding ASCII |
| 157 | |
| 158 | Write-Host "Filter : $Filter" |
| 159 | Write-Host "Runs : $Runs$(if ($Warmup) { ' (plus one untimed warmup)' })" |
| 160 | Write-Host "Output : $OutputPath" |
| 161 | Write-Host '' |
| 162 | |
| 163 | $durations = [System.Collections.Generic.List[double]]::new() |
| 164 | $failedRuns = 0 |
| 165 | |
| 166 | try |
| 167 | { |
| 168 | if ($Warmup) |
| 169 | { |
| 170 | $warmupLog = Join-Path $OutputPath "$Label-warmup.log" |
| 171 | Remove-Item $warmupLog -ErrorAction SilentlyContinue |
| 172 | |
| 173 | $stopwatch = [Diagnostics.Stopwatch]::StartNew() |
| 174 | & cmd.exe /c "call `"$wrapperPath`" > `"$warmupLog`" 2>&1" |
| 175 | $stopwatch.Stop() |
| 176 | |
| 177 | Write-Host ("warmup : {0,8:N2}s (discarded)" -f $stopwatch.Elapsed.TotalSeconds) |
| 178 | } |
| 179 | |
| 180 | for ($run = 1; $run -le $Runs; $run++) |
| 181 | { |
| 182 | $logPath = Join-Path $OutputPath "$Label-run$run.log" |
| 183 | Remove-Item $logPath -ErrorAction SilentlyContinue |
| 184 | |
| 185 | $stopwatch = [Diagnostics.Stopwatch]::StartNew() |
| 186 | & cmd.exe /c "call `"$wrapperPath`" > `"$logPath`" 2>&1" |
| 187 | $exitCode = $LASTEXITCODE |
| 188 | $stopwatch.Stop() |
| 189 | |
| 190 | $total = $passed = $failed = $skipped = '' |
| 191 | $summary = Read-TestLog $logPath | Select-String -SimpleMatch 'Summary: Total=' | Select-Object -First 1 |
| 192 | if ($summary -and $summary.Line -match 'Total=(\d+).*?Passed=(\d+).*?Failed=(\d+).*?Skipped=(\d+)') |
| 193 | { |
| 194 | $total, $passed, $failed, $skipped = $Matches[1], $Matches[2], $Matches[3], $Matches[4] |
| 195 | if ([int]$failed -gt 0) |
| 196 | { |
| 197 | $failedRuns++ |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | $seconds = [math]::Round($stopwatch.Elapsed.TotalSeconds, 2) |
| 202 | $durations.Add($seconds) |
| 203 | "$run,$seconds,$total,$passed,$failed,$skipped,$exitCode,$(Split-Path $logPath -Leaf)" | |
| 204 | Add-Content -Path $csvPath -Encoding ASCII |
| 205 | |
| 206 | Write-Host ("run {0}/{1}: {2,8:N2}s total={3} passed={4} failed={5} skipped={6}" -f ` |
| 207 | $run, $Runs, $seconds, $total, $passed, $failed, $skipped) |
| 208 | } |
| 209 | } |
| 210 | finally |
| 211 | { |
| 212 | Remove-Item $wrapperPath -ErrorAction SilentlyContinue |
| 213 | } |
| 214 | |
| 215 | $statistics = Get-Statistics $durations.ToArray() |
| 216 | |
| 217 | Write-Host '' |
| 218 | Write-Host "=== $Label ===" |
| 219 | Write-Host ("min {0:N2}s | median {1:N2}s | mean {2:N2}s | max {3:N2}s | sd {4:N2}s (n={5})" -f ` |
| 220 | $statistics.Min, $statistics.Median, $statistics.Mean, $statistics.Max, $statistics.StandardDeviation, $statistics.Runs) |
| 221 | |
| 222 | if ($failedRuns -gt 0) |
| 223 | { |
| 224 | Write-Warning "$failedRuns of $Runs run(s) reported test failures. Timings may not be comparable." |
| 225 | } |
| 226 | |
| 227 | Write-Host "CSV: $csvPath" |