@samitouri / QOSAMI-WSL / commits / f4ce6e00

Use alpine instead of debian for registry push/pull test payloads (#41338)

Use alpine instead of debian for registry push/pull test payloads and add test benchmarking script Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

ggarzia-MSFT committed Aug 13, 2026 at 13:04 UTC f4ce6e00e7187ad29bc38af3d485d445b0744bbb
4 files changed +241 -12
test/windows/wslc/e2e/WSLCE2EImageBuildTests.cpp
+3 -1
@@ -173,10 +173,12 @@ class WSLCE2EImageBuildTests
173 WSLC_TEST_METHOD(WSLCE2E_Image_Build_Pull_Success)
174 {
175 // A local registry acts as the private image source that --pull re-resolves the base image from.
176 + EnsureImageIsLoaded(AlpineTestImage());
177 +
178 auto session = OpenDefaultElevatedSession();
179 auto [registryContainer, registryAddress] = StartLocalRegistry(*session, "", "", c_registryPort);
180
179 - auto registryImage = TagImageForRegistry(DebianTestImage().NameAndTag(), string::MultiByteToWide(registryAddress));
181 + auto registryImage = TagImageForRegistry(AlpineTestImage().NameAndTag(), string::MultiByteToWide(registryAddress));
182 auto registryImageCleanup = DeleteImageOnExit(registryImage);
183
184 RunWslcAndVerify(std::format(L"push {}", registryImage), {.Stderr = L"", .ExitCode = 0});
test/windows/wslc/e2e/WSLCE2EPushPullTests.cpp
+6 -6
@@ -55,8 +55,8 @@ class WSLCE2EPushPullTests
55
56 WSLC_TEST_METHOD(WSLCE2E_Image_PushPull)
57 {
58 - const auto& debianImage = DebianTestImage();
59 - EnsureImageIsLoaded(debianImage);
58 + const auto& testImage = AlpineTestImage();
59 + EnsureImageIsLoaded(testImage);
60
61 // Start a local registry without auth.
62 auto session = OpenDefaultElevatedSession();
@@ -66,7 +66,7 @@ class WSLCE2EPushPullTests
66 auto registryAddressW = string::MultiByteToWide(registryAddress);
67
68 // Tag the image for the local registry.
69 - auto registryImage = TagImageForRegistry(debianImage.NameAndTag(), registryAddressW);
69 + auto registryImage = TagImageForRegistry(testImage.NameAndTag(), registryAddressW);
70
71 auto tagCleanup = wil::scope_exit([&]() { RunWslc(std::format(L"image delete --force {}", registryImage)); });
72
@@ -95,8 +95,8 @@ class WSLCE2EPushPullTests
95
96 WSLC_TEST_METHOD(WSLCE2E_Image_Pull_QuietOption)
97 {
98 - const auto& debianImage = DebianTestImage();
99 - EnsureImageIsLoaded(debianImage);
98 + const auto& testImage = AlpineTestImage();
99 + EnsureImageIsLoaded(testImage);
100
101 auto session = OpenDefaultElevatedSession();
102
@@ -105,7 +105,7 @@ class WSLCE2EPushPullTests
105 auto registryAddressW = string::MultiByteToWide(registryAddress);
106
107 // Tag and push the image so it can be pulled back from the registry.
108 - auto registryImage = TagImageForRegistry(debianImage.NameAndTag(), registryAddressW);
108 + auto registryImage = TagImageForRegistry(testImage.NameAndTag(), registryAddressW);
109 auto tagCleanup = wil::scope_exit([&]() { RunWslc(std::format(L"image delete --force {}", registryImage)); });
110
111 RunWslcAndVerify(std::format(L"push {}", registryImage), {.Stderr = L"", .ExitCode = 0});
test/windows/wslc/e2e/WSLCE2ERegistryTests.cpp
+5 -5
@@ -48,8 +48,8 @@ class WSLCE2ERegistryTests
48
49 WSLC_TEST_METHOD(WSLCE2E_Registry_LoginLogout_PushPull_AuthFlow)
50 {
51 - const auto& debianImage = DebianTestImage();
52 - EnsureImageIsLoaded(debianImage);
51 + const auto& testImage = AlpineTestImage();
52 + EnsureImageIsLoaded(testImage);
53
54 auto session = OpenDefaultElevatedSession();
55
@@ -57,7 +57,7 @@ class WSLCE2ERegistryTests
57 auto [registryContainer, registryAddress] = StartLocalRegistry(*session, c_username, c_password, 15001);
58 auto registryAddressW = string::MultiByteToWide(registryAddress);
59
60 - auto registryImageName = TagImageForRegistry(debianImage.NameAndTag(), registryAddressW);
60 + auto registryImageName = TagImageForRegistry(testImage.NameAndTag(), registryAddressW);
61
62 auto cleanup = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&]() {
63 RunWslc(std::format(L"image delete --force {}", registryImageName));
@@ -78,7 +78,7 @@ class WSLCE2ERegistryTests
78 L"login -u {} -p {} {}", string::MultiByteToWide(c_username), string::MultiByteToWide(c_password), registryAddressW));
79 result.Verify({.Stdout = Localization::WSLCCLI_LoginSucceeded() + L"\r\n", .Stderr = L"", .ExitCode = 0});
80
81 - registryImageName = TagImageForRegistry(L"debian:latest", registryAddressW);
81 + registryImageName = TagImageForRegistry(testImage.NameAndTag(), registryAddressW);
82 result = RunWslc(std::format(L"push {}", registryImageName));
83 result.Verify({.ExitCode = 0});
84
@@ -93,7 +93,7 @@ class WSLCE2ERegistryTests
93 result = RunWslc(std::format(L"pull {}", registryImageName));
94 VerifyAuthFailure(result);
95
96 - registryImageName = TagImageForRegistry(L"debian:latest", registryAddressW);
96 + registryImageName = TagImageForRegistry(testImage.NameAndTag(), registryAddressW);
97 result = RunWslc(std::format(L"push {}", registryImageName));
98 VerifyAuthFailure(result);
99
tools/benchmarking/benchmark-tests.ps1 new
+227
@@ -0,0 +1,227 @@
1 +#Requires -RunAsAdministrator
2 +
3 +<#
4 +.SYNOPSIS
5 + Runs the WSL test suite repeatedly with a given filter and reports timing statistics.
6 +
7 +.DESCRIPTION
8 + Wraps bin\<arch>\<config>\test.bat in a timing loop so a change's effect on test runtime can
9 + be measured rather than estimated. Each pass is timed individually, its log is kept, and the
10 + results are written incrementally to a CSV so a run in progress can be monitored.
11 +
12 + Building and branch selection are the caller's responsibility: this script measures whatever
13 + is currently in the bin directory. To compare two revisions, build one, run this, rebuild the
14 + other, and run it again with a different -Label.
15 +
16 + Runtime is dominated by per-invocation setup unless -f is included in the filter, which skips
17 + package and distro installation (requires "wsl --set-default test_distro" to have been run).
18 +
19 +.PARAMETER Filter
20 + Arguments passed verbatim to test.bat, typically a TAEF selection switch.
21 +
22 + They are written into a temporary .cmd rather than forwarded as PowerShell arguments, because
23 + test.bat re-invokes powershell.exe with %*, which would otherwise re-split them. Quotes that
24 + need to survive that hop must be backslash-escaped by the caller:
25 +
26 + /name:WSLCE2ETests::WSLCE2EPushPullTests::WSLCE2E_Image_PushPull
27 + /select:\"@TestCategory='WSLC'\"
28 +
29 + Note that TAEF honours only the first /name switch, so several cannot be OR'd together.
30 +
31 +.PARAMETER Runs
32 + Number of passes to execute.
33 +
34 +.PARAMETER Label
35 + Short name used for the CSV and per-run log file names. Defaults to "benchmark".
36 +
37 +.PARAMETER Warmup
38 + Performs one untimed run before the measured passes and discards its result. The first run of
39 + a series is routinely several seconds slower than the rest because caches, the WSL session and
40 + the test distro are all cold, which skews the mean and inflates the standard deviation.
41 +
42 +.PARAMETER OutputPath
43 + Directory for the CSV and logs. Defaults to a wsl-test-benchmark folder under %TEMP%.
44 +
45 +.PARAMETER Arch
46 + Build architecture to locate test.bat under bin. Defaults to x64.
47 +
48 +.PARAMETER Config
49 + Build configuration to locate test.bat under bin. Defaults to debug.
50 +
51 +.EXAMPLE
52 + .\benchmark-tests.ps1 -Runs 10 -Warmup -Filter '/name:WSLCE2ETests::WSLCE2EPushPullTests::WSLCE2E_Image_PushPull -f'
53 +
54 + Times one test over ten passes in fast mode, discarding a cold first run.
55 +
56 +.EXAMPLE
57 + .\benchmark-tests.ps1 -Runs 3 -Filter '/select:\"@TestCategory=''WSLC''\"' -Label wslc-suite
58 +
59 + Times the whole WSLC category three times.
60 +#>
61 +
62 +[CmdletBinding()]
63 +param(
64 + [Parameter(Mandatory)][string]$Filter,
65 + [ValidateRange(1, 1000)][int]$Runs = 1,
66 + [string]$Label = 'benchmark',
67 + [switch]$Warmup,
68 + [string]$OutputPath = (Join-Path $env:TEMP 'wsl-test-benchmark'),
69 + [ValidateSet('x64', 'arm64')][string]$Arch = 'x64',
70 + [ValidateSet('debug', 'release')][string]$Config = 'debug'
71 +)
72 +
73 +Set-StrictMode -Version Latest
74 +$ErrorActionPreference = 'Stop'
75 +
76 +$repoRoot = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent
77 +$testBat = Join-Path $repoRoot "bin\$Arch\$Config\test.bat"
78 +if (-not (Test-Path $testBat))
79 +{
80 + throw "test.bat not found at $testBat. Build the project first."
81 +}
82 +
83 +# TAEF logs interleave UTF-16 and UTF-8, so decoding as either alone silently loses most lines.
84 +# Dropping NUL bytes yields readable text for both. Shared access allows reading a live log.
85 +function Read-TestLog([string]$Path)
86 +{
87 + if (-not (Test-Path $Path))
88 + {
89 + return @()
90 + }
91 +
92 + $stream = [IO.File]::Open($Path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite)
93 + try
94 + {
95 + $buffer = New-Object IO.MemoryStream
96 + $stream.CopyTo($buffer)
97 + }
98 + finally
99 + {
100 + $stream.Close()
101 + }
102 +
103 + $characters = foreach ($byte in $buffer.ToArray())
104 + {
105 + if ($byte -ne 0)
106 + {
107 + [char]$byte
108 + }
109 + }
110 +
111 + return (($characters -join '') -split "`r?`n")
112 +}
113 +
114 +function Get-Statistics([double[]]$Values)
115 +{
116 + $sorted = @($Values | Sort-Object)
117 + $count = $sorted.Count
118 + $mean = ($sorted | Measure-Object -Average).Average
119 +
120 + $median = if ($count % 2)
121 + {
122 + $sorted[[int][math]::Floor($count / 2)]
123 + }
124 + else
125 + {
126 + ($sorted[$count / 2 - 1] + $sorted[$count / 2]) / 2
127 + }
128 +
129 + $standardDeviation = 0.0
130 + if ($count -gt 1)
131 + {
132 + $sumOfSquares = ($sorted | ForEach-Object { [math]::Pow($_ - $mean, 2) } | Measure-Object -Sum).Sum
133 + $standardDeviation = [math]::Sqrt($sumOfSquares / ($count - 1))
134 + }
135 +
136 + return [pscustomobject]@{
137 + Runs = $count
138 + Min = $sorted[0]
139 + Median = $median
140 + Mean = $mean
141 + Max = $sorted[-1]
142 + StandardDeviation = $standardDeviation
143 + }
144 +}
145 +
146 +New-Item -ItemType Directory -Force -Path $OutputPath | Out-Null
147 +$csvPath = Join-Path $OutputPath "$Label.csv"
148 +$wrapperPath = Join-Path $OutputPath "$Label-wrapper.cmd"
149 +
150 +@"
151 +@echo off
152 +call "$testBat" $Filter
153 +exit /b %ERRORLEVEL%
154 +"@ | Set-Content -Path $wrapperPath -Encoding ASCII
155 +
156 +'Run,Seconds,Total,Passed,Failed,Skipped,ExitCode,Log' | Set-Content -Path $csvPath -Encoding ASCII
157 +
158 +Write-Host "Filter : $Filter"
159 +Write-Host "Runs : $Runs$(if ($Warmup) { ' (plus one untimed warmup)' })"
160 +Write-Host "Output : $OutputPath"
161 +Write-Host ''
162 +
163 +$durations = [System.Collections.Generic.List[double]]::new()
164 +$failedRuns = 0
165 +
166 +try
167 +{
168 + if ($Warmup)
169 + {
170 + $warmupLog = Join-Path $OutputPath "$Label-warmup.log"
171 + Remove-Item $warmupLog -ErrorAction SilentlyContinue
172 +
173 + $stopwatch = [Diagnostics.Stopwatch]::StartNew()
174 + & cmd.exe /c "call `"$wrapperPath`" > `"$warmupLog`" 2>&1"
175 + $stopwatch.Stop()
176 +
177 + Write-Host ("warmup : {0,8:N2}s (discarded)" -f $stopwatch.Elapsed.TotalSeconds)
178 + }
179 +
180 + for ($run = 1; $run -le $Runs; $run++)
181 + {
182 + $logPath = Join-Path $OutputPath "$Label-run$run.log"
183 + Remove-Item $logPath -ErrorAction SilentlyContinue
184 +
185 + $stopwatch = [Diagnostics.Stopwatch]::StartNew()
186 + & cmd.exe /c "call `"$wrapperPath`" > `"$logPath`" 2>&1"
187 + $exitCode = $LASTEXITCODE
188 + $stopwatch.Stop()
189 +
190 + $total = $passed = $failed = $skipped = ''
191 + $summary = Read-TestLog $logPath | Select-String -SimpleMatch 'Summary: Total=' | Select-Object -First 1
192 + if ($summary -and $summary.Line -match 'Total=(\d+).*?Passed=(\d+).*?Failed=(\d+).*?Skipped=(\d+)')
193 + {
194 + $total, $passed, $failed, $skipped = $Matches[1], $Matches[2], $Matches[3], $Matches[4]
195 + if ([int]$failed -gt 0)
196 + {
197 + $failedRuns++
198 + }
199 + }
200 +
201 + $seconds = [math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
202 + $durations.Add($seconds)
203 + "$run,$seconds,$total,$passed,$failed,$skipped,$exitCode,$(Split-Path $logPath -Leaf)" |
204 + Add-Content -Path $csvPath -Encoding ASCII
205 +
206 + Write-Host ("run {0}/{1}: {2,8:N2}s total={3} passed={4} failed={5} skipped={6}" -f `
207 + $run, $Runs, $seconds, $total, $passed, $failed, $skipped)
208 + }
209 +}
210 +finally
211 +{
212 + Remove-Item $wrapperPath -ErrorAction SilentlyContinue
213 +}
214 +
215 +$statistics = Get-Statistics $durations.ToArray()
216 +
217 +Write-Host ''
218 +Write-Host "=== $Label ==="
219 +Write-Host ("min {0:N2}s | median {1:N2}s | mean {2:N2}s | max {3:N2}s | sd {4:N2}s (n={5})" -f `
220 + $statistics.Min, $statistics.Median, $statistics.Mean, $statistics.Max, $statistics.StandardDeviation, $statistics.Runs)
221 +
222 +if ($failedRuns -gt 0)
223 +{
224 + Write-Warning "$failedRuns of $Runs run(s) reported test failures. Timings may not be comparable."
225 +}
226 +
227 +Write-Host "CSV: $csvPath"