@samitouri / QOSAMI-WSL-1 / commits / e2a7b1d1

cleanup: refactor svccommio class (#13843)

* cleanup: refactor svccommio class * add logging if setting code page fails * pr feedback * remove unneeded if * remove unneeded if * pr feedbacK' --------- Co-authored-by: Ben Hillis <benhill@ntdev.microsoft.com>

Ben Hillis committed Jan 14, 2026 at 12:24 UTC e2a7b1d168f3c5cada8c72efb85e2cb9478084f8
2 files changed +155 -214
src/windows/common/svccommio.cpp
+111 -196
@@ -18,7 +18,14 @@ Abstract:
18 #pragma hdrstop
19
20 namespace {
21 -void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
21 +
22 +bool IsConsoleHandle(_In_ HANDLE Handle)
23 +{
24 + DWORD Mode;
25 + return GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode);
26 +}
27 +
28 +void ChangeConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
29 {
30 //
31 // Use the invalid parameter error code to detect the v1 console that does
@@ -29,8 +36,17 @@ void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
36 // disconnected.
37 //
38
32 - if (!SetConsoleMode(File, ConsoleMode))
39 + if (!SetConsoleMode(Handle, Mode))
40 {
41 + // DISABLE_NEWLINE_AUTO_RETURN is not supported everywhere, if the flag was present fall back and try again.
42 + if (WI_IsFlagSet(Mode, DISABLE_NEWLINE_AUTO_RETURN))
43 + {
44 + if (SetConsoleMode(Handle, WI_ClearFlag(Mode, DISABLE_NEWLINE_AUTO_RETURN)))
45 + {
46 + return;
47 + }
48 + }
49 +
50 switch (GetLastError())
51 {
52 case ERROR_INVALID_PARAMETER:
@@ -45,238 +61,137 @@ void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
61 }
62 }
63
48 -void ConfigureStdHandles(_Inout_ LXSS_STD_HANDLES_INFO& StdHandlesInfo)
64 +void TrySetConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
65 +try
66 {
50 - //
51 - // Check stdin to see if it is a console or another device. If it is
52 - // a console, configure it to raw processing mode and VT-100 support. If the
53 - // force console I/O is requested, ignore stdin and get active console input
54 - // handle instead.
55 - //
67 + ChangeConsoleMode(Handle, Mode);
68 +}
69 +CATCH_LOG()
70
57 - UINT NewConsoleInputCP = 0;
58 - DWORD NewConsoleInputMode = 0;
59 - BOOLEAN IsConsoleInput = StdHandlesInfo.IsConsoleInput;
60 - BOOLEAN IsConsoleOutput = StdHandlesInfo.IsConsoleOutput;
61 - BOOLEAN IsConsoleError = StdHandlesInfo.IsConsoleError;
62 - DWORD SavedInputMode = StdHandlesInfo.SavedInputMode;
63 - DWORD SavedOutputMode = StdHandlesInfo.SavedOutputMode;
64 - UINT SavedInputCP = StdHandlesInfo.SavedInputCP;
65 - UINT SavedOutputCP = StdHandlesInfo.SavedOutputCP;
66 - CONSOLE_SCREEN_BUFFER_INFO ScreenBufferInfo;
67 - auto RestoreInputHandle = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&] {
68 - if (NewConsoleInputCP != 0)
69 - {
70 - SetConsoleCP(SavedInputCP);
71 - }
71 +} // namespace
72
73 - if (NewConsoleInputMode != 0)
74 - {
75 - ChangeConsoleMode(StdHandlesInfo.InputHandle, SavedInputMode);
76 - }
77 - });
73 +namespace wsl::windows::common {
74
79 - IsConsoleInput = FALSE;
80 - if ((GetFileType(StdHandlesInfo.InputHandle) == FILE_TYPE_CHAR) && (GetConsoleMode(StdHandlesInfo.InputHandle, &SavedInputMode)))
75 +std::optional<ConsoleInput> ConsoleInput::Create(HANDLE Handle)
76 +{
77 + DWORD Mode;
78 + if (GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode))
79 {
82 - IsConsoleInput = TRUE;
83 - NewConsoleInputMode = SavedInputMode;
84 - WI_SetAllFlags(NewConsoleInputMode, (ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT));
85 - WI_ClearAllFlags(NewConsoleInputMode, (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT));
86 - ChangeConsoleMode(StdHandlesInfo.InputHandle, NewConsoleInputMode);
87 -
88 - //
89 - // Set the console input to the UTF-8 code page.
90 - //
91 -
92 - SavedInputCP = GetConsoleCP();
93 - NewConsoleInputCP = CP_UTF8;
94 - THROW_LAST_ERROR_IF(!::SetConsoleCP(NewConsoleInputCP));
80 + return ConsoleInput(Handle, Mode);
81 }
82
97 - bool RestoreMode = false;
98 - bool RestoreCp = false;
99 - auto RestoreOutput = wil::scope_exit([&] {
100 - if (RestoreMode)
101 - {
102 - SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), SavedOutputMode);
103 - }
104 -
105 - if (RestoreCp)
106 - {
107 - SetConsoleOutputCP(SavedOutputCP);
108 - }
109 - });
110 -
111 - //
112 - // If there is a console output handle, save the output mode and codepage so
113 - // it can be restored.
114 - //
115 -
116 - if (StdHandlesInfo.ConsoleOutputHandle)
117 - {
118 - THROW_LAST_ERROR_IF(!::GetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), &SavedOutputMode));
119 -
120 - //
121 - // Temporarily try both with and without the custom flag to disable newline
122 - // auto return.
123 - //
124 -
125 - DWORD NewConsoleOutputMode = SavedOutputMode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
126 - if (SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode) == FALSE)
127 - {
128 - WI_ClearFlag(NewConsoleOutputMode, DISABLE_NEWLINE_AUTO_RETURN);
129 - ChangeConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode);
130 - }
131 -
132 - RestoreMode = true;
133 -
134 - //
135 - // Set the console output to the UTF-8 code page.
136 - //
137 -
138 - SavedOutputCP = GetConsoleOutputCP();
139 - THROW_LAST_ERROR_IF(!::SetConsoleOutputCP(CP_UTF8));
140 -
141 - RestoreCp = true;
142 - }
83 + return std::nullopt;
84 +}
85
144 - //
145 - // If the force console I/O is requested, ignore stdout and treat the
146 - // console as the output handle.
147 - //
86 +ConsoleInput::ConsoleInput(HANDLE Handle, DWORD SavedMode) : m_Handle(Handle), m_SavedMode(SavedMode)
87 +{
88 + // Save code page.
89 + m_SavedCodePage = GetConsoleCP();
90
149 - IsConsoleOutput = FALSE;
150 - if ((GetFileType(StdHandlesInfo.OutputHandle) == FILE_TYPE_CHAR) &&
151 - (GetConsoleScreenBufferInfo(StdHandlesInfo.OutputHandle, &ScreenBufferInfo)))
152 - {
153 - IsConsoleOutput = TRUE;
154 - }
91 + // Configure for raw input with VT support.
92 + DWORD NewMode = m_SavedMode;
93 + WI_SetAllFlags(NewMode, ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT);
94 + WI_ClearAllFlags(NewMode, ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT);
95 + ChangeConsoleMode(Handle, NewMode);
96
156 - IsConsoleError = FALSE;
157 - if ((GetFileType(StdHandlesInfo.ErrorHandle) == FILE_TYPE_CHAR) &&
158 - (GetConsoleScreenBufferInfo(StdHandlesInfo.ErrorHandle, &ScreenBufferInfo)))
159 - {
160 - IsConsoleError = TRUE;
161 - }
97 + // Set UTF-8 code page.
98 + LOG_IF_WIN32_BOOL_FALSE(SetConsoleCP(CP_UTF8));
99 +}
100
163 - RestoreInputHandle.release();
164 - RestoreOutput.release();
165 - StdHandlesInfo.IsConsoleInput = IsConsoleInput;
166 - StdHandlesInfo.IsConsoleOutput = IsConsoleOutput;
167 - StdHandlesInfo.IsConsoleError = IsConsoleError;
168 - StdHandlesInfo.SavedInputMode = SavedInputMode;
169 - StdHandlesInfo.SavedOutputMode = SavedOutputMode;
170 - StdHandlesInfo.SavedInputCP = SavedInputCP;
171 - StdHandlesInfo.SavedOutputCP = SavedOutputCP;
101 +ConsoleInput::~ConsoleInput()
102 +{
103 + TrySetConsoleMode(m_Handle, m_SavedMode);
104 + LOG_IF_WIN32_BOOL_FALSE(SetConsoleCP(m_SavedCodePage));
105 }
173 -} // namespace
106
175 -wsl::windows::common::SvcCommIo::SvcCommIo()
107 +std::optional<ConsoleOutput> ConsoleOutput::Create()
108 {
177 - _stdHandlesInfo.InputHandle = GetStdHandle(STD_INPUT_HANDLE);
178 - _stdHandlesInfo.OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
179 - _stdHandlesInfo.ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);
180 - _stdHandlesInfo.ConsoleOutputHandle.reset(
109 + wil::unique_hfile ConsoleHandle(
110 CreateFileW(L"CONOUT$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr));
111
183 - ConfigureStdHandles(_stdHandlesInfo);
184 - _stdHandles.StdIn.HandleType = LxssHandleInput;
185 - _stdHandles.StdIn.Handle = HandleToUlong(_stdHandlesInfo.InputHandle);
186 - _stdHandles.StdOut.HandleType = LxssHandleOutput;
187 - _stdHandles.StdOut.Handle = HandleToUlong(_stdHandlesInfo.OutputHandle);
188 - _stdHandles.StdErr.HandleType = LxssHandleOutput;
189 - _stdHandles.StdErr.Handle = HandleToUlong(_stdHandlesInfo.ErrorHandle);
190 -
191 - //
192 - // N.B.: The console handle is not supposed to be closed, it is just copied
193 - // from PEB.
194 - //
195 -
196 - if (_stdHandlesInfo.IsConsoleInput)
197 - {
198 - _stdHandles.StdIn.Handle = LXSS_HANDLE_USE_CONSOLE;
199 - _stdHandles.StdIn.HandleType = LxssHandleConsole;
200 - }
201 -
202 - if (_stdHandlesInfo.IsConsoleOutput)
112 + if (ConsoleHandle)
113 {
204 - _stdHandles.StdOut.Handle = LXSS_HANDLE_USE_CONSOLE;
205 - _stdHandles.StdOut.HandleType = LxssHandleConsole;
114 + DWORD Mode;
115 + if (GetConsoleMode(ConsoleHandle.get(), &Mode))
116 + {
117 + return ConsoleOutput(std::move(ConsoleHandle), Mode);
118 + }
119 }
120
208 - if (_stdHandlesInfo.IsConsoleError)
209 - {
210 - _stdHandles.StdErr.Handle = LXSS_HANDLE_USE_CONSOLE;
211 - _stdHandles.StdErr.HandleType = LxssHandleConsole;
212 - }
121 + return std::nullopt;
122 }
123
215 -wsl::windows::common::SvcCommIo::~SvcCommIo()
124 +ConsoleOutput::ConsoleOutput(wil::unique_hfile&& ConsoleHandle, DWORD SavedMode) :
125 + m_ConsoleHandle(std::move(ConsoleHandle)), m_SavedMode(SavedMode)
126 {
217 - try
218 - {
219 - RestoreConsoleMode();
220 - }
221 - CATCH_LOG()
222 -}
127 + // Save code page.
128 + m_SavedCodePage = GetConsoleOutputCP();
129
224 -PLXSS_STD_HANDLES
225 -wsl::windows::common::SvcCommIo::GetStdHandles()
226 -{
227 - return &_stdHandles;
130 + // Configure for VT output.
131 + DWORD NewMode = m_SavedMode;
132 + WI_SetAllFlags(NewMode, ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN);
133 + ChangeConsoleMode(m_ConsoleHandle.get(), NewMode);
134 +
135 + // Set UTF-8 code page.
136 + LOG_IF_WIN32_BOOL_FALSE(SetConsoleOutputCP(CP_UTF8));
137 }
138
230 -COORD
231 -wsl::windows::common::SvcCommIo::GetWindowSize() const
139 +ConsoleOutput::~ConsoleOutput()
140 {
233 - CONSOLE_SCREEN_BUFFER_INFOEX Info{};
234 - Info.cbSize = sizeof(Info);
235 - if (_stdHandlesInfo.IsConsoleOutput)
236 - {
237 - THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.OutputHandle, &Info));
238 - }
239 - else if (_stdHandlesInfo.IsConsoleError)
240 - {
241 - THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.ErrorHandle, &Info));
242 - }
243 -
244 - return {
245 - static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1), static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
141 + TrySetConsoleMode(m_ConsoleHandle.get(), m_SavedMode);
142 + LOG_IF_WIN32_BOOL_FALSE(SetConsoleOutputCP(m_SavedCodePage));
143 }
144
248 -void wsl::windows::common::SvcCommIo::RestoreConsoleMode() const
249 -
250 -/*++
251 -
252 -Routine Description:
145 +SvcCommIo::SvcCommIo()
146 +{
147 + const HANDLE InputHandle = GetStdHandle(STD_INPUT_HANDLE);
148 + const HANDLE OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
149 + const HANDLE ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);
150
254 - Restores the saved input/output console mode.
151 + // Configure input console
152 + m_ConsoleInput = ConsoleInput::Create(InputHandle);
153
256 -Arguments:
154 + // Configure output console
155 + m_ConsoleOutput = ConsoleOutput::Create();
156
258 - None.
157 + // Initialize the standard handles structure
158 + const bool IsConsoleInput = m_ConsoleInput.has_value();
159 + m_StdHandles.StdIn.HandleType = IsConsoleInput ? LxssHandleConsole : LxssHandleInput;
160 + m_StdHandles.StdIn.Handle = IsConsoleInput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(InputHandle);
161
260 -Return Value:
162 + const bool IsConsoleOutput = IsConsoleHandle(OutputHandle);
163 + m_StdHandles.StdOut.HandleType = IsConsoleOutput ? LxssHandleConsole : LxssHandleOutput;
164 + m_StdHandles.StdOut.Handle = IsConsoleOutput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(OutputHandle);
165
262 - None.
166 + const bool IsConsoleError = IsConsoleHandle(ErrorHandle);
167 + m_StdHandles.StdErr.HandleType = IsConsoleError ? LxssHandleConsole : LxssHandleOutput;
168 + m_StdHandles.StdErr.Handle = IsConsoleError ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(ErrorHandle);
169
264 ---*/
170 + // Cache a console handle for GetWindowSize
171 + m_WindowSizeHandle = IsConsoleOutput ? OutputHandle : (IsConsoleError ? ErrorHandle : nullptr);
172 +}
173
174 +PLXSS_STD_HANDLES
175 +SvcCommIo::GetStdHandles()
176 {
267 - //
268 - // Restore the console input and output modes.
269 - //
177 + return &m_StdHandles;
178 +}
179
271 - if (_stdHandlesInfo.ConsoleOutputHandle)
180 +COORD
181 +SvcCommIo::GetWindowSize() const
182 +{
183 + if (m_WindowSizeHandle)
184 {
273 - ChangeConsoleMode(_stdHandlesInfo.ConsoleOutputHandle.get(), _stdHandlesInfo.SavedOutputMode);
274 - SetConsoleOutputCP(_stdHandlesInfo.SavedOutputCP);
185 + CONSOLE_SCREEN_BUFFER_INFOEX Info{};
186 + Info.cbSize = sizeof(Info);
187 + THROW_IF_WIN32_BOOL_FALSE(GetConsoleScreenBufferInfoEx(m_WindowSizeHandle, &Info));
188 + return {
189 + static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1),
190 + static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
191 }
192
277 - if (_stdHandlesInfo.IsConsoleInput != FALSE)
278 - {
279 - ChangeConsoleMode(_stdHandlesInfo.InputHandle, _stdHandlesInfo.SavedInputMode);
280 - SetConsoleCP(_stdHandlesInfo.SavedInputCP);
281 - }
193 + LOG_HR_MSG(E_UNEXPECTED, "No console handle available for GetWindowSize");
194 + return {80, 24};
195 }
196 +
197 +} // namespace wsl::windows::common
src/windows/common/svccommio.hpp
+44 -18
@@ -14,39 +14,65 @@ Abstract:
14
15 #pragma once
16
17 +#include <optional>
18 #include <wil/filesystem.h>
19 #include <wil/result.h>
20 #include "wslservice.h"
21
21 -typedef struct _LXSS_STD_HANDLES_INFO
22 +namespace wsl::windows::common {
23 +
24 +// RAII wrapper for console input configuration and restoration
25 +class ConsoleInput
26 {
23 - HANDLE InputHandle;
24 - HANDLE OutputHandle;
25 - HANDLE ErrorHandle;
26 - wil::unique_hfile ConsoleOutputHandle;
27 - BOOLEAN IsConsoleInput;
28 - BOOLEAN IsConsoleOutput;
29 - BOOLEAN IsConsoleError;
30 - DWORD SavedInputMode;
31 - DWORD SavedOutputMode;
32 - UINT SavedInputCP;
33 - UINT SavedOutputCP;
34 -} LXSS_STD_HANDLES_INFO, *PLXSS_STD_HANDLES_INFO;
27 +public:
28 + static std::optional<ConsoleInput> Create(HANDLE Handle);
29 + ~ConsoleInput();
30 + ConsoleInput(const ConsoleInput&) = delete;
31 + ConsoleInput& operator=(const ConsoleInput&) = delete;
32 + ConsoleInput(ConsoleInput&&) = default;
33 + ConsoleInput& operator=(ConsoleInput&&) = default;
34 +
35 +private:
36 + ConsoleInput(HANDLE Handle, DWORD SavedMode);
37 +
38 + HANDLE m_Handle = nullptr;
39 + DWORD m_SavedMode = 0;
40 + UINT m_SavedCodePage = 0;
41 +};
42 +
43 +// RAII wrapper for console output configuration and restoration
44 +class ConsoleOutput
45 +{
46 +public:
47 + static std::optional<ConsoleOutput> Create();
48 + ~ConsoleOutput();
49 + ConsoleOutput(const ConsoleOutput&) = delete;
50 + ConsoleOutput& operator=(const ConsoleOutput&) = delete;
51 + ConsoleOutput(ConsoleOutput&&) = default;
52 + ConsoleOutput& operator=(ConsoleOutput&&) = default;
53 +
54 +private:
55 + ConsoleOutput(wil::unique_hfile&& ConsoleHandle, DWORD SavedMode);
56 +
57 + wil::unique_hfile m_ConsoleHandle;
58 + DWORD m_SavedMode = 0;
59 + UINT m_SavedCodePage = 0;
60 +};
61
36 -namespace wsl::windows::common {
62 class SvcCommIo
63 {
64 public:
65 SvcCommIo();
41 - ~SvcCommIo();
66
67 PLXSS_STD_HANDLES GetStdHandles();
68 COORD GetWindowSize() const;
69
70 private:
47 - void RestoreConsoleMode() const;
71 + LXSS_STD_HANDLES m_StdHandles{};
72 + HANDLE m_WindowSizeHandle = nullptr; // Cached console handle for GetWindowSize
73
49 - LXSS_STD_HANDLES _stdHandles{};
50 - LXSS_STD_HANDLES_INFO _stdHandlesInfo{};
74 + // RAII members for automatic restoration
75 + std::optional<ConsoleInput> m_ConsoleInput;
76 + std::optional<ConsoleOutput> m_ConsoleOutput;
77 };
78 } // namespace wsl::windows::common