cleanup: refactor svccommio class (#13843)
* cleanup: refactor svccommio class * add logging if setting code page fails * pr feedback * remove unneeded if * remove unneeded if * pr feedbacK' --------- Co-authored-by: Ben Hillis <benhill@ntdev.microsoft.com>
Ben Hillis committed
Jan 14, 2026 at 12:24 UTC
e2a7b1d168f3c5cada8c72efb85e2cb9478084f8
2 files changed
+155
-214
src/windows/common/svccommio.cpp
+111
-196
@@ -18,7 +18,14 @@ Abstract:
18
#pragma hdrstop
19
20
namespace {
21
-void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
21
+
22
+bool IsConsoleHandle(_In_ HANDLE Handle)
23
+{
24
+ DWORD Mode;
25
+ return GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode);
26
+}
27
+
28
+void ChangeConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
29
{
30
//
31
// Use the invalid parameter error code to detect the v1 console that does
@@ -29,8 +36,17 @@ void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
36
// disconnected.
37
//
38
32
- if (!SetConsoleMode(File, ConsoleMode))
39
+ if (!SetConsoleMode(Handle, Mode))
40
{
41
+ // DISABLE_NEWLINE_AUTO_RETURN is not supported everywhere, if the flag was present fall back and try again.
42
+ if (WI_IsFlagSet(Mode, DISABLE_NEWLINE_AUTO_RETURN))
43
+ {
44
+ if (SetConsoleMode(Handle, WI_ClearFlag(Mode, DISABLE_NEWLINE_AUTO_RETURN)))
45
+ {
46
+ return;
47
+ }
48
+ }
49
+
50
switch (GetLastError())
51
{
52
case ERROR_INVALID_PARAMETER:
@@ -45,238 +61,137 @@ void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
61
}
62
}
63
48
-void ConfigureStdHandles(_Inout_ LXSS_STD_HANDLES_INFO& StdHandlesInfo)
64
+void TrySetConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
65
+try
66
{
50
- //
51
- // Check stdin to see if it is a console or another device. If it is
52
- // a console, configure it to raw processing mode and VT-100 support. If the
53
- // force console I/O is requested, ignore stdin and get active console input
54
- // handle instead.
55
- //
67
+ ChangeConsoleMode(Handle, Mode);
68
+}
69
+CATCH_LOG()
70
57
- UINT NewConsoleInputCP = 0;
58
- DWORD NewConsoleInputMode = 0;
59
- BOOLEAN IsConsoleInput = StdHandlesInfo.IsConsoleInput;
60
- BOOLEAN IsConsoleOutput = StdHandlesInfo.IsConsoleOutput;
61
- BOOLEAN IsConsoleError = StdHandlesInfo.IsConsoleError;
62
- DWORD SavedInputMode = StdHandlesInfo.SavedInputMode;
63
- DWORD SavedOutputMode = StdHandlesInfo.SavedOutputMode;
64
- UINT SavedInputCP = StdHandlesInfo.SavedInputCP;
65
- UINT SavedOutputCP = StdHandlesInfo.SavedOutputCP;
66
- CONSOLE_SCREEN_BUFFER_INFO ScreenBufferInfo;
67
- auto RestoreInputHandle = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&] {
68
- if (NewConsoleInputCP != 0)
69
- {
70
- SetConsoleCP(SavedInputCP);
71
- }
71
+} // namespace
72
73
- if (NewConsoleInputMode != 0)
74
- {
75
- ChangeConsoleMode(StdHandlesInfo.InputHandle, SavedInputMode);
76
- }
77
- });
73
+namespace wsl::windows::common {
74
79
- IsConsoleInput = FALSE;
80
- if ((GetFileType(StdHandlesInfo.InputHandle) == FILE_TYPE_CHAR) && (GetConsoleMode(StdHandlesInfo.InputHandle, &SavedInputMode)))
75
+std::optional<ConsoleInput> ConsoleInput::Create(HANDLE Handle)
76
+{
77
+ DWORD Mode;
78
+ if (GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode))
79
{
82
- IsConsoleInput = TRUE;
83
- NewConsoleInputMode = SavedInputMode;
84
- WI_SetAllFlags(NewConsoleInputMode, (ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT));
85
- WI_ClearAllFlags(NewConsoleInputMode, (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT));
86
- ChangeConsoleMode(StdHandlesInfo.InputHandle, NewConsoleInputMode);
87
-
88
- //
89
- // Set the console input to the UTF-8 code page.
90
- //
91
-
92
- SavedInputCP = GetConsoleCP();
93
- NewConsoleInputCP = CP_UTF8;
94
- THROW_LAST_ERROR_IF(!::SetConsoleCP(NewConsoleInputCP));
80
+ return ConsoleInput(Handle, Mode);
81
}
82
97
- bool RestoreMode = false;
98
- bool RestoreCp = false;
99
- auto RestoreOutput = wil::scope_exit([&] {
100
- if (RestoreMode)
101
- {
102
- SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), SavedOutputMode);
103
- }
104
-
105
- if (RestoreCp)
106
- {
107
- SetConsoleOutputCP(SavedOutputCP);
108
- }
109
- });
110
-
111
- //
112
- // If there is a console output handle, save the output mode and codepage so
113
- // it can be restored.
114
- //
115
-
116
- if (StdHandlesInfo.ConsoleOutputHandle)
117
- {
118
- THROW_LAST_ERROR_IF(!::GetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), &SavedOutputMode));
119
-
120
- //
121
- // Temporarily try both with and without the custom flag to disable newline
122
- // auto return.
123
- //
124
-
125
- DWORD NewConsoleOutputMode = SavedOutputMode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
126
- if (SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode) == FALSE)
127
- {
128
- WI_ClearFlag(NewConsoleOutputMode, DISABLE_NEWLINE_AUTO_RETURN);
129
- ChangeConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode);
130
- }
131
-
132
- RestoreMode = true;
133
-
134
- //
135
- // Set the console output to the UTF-8 code page.
136
- //
137
-
138
- SavedOutputCP = GetConsoleOutputCP();
139
- THROW_LAST_ERROR_IF(!::SetConsoleOutputCP(CP_UTF8));
140
-
141
- RestoreCp = true;
142
- }
83
+ return std::nullopt;
84
+}
85
144
- //
145
- // If the force console I/O is requested, ignore stdout and treat the
146
- // console as the output handle.
147
- //
86
+ConsoleInput::ConsoleInput(HANDLE Handle, DWORD SavedMode) : m_Handle(Handle), m_SavedMode(SavedMode)
87
+{
88
+ // Save code page.
89
+ m_SavedCodePage = GetConsoleCP();
90
149
- IsConsoleOutput = FALSE;
150
- if ((GetFileType(StdHandlesInfo.OutputHandle) == FILE_TYPE_CHAR) &&
151
- (GetConsoleScreenBufferInfo(StdHandlesInfo.OutputHandle, &ScreenBufferInfo)))
152
- {
153
- IsConsoleOutput = TRUE;
154
- }
91
+ // Configure for raw input with VT support.
92
+ DWORD NewMode = m_SavedMode;
93
+ WI_SetAllFlags(NewMode, ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT);
94
+ WI_ClearAllFlags(NewMode, ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT);
95
+ ChangeConsoleMode(Handle, NewMode);
96
156
- IsConsoleError = FALSE;
157
- if ((GetFileType(StdHandlesInfo.ErrorHandle) == FILE_TYPE_CHAR) &&
158
- (GetConsoleScreenBufferInfo(StdHandlesInfo.ErrorHandle, &ScreenBufferInfo)))
159
- {
160
- IsConsoleError = TRUE;
161
- }
97
+ // Set UTF-8 code page.
98
+ LOG_IF_WIN32_BOOL_FALSE(SetConsoleCP(CP_UTF8));
99
+}
100
163
- RestoreInputHandle.release();
164
- RestoreOutput.release();
165
- StdHandlesInfo.IsConsoleInput = IsConsoleInput;
166
- StdHandlesInfo.IsConsoleOutput = IsConsoleOutput;
167
- StdHandlesInfo.IsConsoleError = IsConsoleError;
168
- StdHandlesInfo.SavedInputMode = SavedInputMode;
169
- StdHandlesInfo.SavedOutputMode = SavedOutputMode;
170
- StdHandlesInfo.SavedInputCP = SavedInputCP;
171
- StdHandlesInfo.SavedOutputCP = SavedOutputCP;
101
+ConsoleInput::~ConsoleInput()
102
+{
103
+ TrySetConsoleMode(m_Handle, m_SavedMode);
104
+ LOG_IF_WIN32_BOOL_FALSE(SetConsoleCP(m_SavedCodePage));
105
}
173
-} // namespace
106
175
-wsl::windows::common::SvcCommIo::SvcCommIo()
107
+std::optional<ConsoleOutput> ConsoleOutput::Create()
108
{
177
- _stdHandlesInfo.InputHandle = GetStdHandle(STD_INPUT_HANDLE);
178
- _stdHandlesInfo.OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
179
- _stdHandlesInfo.ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);
180
- _stdHandlesInfo.ConsoleOutputHandle.reset(
109
+ wil::unique_hfile ConsoleHandle(
110
CreateFileW(L"CONOUT$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr));
111
183
- ConfigureStdHandles(_stdHandlesInfo);
184
- _stdHandles.StdIn.HandleType = LxssHandleInput;
185
- _stdHandles.StdIn.Handle = HandleToUlong(_stdHandlesInfo.InputHandle);
186
- _stdHandles.StdOut.HandleType = LxssHandleOutput;
187
- _stdHandles.StdOut.Handle = HandleToUlong(_stdHandlesInfo.OutputHandle);
188
- _stdHandles.StdErr.HandleType = LxssHandleOutput;
189
- _stdHandles.StdErr.Handle = HandleToUlong(_stdHandlesInfo.ErrorHandle);
190
-
191
- //
192
- // N.B.: The console handle is not supposed to be closed, it is just copied
193
- // from PEB.
194
- //
195
-
196
- if (_stdHandlesInfo.IsConsoleInput)
197
- {
198
- _stdHandles.StdIn.Handle = LXSS_HANDLE_USE_CONSOLE;
199
- _stdHandles.StdIn.HandleType = LxssHandleConsole;
200
- }
201
-
202
- if (_stdHandlesInfo.IsConsoleOutput)
112
+ if (ConsoleHandle)
113
{
204
- _stdHandles.StdOut.Handle = LXSS_HANDLE_USE_CONSOLE;
205
- _stdHandles.StdOut.HandleType = LxssHandleConsole;
114
+ DWORD Mode;
115
+ if (GetConsoleMode(ConsoleHandle.get(), &Mode))
116
+ {
117
+ return ConsoleOutput(std::move(ConsoleHandle), Mode);
118
+ }
119
}
120
208
- if (_stdHandlesInfo.IsConsoleError)
209
- {
210
- _stdHandles.StdErr.Handle = LXSS_HANDLE_USE_CONSOLE;
211
- _stdHandles.StdErr.HandleType = LxssHandleConsole;
212
- }
121
+ return std::nullopt;
122
}
123
215
-wsl::windows::common::SvcCommIo::~SvcCommIo()
124
+ConsoleOutput::ConsoleOutput(wil::unique_hfile&& ConsoleHandle, DWORD SavedMode) :
125
+ m_ConsoleHandle(std::move(ConsoleHandle)), m_SavedMode(SavedMode)
126
{
217
- try
218
- {
219
- RestoreConsoleMode();
220
- }
221
- CATCH_LOG()
222
-}
127
+ // Save code page.
128
+ m_SavedCodePage = GetConsoleOutputCP();
129
224
-PLXSS_STD_HANDLES
225
-wsl::windows::common::SvcCommIo::GetStdHandles()
226
-{
227
- return &_stdHandles;
130
+ // Configure for VT output.
131
+ DWORD NewMode = m_SavedMode;
132
+ WI_SetAllFlags(NewMode, ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN);
133
+ ChangeConsoleMode(m_ConsoleHandle.get(), NewMode);
134
+
135
+ // Set UTF-8 code page.
136
+ LOG_IF_WIN32_BOOL_FALSE(SetConsoleOutputCP(CP_UTF8));
137
}
138
230
-COORD
231
-wsl::windows::common::SvcCommIo::GetWindowSize() const
139
+ConsoleOutput::~ConsoleOutput()
140
{
233
- CONSOLE_SCREEN_BUFFER_INFOEX Info{};
234
- Info.cbSize = sizeof(Info);
235
- if (_stdHandlesInfo.IsConsoleOutput)
236
- {
237
- THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.OutputHandle, &Info));
238
- }
239
- else if (_stdHandlesInfo.IsConsoleError)
240
- {
241
- THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.ErrorHandle, &Info));
242
- }
243
-
244
- return {
245
- static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1), static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
141
+ TrySetConsoleMode(m_ConsoleHandle.get(), m_SavedMode);
142
+ LOG_IF_WIN32_BOOL_FALSE(SetConsoleOutputCP(m_SavedCodePage));
143
}
144
248
-void wsl::windows::common::SvcCommIo::RestoreConsoleMode() const
249
-
250
-/*++
251
-
252
-Routine Description:
145
+SvcCommIo::SvcCommIo()
146
+{
147
+ const HANDLE InputHandle = GetStdHandle(STD_INPUT_HANDLE);
148
+ const HANDLE OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
149
+ const HANDLE ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);
150
254
- Restores the saved input/output console mode.
151
+ // Configure input console
152
+ m_ConsoleInput = ConsoleInput::Create(InputHandle);
153
256
-Arguments:
154
+ // Configure output console
155
+ m_ConsoleOutput = ConsoleOutput::Create();
156
258
- None.
157
+ // Initialize the standard handles structure
158
+ const bool IsConsoleInput = m_ConsoleInput.has_value();
159
+ m_StdHandles.StdIn.HandleType = IsConsoleInput ? LxssHandleConsole : LxssHandleInput;
160
+ m_StdHandles.StdIn.Handle = IsConsoleInput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(InputHandle);
161
260
-Return Value:
162
+ const bool IsConsoleOutput = IsConsoleHandle(OutputHandle);
163
+ m_StdHandles.StdOut.HandleType = IsConsoleOutput ? LxssHandleConsole : LxssHandleOutput;
164
+ m_StdHandles.StdOut.Handle = IsConsoleOutput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(OutputHandle);
165
262
- None.
166
+ const bool IsConsoleError = IsConsoleHandle(ErrorHandle);
167
+ m_StdHandles.StdErr.HandleType = IsConsoleError ? LxssHandleConsole : LxssHandleOutput;
168
+ m_StdHandles.StdErr.Handle = IsConsoleError ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(ErrorHandle);
169
264
---*/
170
+ // Cache a console handle for GetWindowSize
171
+ m_WindowSizeHandle = IsConsoleOutput ? OutputHandle : (IsConsoleError ? ErrorHandle : nullptr);
172
+}
173
174
+PLXSS_STD_HANDLES
175
+SvcCommIo::GetStdHandles()
176
{
267
- //
268
- // Restore the console input and output modes.
269
- //
177
+ return &m_StdHandles;
178
+}
179
271
- if (_stdHandlesInfo.ConsoleOutputHandle)
180
+COORD
181
+SvcCommIo::GetWindowSize() const
182
+{
183
+ if (m_WindowSizeHandle)
184
{
273
- ChangeConsoleMode(_stdHandlesInfo.ConsoleOutputHandle.get(), _stdHandlesInfo.SavedOutputMode);
274
- SetConsoleOutputCP(_stdHandlesInfo.SavedOutputCP);
185
+ CONSOLE_SCREEN_BUFFER_INFOEX Info{};
186
+ Info.cbSize = sizeof(Info);
187
+ THROW_IF_WIN32_BOOL_FALSE(GetConsoleScreenBufferInfoEx(m_WindowSizeHandle, &Info));
188
+ return {
189
+ static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1),
190
+ static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
191
}
192
277
- if (_stdHandlesInfo.IsConsoleInput != FALSE)
278
- {
279
- ChangeConsoleMode(_stdHandlesInfo.InputHandle, _stdHandlesInfo.SavedInputMode);
280
- SetConsoleCP(_stdHandlesInfo.SavedInputCP);
281
- }
193
+ LOG_HR_MSG(E_UNEXPECTED, "No console handle available for GetWindowSize");
194
+ return {80, 24};
195
}
196
+
197
+} // namespace wsl::windows::common
src/windows/common/svccommio.hpp
+44
-18
@@ -14,39 +14,65 @@ Abstract:
14
15
#pragma once
16
17
+#include <optional>
18
#include <wil/filesystem.h>
19
#include <wil/result.h>
20
#include "wslservice.h"
21
21
-typedef struct _LXSS_STD_HANDLES_INFO
22
+namespace wsl::windows::common {
23
+
24
+// RAII wrapper for console input configuration and restoration
25
+class ConsoleInput
26
{
23
- HANDLE InputHandle;
24
- HANDLE OutputHandle;
25
- HANDLE ErrorHandle;
26
- wil::unique_hfile ConsoleOutputHandle;
27
- BOOLEAN IsConsoleInput;
28
- BOOLEAN IsConsoleOutput;
29
- BOOLEAN IsConsoleError;
30
- DWORD SavedInputMode;
31
- DWORD SavedOutputMode;
32
- UINT SavedInputCP;
33
- UINT SavedOutputCP;
34
-} LXSS_STD_HANDLES_INFO, *PLXSS_STD_HANDLES_INFO;
27
+public:
28
+ static std::optional<ConsoleInput> Create(HANDLE Handle);
29
+ ~ConsoleInput();
30
+ ConsoleInput(const ConsoleInput&) = delete;
31
+ ConsoleInput& operator=(const ConsoleInput&) = delete;
32
+ ConsoleInput(ConsoleInput&&) = default;
33
+ ConsoleInput& operator=(ConsoleInput&&) = default;
34
+
35
+private:
36
+ ConsoleInput(HANDLE Handle, DWORD SavedMode);
37
+
38
+ HANDLE m_Handle = nullptr;
39
+ DWORD m_SavedMode = 0;
40
+ UINT m_SavedCodePage = 0;
41
+};
42
+
43
+// RAII wrapper for console output configuration and restoration
44
+class ConsoleOutput
45
+{
46
+public:
47
+ static std::optional<ConsoleOutput> Create();
48
+ ~ConsoleOutput();
49
+ ConsoleOutput(const ConsoleOutput&) = delete;
50
+ ConsoleOutput& operator=(const ConsoleOutput&) = delete;
51
+ ConsoleOutput(ConsoleOutput&&) = default;
52
+ ConsoleOutput& operator=(ConsoleOutput&&) = default;
53
+
54
+private:
55
+ ConsoleOutput(wil::unique_hfile&& ConsoleHandle, DWORD SavedMode);
56
+
57
+ wil::unique_hfile m_ConsoleHandle;
58
+ DWORD m_SavedMode = 0;
59
+ UINT m_SavedCodePage = 0;
60
+};
61
36
-namespace wsl::windows::common {
62
class SvcCommIo
63
{
64
public:
65
SvcCommIo();
41
- ~SvcCommIo();
66
67
PLXSS_STD_HANDLES GetStdHandles();
68
COORD GetWindowSize() const;
69
70
private:
47
- void RestoreConsoleMode() const;
71
+ LXSS_STD_HANDLES m_StdHandles{};
72
+ HANDLE m_WindowSizeHandle = nullptr; // Cached console handle for GetWindowSize
73
49
- LXSS_STD_HANDLES _stdHandles{};
50
- LXSS_STD_HANDLES_INFO _stdHandlesInfo{};
74
+ // RAII members for automatic restoration
75
+ std::optional<ConsoleInput> m_ConsoleInput;
76
+ std::optional<ConsoleOutput> m_ConsoleOutput;
77
};
78
} // namespace wsl::windows::common