@samitouri / QOSAMI-WSL-1 / commits / 7d09fb83

Add CSV parsing helpers to shared string library (#41207)

* Add CSV parsing helpers to shared string lib; use in --secret spec parsing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

ggarzia-MSFT committed Jul 30, 2026 at 11:20 UTC 7d09fb838f0de4ae0f89a94dd35ca468e09c1d13
4 files changed +373 -1
src/shared/inc/stringshared.h
+144
@@ -18,6 +18,7 @@ Abstract:
18 #include <string>
19 #include <sstream>
20 #include <fstream>
21 +#include <optional>
22 #include <gsl/gsl>
23 #include <format>
24 #include <source_location>
@@ -129,6 +130,149 @@ inline std::vector<std::basic_string<T>> SplitByMultipleSeparators(const std::ba
130 return Output;
131 }
132
133 +// Splits a single CSV record into fields using the grammar Go's encoding/csv applies to one record
134 +// (which docker buildx relies on via go-csvvalue), so a spec is parsed the way buildx would:
135 +// - fields are separated by commas;
136 +// - a field may be wrapped in double quotes, in which case a comma is a literal character and a
137 +// doubled quote ("") is a single literal quote;
138 +// - an unquoted field may not contain a double quote (Go's non-lazy ErrBareQuote).
139 +// Returns std::nullopt when the record is malformed: an unterminated quoted field, text immediately
140 +// after a closing quote, or a bare quote in an unquoted field.
141 +//
142 +// Deviation from Go/RFC 4180: this parses exactly one record. Go would treat an unquoted CR/LF as a
143 +// record separator; here CR/LF are always ordinary field characters (never a record separator), which
144 +// is what a single-line command-line spec needs.
145 +template <class T>
146 +inline std::optional<std::vector<std::basic_string<T>>> SplitCsvFields(const std::basic_string<T>& Record)
147 +{
148 + constexpr T Quote = static_cast<T>('"');
149 + constexpr T Comma = static_cast<T>(',');
150 +
151 + std::vector<std::basic_string<T>> Fields;
152 + std::basic_string<T> Field;
153 + const size_t Length = Record.size();
154 + size_t Index = 0;
155 +
156 + while (true)
157 + {
158 + Field.clear();
159 + if (Index < Length && Record[Index] == Quote)
160 + {
161 + ++Index;
162 + bool Closed = false;
163 + while (Index < Length)
164 + {
165 + if (Record[Index] == Quote)
166 + {
167 + // A doubled quote inside a quoted field is a single literal quote.
168 + if (Index + 1 < Length && Record[Index + 1] == Quote)
169 + {
170 + Field.push_back(Quote);
171 + Index += 2;
172 + continue;
173 + }
174 +
175 + ++Index;
176 + Closed = true;
177 + break;
178 + }
179 +
180 + Field.push_back(Record[Index]);
181 + ++Index;
182 + }
183 +
184 + if (!Closed)
185 + {
186 + return std::nullopt; // unterminated quoted field
187 + }
188 +
189 + // After a closing quote only a comma (end of field) or end of record is valid.
190 + if (Index < Length && Record[Index] != Comma)
191 + {
192 + return std::nullopt;
193 + }
194 + }
195 + else
196 + {
197 + while (Index < Length && Record[Index] != Comma)
198 + {
199 + // Go (non-lazy) rejects a bare double quote in an unquoted field (ErrBareQuote).
200 + if (Record[Index] == Quote)
201 + {
202 + return std::nullopt;
203 + }
204 +
205 + Field.push_back(Record[Index]);
206 + ++Index;
207 + }
208 + }
209 +
210 + Fields.push_back(Field);
211 + if (Index >= Length)
212 + {
213 + break;
214 + }
215 +
216 + ++Index; // consume the ',' and start the next field
217 + }
218 +
219 + return Fields;
220 +}
221 +
222 +// CSV-escapes a single field for round-tripping through JoinCsvFields/SplitCsvFields: if the field
223 +// contains a comma, a double quote, CR, LF, or a leading/trailing space it is wrapped in double quotes
224 +// with each embedded quote doubled; otherwise it is returned unchanged. This is not a byte-for-byte
225 +// match of Go's encoding/csv writer (which quotes only a leading space and leaves an empty field
226 +// unquoted); it is the minimal quoting needed for every field to parse back via SplitCsvFields.
227 +template <class T>
228 +inline std::basic_string<T> CsvEscapeField(const std::basic_string<T>& Field)
229 +{
230 + constexpr T Quote = static_cast<T>('"');
231 + const T Special[] = {static_cast<T>(','), Quote, static_cast<T>('\r'), static_cast<T>('\n'), static_cast<T>(0)};
232 +
233 + const bool NeedsQuote = Field.find_first_of(Special) != std::basic_string<T>::npos ||
234 + (!Field.empty() && (Field.front() == static_cast<T>(' ') || Field.back() == static_cast<T>(' ')));
235 + if (!NeedsQuote)
236 + {
237 + return Field;
238 + }
239 +
240 + std::basic_string<T> Result;
241 + Result.reserve(Field.size() + 2);
242 + Result.push_back(Quote);
243 + for (const T Ch : Field)
244 + {
245 + if (Ch == Quote)
246 + {
247 + Result.push_back(Quote);
248 + }
249 +
250 + Result.push_back(Ch);
251 + }
252 +
253 + Result.push_back(Quote);
254 + return Result;
255 +}
256 +
257 +// Joins fields into a single CSV record, escaping each field as needed (see CsvEscapeField). The
258 +// result parses back to the original fields via SplitCsvFields.
259 +template <class T>
260 +inline std::basic_string<T> JoinCsvFields(const std::vector<std::basic_string<T>>& Fields)
261 +{
262 + std::basic_string<T> Record;
263 + for (size_t Index = 0; Index < Fields.size(); ++Index)
264 + {
265 + if (Index != 0)
266 + {
267 + Record.push_back(static_cast<T>(','));
268 + }
269 +
270 + Record += CsvEscapeField(Fields[Index]);
271 + }
272 +
273 + return Record;
274 +}
275 +
276 inline const char* FromSpan(gsl::span<gsl::byte> Span, size_t Offset = 0)
277 {
278 THROW_INVALID_ARG_IF(Span.size() < Offset);
src/windows/wslc/arguments/SpecParsing.cpp
+9 -1
@@ -56,7 +56,15 @@ services::BuildSecret ParseSecretSpec(const std::wstring& spec)
56 std::wstring envName;
57 std::wstring srcPath;
58
59 - for (const auto& part : Split(spec, L','))
59 + // Docker parity: buildx parses --secret as a single CSV record (go-csvvalue), so a quoted field
60 + // may contain commas (e.g. a 'src=' path). Malformed quoting is rejected like any other bad spec.
61 + const auto parts = SplitCsvFields(spec);
62 + if (!parts.has_value())
63 + {
64 + throw ArgumentException(Localization::MessageWslcSecretInvalidSpec(spec, L"malformed quoting"));
65 + }
66 +
67 + for (const auto& part : *parts)
68 {
69 const auto kv = SplitKeyValue(part);
70 if (!kv.HadSeparator || kv.Key.empty())
test/windows/wslc/WSLCCLISecretParserUnitTests.cpp
+29
@@ -182,6 +182,35 @@ class WSLCCLISecretParserUnitTests
182 VerifyValidFileSecret(L"id=s,source=" + file.wpath(), L"s", file.wpath());
183 }
184
185 + TEST_METHOD(Secret_File_QuotedFieldWithCommaInSrcPath)
186 + {
187 + // Docker parity: buildx parses --secret as a single CSV record, so a whole 'src=' field can be
188 + // double-quoted to carry a path containing commas; the comma must stay part of the value rather
189 + // than splitting into bogus extra key=value parts. This exercises SplitCsvFields end-to-end
190 + // through secret parsing. Note the entire "src=<path>" field is quoted (Go's CSV grammar), not
191 + // just the value - a bare quote after 'src=' would be an unquoted-field bare quote (malformed).
192 + const auto path =
193 + std::filesystem::temp_directory_path() / (L"wslc_ut_secret_" + std::to_wstring(GetCurrentProcessId()) + L"_a,b,c.bin");
194 + {
195 + std::ofstream file(path, std::ios::binary | std::ios::trunc);
196 + VERIFY_IS_TRUE(file.is_open());
197 + file << 'x';
198 + }
199 + auto cleanup = wil::scope_exit([&]() {
200 + std::error_code ec;
201 + std::filesystem::remove(path, ec);
202 + });
203 +
204 + const std::wstring spec = L"id=s,\"src=" + path.wstring() + L"\"";
205 + auto secret = validation::ParseSecretSpec(spec);
206 + VERIFY_ARE_EQUAL(std::wstring(L"s"), secret.Id);
207 + VERIFY_IS_TRUE(secret.Value.empty());
208 +
209 + std::error_code ec;
210 + const auto expectedCanonical = std::filesystem::weakly_canonical(path, ec);
211 + VERIFY_ARE_EQUAL(expectedCanonical.wstring(), secret.SourcePath);
212 + }
213 +
214 TEST_METHOD(Secret_File_EmptyFileForwardsPath)
215 {
216 // An empty file is still a valid file secret: its path is forwarded and mounted (docker delivers
test/windows/wslc/WSLCCsvSharedUnitTests.cpp new
+191
@@ -0,0 +1,191 @@
1 +// Copyright (C) Microsoft Corporation. All rights reserved.
2 +
3 +/*++
4 +
5 +Module Name:
6 +
7 + WSLCCsvSharedUnitTests.cpp
8 +
9 +Abstract:
10 +
11 + Unit tests for the shared CSV helpers in wsl::shared::string (SplitCsvFields, CsvEscapeField,
12 + JoinCsvFields). These implement the single-record RFC 4180 grammar that docker buildx uses via
13 + go-csvvalue / encoding/csv, and back the --output/--secret spec parsers. The tests pin the grammar
14 + (quoting, "" escaping, embedded commas, malformed-record rejection) and the escape/join round-trip.
15 +
16 +--*/
17 +
18 +#include "precomp.h"
19 +#include "windows/Common.h"
20 +#include "WSLCCLITestHelpers.h"
21 +#include <optional>
22 +#include <string>
23 +#include <vector>
24 +
25 +namespace WSLCCsvSharedUnitTests {
26 +
27 +using WStrings = std::vector<std::wstring>;
28 +
29 +class WSLCCsvSharedUnitTests
30 +{
31 + WSLC_TEST_CLASS(WSLCCsvSharedUnitTests)
32 +
33 + static void VerifySplit(const std::wstring& record, const WStrings& expected)
34 + {
35 + const auto fields = wsl::shared::string::SplitCsvFields(record);
36 + VERIFY_IS_TRUE(fields.has_value());
37 + if (!fields.has_value())
38 + {
39 + return;
40 + }
41 +
42 + VERIFY_ARE_EQUAL(expected.size(), fields->size());
43 + for (size_t i = 0; i < expected.size() && i < fields->size(); ++i)
44 + {
45 + VERIFY_ARE_EQUAL(expected[i], (*fields)[i]);
46 + }
47 + }
48 +
49 + static void VerifyMalformed(const std::wstring& record)
50 + {
51 + VERIFY_IS_FALSE(wsl::shared::string::SplitCsvFields(record).has_value());
52 + }
53 +
54 + // --- SplitCsvFields ---
55 +
56 + TEST_METHOD(Split_SimpleFields)
57 + {
58 + VerifySplit(L"a,b,c", WStrings{L"a", L"b", L"c"});
59 + }
60 +
61 + TEST_METHOD(Split_SingleField)
62 + {
63 + VerifySplit(L"abc", WStrings{L"abc"});
64 + }
65 +
66 + TEST_METHOD(Split_EmptyInputIsOneEmptyField)
67 + {
68 + VerifySplit(L"", WStrings{L""});
69 + }
70 +
71 + TEST_METHOD(Split_EmptyFieldsPreserved)
72 + {
73 + // Unlike wsl::shared::string::Split, empty fields are preserved (they are meaningful in a spec).
74 + VerifySplit(L"a,,c", WStrings{L"a", L"", L"c"});
75 + }
76 +
77 + TEST_METHOD(Split_LeadingAndTrailingCommas)
78 + {
79 + VerifySplit(L",a,", WStrings{L"", L"a", L""});
80 + }
81 +
82 + TEST_METHOD(Split_QuotedFieldWithComma)
83 + {
84 + VerifySplit(L"\"a,b\",c", WStrings{L"a,b", L"c"});
85 + }
86 +
87 + TEST_METHOD(Split_QuotedFieldWithEscapedQuote)
88 + {
89 + // A doubled quote inside a quoted field is a single literal quote.
90 + VerifySplit(L"\"a\"\"b\"", WStrings{L"a\"b"});
91 + }
92 +
93 + TEST_METHOD(Split_QuotedEmptyField)
94 + {
95 + VerifySplit(L"\"\",x", WStrings{L"", L"x"});
96 + }
97 +
98 + TEST_METHOD(Split_QuotedThenPlainField)
99 + {
100 + VerifySplit(L"\"a\",b", WStrings{L"a", L"b"});
101 + }
102 +
103 + TEST_METHOD(Split_UnterminatedQuoteIsMalformed)
104 + {
105 + VerifyMalformed(L"\"abc");
106 + }
107 +
108 + TEST_METHOD(Split_TextAfterClosingQuoteIsMalformed)
109 + {
110 + VerifyMalformed(L"\"a\"b");
111 + }
112 +
113 + TEST_METHOD(Split_BareQuoteInUnquotedFieldIsMalformed)
114 + {
115 + // Go's encoding/csv (non-lazy) rejects a bare double quote in an unquoted field (ErrBareQuote).
116 + VerifyMalformed(L"a\"b,c");
117 + }
118 +
119 + // --- CsvEscapeField ---
120 +
121 + TEST_METHOD(Escape_PlainFieldUnchanged)
122 + {
123 + VERIFY_ARE_EQUAL(std::wstring(L"abc"), wsl::shared::string::CsvEscapeField(std::wstring(L"abc")));
124 + }
125 +
126 + TEST_METHOD(Escape_EmptyFieldUnchanged)
127 + {
128 + VERIFY_ARE_EQUAL(std::wstring(L""), wsl::shared::string::CsvEscapeField(std::wstring(L"")));
129 + }
130 +
131 + TEST_METHOD(Escape_CommaQuoted)
132 + {
133 + VERIFY_ARE_EQUAL(std::wstring(L"\"a,b\""), wsl::shared::string::CsvEscapeField(std::wstring(L"a,b")));
134 + }
135 +
136 + TEST_METHOD(Escape_QuoteDoubledAndQuoted)
137 + {
138 + VERIFY_ARE_EQUAL(std::wstring(L"\"a\"\"b\""), wsl::shared::string::CsvEscapeField(std::wstring(L"a\"b")));
139 + }
140 +
141 + TEST_METHOD(Escape_NewlineQuoted)
142 + {
143 + VERIFY_ARE_EQUAL(std::wstring(L"\"a\nb\""), wsl::shared::string::CsvEscapeField(std::wstring(L"a\nb")));
144 + }
145 +
146 + TEST_METHOD(Escape_LeadingOrTrailingSpaceQuoted)
147 + {
148 + VERIFY_ARE_EQUAL(std::wstring(L"\" a\""), wsl::shared::string::CsvEscapeField(std::wstring(L" a")));
149 + VERIFY_ARE_EQUAL(std::wstring(L"\"a \""), wsl::shared::string::CsvEscapeField(std::wstring(L"a ")));
150 + }
151 +
152 + // --- JoinCsvFields ---
153 +
154 + TEST_METHOD(Join_PlainFields)
155 + {
156 + VERIFY_ARE_EQUAL(std::wstring(L"a,b,c"), wsl::shared::string::JoinCsvFields(WStrings{L"a", L"b", L"c"}));
157 + }
158 +
159 + TEST_METHOD(Join_EscapesFieldContainingComma)
160 + {
161 + VERIFY_ARE_EQUAL(std::wstring(L"\"a,b\",c"), wsl::shared::string::JoinCsvFields(WStrings{L"a,b", L"c"}));
162 + }
163 +
164 + // --- Round-trip: SplitCsvFields(JoinCsvFields(x)) == x ---
165 +
166 + TEST_METHOD(RoundTrip_JoinThenSplitPreservesFields)
167 + {
168 + const WStrings inputs{L"type=image", L"annotation.foo=a,b,c", L"name=x", L"quote=a\"b", L""};
169 + const auto joined = wsl::shared::string::JoinCsvFields(inputs);
170 + VerifySplit(joined, inputs);
171 + }
172 +
173 + // --- Template works for narrow (char) strings too ---
174 +
175 + TEST_METHOD(Narrow_SplitAndJoinRoundTrip)
176 + {
177 + const std::vector<std::string> inputs{"a,b", "c", "d\"e"};
178 + const auto joined = wsl::shared::string::JoinCsvFields(inputs);
179 + VERIFY_ARE_EQUAL(std::string("\"a,b\",c,\"d\"\"e\""), joined);
180 +
181 + const auto fields = wsl::shared::string::SplitCsvFields(joined);
182 + VERIFY_IS_TRUE(fields.has_value());
183 + VERIFY_ARE_EQUAL(inputs.size(), fields->size());
184 + for (size_t i = 0; i < inputs.size() && i < fields->size(); ++i)
185 + {
186 + VERIFY_ARE_EQUAL(inputs[i], (*fields)[i]);
187 + }
188 + }
189 +};
190 +
191 +} // namespace WSLCCsvSharedUnitTests