[Frontend] Enrich chat_parsing streaming events (#58603)
Give serving adapters the delimiter spans and opener captures they need, and keep streaming after a region that fails to parse. Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Fix mypy index types in the event-stream test.
Yoni Gozlan committed
Sep 30, 2026 at 23:13 UTC
2eaa3bc5ac03aa5c0b3282782fd89eb92e074bfb
2 files changed
+157
-18
tests/parser/test_chat_parsing.py
+105
@@ -1160,6 +1160,111 @@ class ResponseEventStreamTest(unittest.TestCase):
1160
self.assertEqual(result, {"role": "assistant"})
1161
self.assertEqual(final_events, [])
1162
1163
+ def test_region_events_expose_delimiter_offsets(self):
1164
+ spec = {
1165
+ "start_anchor": "<assistant>",
1166
+ "fields": {
1167
+ "content": {"content_args": {"strip": False}},
1168
+ "tag": {"open_pattern": r"<tag id=\d+>", "close": "</tag>"},
1169
+ },
1170
+ }
1171
+ parser = ResponseParser(spec, prefix="")
1172
+ events = parser.feed("plain<tag id=7>body</tag>tail")
1173
+ events += parser.finalize()[1]
1174
+
1175
+ boundaries = [
1176
+ (event["type"], event["field"], parser.input_text[event["start"] : event["end"]])
1177
+ for event in events
1178
+ if event["type"] != "region_chunk"
1179
+ ]
1180
+ self.assertEqual(
1181
+ boundaries,
1182
+ [
1183
+ ("region_open", "content", ""),
1184
+ ("region_close", "content", ""),
1185
+ ("region_open", "tag", "<tag id=7>"),
1186
+ ("region_close", "tag", "</tag>"),
1187
+ ("region_open", "content", ""),
1188
+ ("region_close", "content", ""),
1189
+ ],
1190
+ )
1191
+
1192
+ def test_open_captures_and_prefix_boundary(self):
1193
+ spec = {
1194
+ "start_anchor": "[BEGIN]",
1195
+ "fields": {
1196
+ "tool_calls": {
1197
+ "open_pattern": r"<call:(?P<name>\w+)>",
1198
+ "close": "</call>",
1199
+ "content": "json",
1200
+ "transform": {"name": "{name}", "arguments": "{content}"},
1201
+ }
1202
+ },
1203
+ }
1204
+ parser = ResponseParser(spec, prefix="history[BEGIN]<call:get_")
1205
+ prefix_end = len(parser.input_text)
1206
+ events = parser.feed('weather>{"city":"Paris"}</call>')
1207
+ opening = events[0]
1208
+
1209
+ self.assertEqual(prefix_end, len("<call:get_"))
1210
+ self.assertEqual((opening["type"], opening["start"], opening["end"]), ("region_open", 0, len("<call:get_weather>")))
1211
+ self.assertEqual(opening["captures"], {"name": "get_weather"})
1212
+
1213
+ def test_held_prefix_bytes_are_not_rechunked(self):
1214
+ spec = {"start_anchor": "[BEGIN]", "fields": {"content": {}, "tag": {"open": "<tag>", "close": "</tag>"}}}
1215
+ parser = ResponseParser(spec, prefix="[BEGIN]Existing <")
1216
+ chunks = [event["text"] for event in parser.feed("b> tail") if event["type"] == "region_chunk"]
1217
+ message, _ = parser.finalize()
1218
+
1219
+ self.assertEqual(chunks, ["b> tail"])
1220
+ self.assertEqual(message["content"], "Existing <b> tail")
1221
+
1222
+ def test_malformed_region_events_recover_and_continue(self):
1223
+ spec = {
1224
+ "start_anchor": "[BEGIN]",
1225
+ "fields": {
1226
+ "tool_calls": {
1227
+ "open_pattern": r"<call:(?P<name>\w+)>",
1228
+ "close": "</call>",
1229
+ "content": "json",
1230
+ "repeats": True,
1231
+ "transform": {
1232
+ "type": "function",
1233
+ "function": {"name": "{name}", "arguments": "{content}"},
1234
+ },
1235
+ }
1236
+ },
1237
+ }
1238
+ parser = ResponseParser(spec, prefix="")
1239
+ events = parser.feed('<call:bad>{"x":</call><call:good>{"x":1}</call>')
1240
+ message, _ = parser.finalize()
1241
+ opening, malformed = events[0], events[2]
1242
+
1243
+ self.assertEqual(malformed["type"], "region_malformed")
1244
+ self.assertIsInstance(malformed["error"], ValueError)
1245
+ self.assertEqual(parser.input_text[opening["end"] : malformed["start"]], '{"x":')
1246
+ self.assertEqual(parser.input_text[malformed["start"] : malformed["end"]], "</call>")
1247
+ self.assertEqual(
1248
+ message["tool_calls"],
1249
+ [{"type": "function", "function": {"name": "good", "arguments": {"x": 1}}}],
1250
+ )
1251
+
1252
+ parser = ResponseParser(spec, prefix="")
1253
+ parser.feed('<call:bad>{"x":')
1254
+ _, events = parser.finalize()
1255
+ self.assertEqual(events[-1]["type"], "region_malformed")
1256
+ self.assertEqual(events[-1]["start"], len(parser.input_text))
1257
+
1258
+ def test_parse_response_raises_for_malformed_region(self):
1259
+ spec = {
1260
+ "start_anchor": "<|assistant|>",
1261
+ "fields": {"x": {"open": "<x>", "close": "</x>", "content": "json"}},
1262
+ }
1263
+ with self.assertRaisesRegex(ValueError, "could not parse region as JSON"):
1264
+ parse_response('<x>{"name":', spec, prefix="")
1265
+ with self.assertRaisesRegex(ValueError, "could not parse region as JSON"):
1266
+ parse_response("", spec, prefix='<|assistant|><x>{"name":</x>')
1267
+
1268
1269
class PrefixAndTruncationTest(unittest.TestCase):
1270
def test_prefix_lands_inside_explicit_region(self):
vllm/parser/chat_parsing/response_parser.py
+52
-18
@@ -85,8 +85,11 @@ def parse_response(
85
"""
86
response_template = load_response_template(response_template)
87
stream = ResponseParser(response_template, prefix=prefix, tools=tools)
88
- stream.feed(text)
89
- message, _ = stream.finalize()
88
+ events = stream.feed(text)
89
+ message, final_events = stream.finalize()
90
+ for event in stream.initial_events + events + final_events:
91
+ if event["type"] == "region_malformed":
92
+ raise event["error"]
93
return message
94
95
@@ -110,7 +113,12 @@ class ResponseParser:
113
Pass OpenAI-style `tools=` dictionaries to cast tool-call arguments
114
using the calling tool's JSON schema as each region closes.
115
113
- Events can be either "region_open", "region_chunk", or "region_close".
116
+ Events can be "region_open", "region_chunk", "region_close", or "region_malformed".
117
+ Open, close and malformed events carry `start` and `end`, the span of `input_text` the
118
+ event consumed: its opening or closing delimiter, empty for implicit boundaries.
119
+ Explicit opens also carry the opener's named `captures`. A region whose value fails to
120
+ parse ends with "region_malformed" instead of "region_close", carrying the `error`, and
121
+ parsing continues after it.
122
123
ResponseParser requires the chat `prefix` (i.e. the chat history, the prefill before the current generation).
124
This is because chat templates or assistant prefills can sometimes write part of the message, and if we
@@ -155,10 +163,18 @@ class ResponseParser:
163
self._body: str = ""
164
self._opened: bool = False
165
self._finalized: bool = False
166
+ self._prefix_len: int = 0
167
+ self._malformed_fields: set[str] = set()
168
self.initial_events: list[dict] = []
169
if prefix:
170
self._consume_prefix(prefix)
171
172
+ @property
173
+ def input_text(self) -> str:
174
+ """Raw parser input after start-anchor truncation. Before the first `feed()`, this is
175
+ exactly the truncated prefix, so its length marks where generated text begins."""
176
+ return self._buffer
177
+
178
def _consume_prefix(self, prefix: str) -> None:
179
"""Loads the prefix (the chat prefill sent to the model), right-truncates it to the start of the
180
assistant message (as determined by start_anchor) and then runs the remainder through the parser.
@@ -172,6 +188,7 @@ class ResponseParser:
188
return
189
self._buffer = truncated
190
self._process(self.initial_events, eos=False)
191
+ self._prefix_len = len(truncated)
192
193
def feed(self, text: str) -> list[dict]:
194
"""Feeds more text/tokens from the model output into the tokenizer, and returns any events that result
@@ -197,7 +214,11 @@ class ResponseParser:
214
raise RuntimeError("ResponseParser already finalized")
215
events: list[dict] = []
216
self._process(events, eos=True)
200
- missing = [n for n, f in self._spec.fields.items() if not f.optional and n not in self._output]
217
+ missing = [
218
+ n
219
+ for n, f in self._spec.fields.items()
220
+ if not f.optional and n not in self._output and n not in self._malformed_fields
221
+ ]
222
if missing:
223
raise ValueError(f"Required response_template fields missing from parsed output: {missing}")
224
defaults = self._spec.defaults
@@ -216,12 +237,12 @@ class ResponseParser:
237
self._accumulate(events, self._buffer[self._pos : m.start()])
238
self._pos = m.end()
239
if kind == "open":
219
- self._close_current(events)
240
+ self._close_current(events, m.start(), m.start())
241
self._open_explicit(events, field, m)
242
else: # "close" (always the implicit region's close here,
243
# since explicit regions only expose their own close)
244
had_content = self._opened
224
- self._close_current(events)
245
+ self._close_current(events, m.start(), m.end())
246
# Zero-width close on an already-empty region would just
247
# re-fire next iteration -- bail out to make progress.
248
if not had_content and m.start() == m.end():
@@ -233,7 +254,7 @@ class ResponseParser:
254
if self._pos < len(self._buffer):
255
self._accumulate(events, self._buffer[self._pos :])
256
self._pos = len(self._buffer)
236
- self._close_current(events)
257
+ self._close_current(events, self._pos, self._pos)
258
break
259
# Stream everything up to the earliest still-pending delimiter. When
260
# nothing is pending `hold_start == len(self._buffer)`, so this flushes
@@ -341,20 +362,25 @@ class ResponseParser:
362
return
363
field = self._spec.fields[self._current]
364
if not self._opened:
344
- events.append({"type": "region_open", "field": self._current})
365
+ events.append({"type": "region_open", "field": self._current, "start": self._pos, "end": self._pos})
366
self._opened = True
367
self._body += text
347
- dirty = field.content not in STREAMABLE_PARSERS
348
- events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty})
368
+ # Prefix bytes held back as a possible delimiter belong to the prompt, not the chunk.
369
+ text = text[max(0, self._prefix_len - self._pos) :]
370
+ if text:
371
+ dirty = field.content not in STREAMABLE_PARSERS
372
+ events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty})
373
374
def _open_explicit(self, events: list[dict], field: ResponseTemplateField, m: Any) -> None:
375
self._current = field.name
376
self._captures = {k: v for k, v in m.groupdict().items() if v is not None}
377
self._body = ""
378
self._opened = True
355
- events.append({"type": "region_open", "field": field.name})
379
+ events.append(
380
+ {"type": "region_open", "field": field.name, "start": m.start(), "end": m.end(), "captures": self._captures}
381
+ )
382
357
- def _close_current(self, events: list[dict]) -> None:
383
+ def _close_current(self, events: list[dict], close_start: int, close_end: int) -> None:
384
"""Close the current region and reset to the implicit/null region.
385
Skipped (aside from the reset) when the current region never opened --
386
avoids vacuous open/close pairs at every explicit boundary."""
@@ -362,22 +388,30 @@ class ResponseParser:
388
self._reset_to_implicit()
389
return
390
field = self._spec.fields[self._current]
365
- value = process_field(self._body, field, self._captures)
366
- if self._tool_params:
367
- value = self._coerce_tool_calls(value)
368
- if field.join is not None:
369
- if not isinstance(value, str):
391
+ try:
392
+ value = process_field(self._body, field, self._captures)
393
+ if self._tool_params:
394
+ value = self._coerce_tool_calls(value)
395
+ if field.join is not None and not isinstance(value, str):
396
raise ValueError(
397
f"Field '{field.name}': 'join' requires each match to parse to a string, "
398
f"got {type(value).__name__}."
399
)
400
+ except (KeyError, TypeError, ValueError) as error:
401
+ self._malformed_fields.add(self._current)
402
+ events.append(
403
+ {"type": "region_malformed", "field": self._current, "start": close_start, "end": close_end, "error": error}
404
+ )
405
+ self._reset_to_implicit()
406
+ return
407
+ if field.join is not None:
408
previous = self._output.get(self._current)
409
self._output[self._current] = value if previous is None else previous + field.join + value
410
elif field.repeats:
411
self._output.setdefault(self._current, []).append(value)
412
else:
413
self._output[self._current] = value
380
- events.append({"type": "region_close", "field": self._current, "value": value})
414
+ events.append({"type": "region_close", "field": self._current, "start": close_start, "end": close_end, "value": value})
415
self._reset_to_implicit()
416
417
def _coerce_tool_calls(self, value: Any) -> Any: