@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 8db12169a4

fix: stream Qwen3 tool call string arguments (#46351)

Signed-off-by: Rui Yin <2260891073@qq.com> Co-authored-by: abinggo <107740309+abinggo@users.noreply.github.com>

Rui Yin committed Jun 23, 2026 at 10:26 UTC 8db12169a474c0bdbd9be55d6a749cad4ab16caa

4 files changed +177 -9

tests/parser/engine/test_parser_engine.py

+40 -1

@@ -1339,6 +1339,42 @@ class TestArgDeltaWithConverter:

1339 assert parsed == {"count": 5, "name": "test"}

1340 assert isinstance(parsed["count"], int)

1341

1342 + def test_streamable_string_keys_cached_after_name_delta(self, monkeypatch):

1343 + tool = _make_tool(

1344 + "f",

1345 + {

1346 + "name": {"type": "string"},

1347 + "count": {"type": "integer"},

1348 + },

1349 + )

1350 + engine = _make_engine(_converter_config(), tools=[tool])

1351 +

1352 + original = ParserEngine._streamable_string_keys

1353 + calls: list[dict] = []

1354 +

1355 + def wrapped(properties: dict) -> set[str] | None:

1356 + calls.append(properties)

1357 + return original(properties)

1358 +

1359 + monkeypatch.setattr(

1360 + ParserEngine,

1361 + "_streamable_string_keys",

1362 + staticmethod(wrapped),

1363 + )

1364 +

1365 + _run_streaming_tool(

1366 + engine,

1367 + "f",

1368 + ["name=alice ", "count=4", "2"],

1369 + )

1370 +

1371 + assert calls == [

1372 + {

1373 + "name": {"type": "string"},

1374 + "count": {"type": "integer"},

1375 + }

1376 + ]

1377 +

1378

1379 # ── TestSafeArgPrefix ────────────────────────────────────────────

1380

@@ -1351,7 +1387,7 @@ class TestSafeArgPrefix:

1387 [

1388 ('{"a": 1}', '{"a": '),

1389 ('{"a": 1, "b": 2}', '{"a": 1, "b": '),

1354 - ('{"a": "hello", "b": "world"}', '{"a": "hello", "b": '),

1390 + ('{"a": "hello", "b": "world"}', '{"a": "hello", "b": "world'),

1391 ('{"obj": {"x": 1}, "b": 2}', '{"obj": {"x": 1}, "b": '),

1392 ('{"url": "http://x:80", "b": 1}', '{"url": "http://x:80", "b": '),

1393 ('{"a": 1', '{"a": '),

@@ -1360,6 +1396,9 @@ class TestSafeArgPrefix:

1396 ("", ""),

1397 ('{"k":1}', '{"k":'),

1398 ('{"k": 1, "v":2}', '{"k": 1, "v":'),

1399 + ('{"k":"value"}', '{"k":"value'),

1400 + ('{"k":"unterminated', '{"k":"unterminated'),

1401 + (r'{"k":"escaped \" quote"}', r'{"k":"escaped \" quote'),

1402 ],

1403 )

1404 def test_safe_arg_prefix(self, json_str, expected):

tests/parser/engine/test_qwen3.py

+65 -1

@@ -346,6 +346,49 @@ class TestStreaming:

346 parsed = json.loads(concatenated)

347 assert parsed == {"city": "Tokyo", "unit": "celsius", "days": "5"}

348

349 + def test_streaming_long_string_arg_before_parameter_end(self, parser, mock_request):

350 + """Long string arguments should stream before the closing parameter tag."""

351 + chunks = [

352 + "<tool_call>\n",

353 + "<function=write_report>\n",

354 + "<parameter=content>",

355 + "Artificial intelligence has rapidly transformed the way ",

356 + "developers build dynamic applications with external tools.",

357 + "</param",

358 + "eter>\n",

359 + "</function>\n",

360 + "</tool_call>",

361 + ]

362 +

363 + results = simulate_tool_streaming(parser, mock_request, chunks)

364 +

365 + pre_close_arg_deltas: list[str] = []

366 + all_arg_deltas: list[str] = []

367 + for idx, (delta, _) in enumerate(results):

368 + if delta and delta.tool_calls:

369 + for tc in delta.tool_calls:

370 + if tc.function and tc.function.arguments:

371 + all_arg_deltas.append(tc.function.arguments)

372 + if idx < 5:

373 + pre_close_arg_deltas.append(tc.function.arguments)

374 +

375 + assert len(pre_close_arg_deltas) > 1, (

376 + "Expected long string arguments to stream incrementally before "

377 + f"</parameter>, got {pre_close_arg_deltas}"

378 + )

379 + partial_args = "".join(pre_close_arg_deltas)

380 + assert partial_args.startswith('{"content": "Artificial intelligence')

381 + assert partial_args.endswith("external tools.")

382 + assert not partial_args.endswith('"}')

383 +

384 + all_args = "".join(all_arg_deltas)

385 + assert json.loads(all_args) == {

386 + "content": (

387 + "Artificial intelligence has rapidly transformed the way "

388 + "developers build dynamic applications with external tools."

389 + )

390 + }

391 +

392 def test_streaming_text_before_tool(self, parser, mock_request):

393 chunks = [

394 "Let me check ",

@@ -395,6 +438,27 @@ class TestStreaming:

438 parsed = json.loads(args_text)

439 assert parsed["name"] == "Alice"

440

441 + def test_streaming_split_next_parameter_tag_is_buffered(self, parser, mock_request):

442 + """A split opening parameter tag must not leak into previous value."""

443 + chunks = [

444 + "<tool_call>\n",

445 + "<function=search>\n",

446 + "<parameter=query>hello ",

447 + "<param",

448 + "eter=limit>10</parameter>\n",

449 + "</function>\n",

450 + "</tool_call>",

451 + ]

452 +

453 + results = simulate_tool_streaming(parser, mock_request, chunks)

454 +

455 + args_after_partial_tag = collect_tool_arguments(results[:4])

456 + assert "<param" not in args_after_partial_tag

457 + assert args_after_partial_tag == '{"query": "hello'

458 +

459 + args_text = collect_tool_arguments(results)

460 + assert json.loads(args_text) == {"query": "hello", "limit": "10"}

461 +

462 def test_streaming_numeric_values(self, parser, mock_request):

463 chunks = [

464 "<tool_call>\n",

@@ -613,7 +677,7 @@ class TestArgConverter:

677 raw = "<parameter=command>\nls -la</parameter>\n<parameter=desc>\npartial value"

678 result = json.loads(_qwen3_arg_converter(raw, partial=True))

679 assert result["command"] == "ls -la"

616 - assert result["desc"] == "\npartial value"

680 + assert result["desc"] == "partial value"

681

682 def test_partial_value_with_angle_bracket(self):

683 from vllm.parser.qwen3 import (

vllm/parser/engine/parser_engine.py

+59 -6

@@ -52,6 +52,7 @@ class ToolCallSlot:

52 "_args_parts",

53 "_args_joined",

54 "name_sent",

55 + "string_keys",

56 "streamed_json",

57 )

@@ -61,6 +62,7 @@ class ToolCallSlot:

62 self._args_parts: list[str] = []

63 self._args_joined: str | None = ""

64 self.name_sent: bool = False

65 + self.string_keys: set[str] | None = None

66 self.streamed_json: str = ""

68 @property

@@ -264,17 +266,23 @@ class ParserEngine(Parser):

266 return args, changed

267

268 @staticmethod

267 - def _safe_arg_prefix(json_str: str) -> str:

269 + def _safe_arg_prefix(json_str: str, string_keys: set[str] | None = None) -> str:

270 """Return the prefix of *json_str* up to the last top-level value.

271

272 Middle values (followed by a comma) are stable across streaming

271 - ticks and included. The trailing value is excluded because type

272 - coercion may change its serialised form between ticks, which

273 - would violate the ``startswith(prev)`` prefix invariant.

273 + ticks and included. The trailing value is excluded for non-string

274 + values because type coercion may change its serialised form between

275 + ticks, which would violate the ``startswith(prev)`` prefix invariant.

276 + String values for keys in ``string_keys`` are prefix-stable, so stream

277 + their unterminated content instead of buffering long arguments until

278 + the closing tag arrives.

279 """

280 last_colon = -1

281 + last_key: str | None = None

282 + pending_key: str | None = None

283 in_string = False

284 escape = False

285 + string_start = -1

286 depth = 0

287 for i, c in enumerate(json_str):

288 if escape:

@@ -285,21 +293,60 @@ class ParserEngine(Parser):

293 escape = True

294 elif c == '"':

295 in_string = False

296 + if depth == 1 and string_start >= 0:

297 + pending_key = json_str[string_start + 1 : i]

298 continue

299 if c == '"':

300 in_string = True

301 + string_start = i

302 elif c in ("{", "["):

303 depth += 1

304 elif c in ("}", "]"):

305 depth -= 1

306 elif c == ":" and depth == 1:

307 last_colon = i

308 + last_key = pending_key

309 + pending_key = None

310 if last_colon < 0:

311 return ""

312 end = last_colon + 1

313 while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):

314 end += 1

302 - return json_str[:end]

315 + if end >= len(json_str) or json_str[end] != '"':

316 + return json_str[:end]

317 + if string_keys is not None and last_key not in string_keys:

318 + return json_str[:end]

319 +

320 + escape = False

321 + for i in range(end + 1, len(json_str)):

322 + c = json_str[i]

323 + if escape:

324 + escape = False

325 + continue

326 + if c == "\\":

327 + escape = True

328 + continue

329 + if c == '"':

330 + return json_str[:i]

331 + return json_str

332 +

333 + @staticmethod

334 + def _streamable_string_keys(properties: dict) -> set[str] | None:

335 + """Return keys whose trailing string values can safely stream.

336 +

337 + ``None`` means there is no schema, so all string values keep their

338 + JSON representation as strings. With a schema, only fields that can

339 + remain strings are safe to emit before the value is closed; fields

340 + coerced to bool/number/null/object/array may serialize differently.

341 + """

342 + if not properties:

343 + return None

344 +

345 + streamable: set[str] = set()

346 + for key, schema in properties.items():

347 + if set(extract_types_from_schema(schema)) == {"string"}:

348 + streamable.add(key)

349 + return streamable

350

351 def _fix_arg_types(self, args_json: str, func_name: str) -> str:

352 """Correct parameter types using the tool schema.

@@ -736,6 +783,9 @@ class ParserEngine(Parser):

783 slot = self._tool_slots[idx]

784 slot.name = name

785 slot.name_sent = True

786 + slot.string_keys = self._streamable_string_keys(

787 + find_tool_properties(self._tools, name)

788 + )

789 self._ensure_tool_id(slot, name)

790 deltas.append(

791 DeltaToolCall(

@@ -791,6 +841,9 @@ class ParserEngine(Parser):

841 if name and self._is_valid_tool_name(name):

842 slot.name = name

843 slot.name_sent = True

844 + slot.string_keys = self._streamable_string_keys(

845 + find_tool_properties(self._tools, name)

846 + )

847 self._ensure_tool_id(slot, name)

848 deltas.append(

849 DeltaToolCall(

@@ -873,7 +926,7 @@ class ParserEngine(Parser):

926 current_json = self._fix_arg_types(current_json, slot.name)

927

928 prev = slot.streamed_json

876 - safe_json = self._safe_arg_prefix(current_json)

929 + safe_json = self._safe_arg_prefix(current_json, slot.string_keys)

930

931 if not safe_json or safe_json == prev:

932 return None

vllm/parser/qwen3.py

+13 -1

@@ -42,6 +42,8 @@ TOOL_CALL_START = "<tool_call>"

42 TOOL_CALL_END = "</tool_call>"

43 FUNC_PREFIX = "<function="

44 FUNC_END = "</function>"

45 +PARAM_START = "<parameter="

46 +PARAM_END = "</parameter>"

48 _PARAM_RE = re.compile(

49 r"<\s*parameter\s*=\s*([^>]*)>"

@@ -67,7 +69,7 @@ def _qwen3_arg_converter(raw_args: str, partial: bool) -> str:

69 name = m.group(1)

70 value = m.group(2)

71 if name:

70 - params[name] = value

72 + params[name] = value.strip()

74 return json.dumps(params, ensure_ascii=False)

@@ -86,6 +88,8 @@ def qwen3_config(thinking: bool = True) -> ParserEngineConfig:

88 "TOOL_END": TOOL_CALL_END,

89 "FUNC_PREFIX": FUNC_PREFIX,

90 "FUNC_END": FUNC_END,

91 + "PARAM_START": PARAM_START,

92 + "PARAM_END": PARAM_END,

93 "CLOSE_ANGLE": ">",

94 },

95 token_id_terminals={

@@ -146,6 +150,14 @@ def qwen3_config(thinking: bool = True) -> ParserEngineConfig:

150 ParserState.TOOL_BETWEEN,

151 (EventType.TOOL_CALL_END,),

152 ),

153 + (ParserState.TOOL_ARGS, "PARAM_START"): Transition(

154 + ParserState.TOOL_ARGS,

155 + (EventType.ARG_VALUE_CHUNK,),

156 + ),

157 + (ParserState.TOOL_ARGS, "PARAM_END"): Transition(

158 + ParserState.TOOL_ARGS,

159 + (EventType.ARG_VALUE_CHUNK,),

160 + ),

161 (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(

162 ParserState.CONTENT,

163 (),