A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 20:15 UTC
[Bugfix] Fix step3p5 parser when using mtp (#33690)
Signed-off-by: mariohong <mariohong128@gmail.com>
Mario Hong committed
Feb 6, 2026 at 00:04 UTC
82914d2ae8d0362be06700222f4cd4c5f6b0dc36
2 files changed
+1455
-5
tests/tool_parsers/test_step3p5_tool_parser.py
new
+1435
@@ -0,0 +1,1435 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+from collections.abc import Generator
6
+
7
+import pytest
8
+
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
+ ChatCompletionRequest,
11
+ ChatCompletionToolsParam,
12
+)
13
+from vllm.entrypoints.openai.engine.protocol import (
14
+ DeltaMessage,
15
+ FunctionCall,
16
+ ToolCall,
17
+)
18
+from vllm.tokenizers import TokenizerLike, get_tokenizer
19
+from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
20
+from vllm.tool_parsers.step3p5_tool_parser import Step3p5ToolParser
21
+
22
+MODEL = "stepfun-ai/Step-3.5-Flash"
23
+
24
+
25
+@pytest.fixture(scope="module")
26
+def step3p5_tokenizer():
27
+ return get_tokenizer(tokenizer_name=MODEL)
28
+
29
+
30
+@pytest.fixture
31
+def step3p5_tool_parser(step3p5_tokenizer):
32
+ return Step3p5ToolParser(step3p5_tokenizer)
33
+
34
+
35
+@pytest.fixture
36
+def sample_tools():
37
+ return [
38
+ ChatCompletionToolsParam(
39
+ type="function",
40
+ function={
41
+ "name": "get_current_weather",
42
+ "description": "Get the current weather",
43
+ "parameters": {
44
+ "type": "object",
45
+ "properties": {
46
+ "city": {"type": "string", "description": "The city name"},
47
+ "state": {"type": "string", "description": "The state code"},
48
+ "unit": {"type": "string", "enum": ["fahrenheit", "celsius"]},
49
+ },
50
+ "required": ["city", "state"],
51
+ },
52
+ },
53
+ ),
54
+ ChatCompletionToolsParam(
55
+ type="function",
56
+ function={
57
+ "name": "calculate_area",
58
+ "description": "Calculate area of a shape",
59
+ "parameters": {
60
+ "type": "object",
61
+ "properties": {
62
+ "shape": {"type": "string"},
63
+ "dimensions": {"type": "object"},
64
+ "precision": {"type": "integer"},
65
+ },
66
+ },
67
+ },
68
+ ),
69
+ ]
70
+
71
+
72
+def assert_tool_calls(
73
+ actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
74
+):
75
+ assert len(actual_tool_calls) == len(expected_tool_calls)
76
+
77
+ for actual_tool_call, expected_tool_call in zip(
78
+ actual_tool_calls, expected_tool_calls
79
+ ):
80
+ assert actual_tool_call.type == "function"
81
+ assert actual_tool_call.function.name == expected_tool_call.function.name
82
+ assert json.loads(actual_tool_call.function.arguments) == json.loads(
83
+ expected_tool_call.function.arguments
84
+ )
85
+
86
+
87
+def stream_delta_message_generator(
88
+ step3p5_tool_parser,
89
+ step3p5_tokenizer: TokenizerLike,
90
+ model_output: str,
91
+ request: ChatCompletionRequest | None = None,
92
+) -> Generator[DeltaMessage, None, None]:
93
+ all_token_ids = step3p5_tokenizer.encode(model_output, add_special_tokens=False)
94
+
95
+ previous_text = ""
96
+ previous_tokens = None
97
+ prefix_offset = 0
98
+ read_offset = 0
99
+ for i, delta_token in enumerate(all_token_ids):
100
+ delta_token_ids = [delta_token]
101
+ previous_token_ids = all_token_ids[:i]
102
+ current_token_ids = all_token_ids[: i + 1]
103
+
104
+ (new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
105
+ detokenize_incrementally(
106
+ tokenizer=step3p5_tokenizer,
107
+ all_input_ids=current_token_ids,
108
+ prev_tokens=previous_tokens,
109
+ prefix_offset=prefix_offset,
110
+ read_offset=read_offset,
111
+ skip_special_tokens=False,
112
+ spaces_between_special_tokens=True,
113
+ )
114
+ )
115
+
116
+ current_text = previous_text + delta_text
117
+
118
+ delta_message = step3p5_tool_parser.extract_tool_calls_streaming(
119
+ previous_text,
120
+ current_text,
121
+ delta_text,
122
+ previous_token_ids,
123
+ current_token_ids,
124
+ delta_token_ids,
125
+ request=request,
126
+ )
127
+ if delta_message:
128
+ yield delta_message
129
+
130
+ previous_text = current_text
131
+ previous_tokens = (
132
+ previous_tokens + new_tokens if previous_tokens else new_tokens
133
+ )
134
+ prefix_offset = new_prefix_offset
135
+ read_offset = new_read_offset
136
+
137
+
138
+def stream_delta_message_generator_from_chunks(
139
+ step3p5_tool_parser,
140
+ step3p5_tokenizer: TokenizerLike,
141
+ delta_text_chunks: list[str],
142
+ request: ChatCompletionRequest | None = None,
143
+) -> Generator[DeltaMessage, None, None]:
144
+ previous_text = ""
145
+ previous_token_ids: list[int] = []
146
+
147
+ for delta_text in delta_text_chunks:
148
+ delta_token_ids = step3p5_tokenizer.encode(delta_text, add_special_tokens=False)
149
+ current_text = previous_text + delta_text
150
+ current_token_ids = previous_token_ids + delta_token_ids
151
+
152
+ delta_message = step3p5_tool_parser.extract_tool_calls_streaming(
153
+ previous_text,
154
+ current_text,
155
+ delta_text,
156
+ previous_token_ids,
157
+ current_token_ids,
158
+ delta_token_ids,
159
+ request=request,
160
+ )
161
+ if delta_message:
162
+ yield delta_message
163
+
164
+ previous_text = current_text
165
+ previous_token_ids = current_token_ids
166
+
167
+
168
+def test_extract_tool_calls_no_tools(step3p5_tool_parser):
169
+ model_output = "This is a test response without any tool calls"
170
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
171
+ model_output, request=None
172
+ ) # type: ignore[arg-type]
173
+ assert not extracted_tool_calls.tools_called
174
+ assert extracted_tool_calls.tool_calls == []
175
+ assert extracted_tool_calls.content == model_output
176
+
177
+
178
+@pytest.mark.parametrize(
179
+ ids=[
180
+ "single_tool",
181
+ "single_tool_with_content",
182
+ "single_tool_multiline_param",
183
+ "parallel_tools",
184
+ "tool_with_typed_params",
185
+ ],
186
+ argnames=["model_output", "expected_tool_calls", "expected_content"],
187
+ argvalues=[
188
+ (
189
+ """<tool_call>
190
+<function=get_current_weather>
191
+<parameter=city>
192
+Dallas
193
+</parameter>
194
+<parameter=state>
195
+TX
196
+</parameter>
197
+<parameter=unit>
198
+fahrenheit
199
+</parameter>
200
+</function>
201
+</tool_call>""",
202
+ [
203
+ ToolCall(
204
+ function=FunctionCall(
205
+ name="get_current_weather",
206
+ arguments=json.dumps(
207
+ {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
208
+ ),
209
+ )
210
+ )
211
+ ],
212
+ None,
213
+ ),
214
+ (
215
+ """Sure! Let me check the weather for you.<tool_call>
216
+<function=get_current_weather>
217
+<parameter=city>
218
+Dallas
219
+</parameter>
220
+<parameter=state>
221
+TX
222
+</parameter>
223
+<parameter=unit>
224
+fahrenheit
225
+</parameter>
226
+</function>
227
+</tool_call>""",
228
+ [
229
+ ToolCall(
230
+ function=FunctionCall(
231
+ name="get_current_weather",
232
+ arguments=json.dumps(
233
+ {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
234
+ ),
235
+ )
236
+ )
237
+ ],
238
+ "Sure! Let me check the weather for you.",
239
+ ),
240
+ (
241
+ """<tool_call>
242
+<function=calculate_area>
243
+<parameter=shape>
244
+rectangle
245
+</parameter>
246
+<parameter=dimensions>
247
+{"width": 10,
248
+ "height": 20}
249
+</parameter>
250
+<parameter=precision>
251
+2
252
+</parameter>
253
+</function>
254
+</tool_call>""",
255
+ [
256
+ ToolCall(
257
+ function=FunctionCall(
258
+ name="calculate_area",
259
+ arguments=json.dumps(
260
+ {
261
+ "shape": "rectangle",
262
+ "dimensions": {"width": 10, "height": 20},
263
+ "precision": 2,
264
+ }
265
+ ),
266
+ )
267
+ )
268
+ ],
269
+ None,
270
+ ),
271
+ (
272
+ """<tool_call>
273
+<function=get_current_weather>
274
+<parameter=city>
275
+Dallas
276
+</parameter>
277
+<parameter=state>
278
+TX
279
+</parameter>
280
+<parameter=unit>
281
+fahrenheit
282
+</parameter>
283
+</function>
284
+</tool_call>
285
+<tool_call>
286
+<function=get_current_weather>
287
+<parameter=city>
288
+Orlando
289
+</parameter>
290
+<parameter=state>
291
+FL
292
+</parameter>
293
+<parameter=unit>
294
+fahrenheit
295
+</parameter>
296
+</function>
297
+</tool_call>""",
298
+ [
299
+ ToolCall(
300
+ function=FunctionCall(
301
+ name="get_current_weather",
302
+ arguments=json.dumps(
303
+ {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
304
+ ),
305
+ )
306
+ ),
307
+ ToolCall(
308
+ function=FunctionCall(
309
+ name="get_current_weather",
310
+ arguments=json.dumps(
311
+ {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}
312
+ ),
313
+ )
314
+ ),
315
+ ],
316
+ None,
317
+ ),
318
+ (
319
+ """Let me calculate that area for you.<tool_call>
320
+<function=calculate_area>
321
+<parameter=shape>
322
+circle
323
+</parameter>
324
+<parameter=dimensions>
325
+{"radius": 15.5}
326
+</parameter>
327
+<parameter=precision>
328
+3
329
+</parameter>
330
+</function>
331
+</tool_call>""",
332
+ [
333
+ ToolCall(
334
+ function=FunctionCall(
335
+ name="calculate_area",
336
+ arguments=json.dumps(
337
+ {
338
+ "shape": "circle",
339
+ "dimensions": {"radius": 15.5},
340
+ "precision": 3,
341
+ }
342
+ ),
343
+ )
344
+ )
345
+ ],
346
+ "Let me calculate that area for you.",
347
+ ),
348
+ ],
349
+)
350
+def test_extract_tool_calls(
351
+ step3p5_tool_parser,
352
+ sample_tools,
353
+ model_output,
354
+ expected_tool_calls,
355
+ expected_content,
356
+):
357
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
358
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
359
+ model_output, request=request
360
+ )
361
+ assert extracted_tool_calls.tools_called
362
+
363
+ assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
364
+
365
+ assert extracted_tool_calls.content == expected_content
366
+
367
+
368
+def test_extract_tool_calls_fallback_no_tags(step3p5_tool_parser, sample_tools):
369
+ """Test fallback parsing when XML tags are missing"""
370
+ model_output = """<function=get_current_weather>
371
+<parameter=city>
372
+Dallas
373
+</parameter>
374
+<parameter=state>
375
+TX
376
+</parameter>
377
+</function>"""
378
+
379
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
380
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
381
+ model_output, request=request
382
+ )
383
+
384
+ assert extracted_tool_calls.tools_called
385
+ assert len(extracted_tool_calls.tool_calls) == 1
386
+ assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather"
387
+
388
+
389
+def test_extract_tool_calls_type_conversion(step3p5_tool_parser):
390
+ """Test parameter type conversion based on tool schema"""
391
+ tools = [
392
+ ChatCompletionToolsParam(
393
+ type="function",
394
+ function={
395
+ "name": "test_types",
396
+ "parameters": {
397
+ "type": "object",
398
+ "properties": {
399
+ "int_param": {"type": "integer"},
400
+ "float_param": {"type": "float"},
401
+ "bool_param": {"type": "boolean"},
402
+ "str_param": {"type": "string"},
403
+ "obj_param": {"type": "object"},
404
+ },
405
+ },
406
+ },
407
+ )
408
+ ]
409
+
410
+ model_output = """<tool_call>
411
+<function=test_types>
412
+<parameter=int_param>
413
+42
414
+</parameter>
415
+<parameter=float_param>
416
+3.14
417
+</parameter>
418
+<parameter=bool_param>
419
+true
420
+</parameter>
421
+<parameter=str_param>
422
+hello world
423
+</parameter>
424
+<parameter=obj_param>
425
+{"key": "value"}
426
+</parameter>
427
+</function>
428
+</tool_call>"""
429
+
430
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
431
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
432
+ model_output, request=request
433
+ )
434
+
435
+ args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
436
+ assert args["int_param"] == 42
437
+ assert args["float_param"] == 3.14
438
+ assert args["bool_param"] is True
439
+ assert args["str_param"] == "hello world"
440
+ assert args["obj_param"] == {"key": "value"}
441
+
442
+
443
+@pytest.mark.parametrize(
444
+ ids=[
445
+ "no_tools",
446
+ "single_tool",
447
+ "single_tool_with_content",
448
+ "single_tool_multiline_param",
449
+ "parallel_tools",
450
+ "tool_with_typed_params", # Added this test case
451
+ ],
452
+ argnames=["model_output", "expected_tool_calls", "expected_content"],
453
+ argvalues=[
454
+ ("This is a test without tools", [], "This is a test without tools"),
455
+ (
456
+ """<tool_call>
457
+<function=get_current_weather>
458
+<parameter=city>
459
+Dallas
460
+</parameter>
461
+<parameter=state>
462
+TX
463
+</parameter>
464
+<parameter=unit>
465
+fahrenheit
466
+</parameter>
467
+</function>
468
+</tool_call>""",
469
+ [
470
+ ToolCall(
471
+ function=FunctionCall(
472
+ name="get_current_weather",
473
+ arguments=json.dumps(
474
+ {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
475
+ ),
476
+ )
477
+ )
478
+ ],
479
+ None,
480
+ ),
481
+ (
482
+ """Sure! Let me check the weather for you.<tool_call>
483
+<function=get_current_weather>
484
+<parameter=city>
485
+Dallas
486
+</parameter>
487
+<parameter=state>
488
+TX
489
+</parameter>
490
+<parameter=unit>
491
+fahrenheit
492
+</parameter>
493
+</function>
494
+</tool_call>""",
495
+ [
496
+ ToolCall(
497
+ function=FunctionCall(
498
+ name="get_current_weather",
499
+ arguments=json.dumps(
500
+ {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
501
+ ),
502
+ )
503
+ )
504
+ ],
505
+ "Sure! Let me check the weather for you.",
506
+ ),
507
+ (
508
+ """<tool_call>
509
+<function=calculate_area>
510
+<parameter=shape>
511
+rectangle
512
+</parameter>
513
+<parameter=dimensions>
514
+{"width": 10,
515
+ "height": 20}
516
+</parameter>
517
+<parameter=precision>
518
+2
519
+</parameter>
520
+</function>
521
+</tool_call>""",
522
+ [
523
+ ToolCall(
524
+ function=FunctionCall(
525
+ name="calculate_area",
526
+ arguments=json.dumps(
527
+ {
528
+ "shape": "rectangle",
529
+ "dimensions": {"width": 10, "height": 20},
530
+ "precision": 2,
531
+ }
532
+ ),
533
+ )
534
+ )
535
+ ],
536
+ None,
537
+ ),
538
+ (
539
+ """<tool_call>
540
+<function=get_current_weather>
541
+<parameter=city>
542
+Dallas
543
+</parameter>
544
+<parameter=state>
545
+TX
546
+</parameter>
547
+<parameter=unit>
548
+fahrenheit
549
+</parameter>
550
+</function>
551
+</tool_call>
552
+<tool_call>
553
+<function=get_current_weather>
554
+<parameter=city>
555
+Orlando
556
+</parameter>
557
+<parameter=state>
558
+FL
559
+</parameter>
560
+<parameter=unit>
561
+celsius
562
+</parameter>
563
+</function>
564
+</tool_call>""",
565
+ [
566
+ ToolCall(
567
+ function=FunctionCall(
568
+ name="get_current_weather",
569
+ arguments=json.dumps(
570
+ {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
571
+ ),
572
+ )
573
+ ),
574
+ ToolCall(
575
+ function=FunctionCall(
576
+ name="get_current_weather",
577
+ arguments=json.dumps(
578
+ {"city": "Orlando", "state": "FL", "unit": "celsius"}
579
+ ),
580
+ )
581
+ ),
582
+ ],
583
+ None,
584
+ ),
585
+ # Added tool_with_typed_params test case
586
+ (
587
+ """Let me calculate that area for you.<tool_call>
588
+<function=calculate_area>
589
+<parameter=shape>
590
+circle
591
+</parameter>
592
+<parameter=dimensions>
593
+{"radius": 15.5}
594
+</parameter>
595
+<parameter=precision>
596
+3
597
+</parameter>
598
+</function>
599
+</tool_call>""",
600
+ [
601
+ ToolCall(
602
+ function=FunctionCall(
603
+ name="calculate_area",
604
+ arguments=json.dumps(
605
+ {
606
+ "shape": "circle",
607
+ "dimensions": {"radius": 15.5},
608
+ "precision": 3,
609
+ }
610
+ ),
611
+ )
612
+ )
613
+ ],
614
+ "Let me calculate that area for you.",
615
+ ),
616
+ ],
617
+)
618
+def test_extract_tool_calls_streaming(
619
+ step3p5_tool_parser,
620
+ step3p5_tokenizer,
621
+ sample_tools,
622
+ model_output,
623
+ expected_tool_calls,
624
+ expected_content,
625
+):
626
+ """Test incremental streaming behavior including typed parameters"""
627
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
628
+
629
+ other_content = ""
630
+ tool_states = {} # Track state per tool index
631
+
632
+ for delta_message in stream_delta_message_generator(
633
+ step3p5_tool_parser, step3p5_tokenizer, model_output, request
634
+ ):
635
+ # role should never be streamed from tool parser
636
+ assert not delta_message.role
637
+
638
+ if delta_message.content:
639
+ other_content += delta_message.content
640
+
641
+ if delta_message.tool_calls:
642
+ for tool_call in delta_message.tool_calls:
643
+ idx = tool_call.index
644
+
645
+ # Initialize state for new tool
646
+ if idx not in tool_states:
647
+ tool_states[idx] = {
648
+ "id": None,
649
+ "name": None,
650
+ "arguments": "",
651
+ "type": None,
652
+ }
653
+
654
+ # First chunk should have id, name, and type
655
+ if tool_call.id:
656
+ tool_states[idx]["id"] = tool_call.id
657
+
658
+ if tool_call.type:
659
+ assert tool_call.type == "function"
660
+ tool_states[idx]["type"] = tool_call.type
661
+
662
+ if tool_call.function:
663
+ if tool_call.function.name:
664
+ # Should only be set once
665
+ assert tool_states[idx]["name"] is None
666
+ tool_states[idx]["name"] = tool_call.function.name
667
+
668
+ if tool_call.function.arguments is not None:
669
+ # Accumulate arguments incrementally
670
+ tool_states[idx]["arguments"] += tool_call.function.arguments
671
+
672
+ # Verify final content
673
+ assert other_content == (expected_content or "") # Handle None case
674
+
675
+ # Verify we got all expected tool calls
676
+ assert len(tool_states) == len(expected_tool_calls)
677
+
678
+ # Verify each tool call
679
+ for idx, expected_tool in enumerate(expected_tool_calls):
680
+ state = tool_states[idx]
681
+ assert state["id"] is not None
682
+ assert state["type"] == "function"
683
+ assert state["name"] == expected_tool.function.name
684
+
685
+ # Parse accumulated arguments
686
+ arguments_str = state["arguments"]
687
+ assert arguments_str is not None
688
+ actual_args = json.loads(arguments_str)
689
+ expected_args = json.loads(expected_tool.function.arguments)
690
+ assert actual_args == expected_args
691
+
692
+
693
+def test_extract_tool_calls_missing_closing_parameter_tag(
694
+ step3p5_tool_parser, sample_tools
695
+):
696
+ """Test handling of missing closing </parameter> tag"""
697
+ # Using get_current_weather from sample_tools but with malformed XML
698
+ model_output = """Let me check the weather for you:
699
+<tool_call>
700
+<function=get_current_weather>
701
+<parameter=city>
702
+Dallas
703
+<parameter=state>
704
+TX
705
+</parameter>
706
+<parameter=unit>
707
+fahrenheit
708
+</parameter>
709
+</function>
710
+</tool_call>"""
711
+
712
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
713
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
714
+ model_output, request=request
715
+ )
716
+
717
+ # The parser should handle the malformed XML gracefully
718
+ assert extracted_tool_calls.tools_called
719
+ assert len(extracted_tool_calls.tool_calls) == 1
720
+
721
+ # Verify the function name is correct
722
+ assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather"
723
+
724
+ # Verify the arguments are parsed despite the missing closing tag
725
+ args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
726
+ assert "city" in args
727
+ assert args["city"] == "Dallas"
728
+ assert args["state"] == "TX"
729
+ assert args["unit"] == "fahrenheit"
730
+
731
+ # Check that content before the tool call is preserved
732
+ assert "Let me check the weather for you:" in extracted_tool_calls.content
733
+
734
+
735
+def test_extract_tool_calls_streaming_missing_closing_tag(
736
+ step3p5_tool_parser, step3p5_tokenizer, sample_tools
737
+):
738
+ """Test streaming with missing closing </parameter> tag"""
739
+ # Using get_current_weather from sample_tools but with malformed XML
740
+ model_output = """Let me check the weather for you:
741
+<tool_call>
742
+<function=get_current_weather>
743
+<parameter=city>
744
+Dallas
745
+<parameter=state>
746
+TX
747
+</parameter>
748
+<parameter=unit>
749
+fahrenheit
750
+</parameter>
751
+</function>
752
+</tool_call>"""
753
+
754
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
755
+
756
+ other_content = ""
757
+ tool_states = {}
758
+
759
+ for delta_message in stream_delta_message_generator(
760
+ step3p5_tool_parser, step3p5_tokenizer, model_output, request
761
+ ):
762
+ if delta_message.content:
763
+ other_content += delta_message.content
764
+
765
+ if delta_message.tool_calls:
766
+ for tool_call in delta_message.tool_calls:
767
+ idx = tool_call.index
768
+
769
+ if idx not in tool_states:
770
+ tool_states[idx] = {
771
+ "id": None,
772
+ "name": None,
773
+ "arguments": "",
774
+ "type": None,
775
+ }
776
+
777
+ if tool_call.id:
778
+ tool_states[idx]["id"] = tool_call.id
779
+
780
+ if tool_call.type:
781
+ assert tool_call.type == "function"
782
+ tool_states[idx]["type"] = tool_call.type
783
+
784
+ if tool_call.function:
785
+ if tool_call.function.name:
786
+ tool_states[idx]["name"] = tool_call.function.name
787
+
788
+ if tool_call.function.arguments is not None:
789
+ tool_states[idx]["arguments"] += tool_call.function.arguments
790
+
791
+ # Verify content was streamed
792
+ assert "Let me check the weather for you:" in other_content
793
+
794
+ # Verify we got the tool call
795
+ assert len(tool_states) == 1
796
+ state = tool_states[0]
797
+ assert state["id"] is not None
798
+ assert state["type"] == "function"
799
+ assert state["name"] == "get_current_weather"
800
+
801
+ # Verify arguments were parsed correctly despite missing closing tag
802
+ assert state["arguments"] is not None
803
+ args = json.loads(state["arguments"])
804
+ assert args["city"] == "Dallas"
805
+ assert args["state"] == "TX"
806
+ assert args["unit"] == "fahrenheit"
807
+
808
+
809
+def test_extract_tool_calls_streaming_incremental(
810
+ step3p5_tool_parser, step3p5_tokenizer, sample_tools
811
+):
812
+ """Test that streaming is truly incremental"""
813
+ model_output = """I'll check the weather.<tool_call>
814
+<function=get_current_weather>
815
+<parameter=city>
816
+Dallas
817
+</parameter>
818
+<parameter=state>
819
+TX
820
+</parameter>
821
+</function>
822
+</tool_call>"""
823
+
824
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
825
+
826
+ chunks = []
827
+ for delta_message in stream_delta_message_generator(
828
+ step3p5_tool_parser, step3p5_tokenizer, model_output, request
829
+ ):
830
+ chunks.append(delta_message)
831
+
832
+ # Should have multiple chunks
833
+ assert len(chunks) > 3
834
+
835
+ # First chunk(s) should be content
836
+ assert chunks[0].content is not None
837
+ assert chunks[0].tool_calls is None or chunks[0].tool_calls == []
838
+
839
+ # Should have a chunk with tool header (id, name, type)
840
+ header_found = False
841
+ for chunk in chunks:
842
+ if chunk.tool_calls and chunk.tool_calls[0].id:
843
+ header_found = True
844
+ assert chunk.tool_calls[0].function.name == "get_current_weather"
845
+ assert chunk.tool_calls[0].type == "function"
846
+ # Empty initially
847
+ assert chunk.tool_calls[0].function.arguments == ""
848
+ break
849
+ assert header_found
850
+
851
+ # Should have chunks with incremental arguments
852
+ arg_chunks = []
853
+ for chunk in chunks:
854
+ if chunk.tool_calls and chunk.tool_calls[0].function.arguments:
855
+ arg_chunks.append(chunk.tool_calls[0].function.arguments)
856
+
857
+ # Arguments should be streamed incrementally
858
+ assert len(arg_chunks) > 1
859
+
860
+ # Concatenated arguments should form valid JSON
861
+ full_args = "".join(arg_chunks)
862
+ parsed_args = json.loads(full_args)
863
+ assert parsed_args["city"] == "Dallas"
864
+ assert parsed_args["state"] == "TX"
865
+
866
+
867
+def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tool_parser):
868
+ """Test parameter type conversion based on tool schema"""
869
+ tools = [
870
+ ChatCompletionToolsParam(
871
+ type="function",
872
+ function={
873
+ "name": "test_types",
874
+ "parameters": {
875
+ "type": "object",
876
+ "properties": {
877
+ "int_param": {"type": "integer"},
878
+ "float_param": {"type": "float"},
879
+ "bool_param": {"type": "boolean"},
880
+ "str_param": {"type": "string"},
881
+ "obj_param": {"type": "object"},
882
+ },
883
+ },
884
+ },
885
+ )
886
+ ]
887
+
888
+ model_output = """<tool_call>
889
+<function=test_types>
890
+<parameter=obj_param>
891
+{'key': 'value'}
892
+</parameter>
893
+</function>
894
+</tool_call>"""
895
+
896
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
897
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
898
+ model_output, request=request
899
+ )
900
+
901
+ args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
902
+ assert args["obj_param"] == {"key": "value"}
903
+
904
+
905
+def test_extract_tool_calls_streaming_mixed_content_and_multiple_tool_calls(
906
+ step3p5_tool_parser, step3p5_tokenizer, sample_tools
907
+):
908
+ """Test mixed content with multiple complete tool calls.
909
+
910
+ Scenario: Model outputs "hello" + complete tool call + "hi" + complete tool call.
911
+ Expected: "hello" as content, first tool call parsed (index=0), "hi" as content,
912
+ second tool call parsed (index=1).
913
+ """
914
+ # Model output: hello + complete tool call + hi + complete tool call
915
+ model_output = """hello<tool_call>
916
+<function=get_current_weather>
917
+<parameter=city>
918
+Dallas
919
+</parameter>
920
+<parameter=state>
921
+TX
922
+</parameter>
923
+</function>
924
+</tool_call>hi<tool_call>
925
+<function=calculate_area>
926
+<parameter=shape>
927
+rectangle
928
+</parameter>
929
+<parameter=dimensions>
930
+{"width": 10, "height": 5}
931
+</parameter>
932
+</function>
933
+</tool_call>"""
934
+
935
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
936
+
937
+ other_content = ""
938
+ tool_states = {}
939
+
940
+ for delta_message in stream_delta_message_generator(
941
+ step3p5_tool_parser, step3p5_tokenizer, model_output, request
942
+ ):
943
+ if delta_message.content:
944
+ other_content += delta_message.content
945
+
946
+ if delta_message.tool_calls:
947
+ for tool_call in delta_message.tool_calls:
948
+ idx = tool_call.index
949
+
950
+ if idx not in tool_states:
951
+ tool_states[idx] = {
952
+ "id": None,
953
+ "name": None,
954
+ "arguments": "",
955
+ "type": None,
956
+ }
957
+
958
+ if tool_call.id:
959
+ tool_states[idx]["id"] = tool_call.id
960
+
961
+ if tool_call.type:
962
+ assert tool_call.type == "function"
963
+ tool_states[idx]["type"] = tool_call.type
964
+
965
+ if tool_call.function:
966
+ if tool_call.function.name:
967
+ tool_states[idx]["name"] = tool_call.function.name
968
+
969
+ if tool_call.function.arguments is not None:
970
+ tool_states[idx]["arguments"] += tool_call.function.arguments
971
+
972
+ # Should have exactly two complete tool calls
973
+ assert len(tool_states) == 2, "Should have exactly two complete tool calls"
974
+
975
+ # Verify the first tool call (index=0)
976
+ assert tool_states[0]["name"] == "get_current_weather"
977
+ assert tool_states[0]["arguments"]
978
+ args_dict_0 = json.loads(tool_states[0]["arguments"])
979
+ assert args_dict_0["city"] == "Dallas"
980
+ assert args_dict_0["state"] == "TX"
981
+
982
+ # Verify the second tool call (index=1)
983
+ assert tool_states[1]["name"] == "calculate_area"
984
+ assert tool_states[1]["arguments"]
985
+ args_dict_1 = json.loads(tool_states[1]["arguments"])
986
+ assert args_dict_1["shape"] == "rectangle"
987
+ assert isinstance(args_dict_1["dimensions"], dict), "dimensions should be a dict"
988
+ assert args_dict_1["dimensions"]["width"] == 10
989
+ assert args_dict_1["dimensions"]["height"] == 5
990
+ # Verify content: should contain "hello", "hi"
991
+ assert "hello" in other_content, "Should contain 'hello' as content"
992
+ assert "hi" in other_content, "Should contain 'hi' as content"
993
+
994
+ # Verify the order: hello should come first, then hi
995
+ hello_index = other_content.find("hello")
996
+ hi_index = other_content.find("hi")
997
+
998
+ assert hello_index >= 0, "'hello' should be in content"
999
+ assert hi_index > hello_index, "'hi' should come after 'hello'"
1000
+
1001
+ # Verify that tool call tags are NOT in the content
1002
+ # We should not see complete tool call structures in content
1003
+ assert "<function=get_current_weather>" not in other_content, (
1004
+ "First tool call should not be in content"
1005
+ )
1006
+ assert "<function=calculate_area>" not in other_content, (
1007
+ "Second tool call should not be in content"
1008
+ )
1009
+
1010
+
1011
+def test_extract_tool_calls_non_streaming_mixed_content_and_multiple_tool_calls(
1012
+ step3p5_tool_parser, sample_tools
1013
+):
1014
+ """Test non-streaming extraction with mixed content and multiple tool calls.
1015
+
1016
+ Scenario: Model outputs "hello" + complete tool call + "hi" + complete tool call.
1017
+ Expected: "hello" as content, first tool call parsed (index=0), "hi" as content,
1018
+ second tool call parsed (index=1)
1019
+ """
1020
+ # Model output: hello + complete tool call + hi + complete tool call
1021
+ model_output = """hello<tool_call>
1022
+<function=get_current_weather>
1023
+<parameter=city>
1024
+Dallas
1025
+</parameter>
1026
+<parameter=state>
1027
+TX
1028
+</parameter>
1029
+</function>
1030
+</tool_call>hi<tool_call>
1031
+<function=calculate_area>
1032
+<parameter=shape>
1033
+rectangle
1034
+</parameter>
1035
+<parameter=dimensions>
1036
+{"width": 10, "height": 5}
1037
+</parameter>
1038
+</function>
1039
+</tool_call>"""
1040
+
1041
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1042
+
1043
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
1044
+ model_output, request=request
1045
+ )
1046
+
1047
+ # Should have exactly two complete tool calls
1048
+ assert extracted_tool_calls.tools_called
1049
+ assert len(extracted_tool_calls.tool_calls) == 2, (
1050
+ "Should have exactly two complete tool calls"
1051
+ )
1052
+
1053
+ # Verify the first tool call (index=0)
1054
+ assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather"
1055
+ args_dict_0 = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
1056
+ assert args_dict_0["city"] == "Dallas"
1057
+ assert args_dict_0["state"] == "TX"
1058
+
1059
+ # Verify the second tool call (index=1)
1060
+ assert extracted_tool_calls.tool_calls[1].function.name == "calculate_area"
1061
+ args_dict_1 = json.loads(extracted_tool_calls.tool_calls[1].function.arguments)
1062
+ assert args_dict_1["shape"] == "rectangle"
1063
+ assert isinstance(args_dict_1["dimensions"], dict), "dimensions should be a dict"
1064
+ assert args_dict_1["dimensions"]["width"] == 10
1065
+ assert args_dict_1["dimensions"]["height"] == 5
1066
+
1067
+ # Verify content: should contain "hello", "hi"
1068
+ assert extracted_tool_calls.content is not None
1069
+ assert "hello" in extracted_tool_calls.content, "Should contain 'hello' as content"
1070
+ assert "hi" in extracted_tool_calls.content, "Should contain 'hi' as content"
1071
+
1072
+ # Verify the order: hello should come first, then hi
1073
+ hello_index = extracted_tool_calls.content.find("hello")
1074
+ hi_index = extracted_tool_calls.content.find("hi")
1075
+
1076
+ assert hello_index >= 0, "'hello' should be in content"
1077
+ assert hi_index > hello_index, "'hi' should come after 'hello'"
1078
+
1079
+ # Verify that tool call tags are NOT in the content
1080
+ assert "<function=get_current_weather>" not in extracted_tool_calls.content, (
1081
+ "First tool call should not be in content"
1082
+ )
1083
+ assert "<function=calculate_area>" not in extracted_tool_calls.content, (
1084
+ "Second tool call should not be in content"
1085
+ )
1086
+
1087
+
1088
+def test_extract_tool_calls_streaming_full_input_mixed_content_and_multiple_tool_calls(
1089
+ step3p5_tool_parser, step3p5_tokenizer, sample_tools
1090
+):
1091
+ """Test streaming with entire input as single delta_text.
1092
+
1093
+ Scenario: Model outputs "hello" + complete tool call + "hi" + complete tool call.
1094
+ This test simulates the case where the entire input is sent as a single delta_text.
1095
+ Expected: "hello" as content, first tool call parsed (index=0), "hi" as content,
1096
+ second tool call parsed (index=1).
1097
+ """
1098
+ # Model output: hello + complete tool call + hi + complete tool call
1099
+ model_output = """hello<tool_call>
1100
+<function=get_current_weather>
1101
+<parameter=city>
1102
+Dallas
1103
+</parameter>
1104
+<parameter=state>
1105
+TX
1106
+</parameter>
1107
+</function>
1108
+</tool_call>hi<tool_call>
1109
+<function=calculate_area>
1110
+<parameter=shape>
1111
+rectangle
1112
+</parameter>
1113
+<parameter=dimensions>
1114
+{"width": 10, "height": 5}
1115
+</parameter>
1116
+</function>
1117
+</tool_call>"""
1118
+
1119
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1120
+
1121
+ other_content = ""
1122
+ tool_states = {}
1123
+
1124
+ # Encode all content tokens at once
1125
+ all_token_ids = step3p5_tokenizer.encode(model_output, add_special_tokens=False)
1126
+ eos_token_id = getattr(step3p5_tokenizer, "eos_token_id", None)
1127
+
1128
+ # Include EOS token in delta_token_ids if available
1129
+ if eos_token_id is not None:
1130
+ delta_token_ids = all_token_ids + [eos_token_id]
1131
+ else:
1132
+ delta_token_ids = all_token_ids
1133
+
1134
+ # current_token_ids includes all content tokens (EOS is not part of the text)
1135
+ current_token_ids = all_token_ids
1136
+ previous_token_ids: list[int] = []
1137
+
1138
+ # Decode all tokens to get the full text
1139
+ current_text = step3p5_tokenizer.decode(
1140
+ current_token_ids, skip_special_tokens=False
1141
+ )
1142
+ previous_text = ""
1143
+ delta_text = current_text
1144
+
1145
+ # Call parser once with all tokens including EOS
1146
+ delta_result = step3p5_tool_parser.extract_tool_calls_streaming(
1147
+ previous_text,
1148
+ current_text,
1149
+ delta_text,
1150
+ previous_token_ids,
1151
+ current_token_ids,
1152
+ delta_token_ids,
1153
+ request=request,
1154
+ )
1155
+
1156
+ # Process delta result
1157
+ if delta_result:
1158
+ if delta_result.content:
1159
+ other_content += delta_result.content
1160
+ if delta_result.tool_calls:
1161
+ for tool_call in delta_result.tool_calls:
1162
+ idx = tool_call.index
1163
+ if idx not in tool_states:
1164
+ tool_states[idx] = {
1165
+ "id": None,
1166
+ "name": None,
1167
+ "arguments": "",
1168
+ "type": None,
1169
+ }
1170
+ if tool_call.id:
1171
+ tool_states[idx]["id"] = tool_call.id
1172
+ if tool_call.type:
1173
+ tool_states[idx]["type"] = tool_call.type
1174
+ if tool_call.function:
1175
+ if tool_call.function.name:
1176
+ tool_states[idx]["name"] = tool_call.function.name
1177
+ if tool_call.function.arguments is not None:
1178
+ tool_states[idx]["arguments"] += tool_call.function.arguments
1179
+
1180
+ # Should have exactly two complete tool calls
1181
+ assert len(tool_states) == 2, "Should have exactly two complete tool calls"
1182
+
1183
+ # Verify the first tool call (index=0)
1184
+ assert tool_states[0]["name"] == "get_current_weather"
1185
+ assert tool_states[0]["arguments"]
1186
+ args_dict_0 = json.loads(tool_states[0]["arguments"])
1187
+ assert args_dict_0["city"] == "Dallas"
1188
+ assert args_dict_0["state"] == "TX"
1189
+
1190
+ # Verify the second tool call (index=1)
1191
+ assert tool_states[1]["name"] == "calculate_area"
1192
+ assert tool_states[1]["arguments"]
1193
+ args_dict_1 = json.loads(tool_states[1]["arguments"])
1194
+ assert args_dict_1["shape"] == "rectangle"
1195
+ assert isinstance(args_dict_1["dimensions"], dict), "dimensions should be a dict"
1196
+ assert args_dict_1["dimensions"]["width"] == 10
1197
+ assert args_dict_1["dimensions"]["height"] == 5
1198
+
1199
+ # Verify content: should contain "hello", "hi"
1200
+ assert "hello" in other_content, "Should contain 'hello' as content"
1201
+ assert "hi" in other_content, "Should contain 'hi' as content"
1202
+
1203
+ # Verify the order: hello should come first, then hi
1204
+ hello_index = other_content.find("hello")
1205
+ hi_index = other_content.find("hi")
1206
+
1207
+ assert hello_index >= 0, "'hello' should be in content"
1208
+ assert hi_index > hello_index, "'hi' should come after 'hello'"
1209
+
1210
+ # Verify that tool call tags are NOT in the content
1211
+ assert "<function=get_current_weather>" not in other_content, (
1212
+ "First tool call should not be in content"
1213
+ )
1214
+ assert "<function=calculate_area>" not in other_content, (
1215
+ "Second tool call should not be in content"
1216
+ )
1217
+
1218
+
1219
+def test_extract_tool_calls_streaming_multiple_tool_calls_no_content_between(
1220
+ step3p5_tool_parser, step3p5_tokenizer, sample_tools
1221
+):
1222
+ """Test multiple tool calls with no content between them.
1223
+
1224
+ Scenario: Model outputs "hello" + tool call + tool call
1225
+ Expected: "hello" as content, first tool call parsed (index=0),
1226
+ second tool call parsed (index=1).
1227
+ No content should appear between the two tool calls.
1228
+ """
1229
+ # Model output: hello + tool call + tool call (no content between tool calls)
1230
+ model_output = """hello<tool_call>
1231
+<function=get_current_weather>
1232
+<parameter=city>
1233
+Dallas
1234
+</parameter>
1235
+<parameter=state>
1236
+TX
1237
+</parameter>
1238
+</function>
1239
+</tool_call><tool_call>
1240
+<function=calculate_area>
1241
+<parameter=shape>
1242
+rectangle
1243
+</parameter>
1244
+<parameter=dimensions>
1245
+{"width": 10, "height": 5}
1246
+</parameter>
1247
+</function>
1248
+</tool_call>"""
1249
+
1250
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1251
+
1252
+ other_content = ""
1253
+ tool_states = {}
1254
+
1255
+ for delta_message in stream_delta_message_generator(
1256
+ step3p5_tool_parser, step3p5_tokenizer, model_output, request
1257
+ ):
1258
+ if delta_message.content:
1259
+ other_content += delta_message.content
1260
+
1261
+ if delta_message.tool_calls:
1262
+ for tool_call in delta_message.tool_calls:
1263
+ idx = tool_call.index
1264
+
1265
+ if idx not in tool_states:
1266
+ tool_states[idx] = {
1267
+ "id": None,
1268
+ "name": None,
1269
+ "arguments": "",
1270
+ "type": None,
1271
+ }
1272
+
1273
+ if tool_call.id:
1274
+ tool_states[idx]["id"] = tool_call.id
1275
+
1276
+ if tool_call.type:
1277
+ assert tool_call.type == "function"
1278
+ tool_states[idx]["type"] = tool_call.type
1279
+
1280
+ if tool_call.function:
1281
+ if tool_call.function.name:
1282
+ tool_states[idx]["name"] = tool_call.function.name
1283
+
1284
+ if tool_call.function.arguments is not None:
1285
+ tool_states[idx]["arguments"] += tool_call.function.arguments
1286
+
1287
+ # Should have exactly two complete tool calls
1288
+ assert len(tool_states) == 2, "Should have exactly two complete tool calls"
1289
+
1290
+ # Verify the first tool call (index=0)
1291
+ assert tool_states[0]["name"] == "get_current_weather"
1292
+ assert tool_states[0]["arguments"]
1293
+ args_dict_0 = json.loads(tool_states[0]["arguments"])
1294
+ assert args_dict_0["city"] == "Dallas"
1295
+ assert args_dict_0["state"] == "TX"
1296
+
1297
+ # Verify the second tool call (index=1)
1298
+ assert tool_states[1]["name"] == "calculate_area"
1299
+ assert tool_states[1]["arguments"]
1300
+ args_dict_1 = json.loads(tool_states[1]["arguments"])
1301
+ assert args_dict_1["shape"] == "rectangle"
1302
+ assert isinstance(args_dict_1["dimensions"], dict), "dimensions should be a dict"
1303
+ assert args_dict_1["dimensions"]["width"] == 10
1304
+ assert args_dict_1["dimensions"]["height"] == 5
1305
+
1306
+ assert "hello" in other_content, "Should contain 'hello' as content"
1307
+
1308
+ # Verify that tool call tags are NOT in the content
1309
+ assert "<function=get_current_weather>" not in other_content, (
1310
+ "First tool call should not be in content"
1311
+ )
1312
+ assert "<function=calculate_area>" not in other_content, (
1313
+ "Second tool call should not be in content"
1314
+ )
1315
+
1316
+
1317
+def test_extract_tool_calls_streaming_multi_token_chunk_boundary(
1318
+ step3p5_tool_parser, step3p5_tokenizer, sample_tools
1319
+):
1320
+ """Ensure fallback doesn't close a new tool_call when boundary is in one chunk."""
1321
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1322
+ delta_text_chunks = [
1323
+ """<tool_call>
1324
+<function=get_current_weather>
1325
+<parameter=city>
1326
+Sys""",
1327
+ """
1328
+</parameter>
1329
+</function>
1330
+""",
1331
+ """</tool_call><tool_call>
1332
+<""",
1333
+ """function=calculate_area>
1334
+<parameter=shape>
1335
+rectangle""",
1336
+ """</parameter>
1337
+</function>
1338
+</tool_call>""",
1339
+ ]
1340
+ boundary_chunk = delta_text_chunks[1]
1341
+ assert len(step3p5_tokenizer.encode(boundary_chunk, add_special_tokens=False)) > 1
1342
+
1343
+ tool_states = {}
1344
+ for delta_message in stream_delta_message_generator_from_chunks(
1345
+ step3p5_tool_parser, step3p5_tokenizer, delta_text_chunks, request
1346
+ ):
1347
+ print(delta_message)
1348
+ if delta_message.tool_calls:
1349
+ for tool_call in delta_message.tool_calls:
1350
+ idx = tool_call.index
1351
+ if idx not in tool_states:
1352
+ tool_states[idx] = {
1353
+ "name": None,
1354
+ "arguments": "",
1355
+ }
1356
+ if tool_call.function:
1357
+ if tool_call.function.name:
1358
+ tool_states[idx]["name"] = tool_call.function.name
1359
+ if tool_call.function.arguments is not None:
1360
+ tool_states[idx]["arguments"] += tool_call.function.arguments
1361
+
1362
+ assert len(tool_states) == 2
1363
+ assert all(state["name"] for state in tool_states.values())
1364
+ assert tool_states[0]["name"] == "get_current_weather"
1365
+ assert tool_states[1]["name"] == "calculate_area"
1366
+
1367
+
1368
+def test_extract_tool_calls_non_streaming_multiple_tool_calls_no_content_between(
1369
+ step3p5_tool_parser, sample_tools
1370
+):
1371
+ """Test non-streaming extraction with tool calls and no content between them.
1372
+
1373
+ Scenario: Model outputs "hello" + tool call + tool call.
1374
+ Expected: "hello" as content, first tool call parsed (index=0),
1375
+ second tool call parsed (index=1).
1376
+ No content should appear between the two tool calls.
1377
+ """
1378
+ # Model output: hello + tool call + tool call (no content between tool calls)
1379
+ model_output = """hello<tool_call>
1380
+<function=get_current_weather>
1381
+<parameter=city>
1382
+Dallas
1383
+</parameter>
1384
+<parameter=state>
1385
+TX
1386
+</parameter>
1387
+</function>
1388
+</tool_call><tool_call>
1389
+<function=calculate_area>
1390
+<parameter=shape>
1391
+rectangle
1392
+</parameter>
1393
+<parameter=dimensions>
1394
+{"width": 10, "height": 5}
1395
+</parameter>
1396
+</function>
1397
+</tool_call>"""
1398
+
1399
+ request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1400
+
1401
+ extracted_tool_calls = step3p5_tool_parser.extract_tool_calls(
1402
+ model_output, request=request
1403
+ )
1404
+
1405
+ # Should have exactly two complete tool calls
1406
+ assert extracted_tool_calls.tools_called
1407
+ assert len(extracted_tool_calls.tool_calls) == 2, (
1408
+ "Should have exactly two complete tool calls"
1409
+ )
1410
+
1411
+ # Verify the first tool call (index=0)
1412
+ assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather"
1413
+ args_dict_0 = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
1414
+ assert args_dict_0["city"] == "Dallas"
1415
+ assert args_dict_0["state"] == "TX"
1416
+
1417
+ # Verify the second tool call (index=1)
1418
+ assert extracted_tool_calls.tool_calls[1].function.name == "calculate_area"
1419
+ args_dict_1 = json.loads(extracted_tool_calls.tool_calls[1].function.arguments)
1420
+ assert args_dict_1["shape"] == "rectangle"
1421
+ assert isinstance(args_dict_1["dimensions"], dict), "dimensions should be a dict"
1422
+ assert args_dict_1["dimensions"]["width"] == 10
1423
+ assert args_dict_1["dimensions"]["height"] == 5
1424
+
1425
+ # Verify content: should contain "hello"
1426
+ assert extracted_tool_calls.content is not None
1427
+ assert "hello" in extracted_tool_calls.content, "Should contain 'hello' as content"
1428
+
1429
+ # Verify that tool call tags are NOT in the content
1430
+ assert "<function=get_current_weather>" not in extracted_tool_calls.content, (
1431
+ "First tool call should not be in content"
1432
+ )
1433
+ assert "<function=calculate_area>" not in extracted_tool_calls.content, (
1434
+ "Second tool call should not be in content"
1435
+ )
vllm/tool_parsers/step3p5_tool_parser.py
+20
-5
@@ -97,11 +97,26 @@ class StreamingXMLToolCallParser:
97
"""
98
# Record delta count before processing
99
initial_delta_count = len(self.deltas)
100
+ entry_call_id = self.current_call_id
101
+ entry_tool_call_index = self.tool_call_index
102
103
self.streaming_buffer += xml_chunk
104
105
found_elements = self._process_complete_xml_elements()
106
107
+ fallback_call_id = None
108
+ if entry_call_id is not None:
109
+ if (
110
+ self.current_call_id == entry_call_id
111
+ and self.tool_call_index == entry_tool_call_index
112
+ ):
113
+ fallback_call_id = entry_call_id
114
+ elif (
115
+ self.current_call_id is not None
116
+ and self.tool_call_index == entry_tool_call_index + 1
117
+ ):
118
+ fallback_call_id = self.current_call_id
119
+
120
if found_elements:
121
# If complete elements found, check if end events were missed
122
# some tags may not have been triggered
@@ -110,7 +125,7 @@ class StreamingXMLToolCallParser:
125
# If this chunk contains </function>
126
# but didn't generate '}', then complete it
127
if (
113
- self.current_call_id is not None
128
+ fallback_call_id is not None
129
and self.function_end_token in xml_chunk
130
):
131
# - Added '}' (non-empty parameter ending)
@@ -121,7 +136,7 @@ class StreamingXMLToolCallParser:
136
and any(
137
(
138
tc.function
124
- and tc.id == self.current_call_id
139
+ and tc.id == fallback_call_id
140
and isinstance(tc.function.arguments, str)
141
and (tc.function.arguments in ("}", "{}"))
142
)
@@ -139,7 +154,7 @@ class StreamingXMLToolCallParser:
154
# If this chunk contains </tool_call>
155
# but didn't generate final empty delta, then complete it
156
if (
142
- self.current_call_id is not None
157
+ fallback_call_id is not None
158
and self.tool_call_end_token in xml_chunk
159
):
160
has_toolcall_close = any(
@@ -150,7 +165,7 @@ class StreamingXMLToolCallParser:
165
tc.type == "function"
166
and tc.function
167
and tc.function.arguments == ""
153
- and tc.id == self.current_call_id
168
+ and tc.id == fallback_call_id
169
)
170
for tc in td.tool_calls
171
)
@@ -186,7 +201,7 @@ class StreamingXMLToolCallParser:
201
# Only execute when still on the same call as when entered,
202
# to prevent accidentally closing new calls
203
# in multi <tool_call> scenarios
189
- if self.current_call_id is not None and (
204
+ if fallback_call_id is not None and (
205
self.function_end_token in xml_chunk
206
or self.tool_call_end_token in xml_chunk
207
):