@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e9455ae8f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Bugfix]: Fix the streaming output for function calls in the minimax (#22015)

Signed-off-by: QscQ <qscqesze@gmail.com> Signed-off-by: qingjun <qingjun@minimaxi.com>

qscqesze committed Aug 7, 2025 at 11:30 UTC 5e9455ae8f33599865f8855b28db2d074ea04eb5
2 files changed +1494 -204
tests/tool_use/test_minimax_tool_parser.py
+845 -1
@@ -3,10 +3,12 @@
3 # ruff: noqa: E501
4
5 import json
6 +from typing import Any
7
8 import pytest
9
9 -from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
10 +from vllm.entrypoints.openai.protocol import (ChatCompletionToolsParam,
11 + FunctionCall, ToolCall)
12 from vllm.entrypoints.openai.tool_parsers import MinimaxToolParser
13 from vllm.transformers_utils.tokenizer import get_tokenizer
14
@@ -24,6 +26,57 @@ def minimax_tool_parser(minimax_tokenizer):
26 return MinimaxToolParser(minimax_tokenizer)
27
28
29 +@pytest.fixture
30 +def sample_tools():
31 + return [
32 + ChatCompletionToolsParam(type="function",
33 + function={
34 + "name": "get_current_weather",
35 + "description": "Get the current weather",
36 + "parameters": {
37 + "type": "object",
38 + "properties": {
39 + "city": {
40 + "type": "string",
41 + "description": "The city name"
42 + },
43 + "state": {
44 + "type": "string",
45 + "description":
46 + "The state code"
47 + },
48 + "unit": {
49 + "type": "string",
50 + "enum":
51 + ["fahrenheit", "celsius"]
52 + }
53 + },
54 + "required": ["city", "state"]
55 + }
56 + }),
57 + ChatCompletionToolsParam(type="function",
58 + function={
59 + "name": "calculate_area",
60 + "description":
61 + "Calculate area of a shape",
62 + "parameters": {
63 + "type": "object",
64 + "properties": {
65 + "shape": {
66 + "type": "string"
67 + },
68 + "dimensions": {
69 + "type": "object"
70 + },
71 + "precision": {
72 + "type": "integer"
73 + }
74 + }
75 + }
76 + })
77 + ]
78 +
79 +
80 def assert_tool_calls(actual_tool_calls: list[ToolCall],
81 expected_tool_calls: list[ToolCall]):
82 assert len(actual_tool_calls) == len(expected_tool_calls)
@@ -370,3 +423,794 @@ def test_extract_tool_calls_multiline_json_not_supported(minimax_tool_parser):
423 assert not extracted_tool_calls.tools_called
424 assert extracted_tool_calls.tool_calls == []
425 assert extracted_tool_calls.content is None
426 +
427 +
428 +def test_streaming_arguments_incremental_output(minimax_tool_parser):
429 + """Test that streaming arguments are returned incrementally, not cumulatively."""
430 + # Reset streaming state
431 + minimax_tool_parser.current_tool_name_sent = False
432 + minimax_tool_parser.prev_tool_call_arr = []
433 + minimax_tool_parser.current_tool_id = -1
434 + minimax_tool_parser.streamed_args_for_tool = []
435 +
436 + # Simulate progressive tool call building
437 + stages = [
438 + # Stage 1: Function name complete
439 + '<tool_calls>\n{"name": "get_current_weather", "arguments": ',
440 + # Stage 2: Arguments object starts with first key
441 + '<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": ',
442 + # Stage 3: First parameter value added
443 + '<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle"',
444 + # Stage 4: Second parameter added
445 + '<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA"',
446 + # Stage 5: Third parameter added, arguments complete
447 + '<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}',
448 + # Stage 6: Tool calls closed
449 + '<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}\n</tool',
450 + '<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}\n</tool_calls>'
451 + ]
452 +
453 + function_name_sent = False
454 + previous_args_content = ""
455 +
456 + for i, current_text in enumerate(stages):
457 + previous_text = stages[i - 1] if i > 0 else ""
458 + delta_text = current_text[len(previous_text
459 + ):] if i > 0 else current_text
460 +
461 + result = minimax_tool_parser.extract_tool_calls_streaming(
462 + previous_text=previous_text,
463 + current_text=current_text,
464 + delta_text=delta_text,
465 + previous_token_ids=[],
466 + current_token_ids=[],
467 + delta_token_ids=[],
468 + request=None,
469 + )
470 +
471 + print(f"Stage {i}: Current text: {repr(current_text)}")
472 + print(f"Stage {i}: Delta text: {repr(delta_text)}")
473 +
474 + if result is not None and hasattr(result,
475 + 'tool_calls') and result.tool_calls:
476 + tool_call = result.tool_calls[0]
477 +
478 + # Check if function name is sent (should happen only once)
479 + if tool_call.function and tool_call.function.name:
480 + assert tool_call.function.name == "get_current_weather"
481 + function_name_sent = True
482 + print(
483 + f"Stage {i}: Function name sent: {tool_call.function.name}"
484 + )
485 +
486 + # Check if arguments are sent incrementally
487 + if tool_call.function and tool_call.function.arguments:
488 + args_fragment = tool_call.function.arguments
489 + print(
490 + f"Stage {i}: Got arguments fragment: {repr(args_fragment)}"
491 + )
492 +
493 + # For incremental output, each fragment should be new content only
494 + # The fragment should not contain all previous content
495 + if i >= 2 and previous_args_content: # After we start getting arguments
496 + # The new fragment should not be identical to or contain all previous content
497 + assert args_fragment != previous_args_content, f"Fragment should be incremental, not cumulative: {args_fragment}"
498 +
499 + # If this is truly incremental, the fragment should be relatively small
500 + # compared to the complete arguments so far
501 + if len(args_fragment) > len(previous_args_content):
502 + print(
503 + "Warning: Fragment seems cumulative rather than incremental"
504 + )
505 +
506 + previous_args_content = args_fragment
507 +
508 + # Verify function name was sent at least once
509 + assert function_name_sent, "Function name should have been sent"
510 +
511 +
512 +def test_streaming_arguments_delta_only(minimax_tool_parser):
513 + """Test that each streaming call returns only the delta (new part) of arguments."""
514 + # Reset streaming state
515 + minimax_tool_parser.current_tool_name_sent = False
516 + minimax_tool_parser.prev_tool_call_arr = []
517 + minimax_tool_parser.current_tool_id = -1
518 + minimax_tool_parser.streamed_args_for_tool = []
519 +
520 + # Simulate two consecutive calls with growing arguments
521 + call1_text = '<tool_calls>\n{"name": "test_tool", "arguments": {"param1": "value1"}}'
522 + call2_text = '<tool_calls>\n{"name": "test_tool", "arguments": {"param1": "value1", "param2": "value2"}}'
523 +
524 + print(f"Call 1 text: {repr(call1_text)}")
525 + print(f"Call 2 text: {repr(call2_text)}")
526 +
527 + # First call - should get the function name and initial arguments
528 + result1 = minimax_tool_parser.extract_tool_calls_streaming(
529 + previous_text="",
530 + current_text=call1_text,
531 + delta_text=call1_text,
532 + previous_token_ids=[],
533 + current_token_ids=[],
534 + delta_token_ids=[],
535 + request=None,
536 + )
537 +
538 + print(f"Result 1: {result1}")
539 + if result1 and hasattr(result1, 'tool_calls') and result1.tool_calls:
540 + for i, tc in enumerate(result1.tool_calls):
541 + print(f" Tool call {i}: {tc}")
542 +
543 + # Second call - should only get the delta (new part) of arguments
544 + result2 = minimax_tool_parser.extract_tool_calls_streaming(
545 + previous_text=call1_text,
546 + current_text=call2_text,
547 + delta_text=', "param2": "value2"}',
548 + previous_token_ids=[],
549 + current_token_ids=[],
550 + delta_token_ids=[],
551 + request=None,
552 + )
553 +
554 + print(f"Result 2: {result2}")
555 + if result2 and hasattr(result2, 'tool_calls') and result2.tool_calls:
556 + for i, tc in enumerate(result2.tool_calls):
557 + print(f" Tool call {i}: {tc}")
558 +
559 + # Verify the second call only returns the delta
560 + if result2 is not None and hasattr(result2,
561 + 'tool_calls') and result2.tool_calls:
562 + tool_call = result2.tool_calls[0]
563 + if tool_call.function and tool_call.function.arguments:
564 + args_delta = tool_call.function.arguments
565 + print(f"Arguments delta from second call: {repr(args_delta)}")
566 +
567 + # Should only contain the new part, not the full arguments
568 + # The delta should be something like ', "param2": "value2"}' or just '"param2": "value2"'
569 + assert ', "param2": "value2"}' in args_delta or '"param2": "value2"' in args_delta, f"Expected delta containing param2, got: {args_delta}"
570 +
571 + # Should NOT contain the previous parameter data
572 + assert '"param1": "value1"' not in args_delta, f"Arguments delta should not contain previous data: {args_delta}"
573 +
574 + # The delta should be relatively short (incremental, not cumulative)
575 + expected_max_length = len(
576 + ', "param2": "value2"}') + 10 # Some tolerance
577 + assert len(
578 + args_delta
579 + ) <= expected_max_length, f"Delta seems too long (possibly cumulative): {args_delta}"
580 +
581 + print("✓ Delta validation passed")
582 + else:
583 + print("No arguments in result2 tool call")
584 + else:
585 + print("No tool calls in result2 or result2 is None")
586 + # This might be acceptable if no incremental update is needed
587 + # But let's at least verify that result1 had some content
588 + assert result1 is not None, "At least the first call should return something"
589 +
590 +
591 +def test_streaming_openai_compatibility(minimax_tool_parser):
592 + """Test that streaming behavior with buffering works correctly."""
593 + # Reset streaming state
594 + minimax_tool_parser.current_tool_name_sent = False
595 + minimax_tool_parser.prev_tool_call_arr = []
596 + minimax_tool_parser.current_tool_id = -1
597 + minimax_tool_parser.streamed_args_for_tool = []
598 + # Reset buffering state
599 + minimax_tool_parser.pending_buffer = ""
600 + minimax_tool_parser.in_thinking_tag = False
601 + minimax_tool_parser.thinking_depth = 0
602 +
603 + # Test scenario: simple buffering without complex tool call context
604 + test_cases: list[dict[str, Any]] = [
605 + {
606 + 'stage': 'Token: <',
607 + 'previous': '',
608 + 'current': '<',
609 + 'delta': '<',
610 + 'expected_content': None, # Should be buffered
611 + },
612 + {
613 + 'stage': 'Token: tool_calls>',
614 + 'previous': '<',
615 + 'current': '<tool_calls>',
616 + 'delta': 'tool_calls>',
617 + 'expected_content': None, # Complete tag, should not output
618 + },
619 + {
620 + 'stage': 'Regular content',
621 + 'previous': 'Hello',
622 + 'current': 'Hello world',
623 + 'delta': ' world',
624 + 'expected_content': ' world', # Normal content should pass through
625 + },
626 + {
627 + 'stage': 'Content with end tag start',
628 + 'previous': 'Text',
629 + 'current': 'Text content</tool_',
630 + 'delta': ' content</tool_',
631 + 'expected_content':
632 + ' content', # Content part output, </tool_ buffered
633 + },
634 + {
635 + 'stage': 'Complete end tag',
636 + 'previous': 'Text content</tool_',
637 + 'current': 'Text content</tool_calls>',
638 + 'delta': 'calls>',
639 + 'expected_content': None, # Complete close tag, should not output
640 + },
641 + ]
642 +
643 + for i, test_case in enumerate(test_cases):
644 + print(f"\n--- Stage {i}: {test_case['stage']} ---")
645 + print(f"Previous: {repr(test_case['previous'])}")
646 + print(f"Current: {repr(test_case['current'])}")
647 + print(f"Delta: {repr(test_case['delta'])}")
648 +
649 + result = minimax_tool_parser.extract_tool_calls_streaming(
650 + previous_text=test_case['previous'],
651 + current_text=test_case['current'],
652 + delta_text=test_case['delta'],
653 + previous_token_ids=[],
654 + current_token_ids=[],
655 + delta_token_ids=[],
656 + request=None,
657 + )
658 +
659 + print(f"Result: {result}")
660 +
661 + # Check expected content
662 + if test_case['expected_content'] is None:
663 + assert result is None or not getattr(result, 'content', None), \
664 + f"Stage {i}: Expected no content, got {result}"
665 + print("✓ No content output as expected")
666 + else:
667 + assert result is not None and hasattr(result, 'content'), \
668 + f"Stage {i}: Expected content, got {result}"
669 + assert result.content == test_case['expected_content'], \
670 + f"Stage {i}: Expected content {test_case['expected_content']}, got {result.content}"
671 + print(f"✓ Content matches: {repr(result.content)}")
672 +
673 + print("✓ Streaming test with buffering completed successfully")
674 +
675 +
676 +def test_streaming_thinking_tag_buffering(minimax_tool_parser):
677 + """Test that tool calls within thinking tags are properly handled during streaming."""
678 + # Reset streaming state
679 + minimax_tool_parser.current_tool_name_sent = False
680 + minimax_tool_parser.prev_tool_call_arr = []
681 + minimax_tool_parser.current_tool_id = -1
682 + minimax_tool_parser.streamed_args_for_tool = []
683 + # Reset buffering state
684 + minimax_tool_parser.pending_buffer = ""
685 + minimax_tool_parser.in_thinking_tag = False
686 + minimax_tool_parser.thinking_depth = 0
687 +
688 + # Test scenario: tool calls within thinking tags should be ignored
689 + test_cases: list[dict[str, Any]] = [
690 + {
691 + 'stage': 'Start thinking',
692 + 'previous': '',
693 + 'current': '<think>I need to use a tool. <tool_calls>',
694 + 'delta': '<think>I need to use a tool. <tool_calls>',
695 + 'expected_content':
696 + '<think>I need to use a tool. <tool_calls>', # Should pass through as content
697 + },
698 + {
699 + 'stage':
700 + 'Tool call in thinking',
701 + 'previous':
702 + '<think>I need to use a tool. <tool_calls>',
703 + 'current':
704 + '<think>I need to use a tool. <tool_calls>\n{"name": "ignored_tool", "arguments": {"param": "value"}}\n</tool_calls>',
705 + 'delta':
706 + '\n{"name": "ignored_tool", "arguments": {"param": "value"}}\n</tool_calls>',
707 + 'expected_content':
708 + '\n{"name": "ignored_tool", "arguments": {"param": "value"}}\n</tool_calls>', # </tool_calls> should be preserved in thinking tags
709 + },
710 + {
711 + 'stage': 'Real tool call after thinking',
712 + 'previous':
713 + '<think>I need to use a tool. <tool_calls>\n{"name": "ignored_tool", "arguments": {"param": "value"}}\n</tool_calls></think>',
714 + 'current':
715 + '<think>I need to use a tool. <tool_calls>\n{"name": "ignored_tool", "arguments": {"param": "value"}}\n</tool_calls></think>\n<tool_calls>',
716 + 'delta': '\n<tool_calls>',
717 + 'expected_content':
718 + '\n', # Should output '\n' and suppress <tool_calls>
719 + }
720 + ]
721 +
722 + for i, test_case in enumerate(test_cases):
723 + print(f"\n--- Stage {i}: {test_case['stage']} ---")
724 + print(f"Previous: {repr(test_case['previous'])}")
725 + print(f"Current: {repr(test_case['current'])}")
726 + print(f"Delta: {repr(test_case['delta'])}")
727 +
728 + result = minimax_tool_parser.extract_tool_calls_streaming(
729 + previous_text=test_case['previous'],
730 + current_text=test_case['current'],
731 + delta_text=test_case['delta'],
732 + previous_token_ids=[],
733 + current_token_ids=[],
734 + delta_token_ids=[],
735 + request=None,
736 + )
737 +
738 + print(f"Result: {result}")
739 +
740 + # Check expected content
741 + if 'expected_content' in test_case:
742 + if test_case['expected_content'] is None:
743 + assert result is None or not getattr(result, 'content', None), \
744 + f"Stage {i}: Expected no content, got {result}"
745 + else:
746 + assert result is not None and hasattr(result, 'content'), \
747 + f"Stage {i}: Expected content, got {result}"
748 + assert result.content == test_case['expected_content'], \
749 + f"Stage {i}: Expected content {test_case['expected_content']}, got {result.content}"
750 + print(f"✓ Content matches: {repr(result.content)}")
751 +
752 + # Check tool calls
753 + if test_case.get('expected_tool_call'):
754 + assert result is not None and hasattr(result, 'tool_calls') and result.tool_calls, \
755 + f"Stage {i}: Expected tool call, got {result}"
756 +
757 + tool_call = result.tool_calls[0]
758 + assert tool_call.function.name == "real_tool", \
759 + f"Expected real_tool, got {tool_call.function.name}"
760 + print(f"✓ Real tool call detected: {tool_call.function.name}")
761 +
762 + print("✓ Thinking tag buffering test completed successfully")
763 +
764 +
765 +def reset_streaming_state(minimax_tool_parser):
766 + """Helper function to properly reset the streaming state for MinimaxToolParser."""
767 + # Reset minimax-specific state
768 + minimax_tool_parser._reset_streaming_state()
769 +
770 + # Reset base class state (these should still be reset for compatibility)
771 + minimax_tool_parser.prev_tool_call_arr = []
772 + minimax_tool_parser.current_tool_id = -1
773 + minimax_tool_parser.current_tool_name_sent = False
774 + minimax_tool_parser.streamed_args_for_tool = []
775 +
776 +
777 +def test_streaming_complex_scenario_with_multiple_tools(minimax_tool_parser):
778 + """Test complex streaming scenario: tools inside <think> tags and multiple tool calls in one group."""
779 + # Reset streaming state
780 + reset_streaming_state(minimax_tool_parser)
781 +
782 + # Complex scenario: tools inside thinking tags and multiple tools in one group
783 + test_stages: list[dict[str, Any]] = [
784 + {
785 + 'stage': 'Initial content',
786 + 'previous': '',
787 + 'current': 'Let me help you with this task.',
788 + 'delta': 'Let me help you with this task.',
789 + 'expected_content': 'Let me help you with this task.',
790 + 'expected_tool_calls': 0,
791 + },
792 + {
793 + 'stage': 'Start thinking tag',
794 + 'previous': 'Let me help you with this task.',
795 + 'current':
796 + 'Let me help you with this task.<think>I need to analyze this situation first.',
797 + 'delta': '<think>I need to analyze this situation first.',
798 + 'expected_content':
799 + '<think>I need to analyze this situation first.',
800 + 'expected_tool_calls': 0,
801 + },
802 + {
803 + 'stage': 'Tool call inside thinking tag starts',
804 + 'previous':
805 + 'Let me help you with this task.<think>I need to analyze this situation first.',
806 + 'current':
807 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>',
808 + 'delta': '<tool_calls>',
809 + 'expected_content':
810 + '<tool_calls>', # Inside thinking tags, tool tags should be preserved as content
811 + 'expected_tool_calls': 0,
812 + },
813 + {
814 + 'stage': 'Complete tool call inside thinking tag',
815 + 'previous':
816 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>',
817 + 'current':
818 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls>',
819 + 'delta':
820 + '\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls>',
821 + 'expected_content':
822 + '\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls>',
823 + 'expected_tool_calls':
824 + 0, # Tools inside thinking tags should be ignored
825 + },
826 + {
827 + 'stage': 'End thinking tag',
828 + 'previous':
829 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls>',
830 + 'current':
831 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>',
832 + 'delta': '</think>',
833 + 'expected_content': '</think>',
834 + 'expected_tool_calls': 0,
835 + },
836 + {
837 + 'stage': 'Multiple tools group starts',
838 + 'previous':
839 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>',
840 + 'current':
841 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>',
842 + 'delta':
843 + '\nNow I need to get weather information and calculate area.<tool_calls>',
844 + 'expected_content':
845 + '\nNow I need to get weather information and calculate area.', # <tool_calls> should be filtered
846 + 'expected_tool_calls': 0,
847 + },
848 + {
849 + 'stage': 'First tool in group',
850 + 'previous':
851 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>',
852 + 'current':
853 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}',
854 + 'delta':
855 + '\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}',
856 + 'expected_content':
857 + None, # No content should be output when tool call is in progress
858 + 'expected_tool_calls': 1,
859 + 'expected_tool_name': 'get_current_weather',
860 + },
861 + {
862 + 'stage': 'Second tool in group',
863 + 'previous':
864 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}',
865 + 'current':
866 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}\n{"name": "calculate_area", "arguments": {"shape": "rectangle", "dimensions": {"width": 10, "height": 5}}}',
867 + 'delta':
868 + '\n{"name": "calculate_area", "arguments": {"shape": "rectangle", "dimensions": {"width": 10, "height": 5}}}',
869 + 'expected_content': None,
870 + 'expected_tool_calls': 1,
871 + 'expected_tool_name': 'calculate_area',
872 + },
873 + {
874 + 'stage': 'Complete tool calls group',
875 + 'previous':
876 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}\n{"name": "calculate_area", "arguments": {"shape": "rectangle", "dimensions": {"width": 10, "height": 5}}}',
877 + 'current':
878 + 'Let me help you with this task.<think>I need to analyze this situation first.<tool_calls>\n{"name": "internal_analysis", "arguments": {"query": "analyze situation"}}\n</tool_calls></think>\nNow I need to get weather information and calculate area.<tool_calls>\n{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}\n{"name": "calculate_area", "arguments": {"shape": "rectangle", "dimensions": {"width": 10, "height": 5}}}</tool_calls>',
879 + 'delta': '</tool_calls>',
880 + 'expected_content': None,
881 + 'expected_tool_calls': 0,
882 + }
883 + ]
884 +
885 + tool_calls_count = 0
886 +
887 + for i, test_case in enumerate(test_stages):
888 + print(f"\n--- Stage {i}: {test_case['stage']} ---")
889 + print(
890 + f"Previous: {repr(test_case['previous'][:100])}{'...' if len(test_case['previous']) > 100 else ''}"
891 + )
892 + print(f"Current: {repr(test_case['current'][-100:])}")
893 + print(f"Delta: {repr(test_case['delta'])}")
894 +
895 + result = minimax_tool_parser.extract_tool_calls_streaming(
896 + previous_text=test_case['previous'],
897 + current_text=test_case['current'],
898 + delta_text=test_case['delta'],
899 + previous_token_ids=[],
900 + current_token_ids=[],
901 + delta_token_ids=[],
902 + request=None,
903 + )
904 +
905 + print(f"Result: {result}")
906 +
907 + # Check expected content
908 + if test_case['expected_content'] is None:
909 + assert result is None or not getattr(result, 'content', None), \
910 + f"Stage {i}: Expected no content output, got {result}"
911 + print("✓ No content output as expected")
912 + else:
913 + assert result is not None and hasattr(result, 'content'), \
914 + f"Stage {i}: Expected content output, got {result}"
915 + assert result.content == test_case['expected_content'], \
916 + f"Stage {i}: Expected content {repr(test_case['expected_content'])}, got {repr(result.content)}"
917 + print(f"✓ Content matches: {repr(result.content)}")
918 +
919 + # Check tool calls
920 + expected_tool_calls = test_case['expected_tool_calls']
921 + actual_tool_calls = len(result.tool_calls) if result and hasattr(
922 + result, 'tool_calls') and result.tool_calls else 0
923 +
924 + if expected_tool_calls > 0:
925 + assert actual_tool_calls >= expected_tool_calls, \
926 + f"Stage {i}: Expected at least {expected_tool_calls} tool calls, got {actual_tool_calls}"
927 +
928 + if 'expected_tool_name' in test_case:
929 + # Find the tool call with the expected name
930 + found_tool_call = None
931 + for tool_call in result.tool_calls:
932 + if tool_call.function.name == test_case[
933 + 'expected_tool_name']:
934 + found_tool_call = tool_call
935 + break
936 +
937 + assert found_tool_call is not None, \
938 + f"Stage {i}: Expected tool name {test_case['expected_tool_name']} not found in tool calls: {[tc.function.name for tc in result.tool_calls]}"
939 + print(f"✓ Tool call correct: {found_tool_call.function.name}")
940 +
941 + # Ensure tools inside thinking tags are not called
942 + assert found_tool_call.function.name != "internal_analysis", \
943 + f"Stage {i}: Tool 'internal_analysis' inside thinking tags should not be called"
944 +
945 + tool_calls_count += actual_tool_calls
946 + print(f"✓ Detected {actual_tool_calls} tool calls")
947 + else:
948 + assert actual_tool_calls == 0, \
949 + f"Stage {i}: Expected no tool calls, got {actual_tool_calls}"
950 +
951 + # Verify overall results
952 + print("\n=== Test Summary ===")
953 + print(f"Total tool calls count: {tool_calls_count}")
954 + assert tool_calls_count >= 2, f"Expected at least 2 valid tool calls (outside thinking tags), but got {tool_calls_count}"
955 +
956 + print("✓ Complex streaming test completed:")
957 + print(" - ✓ Tools inside thinking tags correctly ignored")
958 + print(" - ✓ Two tool groups outside thinking tags correctly parsed")
959 + print(" - ✓ Content and tool call streaming correctly handled")
960 + print(" - ✓ Buffering mechanism works correctly")
961 +
962 +
963 +def test_streaming_character_by_character_output(minimax_tool_parser):
964 + """Test character-by-character streaming output to simulate real streaming scenarios."""
965 + # Reset streaming state
966 + reset_streaming_state(minimax_tool_parser)
967 +
968 + # Complete text that will be streamed character by character
969 + complete_text = """I'll help you with the weather analysis. <think>Let me think about this. <tool_calls>
970 +{"name": "internal_analysis", "arguments": {"type": "thinking"}}
971 +</tool_calls>This tool should be ignored.</think>
972 +
973 +Now I'll get the weather information for you. <tool_calls>
974 +{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}
975 +{"name": "calculate_area", "arguments": {"shape": "rectangle", "dimensions": {"width": 10, "height": 5}}}
976 +</tool_calls>Here are the results."""
977 +
978 + print("\n=== Starting character-by-character streaming test ===")
979 + print(f"Complete text length: {len(complete_text)} characters")
980 +
981 + # Track the streaming results
982 + content_fragments = []
983 + tool_calls_detected = []
984 +
985 + # Stream character by character
986 + for i in range(1, len(complete_text) + 1):
987 + current_text = complete_text[:i]
988 + previous_text = complete_text[:i - 1] if i > 1 else ""
989 + delta_text = complete_text[i - 1:i]
990 +
991 + # Show progress every 50 characters
992 + if i % 50 == 0 or i == len(complete_text):
993 + print(f"Progress: {i}/{len(complete_text)} characters")
994 +
995 + # Call the streaming parser
996 + result = minimax_tool_parser.extract_tool_calls_streaming(
997 + previous_text=previous_text,
998 + current_text=current_text,
999 + delta_text=delta_text,
1000 + previous_token_ids=[],
1001 + current_token_ids=[],
1002 + delta_token_ids=[],
1003 + request=None,
1004 + )
1005 +
1006 + # Collect results
1007 + if result is not None:
1008 + if hasattr(result, 'content') and result.content:
1009 + content_fragments.append(result.content)
1010 + # Log important content fragments
1011 + if any(
1012 + keyword in result.content for keyword in
1013 + ['<think>', '</think>', '<tool_calls>', '</tool_calls>']):
1014 + print(
1015 + f" Char {i}: Content fragment: {repr(result.content)}"
1016 + )
1017 +
1018 + if hasattr(result, 'tool_calls') and result.tool_calls:
1019 + for tool_call in result.tool_calls:
1020 + tool_info = {
1021 + 'character_position':
1022 + i,
1023 + 'function_name':
1024 + tool_call.function.name
1025 + if tool_call.function else None,
1026 + 'arguments':
1027 + tool_call.function.arguments
1028 + if tool_call.function else None,
1029 + }
1030 + tool_calls_detected.append(tool_info)
1031 + print(
1032 + f" Char {i}: Tool call detected: {tool_call.function.name}"
1033 + )
1034 + if tool_call.function.arguments:
1035 + print(
1036 + f" Arguments: {repr(tool_call.function.arguments)}"
1037 + )
1038 +
1039 + # Verify results
1040 + print("\n=== Streaming Test Results ===")
1041 + print(f"Total content fragments: {len(content_fragments)}")
1042 + print(f"Total tool calls detected: {len(tool_calls_detected)}")
1043 +
1044 + # Reconstruct content from fragments
1045 + reconstructed_content = ''.join(content_fragments)
1046 + print(f"Reconstructed content length: {len(reconstructed_content)}")
1047 +
1048 + # Verify thinking tags content is preserved
1049 + assert '<think>' in reconstructed_content, "Opening thinking tag should be preserved in content"
1050 + assert '</think>' in reconstructed_content, "Closing thinking tag should be preserved in content"
1051 +
1052 + # Verify that tool calls inside thinking tags are NOT extracted as actual tool calls
1053 + thinking_tool_calls = [
1054 + tc for tc in tool_calls_detected
1055 + if tc['function_name'] == 'internal_analysis'
1056 + ]
1057 + assert len(
1058 + thinking_tool_calls
1059 + ) == 0, f"Tool calls inside thinking tags should be ignored, but found: {thinking_tool_calls}"
1060 +
1061 + # Verify that real tool calls outside thinking tags ARE extracted
1062 + weather_tool_calls = [
1063 + tc for tc in tool_calls_detected
1064 + if tc['function_name'] == 'get_current_weather'
1065 + ]
1066 + area_tool_calls = [
1067 + tc for tc in tool_calls_detected
1068 + if tc['function_name'] == 'calculate_area'
1069 + ]
1070 + print(tool_calls_detected)
1071 + assert len(weather_tool_calls
1072 + ) > 0, "get_current_weather tool call should be detected"
1073 + assert len(
1074 + area_tool_calls) > 0, "calculate_area tool call should be detected"
1075 +
1076 + # Verify tool call arguments are properly streamed
1077 + weather_args_found = any(tc['arguments'] for tc in weather_tool_calls
1078 + if tc['arguments'])
1079 + area_args_found = any(tc['arguments'] for tc in area_tool_calls
1080 + if tc['arguments'])
1081 +
1082 + print(f"Weather tool call with arguments: {weather_args_found}")
1083 + print(f"Area tool call with arguments: {area_args_found}")
1084 +
1085 + # Verify content before and after tool calls
1086 + assert 'I\'ll help you with the weather analysis.' in reconstructed_content, "Initial content should be preserved"
1087 + assert 'Here are the results.' in reconstructed_content, "Final content should be preserved"
1088 +
1089 + # Verify that <tool_calls> and </tool_calls> tags are not included in the final content
1090 + # (they should be filtered out when not inside thinking tags)
1091 + content_outside_thinking = reconstructed_content
1092 + # Remove thinking tag content to check content outside
1093 + if '<think>' in content_outside_thinking and '</think>' in content_outside_thinking:
1094 + start_think = content_outside_thinking.find('<think>')
1095 + end_think = content_outside_thinking.find('</think>') + len('</think>')
1096 + content_outside_thinking = content_outside_thinking[:
1097 + start_think] + content_outside_thinking[
1098 + end_think:]
1099 +
1100 + # Outside thinking tags, tool_calls tags should be filtered
1101 + tool_calls_in_content = content_outside_thinking.count('<tool_calls>')
1102 + assert tool_calls_in_content == 0, f"<tool_calls> tags should be filtered from content outside thinking tags, but found {tool_calls_in_content}"
1103 +
1104 + print(
1105 + "\n=== Character-by-character streaming test completed successfully ==="
1106 + )
1107 + print("✓ Tool calls inside thinking tags correctly ignored")
1108 + print("✓ Tool calls outside thinking tags correctly detected")
1109 + print("✓ Content properly streamed and reconstructed")
1110 + print("✓ Tool call tags properly filtered from content")
1111 + print("✓ Character-level streaming works correctly")
1112 +
1113 +
1114 +def test_streaming_character_by_character_simple_tool_call(
1115 + minimax_tool_parser):
1116 + """Test character-by-character streaming for a simple tool call scenario."""
1117 + # Reset streaming state
1118 + reset_streaming_state(minimax_tool_parser)
1119 +
1120 + # Simple tool call text
1121 + simple_text = 'Let me check the weather. <tool_calls>\n{"name": "get_weather", "arguments": {"city": "NYC"}}\n</tool_calls>'
1122 +
1123 + print("\n=== Simple character-by-character test ===")
1124 + print(f"Text: {repr(simple_text)}")
1125 +
1126 + content_parts = []
1127 + tool_name_sent = False
1128 + tool_args_sent = False
1129 +
1130 + for i in range(1, len(simple_text) + 1):
1131 + current_text = simple_text[:i]
1132 + previous_text = simple_text[:i - 1] if i > 1 else ""
1133 + delta_text = simple_text[i - 1:i]
1134 +
1135 + result = minimax_tool_parser.extract_tool_calls_streaming(
1136 + previous_text=previous_text,
1137 + current_text=current_text,
1138 + delta_text=delta_text,
1139 + previous_token_ids=[],
1140 + current_token_ids=[],
1141 + delta_token_ids=[],
1142 + request=None,
1143 + )
1144 +
1145 + if result:
1146 + if hasattr(result, 'content') and result.content:
1147 + content_parts.append(result.content)
1148 + print(
1149 + f" Char {i} ({repr(delta_text)}): Content: {repr(result.content)}"
1150 + )
1151 +
1152 + if hasattr(result, 'tool_calls') and result.tool_calls:
1153 + for tool_call in result.tool_calls:
1154 + if tool_call.function and tool_call.function.name:
1155 + tool_name_sent = True
1156 + print(
1157 + f" Char {i}: Tool name: {tool_call.function.name}"
1158 + )
1159 + if tool_call.function and tool_call.function.arguments:
1160 + tool_args_sent = True
1161 + print(
1162 + f" Char {i}: Tool args: {repr(tool_call.function.arguments)}"
1163 + )
1164 +
1165 + # Verify basic expectations
1166 + reconstructed_content = ''.join(content_parts)
1167 + print(f"Final reconstructed content: {repr(reconstructed_content)}")
1168 +
1169 + assert tool_name_sent, "Tool name should be sent during streaming"
1170 + assert tool_args_sent, "Tool arguments should be sent during streaming"
1171 + assert "Let me check the weather." in reconstructed_content, "Initial content should be preserved"
1172 +
1173 + print("✓ Simple character-by-character test passed")
1174 +
1175 +
1176 +def test_streaming_character_by_character_with_buffering(minimax_tool_parser):
1177 + """Test character-by-character streaming with edge cases that trigger buffering."""
1178 + # Reset streaming state
1179 + reset_streaming_state(minimax_tool_parser)
1180 +
1181 + # Text that includes potential buffering scenarios
1182 + buffering_text = 'Hello world<tool_calls>\n{"name": "test"}\n</tool_calls>done'
1183 +
1184 + print("\n=== Buffering character-by-character test ===")
1185 + print(f"Text: {repr(buffering_text)}")
1186 +
1187 + all_content = []
1188 +
1189 + for i in range(1, len(buffering_text) + 1):
1190 + current_text = buffering_text[:i]
1191 + previous_text = buffering_text[:i - 1] if i > 1 else ""
1192 + delta_text = buffering_text[i - 1:i]
1193 +
1194 + result = minimax_tool_parser.extract_tool_calls_streaming(
1195 + previous_text=previous_text,
1196 + current_text=current_text,
1197 + delta_text=delta_text,
1198 + previous_token_ids=[],
1199 + current_token_ids=[],
1200 + delta_token_ids=[],
1201 + request=None,
1202 + )
1203 +
1204 + if result and hasattr(result, 'content') and result.content:
1205 + all_content.append(result.content)
1206 + print(f" Char {i} ({repr(delta_text)}): {repr(result.content)}")
1207 +
1208 + final_content = ''.join(all_content)
1209 + print(f"Final content: {repr(final_content)}")
1210 +
1211 + # The parser should handle the edge case where </tool_calls> appears before <tool_calls>
1212 + assert "Hello" in final_content, "Initial 'Hello' should be preserved"
1213 + assert "world" in final_content, "Content after false closing tag should be preserved"
1214 + assert "done" in final_content, "Final content should be preserved"
1215 +
1216 + print("✓ Buffering character-by-character test passed")
vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py
+649 -203
@@ -3,11 +3,9 @@
3
4 import json
5 from collections.abc import Sequence
6 -from typing import Union
6 +from typing import Any, Optional, Union
7
8 -import partial_json_parser
8 import regex as re
10 -from partial_json_parser.core.options import Allow
9
10 from vllm.entrypoints.chat_utils import random_tool_call_id
11 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -17,6 +15,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
15 FunctionCall, ToolCall)
16 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
17 ToolParser, ToolParserManager)
18 +from vllm.entrypoints.openai.tool_parsers.utils import (
19 + extract_intermediate_diff)
20 from vllm.logger import init_logger
21 from vllm.transformers_utils.tokenizer import AnyTokenizer
22
@@ -29,25 +29,32 @@ class MinimaxToolParser(ToolParser):
29 def __init__(self, tokenizer: AnyTokenizer):
30 super().__init__(tokenizer)
31
32 - self.current_tool_name_sent: bool = False
33 - self.prev_tool_call_arr: list[dict] = []
34 - self.current_tool_id: int = -1
35 - self.streamed_args_for_tool: list[str] = []
36 -
37 - self.tool_call_start_token: str = "<tool_calls>"
38 - self.tool_call_end_token: str = "</tool_calls>"
32 + # Initialize streaming state for tracking tool call progress
33 + self.streaming_state: dict[str, Any] = {
34 + "current_tool_index": -1, # Index of current tool being processed
35 + "tool_ids": [], # List of tool call IDs
36 + "sent_tools": [], # List of tools that have been sent
37 + }
38
39 + # Define tool call tokens and patterns
40 + self.tool_call_start_token = "<tool_calls>"
41 + self.tool_call_end_token = "</tool_calls>"
42 self.tool_call_regex = re.compile(
43 r"<tool_calls>(.*?)</tool_calls>|<tool_calls>(.*)", re.DOTALL)
42 -
43 - # Add regex pattern for thinking tag
44 self.thinking_tag_pattern = r"<think>(.*?)</think>"
45 + self.tool_name_pattern = re.compile(r'"name":\s*"([^"]+)"')
46 + self.tool_args_pattern = re.compile(r'"arguments":\s*')
47 +
48 + # Buffer for handling partial tool calls during streaming
49 + self.pending_buffer = ""
50 + self.in_thinking_tag = False
51
52 if not self.model_tokenizer:
53 raise ValueError(
54 "The model tokenizer must be passed to the ToolParser "
55 "constructor during construction.")
56
57 + # Get token IDs for tool call start/end tokens
58 self.tool_call_start_token_id = self.vocab.get(
59 self.tool_call_start_token)
60 self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
@@ -60,33 +67,95 @@ class MinimaxToolParser(ToolParser):
67
68 def preprocess_model_output(self, model_output: str) -> str:
69 """
63 - Remove tool calls from within thinking tags to avoid processing them.
70 + Preprocess model output by removing tool calls from thinking tags.
71 +
72 + Args:
73 + model_output: Raw model output string
74 +
75 + Returns:
76 + Preprocessed model output with tool calls removed from thinking tags
77 """
78
79 def remove_tool_calls_from_think(match):
80 think_content = match.group(1)
68 - # Remove tool_calls from within the think tag
81 cleaned_content = re.sub(r"<tool_calls>.*?</tool_calls>",
82 "",
83 think_content,
84 flags=re.DOTALL)
85 return f"<think>{cleaned_content}</think>"
86
75 - # Process thinking tags and remove tool_calls from within them
76 - processed_output = re.sub(self.thinking_tag_pattern,
77 - remove_tool_calls_from_think,
78 - model_output,
79 - flags=re.DOTALL)
87 + return re.sub(self.thinking_tag_pattern,
88 + remove_tool_calls_from_think,
89 + model_output,
90 + flags=re.DOTALL)
91
81 - return processed_output
92 + def _clean_duplicate_braces(self, args_text: str) -> str:
93 + """
94 + Clean duplicate closing braces from arguments text.
95 +
96 + Args:
97 + args_text: Raw arguments text
98 +
99 + Returns:
100 + Cleaned arguments text with proper JSON formatting
101 + """
102 + args_text = args_text.strip()
103 + if not args_text:
104 + return args_text
105 +
106 + try:
107 + json.loads(args_text)
108 + return args_text
109 + except json.JSONDecodeError:
110 + pass
111 +
112 + while args_text.endswith('}}'):
113 + candidate = args_text[:-1]
114 + try:
115 + json.loads(candidate)
116 + return candidate
117 + except json.JSONDecodeError:
118 + args_text = candidate
119 +
120 + return args_text
121 +
122 + def _clean_delta_braces(self, delta_text: str) -> str:
123 + """
124 + Clean delta text by removing excessive closing braces.
125 +
126 + Args:
127 + delta_text: Delta text to clean
128 +
129 + Returns:
130 + Cleaned delta text
131 + """
132 + if not delta_text:
133 + return delta_text
134 +
135 + delta_stripped = delta_text.strip()
136 +
137 + if delta_stripped and all(c in '}\n\r\t ' for c in delta_stripped):
138 + brace_count = delta_stripped.count('}')
139 + if brace_count > 1:
140 + return '}\n' if delta_text.endswith('\n') else '}'
141 +
142 + return delta_text
143
144 def extract_tool_calls(
145 self,
146 model_output: str,
147 request: ChatCompletionRequest,
148 ) -> ExtractedToolCallInformation:
88 -
89 - # Preprocess to remove tool calls from thinking tags
149 + """
150 + Extract tool calls from model output for non-streaming mode.
151 +
152 + Args:
153 + model_output: Complete model output
154 + request: Chat completion request
155 +
156 + Returns:
157 + ExtractedToolCallInformation containing tool calls and content
158 + """
159 processed_output = self.preprocess_model_output(model_output)
160
161 if self.tool_call_start_token not in processed_output:
@@ -95,8 +164,8 @@ class MinimaxToolParser(ToolParser):
164 content=model_output)
165
166 try:
98 - function_call_tuples = (
99 - self.tool_call_regex.findall(processed_output))
167 + function_call_tuples = self.tool_call_regex.findall(
168 + processed_output)
169
170 raw_function_calls = []
171 for match in function_call_tuples:
@@ -124,21 +193,15 @@ class MinimaxToolParser(ToolParser):
193 function_call["arguments"],
194 ensure_ascii=False))))
195
127 - # Extract content before the first valid tool call
128 - # Find the position in processed output, then map back to original
196 processed_pos = processed_output.find(self.tool_call_start_token)
197 if processed_pos != -1:
131 - # Get the content before tool calls in processed output
198 processed_content = processed_output[:processed_pos].strip()
199
200 if processed_content:
135 - # Find the end of this content in the original output
136 - # Look for the last non-empty line of processed content
201 lines = processed_content.split('\n')
202 for line in reversed(lines):
203 line = line.strip()
204 if line:
141 - # Find this line in original output
205 pos = model_output.find(line)
206 if pos != -1:
207 content = model_output[:pos + len(line)]
@@ -162,6 +225,445 @@ class MinimaxToolParser(ToolParser):
225 tool_calls=[],
226 content=model_output)
227
228 + def _update_thinking_state(self, text: str) -> None:
229 + """
230 + Update the thinking tag state based on text content.
231 +
232 + Args:
233 + text: Text to analyze for thinking tags
234 + """
235 + open_count = text.count("<think>")
236 + close_count = text.count("</think>")
237 + self.in_thinking_tag = open_count > close_count or (
238 + open_count == close_count and text.endswith("</think>"))
239 +
240 + def _is_potential_tag_start(self, text: str) -> bool:
241 + """
242 + Check if text might be the start of a tool call tag.
243 +
244 + Args:
245 + text: Text to check
246 +
247 + Returns:
248 + True if text could be the start of a tool call tag
249 + """
250 + for tag in [self.tool_call_start_token, self.tool_call_end_token]:
251 + if any(
252 + tag.startswith(text[-i:])
253 + for i in range(1, min(len(text) + 1, len(tag)))):
254 + return True
255 + return False
256 +
257 + def _should_buffer_content(self, delta_text: str) -> bool:
258 + """
259 + Determine if content should be buffered for later processing.
260 +
261 + Args:
262 + delta_text: Delta text to check
263 +
264 + Returns:
265 + True if content should be buffered
266 + """
267 + if self.in_thinking_tag:
268 + return False
269 + return bool(self.pending_buffer
270 + or self.tool_call_start_token in delta_text
271 + or self.tool_call_end_token in delta_text
272 + or delta_text.startswith('<'))
273 +
274 + def _split_content_for_buffering(self, delta_text: str) -> tuple[str, str]:
275 + """
276 + Split delta text into safe content and potential tag content.
277 +
278 + Args:
279 + delta_text: Delta text to split
280 +
281 + Returns:
282 + Tuple of (safe_content, potential_tag_content)
283 + """
284 + if self.in_thinking_tag:
285 + return delta_text, ""
286 +
287 + for tag in [self.tool_call_start_token, self.tool_call_end_token]:
288 + for i in range(1, len(tag)):
289 + tag_prefix = tag[:i]
290 + pos = delta_text.rfind(tag_prefix)
291 + if pos != -1 and tag.startswith(delta_text[pos:]):
292 + return delta_text[:pos], delta_text[pos:]
293 + return delta_text, ""
294 +
295 + def _process_buffer(self, new_content: str) -> str:
296 + """
297 + Process buffered content and return output content.
298 +
299 + Args:
300 + new_content: New content to add to buffer
301 +
302 + Returns:
303 + Processed output content
304 + """
305 + self.pending_buffer += new_content
306 + output_content = ""
307 +
308 + if self.in_thinking_tag:
309 + output_content = self.pending_buffer
310 + self.pending_buffer = ""
311 + return output_content
312 +
313 + while self.pending_buffer:
314 + start_pos = self.pending_buffer.find(self.tool_call_start_token)
315 + end_pos = self.pending_buffer.find(self.tool_call_end_token)
316 +
317 + if start_pos != -1 and (end_pos == -1 or start_pos < end_pos):
318 + tag_pos, tag_len = start_pos, len(self.tool_call_start_token)
319 + elif end_pos != -1:
320 + tag_pos, tag_len = end_pos, len(self.tool_call_end_token)
321 + else:
322 + if self._is_potential_tag_start(self.pending_buffer):
323 + break
324 + output_content += self.pending_buffer
325 + self.pending_buffer = ""
326 + break
327 +
328 + output_content += self.pending_buffer[:tag_pos]
329 + self.pending_buffer = self.pending_buffer[tag_pos + tag_len:]
330 +
331 + return output_content
332 +
333 + def _reset_streaming_state(self) -> None:
334 + """Reset the streaming state to initial values."""
335 + self.streaming_state = {
336 + "current_tool_index": -1,
337 + "tool_ids": [],
338 + "sent_tools": [],
339 + }
340 +
341 + def _advance_to_next_tool(self) -> None:
342 + """Advance to the next tool in the streaming sequence."""
343 + self.streaming_state["current_tool_index"] = int(
344 + self.streaming_state["current_tool_index"]) + 1
345 +
346 + def _set_current_tool_index(self, index: int) -> None:
347 + """
348 + Set the current tool index.
349 +
350 + Args:
351 + index: Tool index to set
352 + """
353 + self.streaming_state["current_tool_index"] = index
354 +
355 + def _get_current_tool_index(self) -> int:
356 + """
357 + Get the current tool index.
358 +
359 + Returns:
360 + Current tool index
361 + """
362 + return int(self.streaming_state["current_tool_index"])
363 +
364 + def _get_next_unsent_tool_index(self, tool_count: int) -> int:
365 + """
366 + Get the index of the next unsent tool.
367 +
368 + Args:
369 + tool_count: Total number of tools
370 +
371 + Returns:
372 + Index of next unsent tool, or -1 if all tools sent
373 + """
374 + sent_tools = list(self.streaming_state["sent_tools"])
375 + for i in range(tool_count):
376 + if i < len(sent_tools):
377 + if not sent_tools[i]["sent_name"]:
378 + return i
379 + else:
380 + return i
381 + return -1
382 +
383 + def _ensure_state_arrays(self, tool_count: int) -> None:
384 + """
385 + Ensure state arrays have sufficient capacity for tool_count tools.
386 +
387 + Args:
388 + tool_count: Number of tools to prepare for
389 + """
390 + sent_tools = list(self.streaming_state["sent_tools"])
391 + tool_ids = list(self.streaming_state["tool_ids"])
392 +
393 + while len(sent_tools) < tool_count:
394 + sent_tools.append({
395 + "sent_name": False,
396 + "sent_arguments": "",
397 + "id": random_tool_call_id(),
398 + })
399 +
400 + while len(tool_ids) < tool_count:
401 + tool_ids.append(None)
402 +
403 + self.streaming_state["sent_tools"] = sent_tools
404 + self.streaming_state["tool_ids"] = tool_ids
405 +
406 + def _detect_tools_in_text(self, text: str) -> int:
407 + """
408 + Detect the number of tools in text by counting name patterns.
409 +
410 + Args:
411 + text: Text to analyze
412 +
413 + Returns:
414 + Number of tools detected
415 + """
416 + matches = self.tool_name_pattern.findall(text)
417 + return len(matches)
418 +
419 + def _find_tool_boundaries(self, text: str) -> list[tuple[int, int]]:
420 + """
421 + Find the boundaries of tool calls in text.
422 +
423 + Args:
424 + text: Text to analyze
425 +
426 + Returns:
427 + List of (start, end) positions for tool calls
428 + """
429 + boundaries = []
430 + i = 0
431 + while i < len(text):
432 + if text[i] == '{':
433 + start = i
434 + depth = 0
435 + has_name = False
436 + has_arguments = False
437 +
438 + while i < len(text):
439 + if text[i] == '{':
440 + depth += 1
441 + elif text[i] == '}':
442 + depth -= 1
443 + if depth == 0:
444 + end = i + 1
445 + segment = text[start:end]
446 + if '"name"' in segment and '"arguments"' in segment:
447 + boundaries.append((start, end))
448 + break
449 +
450 + if not has_name and '"name"' in text[start:i + 1]:
451 + has_name = True
452 + if not has_arguments and '"arguments"' in text[start:i +
453 + 1]:
454 + has_arguments = True
455 +
456 + i += 1
457 +
458 + if depth > 0 and has_name:
459 + boundaries.append((start, i))
460 + else:
461 + i += 1
462 + return boundaries
463 +
464 + def _extract_tool_args(self, tool_content: str, args_match) -> str:
465 + """
466 + Extract tool arguments from tool content.
467 +
468 + Args:
469 + tool_content: Tool call content
470 + args_match: Regex match for arguments pattern
471 +
472 + Returns:
473 + Extracted arguments as string
474 + """
475 + args_start_pos = args_match.end()
476 + remaining_content = tool_content[args_start_pos:]
477 +
478 + if remaining_content.strip().startswith('{'):
479 + depth = 0
480 + for i, char in enumerate(remaining_content):
481 + if char == '{':
482 + depth += 1
483 + elif char == '}':
484 + depth -= 1
485 + if depth == 0:
486 + return remaining_content[:i + 1]
487 + else:
488 + args_end = remaining_content.find('}')
489 + if args_end > 0:
490 + return remaining_content[:args_end].strip()
491 +
492 + return remaining_content.rstrip('}').strip()
493 +
494 + def _get_current_tool_content(
495 + self, text: str,
496 + tool_index: int) -> tuple[Optional[str], Optional[str]]:
497 + """
498 + Get the content of a specific tool by index.
499 +
500 + Args:
501 + text: Text containing tool calls
502 + tool_index: Index of tool to extract
503 +
504 + Returns:
505 + Tuple of (tool_name, tool_arguments) or (None, None) if not found
506 + """
507 + boundaries = self._find_tool_boundaries(text)
508 +
509 + if tool_index >= len(boundaries):
510 + return None, None
511 +
512 + start, end = boundaries[tool_index]
513 + tool_content = text[start:end]
514 +
515 + name_match = self.tool_name_pattern.search(tool_content)
516 + name = name_match.group(1) if name_match else None
517 +
518 + args_match = self.tool_args_pattern.search(tool_content)
519 + if args_match:
520 + try:
521 + args_text = self._extract_tool_args(tool_content, args_match)
522 + return name, args_text
523 + except Exception:
524 + remaining_content = tool_content[args_match.end():]
525 + args_text = remaining_content.rstrip('}').strip()
526 + return name, args_text
527 +
528 + return name, None
529 +
530 + def _handle_tool_name_streaming(
531 + self, tool_content: str,
532 + tool_count: int) -> Union[DeltaMessage, None]:
533 + """
534 + Handle streaming of tool names.
535 +
536 + Args:
537 + tool_content: Content containing tool calls
538 + tool_count: Total number of tools
539 +
540 + Returns:
541 + DeltaMessage with tool name or None if no tool to stream
542 + """
543 + next_idx = self._get_next_unsent_tool_index(tool_count)
544 +
545 + if next_idx == -1:
546 + return None
547 +
548 + boundaries = self._find_tool_boundaries(tool_content)
549 + if next_idx >= len(boundaries):
550 + return None
551 +
552 + tool_name, _ = self._get_current_tool_content(tool_content, next_idx)
553 + if not tool_name:
554 + return None
555 +
556 + self._set_current_tool_index(next_idx)
557 + sent_tools = list(self.streaming_state["sent_tools"])
558 + tool_ids = list(self.streaming_state["tool_ids"])
559 +
560 + tool_id = sent_tools[next_idx]["id"]
561 + tool_ids[next_idx] = tool_id
562 + sent_tools[next_idx]["sent_name"] = True
563 +
564 + self.streaming_state["sent_tools"] = sent_tools
565 + self.streaming_state["tool_ids"] = tool_ids
566 +
567 + return DeltaMessage(tool_calls=[
568 + DeltaToolCall(index=next_idx,
569 + type="function",
570 + id=tool_id,
571 + function=DeltaFunctionCall(
572 + name=tool_name).model_dump(exclude_none=True))
573 + ])
574 +
575 + def _handle_tool_args_streaming(
576 + self, tool_content: str,
577 + tool_count: int) -> Union[DeltaMessage, None]:
578 + """
579 + Handle streaming of tool arguments.
580 +
581 + Args:
582 + tool_content: Content containing tool calls
583 + tool_count: Total number of tools
584 +
585 + Returns:
586 + DeltaMessage with tool arguments or None if no arguments to stream
587 + """
588 + current_idx = self._get_current_tool_index()
589 +
590 + if current_idx < 0 or current_idx >= tool_count:
591 + return None
592 +
593 + tool_name, tool_args = self._get_current_tool_content(
594 + tool_content, current_idx)
595 + if not tool_name or tool_args is None:
596 + return None
597 +
598 + sent_tools = list(self.streaming_state["sent_tools"])
599 +
600 + if not sent_tools[current_idx]["sent_name"]:
601 + return None
602 +
603 + clean_args = self._clean_duplicate_braces(tool_args)
604 + sent_args = sent_tools[current_idx]["sent_arguments"]
605 +
606 + if clean_args != sent_args:
607 + if sent_args and clean_args.startswith(sent_args):
608 + args_delta = extract_intermediate_diff(clean_args, sent_args)
609 + if args_delta:
610 + args_delta = self._clean_delta_braces(args_delta)
611 + sent_tools[current_idx]["sent_arguments"] = clean_args
612 + self.streaming_state["sent_tools"] = sent_tools
613 +
614 + if clean_args.endswith('}'):
615 + self._advance_to_next_tool()
616 +
617 + return DeltaMessage(tool_calls=[
618 + DeltaToolCall(index=current_idx,
619 + function=DeltaFunctionCall(
620 + arguments=args_delta).model_dump(
621 + exclude_none=True))
622 + ])
623 + elif not sent_args and clean_args:
624 + clean_args_delta = self._clean_delta_braces(clean_args)
625 + sent_tools[current_idx]["sent_arguments"] = clean_args
626 + self.streaming_state["sent_tools"] = sent_tools
627 +
628 + if clean_args.endswith('}'):
629 + self._advance_to_next_tool()
630 +
631 + return DeltaMessage(tool_calls=[
632 + DeltaToolCall(index=current_idx,
633 + function=DeltaFunctionCall(
634 + arguments=clean_args_delta).model_dump(
635 + exclude_none=True))
636 + ])
637 +
638 + return None
639 +
640 + def _is_end_tool_calls(self, current_text: str) -> bool:
641 + if self.tool_call_end_token not in current_text:
642 + return False
643 +
644 + end_token_positions = []
645 + search_start = 0
646 + while True:
647 + pos = current_text.find(self.tool_call_end_token, search_start)
648 + if pos == -1:
649 + break
650 + end_token_positions.append(pos)
651 + search_start = pos + 1
652 +
653 + think_regions = []
654 + for match in re.finditer(self.thinking_tag_pattern,
655 + current_text,
656 + flags=re.DOTALL):
657 + think_regions.append((match.start(), match.end()))
658 +
659 + for pos in end_token_positions:
660 + in_think = any(pos >= t_start and pos < t_end
661 + for t_start, t_end in think_regions)
662 + if not in_think:
663 + return True
664 +
665 + return False
666 +
667 def extract_tool_calls_streaming(
668 self,
669 previous_text: str,
@@ -172,13 +674,37 @@ class MinimaxToolParser(ToolParser):
674 delta_token_ids: Sequence[int],
675 request: ChatCompletionRequest,
676 ) -> Union[DeltaMessage, None]:
175 - logger.debug("delta_text: %s", delta_text)
176 - logger.debug("delta_token_ids: %s", delta_token_ids)
677 + self._update_thinking_state(current_text)
678 +
679 + if self.in_thinking_tag:
680 + return DeltaMessage(content=delta_text)
681 +
682 + if self._should_buffer_content(delta_text):
683 + buffered_output = self._process_buffer(delta_text)
684 + return DeltaMessage(
685 + content=buffered_output) if buffered_output else None
686 +
687 + if self._is_end_tool_calls(current_text):
688 + return DeltaMessage(content=delta_text)
689 +
690 + safe_content, potential_tag = self._split_content_for_buffering(
691 + delta_text)
692 + if potential_tag:
693 + self.pending_buffer += potential_tag
694 + return DeltaMessage(content=safe_content) if safe_content else None
695
178 - # Preprocess to remove tool calls from thinking tags
696 processed_current_text = self.preprocess_model_output(current_text)
697
698 if self.tool_call_start_token not in processed_current_text:
699 + if (self.tool_call_end_token in delta_text
700 + and self.tool_call_start_token in current_text):
701 + return None
702 + if delta_text.strip(
703 + ) == '' and self.tool_call_start_token in current_text:
704 + return None
705 + if (self._get_current_tool_index() != -1
706 + and self.tool_call_end_token in current_text):
707 + self._reset_streaming_state()
708 return DeltaMessage(content=delta_text)
709
710 if (self.tool_call_start_token_id is not None
@@ -186,184 +712,104 @@ class MinimaxToolParser(ToolParser):
712 and len(delta_token_ids) == 1):
713 return None
714
189 - original_tool_call_start_pos = current_text.find(
190 - self.tool_call_start_token)
191 - if original_tool_call_start_pos > 0:
192 - delta_start_pos = len(current_text) - len(delta_text)
193 - if delta_start_pos < original_tool_call_start_pos:
194 - content_part = delta_text
195 - if delta_start_pos + len(
196 - delta_text) > original_tool_call_start_pos:
197 - content_part = delta_text[:original_tool_call_start_pos -
198 - delta_start_pos]
199 - if content_part:
200 - return DeltaMessage(content=content_part)
715 + original_tool_start = self._find_tool_start_outside_thinking(
716 + current_text)
717 + if original_tool_start is None:
718 + return None
719
202 - flags = Allow.ALL if self.current_tool_name_sent \
203 - else Allow.ALL & ~Allow.STR
720 + content_before_tools = self._extract_content_before_tools(
721 + current_text, delta_text, original_tool_start)
722 + if content_before_tools:
723 + return DeltaMessage(content=content_before_tools)
724
725 try:
206 - parsable_content = processed_current_text.split(
207 - self.tool_call_start_token)[-1].split(
208 - self.tool_call_end_token)[0]
209 -
210 - tool_call_arr = []
211 - if parsable_content.strip():
212 - lines = parsable_content.strip().split('\n')
213 - for line in lines:
214 - line = line.strip()
215 - if line and (line.startswith('{') or '"name"' in line):
216 - try:
217 - if line.endswith('}'):
218 - parsed_call = json.loads(line)
219 - tool_call_arr.append(parsed_call)
220 - else:
221 - parsed_call = partial_json_parser.loads(
222 - line, flags)
223 - if parsed_call and isinstance(
224 - parsed_call, dict):
225 - tool_call_arr.append(parsed_call)
226 - except (json.JSONDecodeError, partial_json_parser.core.
227 - exceptions.MalformedJSON):
228 - continue
229 -
230 - current_tool_call: dict = tool_call_arr[self.current_tool_id] \
231 - if len(tool_call_arr) > self.current_tool_id >= 0 else {}
232 -
233 - if len(tool_call_arr) == 0:
726 + tool_content = self._extract_tool_content(current_text,
727 + original_tool_start)
728 + current_tools_count = self._detect_tools_in_text(tool_content)
729 +
730 + if current_tools_count == 0:
731 return None
732
236 - # Starting a new tool in the array
237 - elif (len(tool_call_arr) > 0
238 - and len(tool_call_arr) > self.current_tool_id + 1):
239 -
240 - # Handle any missed arguments from previous tool
241 - if self.current_tool_id >= 0 and self.current_tool_id < len(
242 - self.prev_tool_call_arr):
243 - prev_tool_call = self.prev_tool_call_arr[
244 - self.current_tool_id]
245 - diff_arguments = prev_tool_call.get("arguments")
246 -
247 - if diff_arguments:
248 - diff_arguments_json = json.dumps(diff_arguments,
249 - ensure_ascii=False)
250 - already_streamed = self.streamed_args_for_tool[
251 - self.
252 - current_tool_id] if self.current_tool_id < len(
253 - self.streamed_args_for_tool) else ""
254 -
255 - if diff_arguments_json != already_streamed:
256 - diff = diff_arguments_json[len(already_streamed):]
257 - delta = DeltaMessage(tool_calls=[
258 - DeltaToolCall(index=self.current_tool_id,
259 - function=DeltaFunctionCall(
260 - arguments=diff).model_dump(
261 - exclude_none=True))
262 - ])
263 - if self.current_tool_id < len(
264 - self.streamed_args_for_tool):
265 - self.streamed_args_for_tool[
266 - self.current_tool_id] = diff_arguments_json
267 - else:
268 - delta = None
269 - else:
270 - delta = None
271 - else:
272 - delta = None
273 -
274 - self.current_tool_id = len(tool_call_arr) - 1
275 - self.current_tool_name_sent = False
276 - self.streamed_args_for_tool.append("")
277 - logger.debug("starting on new tool %d", self.current_tool_id)
278 - return delta
279 -
280 - # Send tool name if not sent yet
281 - if not self.current_tool_name_sent:
282 - function_name = current_tool_call.get("name")
283 - if function_name:
284 - delta = DeltaMessage(tool_calls=[
285 - DeltaToolCall(index=self.current_tool_id,
286 - type="function",
287 - id=random_tool_call_id(),
288 - function=DeltaFunctionCall(
289 - name=function_name).model_dump(
290 - exclude_none=True))
291 - ])
292 - self.current_tool_name_sent = True
293 - else:
294 - delta = None
733 + if self._get_current_tool_index() == -1:
734 + self._reset_streaming_state()
735
296 - # Stream arguments
297 - else:
298 - prev_arguments = None
299 - if (self.current_tool_id < len(self.prev_tool_call_arr)
300 - and self.prev_tool_call_arr[self.current_tool_id]):
301 - prev_arguments = self.prev_tool_call_arr[
302 - self.current_tool_id].get("arguments")
303 -
304 - cur_arguments = current_tool_call.get("arguments")
305 -
306 - if not cur_arguments and not prev_arguments:
307 - delta = None
308 - elif not cur_arguments and prev_arguments:
309 - logger.error(
310 - "Arguments reset mid-call, skipping streaming")
311 - delta = None
312 - elif cur_arguments and not prev_arguments:
313 - cur_arguments_json = json.dumps(cur_arguments,
314 - ensure_ascii=False)
315 - logger.debug("First tokens in arguments received: %s",
316 - cur_arguments_json)
317 -
318 - delta = DeltaMessage(tool_calls=[
319 - DeltaToolCall(index=self.current_tool_id,
320 - function=DeltaFunctionCall(
321 - arguments=cur_arguments_json).
322 - model_dump(exclude_none=True))
323 - ])
324 - self.streamed_args_for_tool[
325 - self.current_tool_id] = cur_arguments_json
326 -
327 - elif cur_arguments and prev_arguments:
328 - cur_args_json = json.dumps(cur_arguments,
329 - ensure_ascii=False)
330 - prev_args_json = json.dumps(prev_arguments,
331 - ensure_ascii=False)
332 -
333 - logger.debug("Searching for diff between \n%s\n%s",
334 - cur_args_json, prev_args_json)
335 -
336 - already_streamed = self.streamed_args_for_tool[
337 - self.current_tool_id] if self.current_tool_id < len(
338 - self.streamed_args_for_tool) else ""
339 -
340 - if cur_args_json.startswith(already_streamed):
341 - argument_diff = cur_args_json[len(already_streamed):]
342 - elif cur_args_json != already_streamed:
343 - argument_diff = cur_args_json
344 - self.streamed_args_for_tool[self.current_tool_id] = ""
345 - else:
346 - argument_diff = ""
347 -
348 - if argument_diff:
349 - logger.debug("got arguments diff: %s", argument_diff)
350 - delta = DeltaMessage(tool_calls=[
351 - DeltaToolCall(index=self.current_tool_id,
352 - function=DeltaFunctionCall(
353 - arguments=argument_diff).
354 - model_dump(exclude_none=True))
355 - ])
356 - self.streamed_args_for_tool[
357 - self.current_tool_id] += argument_diff
358 - else:
359 - delta = None
360 - else:
361 - delta = None
736 + self._ensure_state_arrays(current_tools_count)
737
363 - self.prev_tool_call_arr = tool_call_arr
364 - return delta
738 + return (self._handle_tool_name_streaming(tool_content,
739 + current_tools_count)
740 + or self._handle_tool_args_streaming(
741 + tool_content, current_tools_count))
742
743 except Exception:
367 - logger.exception("An unexpected error occurred",
744 + logger.exception("An unexpected error occurred ",
745 "during streaming tool call handling.")
746 return None
747 +
748 + def _find_tool_start_outside_thinking(self,
749 + current_text: str) -> Optional[int]:
750 + """
751 + Find the start position of tool calls outside of thinking tags.
752 +
753 + Args:
754 + current_text: Current text to search
755 +
756 + Returns:
757 + Position of tool call start or None if not found
758 + """
759 + search_start = 0
760 + while True:
761 + pos = current_text.find(self.tool_call_start_token, search_start)
762 + if pos == -1:
763 + return None
764 +
765 + think_regions = [(m.start(), m.end()) for m in re.finditer(
766 + r"<think>(.*?)</think>", current_text, flags=re.DOTALL)]
767 + in_think = any(pos >= t_start and pos < t_end
768 + for t_start, t_end in think_regions)
769 +
770 + if not in_think:
771 + return pos
772 +
773 + search_start = pos + 1
774 +
775 + def _extract_content_before_tools(self, current_text: str, delta_text: str,
776 + tool_start: int) -> Optional[str]:
777 + """
778 + Extract content that appears before tool calls.
779 +
780 + Args:
781 + current_text: Current text
782 + delta_text: Delta text
783 + tool_start: Start position of tools
784 +
785 + Returns:
786 + Content before tools or None
787 + """
788 + if tool_start > 0:
789 + delta_start_pos = len(current_text) - len(delta_text)
790 + if delta_start_pos < tool_start:
791 + content_part = delta_text
792 + if delta_start_pos + len(delta_text) > tool_start:
793 + content_part = delta_text[:tool_start - delta_start_pos]
794 + return content_part if content_part else None
795 + return None
796 +
797 + def _extract_tool_content(self, current_text: str, tool_start: int) -> str:
798 + """
799 + Extract tool content from current text starting at tool_start.
800 +
801 + Args:
802 + current_text: Current text
803 + tool_start: Start position of tool calls
804 +
805 + Returns:
806 + Extracted tool content
807 + """
808 + tool_content_start = tool_start + len(self.tool_call_start_token)
809 + tool_content = current_text[tool_content_start:]
810 +
811 + end_pos = tool_content.find(self.tool_call_end_token)
812 + if end_pos != -1:
813 + tool_content = tool_content[:end_pos]
814 +
815 + return tool_content