A high-throughput and memory-efficient inference and serving engine for LLMs
[Cohere] Cohere2 moe parser fix (#44907)
Signed-off-by: Terrencezzj <terrence@cohere.ai>
Terrence Zhao committed
Jun 9, 2026 at 09:32 UTC
ee4d7df2b5211448a4f9bb34e807c1dc1a360de1
1 file changed
+33
-12
vllm/tool_parsers/cohere_command_tool_parser.py
+33
-12
@@ -41,6 +41,9 @@ class BaseCohereCommandToolParser(ToolParser):
41
super().__init__(tokenizer)
42
self.melody_streaming = PyFilter(streaming_opts)
43
self.melody_unary = PyFilter(unary_opts)
44
+ # Melody can emit the tool-call id before the function name. Keep it
45
+ # until the first real name delta so clients receive both together.
46
+ self._pending_streaming_tool_call_ids: dict[int, str] = {}
47
48
def adjust_request(
49
self, request: ChatCompletionRequest | ResponsesRequest
@@ -65,19 +68,37 @@ class BaseCohereCommandToolParser(ToolParser):
68
if r.reasoning is not None:
69
return DeltaMessage(reasoning=r.reasoning)
70
if r.tool_calls:
68
- return DeltaMessage(
69
- tool_calls=[
70
- DeltaToolCall(
71
- id=tc.id,
72
- index=tc.index,
73
- type="function",
74
- function=DeltaFunctionCall(
75
- name=tc.name, arguments=tc.arguments
76
- ),
71
+ tool_calls: list[DeltaToolCall] = []
72
+ for tc in r.tool_calls:
73
+ if tc.id:
74
+ self._pending_streaming_tool_call_ids[tc.index] = tc.id
75
+ name = tc.name or None
76
+ arguments = tc.arguments or None
77
+ # Empty strings are placeholders in Melody's streaming output;
78
+ # omit them from OpenAI-compatible deltas instead of sending
79
+ # invalid tool names or empty argument fragments.
80
+ if name is None and arguments is None:
81
+ continue
82
+
83
+ function_kwargs = {}
84
+ if name is not None:
85
+ function_kwargs["name"] = name
86
+ if arguments is not None:
87
+ function_kwargs["arguments"] = arguments
88
+ tool_call_kwargs = {
89
+ "index": tc.index,
90
+ "function": DeltaFunctionCall(**function_kwargs),
91
+ }
92
+ if name is not None:
93
+ tool_call_id = tc.id or self._pending_streaming_tool_call_ids.pop(
94
+ tc.index, None
95
)
78
- for tc in r.tool_calls
79
- ]
80
- )
96
+ if tool_call_id is not None:
97
+ tool_call_kwargs["id"] = tool_call_id
98
+ tool_call_kwargs["type"] = "function"
99
+ tool_calls.append(DeltaToolCall(**tool_call_kwargs))
100
+ if tool_calls:
101
+ return DeltaMessage(tool_calls=tool_calls)
102
return None
103
104
def extract_tool_calls(