@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 5fee0a872d

chore: Upstream Cohere parser fixes + tests (#51998)

Signed-off-by: Jason Ozuzu <jasonozuzu@cohere.com>

JasonCohere committed Aug 13, 2026 at 13:05 UTC 5fee0a872dc31dd3476d61bb72b782b2d9d47492
10 files changed +533 -54
requirements/test/cpu.txt
+1 -1
@@ -126,7 +126,7 @@ cloudpickle==3.1.2
126 # via -r requirements/test/../common.txt
127 cohere==7.0.8
128 # via -r requirements/test/cuda.in
129 -cohere-melody==0.9.0
129 +cohere-melody==0.11.1
130 # via -r requirements/test/cuda.in
131 colorama==0.4.6
132 # via
requirements/test/cuda.in
+1 -1
@@ -71,7 +71,7 @@ gpt-oss >= 0.0.7; python_version > '3.11'
71
72 perceptron # required for isaac test
73 kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
74 -cohere_melody>=0.9.0 # required for cohere command reasoning parser test
74 +cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests
75 cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types)
76
77 # Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
requirements/test/cuda.txt
+1 -1
@@ -131,7 +131,7 @@ cloudpickle==3.1.2
131 # via -r requirements/test/../common.txt
132 cohere==7.0.8
133 # via -r requirements/test/cuda.in
134 -cohere-melody==0.9.0
134 +cohere-melody==0.11.1
135 # via -r requirements/test/cuda.in
136 colorama==0.4.6
137 # via
requirements/test/rocm.in
+1 -1
@@ -69,7 +69,7 @@ gpt-oss>=0.0.7; python_version > '3.11'
69
70 perceptron # required for isaac test
71 kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
72 -cohere_melody>=0.9.0 # required for cohere command reasoning parser test
72 +cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests
73 cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types)
74
75 # Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
requirements/test/rocm.txt
+1 -1
@@ -131,7 +131,7 @@ cloudpickle==3.1.2
131 # tilelang
132 cohere==7.0.8
133 # via -r requirements/test/rocm.in
134 -cohere-melody==0.9.0
134 +cohere-melody==0.11.1
135 # via -r requirements/test/rocm.in
136 colorama==0.4.6
137 # via
tests/reasoning/test_cohere_command_reasoning_parser.py
+3 -38
@@ -5,7 +5,7 @@ from __future__ import annotations
5
6 import json
7 from collections import UserDict
8 -from dataclasses import dataclass, field
8 +from dataclasses import dataclass
9 from types import SimpleNamespace
10 from typing import Any
11
@@ -34,20 +34,12 @@ from vllm.reasoning.cohere_command_reasoning_parser import (
34 from vllm.sampling_params import StructuredOutputsParams
35
36
37 -@dataclass
38 -class ExpectedToolCall:
39 - id: str
40 - name: str
41 - arguments: dict
42 -
43 -
37 @dataclass
38 class ReasoningCase:
39 parser_cls: Any
40 model_output: str
41 expected_reasoning: str | None
42 expected_content: str | None
50 - expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list)
43
44
45 REASONING_CASES = [
@@ -67,9 +59,6 @@ REASONING_CASES = [
59 {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
60 ]
61 <|END_ACTION|>""",
70 - expected_tool_calls=[
71 - ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
72 - ],
62 ),
63 id="cmd3-single_tool_call",
64 ),
@@ -89,9 +78,6 @@ REASONING_CASES = [
78 {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
79 ]
80 <|END_ACTION|>""",
92 - expected_tool_calls=[
93 - ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
94 - ],
81 ),
82 id="cmd4-single_tool_call",
83 ),
@@ -236,29 +222,8 @@ class TestExtractReasoning:
222 assert reasoning == case.expected_reasoning
223
224 content = "".join(content_parts) if content_parts else None
239 - if case.expected_tool_calls:
240 - assert content is None or content == ""
241 - else:
242 - assert content == case.expected_content
243 -
244 - accumulated: dict[int, dict] = {}
245 - for d in tool_call_deltas:
246 - idx = d["index"]
247 - if idx not in accumulated:
248 - accumulated[idx] = {"id": "", "name": "", "arguments": ""}
249 - if d["id"]:
250 - accumulated[idx]["id"] = d["id"]
251 - if d["name"]:
252 - accumulated[idx]["name"] = d["name"]
253 - if d["arguments"]:
254 - accumulated[idx]["arguments"] += d["arguments"]
255 -
256 - assert len(accumulated) == len(case.expected_tool_calls)
257 - for i, expected_tc in enumerate(case.expected_tool_calls):
258 - tc = accumulated[i]
259 - assert tc["id"] == expected_tc.id
260 - assert tc["name"] == expected_tc.name
261 - assert json.loads(tc["arguments"]) == expected_tc.arguments
225 + assert content == case.expected_content
226 + assert tool_call_deltas == []
227
228
229 class TestIsReasoningEnd:
tests/tool_parsers/test_cohere_command_tool_parser.py new
+510
@@ -0,0 +1,510 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from __future__ import annotations
5 +
6 +import json
7 +from dataclasses import dataclass, field
8 +from typing import Any
9 +
10 +import pytest
11 +
12 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
13 +from vllm.tool_parsers.cohere_command_tool_parser import (
14 + CohereCommand3ToolParser,
15 + CohereCommand4ToolParser,
16 +)
17 +
18 +
19 +@dataclass
20 +class ExpectedToolCall:
21 + id: str
22 + name: str
23 + arguments: dict
24 +
25 +
26 +@dataclass
27 +class ToolCallCase:
28 + parser_cls: Any
29 + model_output: str
30 + expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list)
31 + expected_reasoning: str | None = None
32 + expected_content: str | None = None
33 +
34 +
35 +TOOL_CALL_CASES = [
36 + pytest.param(
37 + ToolCallCase(
38 + parser_cls=CohereCommand3ToolParser,
39 + model_output="""\
40 +<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
41 +[
42 + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
43 +]
44 +<|END_ACTION|>""",
45 + expected_tool_calls=[
46 + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
47 + ],
48 + expected_reasoning="i will call foo with query1",
49 + ),
50 + id="cmd3-single_tool_call",
51 + ),
52 + pytest.param(
53 + ToolCallCase(
54 + parser_cls=CohereCommand4ToolParser,
55 + model_output="""\
56 +<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
57 +[
58 + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
59 +]
60 +<|END_ACTION|>""",
61 + expected_tool_calls=[
62 + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
63 + ],
64 + expected_reasoning="i will call foo with query1",
65 + ),
66 + id="cmd4-single_tool_call",
67 + ),
68 + pytest.param(
69 + ToolCallCase(
70 + parser_cls=CohereCommand3ToolParser,
71 + model_output="""\
72 +<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
73 +<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
74 + expected_reasoning="This is a rainbow emoji: 🌈",
75 + expected_content="foo bar",
76 + ),
77 + id="cmd3-citations_no_tool_calls",
78 + ),
79 + pytest.param(
80 + ToolCallCase(
81 + parser_cls=CohereCommand4ToolParser,
82 + model_output="""\
83 +<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
84 +<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
85 + expected_reasoning="This is a rainbow emoji: 🌈",
86 + expected_content="foo bar",
87 + ),
88 + id="cmd4-citations_no_tool_calls",
89 + ),
90 + pytest.param(
91 + ToolCallCase(
92 + parser_cls=CohereCommand3ToolParser,
93 + model_output="""\
94 +<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|>
95 +[
96 + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}},
97 + {"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}}
98 +]
99 +<|END_ACTION|>""",
100 + expected_tool_calls=[
101 + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
102 + ExpectedToolCall(id="1", name="bar", arguments={"x": 42}),
103 + ],
104 + expected_reasoning="first I think about foo",
105 + ),
106 + id="cmd3-multiple_tool_calls",
107 + ),
108 + pytest.param(
109 + ToolCallCase(
110 + parser_cls=CohereCommand4ToolParser,
111 + model_output="""\
112 +<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|>
113 +[
114 + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}},
115 + {"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}}
116 +]
117 +<|END_ACTION|>""",
118 + expected_tool_calls=[
119 + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
120 + ExpectedToolCall(id="1", name="bar", arguments={"x": 42}),
121 + ],
122 + expected_reasoning="first I think about foo",
123 + ),
124 + id="cmd4-multiple_tool_calls",
125 + ),
126 + pytest.param(
127 + ToolCallCase(
128 + parser_cls=CohereCommand3ToolParser,
129 + model_output="""\
130 +<|START_THINKING|>just think, no response<|END_THINKING|>""",
131 + expected_reasoning="just think, no response",
132 + ),
133 + id="cmd3-reasoning_only",
134 + ),
135 + pytest.param(
136 + ToolCallCase(
137 + parser_cls=CohereCommand4ToolParser,
138 + model_output="""\
139 +<|START_THINKING|>just think, no response<|END_THINKING|>""",
140 + expected_reasoning="just think, no response",
141 + ),
142 + id="cmd4-reasoning_only",
143 + ),
144 +]
145 +
146 +
147 +class MockCohereTokenizer:
148 + """Minimal byte-level stand-in for the Cohere tokenizer."""
149 +
150 + def get_vocab(self) -> dict[str, int]:
151 + return {}
152 +
153 + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
154 + return list(text.encode("utf-8"))
155 +
156 + def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
157 + return bytes(ids).decode("utf-8", errors="replace")
158 +
159 +
160 +@pytest.fixture(scope="module")
161 +def tokenizer() -> MockCohereTokenizer:
162 + return MockCohereTokenizer()
163 +
164 +
165 +@pytest.fixture
166 +def request_obj() -> ChatCompletionRequest:
167 + return ChatCompletionRequest(messages=[], model="test-model")
168 +
169 +
170 +REPLACEMENT_CHAR = "\ufffd"
171 +
172 +
173 +def _token_deltas(tokenizer: MockCohereTokenizer, text: str) -> list[str]:
174 + """Decode per-token string deltas, buffering incomplete multi-byte chars."""
175 + ids = tokenizer.encode(text, add_special_tokens=False)
176 + deltas: list[str] = []
177 + prev = ""
178 + for i in range(1, len(ids) + 1):
179 + current = tokenizer.decode(ids[:i], skip_special_tokens=False)
180 + if current.endswith(REPLACEMENT_CHAR):
181 + continue
182 + delta = current[len(prev) :]
183 + if delta:
184 + deltas.append(delta)
185 + prev = current
186 + return deltas
187 +
188 +
189 +@dataclass
190 +class StreamingResult:
191 + tool_calls: dict[int, dict]
192 + reasoning: str | None
193 + content: str | None
194 +
195 +
196 +def _run_streaming_over_deltas(
197 + parser,
198 + deltas: list[str],
199 + request_obj: ChatCompletionRequest,
200 +) -> StreamingResult:
201 + accumulated: dict[int, dict] = {}
202 + reasoning_parts: list[str] = []
203 + content_parts: list[str] = []
204 + previous_text = ""
205 +
206 + for token_str in deltas:
207 + current_text = previous_text + token_str
208 + delta = parser.extract_tool_calls_streaming(
209 + previous_text=previous_text,
210 + current_text=current_text,
211 + delta_text=token_str,
212 + previous_token_ids=[],
213 + current_token_ids=[],
214 + delta_token_ids=[],
215 + request=request_obj,
216 + )
217 + if delta is not None:
218 + if delta.reasoning is not None:
219 + reasoning_parts.append(delta.reasoning)
220 + if delta.content is not None:
221 + content_parts.append(delta.content)
222 + for tc in delta.tool_calls:
223 + idx = tc.index
224 + if idx not in accumulated:
225 + accumulated[idx] = {"id": "", "name": "", "arguments": ""}
226 + if tc.id:
227 + accumulated[idx]["id"] = tc.id
228 + if tc.function and tc.function.name:
229 + accumulated[idx]["name"] = tc.function.name
230 + if tc.function and tc.function.arguments:
231 + accumulated[idx]["arguments"] += tc.function.arguments
232 + previous_text = current_text
233 +
234 + return StreamingResult(
235 + tool_calls=accumulated,
236 + reasoning="".join(reasoning_parts) if reasoning_parts else None,
237 + content="".join(content_parts) if content_parts else None,
238 + )
239 +
240 +
241 +def _run_streaming(
242 + parser,
243 + tokenizer: MockCohereTokenizer,
244 + model_output: str,
245 + request_obj: ChatCompletionRequest,
246 +) -> StreamingResult:
247 + return _run_streaming_over_deltas(
248 + parser,
249 + _token_deltas(tokenizer, model_output),
250 + request_obj,
251 + )
252 +
253 +
254 +@pytest.mark.parametrize("case", TOOL_CALL_CASES)
255 +class TestExtractToolCalls:
256 + def test_streaming(
257 + self,
258 + tokenizer: MockCohereTokenizer,
259 + request_obj: ChatCompletionRequest,
260 + case: ToolCallCase,
261 + ):
262 + parser = case.parser_cls(tokenizer)
263 + streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj)
264 +
265 + assert len(streamed.tool_calls) == len(case.expected_tool_calls)
266 +
267 + for i, expected_tc in enumerate(case.expected_tool_calls):
268 + tc = streamed.tool_calls[i]
269 + assert tc["id"] == expected_tc.id
270 + assert tc["name"] == expected_tc.name
271 + assert json.loads(tc["arguments"]) == expected_tc.arguments
272 +
273 + def test_streaming_reasoning(
274 + self,
275 + tokenizer: MockCohereTokenizer,
276 + request_obj: ChatCompletionRequest,
277 + case: ToolCallCase,
278 + ):
279 + parser = case.parser_cls(tokenizer)
280 + streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj)
281 +
282 + assert streamed.reasoning == case.expected_reasoning
283 +
284 + def test_streaming_content(
285 + self,
286 + tokenizer: MockCohereTokenizer,
287 + request_obj: ChatCompletionRequest,
288 + case: ToolCallCase,
289 + ):
290 + parser = case.parser_cls(tokenizer)
291 + streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj)
292 +
293 + assert streamed.content == case.expected_content
294 +
295 + def test_nonstreaming(
296 + self,
297 + request_obj: ChatCompletionRequest,
298 + tokenizer: MockCohereTokenizer,
299 + case: ToolCallCase,
300 + ):
301 + parser = case.parser_cls(tokenizer)
302 + result = parser.extract_tool_calls(case.model_output, request_obj)
303 +
304 + assert result.tools_called == (len(case.expected_tool_calls) > 0)
305 + assert len(result.tool_calls) == len(case.expected_tool_calls)
306 +
307 + for actual_tc, expected_tc in zip(result.tool_calls, case.expected_tool_calls):
308 + assert actual_tc.type == "function"
309 + assert actual_tc.function.name == expected_tc.name
310 + assert json.loads(actual_tc.function.arguments) == expected_tc.arguments
311 +
312 + def test_streaming_nonstreaming_agree(
313 + self,
314 + tokenizer: MockCohereTokenizer,
315 + request_obj: ChatCompletionRequest,
316 + case: ToolCallCase,
317 + ):
318 + parser_streaming = case.parser_cls(tokenizer)
319 + parser_nonstreaming = case.parser_cls(tokenizer)
320 +
321 + streamed = _run_streaming(
322 + parser_streaming,
323 + tokenizer,
324 + case.model_output,
325 + request_obj,
326 + )
327 + result = parser_nonstreaming.extract_tool_calls(
328 + case.model_output,
329 + request_obj,
330 + )
331 +
332 + assert len(streamed.tool_calls) == len(result.tool_calls)
333 +
334 + for i, actual_tc in enumerate(result.tool_calls):
335 + assert streamed.tool_calls[i]["name"] == actual_tc.function.name
336 + assert json.loads(streamed.tool_calls[i]["arguments"]) == json.loads(
337 + actual_tc.function.arguments
338 + )
339 +
340 +
341 +SPECIAL_TOKEN_MARKERS = (
342 + "<|START_THINKING|>",
343 + "<|END_THINKING|>",
344 + "<|START_RESPONSE|>",
345 + "<|END_RESPONSE|>",
346 + "<|START_ACTION|>",
347 + "<|END_ACTION|>",
348 + "<|START_TEXT|>",
349 + "<|END_TEXT|>",
350 +)
351 +
352 +
353 +def _multi_token_deltas(
354 + tokenizer: MockCohereTokenizer,
355 + text: str,
356 + chunk_size: int,
357 +) -> list[str]:
358 + ids = tokenizer.encode(text, add_special_tokens=False)
359 + deltas: list[str] = []
360 + prev = ""
361 + i = 0
362 + while i < len(ids):
363 + end = min(len(ids), i + chunk_size)
364 + current = tokenizer.decode(ids[:end], skip_special_tokens=False)
365 + i = end
366 + if current.endswith(REPLACEMENT_CHAR):
367 + continue
368 + delta = current[len(prev) :]
369 + if delta:
370 + deltas.append(delta)
371 + prev = current
372 + return deltas
373 +
374 +
375 +class TestSpeculativeDecodingMultiTokenDelta:
376 + MODEL_OUTPUT = (
377 + "<|START_THINKING|> i will call foo with query1<|END_THINKING|>"
378 + "<|START_ACTION|>\n"
379 + '[\n {"tool_call_id": "0", "tool_name": "foo", '
380 + '"parameters": {"query": "query1"}}\n]\n'
381 + "<|END_ACTION|>"
382 + )
383 +
384 + @pytest.mark.parametrize(
385 + "parser_cls",
386 + [CohereCommand3ToolParser, CohereCommand4ToolParser],
387 + ids=["cmd3", "cmd4"],
388 + )
389 + @pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
390 + def test_no_special_token_leak_in_streaming_deltas(
391 + self,
392 + tokenizer: MockCohereTokenizer,
393 + request_obj: ChatCompletionRequest,
394 + parser_cls,
395 + chunk_size: int,
396 + ):
397 + parser = parser_cls(tokenizer)
398 + chunked = _multi_token_deltas(tokenizer, self.MODEL_OUTPUT, chunk_size)
399 +
400 + previous_text = ""
401 + for token_str in chunked:
402 + current_text = previous_text + token_str
403 + delta = parser.extract_tool_calls_streaming(
404 + previous_text=previous_text,
405 + current_text=current_text,
406 + delta_text=token_str,
407 + previous_token_ids=[],
408 + current_token_ids=[],
409 + delta_token_ids=[],
410 + request=request_obj,
411 + )
412 + previous_text = current_text
413 + if delta is None:
414 + continue
415 +
416 + fields: list[tuple[str, str | None]] = [
417 + ("reasoning", delta.reasoning),
418 + ("content", delta.content),
419 + ]
420 + for tc in delta.tool_calls or []:
421 + if tc.function:
422 + fields.append(("tool_call.name", tc.function.name))
423 + fields.append(("tool_call.arguments", tc.function.arguments))
424 +
425 + for marker in SPECIAL_TOKEN_MARKERS:
426 + for field_name, value in fields:
427 + assert value is None or marker not in value, (
428 + f"special token {marker!r} leaked into {field_name} "
429 + f"with chunk_size={chunk_size} delta={delta!r}"
430 + )
431 +
432 + @pytest.mark.parametrize(
433 + "parser_cls",
434 + [CohereCommand3ToolParser, CohereCommand4ToolParser],
435 + ids=["cmd3", "cmd4"],
436 + )
437 + @pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
438 + def test_multi_token_chunks_still_produce_correct_tool_call(
439 + self,
440 + tokenizer: MockCohereTokenizer,
441 + request_obj: ChatCompletionRequest,
442 + parser_cls,
443 + chunk_size: int,
444 + ):
445 + parser = parser_cls(tokenizer)
446 + chunked = _multi_token_deltas(tokenizer, self.MODEL_OUTPUT, chunk_size)
447 + streamed = _run_streaming_over_deltas(parser, chunked, request_obj)
448 +
449 + assert len(streamed.tool_calls) == 1
450 + tc = streamed.tool_calls[0]
451 + assert tc["id"] == "0"
452 + assert tc["name"] == "foo"
453 + assert json.loads(tc["arguments"]) == {"query": "query1"}
454 +
455 +
456 +class TestStreamingDeltaShape:
457 + @pytest.mark.parametrize(
458 + "parser_cls",
459 + [CohereCommand3ToolParser, CohereCommand4ToolParser],
460 + ids=["cmd3", "cmd4"],
461 + )
462 + def test_reasoning_and_tool_calls_are_separate_deltas(
463 + self,
464 + tokenizer: MockCohereTokenizer,
465 + request_obj: ChatCompletionRequest,
466 + parser_cls,
467 + ):
468 + parser = parser_cls(tokenizer)
469 + model_output = (
470 + "<|START_THINKING|> i will call foo with query1<|END_THINKING|>"
471 + "<|START_ACTION|>\n"
472 + '[\n {"tool_call_id": "0", "tool_name": "foo", '
473 + '"parameters": {"query": "query1"}}\n]\n'
474 + "<|END_ACTION|>"
475 + )
476 +
477 + token_strings = _token_deltas(tokenizer, model_output)
478 + previous_text = ""
479 + saw_reasoning = False
480 + saw_tool_call = False
481 +
482 + for token_str in token_strings:
483 + current_text = previous_text + token_str
484 + delta = parser.extract_tool_calls_streaming(
485 + previous_text=previous_text,
486 + current_text=current_text,
487 + delta_text=token_str,
488 + previous_token_ids=[],
489 + current_token_ids=[],
490 + delta_token_ids=[],
491 + request=request_obj,
492 + )
493 + if delta is not None:
494 + populated = [
495 + delta.content is not None,
496 + delta.reasoning is not None,
497 + bool(delta.tool_calls),
498 + ]
499 + assert sum(populated) == 1, (
500 + "A single streaming delta must carry exactly one of "
501 + f"content/reasoning/tool_calls, got {delta!r}"
502 + )
503 + if delta.reasoning is not None:
504 + saw_reasoning = True
505 + if delta.tool_calls:
506 + saw_tool_call = True
507 + previous_text = current_text
508 +
509 + assert saw_reasoning, "expected at least one reasoning delta"
510 + assert saw_tool_call, "expected at least one tool-call delta"
vllm/reasoning/cohere_command_reasoning_parser.py
+9 -5
@@ -16,7 +16,7 @@ except ImportError as e:
16 raise ImportError(
17 "The Cohere reasoning parser requires the `cohere_melody` "
18 "package, which is not installed. Install it with:\n"
19 - " pip install cohere_melody"
19 + " pip install 'cohere-melody>=0.11.1'"
20 ) from e
21
22
@@ -132,7 +132,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
132 tool_dictionary[tool_name] = f"{tool_name} ::= {tool_name}root\n{tool_grammar}"
133 # Emitted grammar shape:
134 # root ::= tools
135 - # tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws
135 + # tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws
136 # ws ::= (" " | "\t" | "\n")*
137 # tool ::= <tool_a> | <tool_b> | ... (one alternative per input)
138 # <tool_x> ::= <tool_x>root (per-tool xgrammar rules)
@@ -140,7 +140,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
140 tool_alternatives = "tool ::= " + " | ".join(tool_dictionary.keys())
141 tool_rules = "\n ".join(tool_dictionary.values())
142 grammar = f"""root ::= tools
143 - tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws
143 + tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws
144 ws ::= (" " | "\\t" | "\\n")*
145 {tool_alternatives}
146 {tool_rules}
@@ -699,7 +699,9 @@ class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
699 super().__init__(
700 tokenizer,
701 *args,
702 - streaming_opts=PyFilterOptions().cmd3().stream_non_grounded_answer(),
702 + streaming_opts=(
703 + PyFilterOptions().cmd3().no_tools().stream_non_grounded_answer()
704 + ),
705 unary_opts=PyFilterOptions().cmd3().no_tools(),
706 **kwargs,
707 )
@@ -710,7 +712,9 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
712 super().__init__(
713 tokenizer,
714 *args,
713 - streaming_opts=PyFilterOptions().cmd4().stream_non_grounded_answer(),
715 + streaming_opts=(
716 + PyFilterOptions().cmd4().no_tools().stream_non_grounded_answer()
717 + ),
718 unary_opts=PyFilterOptions().cmd4().no_tools(),
719 **kwargs,
720 )
vllm/renderers/cohere.py
+1 -1
@@ -209,7 +209,7 @@ def _try_import_melody():
209 raise ImportError(
210 "The `cohere` tokenizer/renderer mode requires the "
211 "`cohere_melody` package. Install it via "
212 - "`pip install cohere-melody` or build from "
212 + "`pip install 'cohere-melody>=0.11.1'` or build from "
213 "https://github.com/cohere-ai/melody."
214 ) from e
215
vllm/tool_parsers/cohere_command_tool_parser.py
+5 -5
@@ -9,7 +9,7 @@ except ImportError as e:
9 raise ImportError(
10 "The Cohere tool parser requires the `cohere_melody` "
11 "package, which is not installed. Install it with:\n"
12 - " pip install cohere_melody"
12 + " pip install 'cohere-melody>=0.11.1'"
13 ) from e
14
15 from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -130,8 +130,8 @@ class CohereCommand3ToolParser(BaseCohereCommandToolParser):
130 ):
131 super().__init__(
132 tokenizer,
133 - streaming_opts=PyFilterOptions().cmd3(),
134 - unary_opts=PyFilterOptions().cmd3(),
133 + streaming_opts=PyFilterOptions().cmd3().start_in_answer(),
134 + unary_opts=PyFilterOptions().cmd3().start_in_answer(),
135 )
136
137
@@ -143,6 +143,6 @@ class CohereCommand4ToolParser(BaseCohereCommandToolParser):
143 ):
144 super().__init__(
145 tokenizer,
146 - streaming_opts=PyFilterOptions().cmd4(),
147 - unary_opts=PyFilterOptions().cmd4(),
146 + streaming_opts=PyFilterOptions().cmd4().start_in_answer(),
147 + unary_opts=PyFilterOptions().cmd4().start_in_answer(),
148 )