A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 12:30 UTC
[Feature] add cohere reasoning and tool parsers (#40422)
Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Walter Beller-Morales committed
Apr 29, 2026 at 00:07 UTC
1312f0753115cb36410334e3667961d1237a287b
6 files changed
+700
docs/features/reasoning_outputs.md
+1
@@ -13,6 +13,7 @@ vLLM currently supports the following reasoning models:
13
14
| Model Series | Parser Name | Structured Output Support | Tool Calling |
15
| ------------ | ----------- | ---------------- | ----------- |
16
+| [Cohere Command A Reasoning](https://huggingface.co/CohereLabs/command-a-reasoning-08-2025) | `cohere_command3` | `json`, `regex` | ✅ |
17
| [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `json`, `regex` | ❌ |
18
| [DeepSeek-V3.1](https://huggingface.co/collections/deepseek-ai/deepseek-v31-68a491bed32bd77e7fca048f) | `deepseek_v3` | `json`, `regex` | ❌ |
19
| [ERNIE-4.5-VL series](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-PT) | `ernie45` | `json`, `regex` | ❌ |
docs/features/tool_calling.md
+10
@@ -369,6 +369,16 @@ Flags:
369
* For non-reasoning: `--tool-call-parser hunyuan_a13b`
370
* For reasoning: `--tool-call-parser hunyuan_a13b --reasoning-parser hunyuan_a13b`
371
372
+### Cohere Command A Reasoning (`cohere_command3`)
373
+
374
+Supported models:
375
+
376
+* [`CohereLabs/command-a-reasoning-08-2025`](https://huggingface.co/CohereLabs/command-a-reasoning-08-2025)
377
+
378
+Flags: `--tool-call-parser cohere_command3 --reasoning-parser cohere_command3`
379
+
380
+Note: the Cohere tool parser requires the `cohere_melody` package, which is not installed by default. Before using this parser please install the [cohere_melody](https://pypi.org/project/cohere-melody/) package.
381
+
382
### LongCat-Flash-Chat Models (`longcat`)
383
384
Supported models:
vllm/reasoning/__init__.py
+8
@@ -36,6 +36,14 @@ _REASONING_PARSERS_TO_REGISTER = {
36
"poolside_v1_reasoning_parser",
37
"PoolsideV1ReasoningParser",
38
),
39
+ "cohere_command3": (
40
+ "cohere_command_reasoning_parser",
41
+ "CohereCommand3ReasoningParser",
42
+ ),
43
+ "cohere_command4": (
44
+ "cohere_command_reasoning_parser",
45
+ "CohereCommand4ReasoningParser",
46
+ ),
47
"ernie45": (
48
"ernie45_reasoning_parser",
49
"Ernie45ReasoningParser",
vllm/reasoning/cohere_command_reasoning_parser.py
new
+546
@@ -0,0 +1,546 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from __future__ import annotations
5
+
6
+import json
7
+from collections.abc import Mapping, Sequence
8
+from typing import Any, NamedTuple, TypedDict, TypeGuard
9
+
10
+import regex as re
11
+import xgrammar as xgr
12
+
13
+try:
14
+ from cohere_melody import PyFilter, PyFilterOptions
15
+except ImportError as e:
16
+ raise ImportError(
17
+ "The Cohere reasoning parser requires the `cohere_melody` "
18
+ "package, which is not installed. Install it with:\n"
19
+ " pip install cohere_melody"
20
+ ) from e
21
+
22
+
23
+from vllm.entrypoints.mcp.tool_server import ToolServer
24
+from vllm.entrypoints.openai.chat_completion.protocol import (
25
+ ChatCompletionRequest,
26
+)
27
+from vllm.entrypoints.openai.engine.protocol import (
28
+ AnyResponseFormat,
29
+ DeltaFunctionCall,
30
+ DeltaMessage,
31
+ DeltaToolCall,
32
+)
33
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
34
+from vllm.reasoning import ReasoningParser
35
+from vllm.sampling_params import StructuredOutputsParams
36
+from vllm.tokenizers import TokenizerLike
37
+
38
+REPLACEMENT_CHAR = "\ufffd"
39
+
40
+
41
+class CohereTagRegistry(NamedTuple):
42
+ """A single ``structural_tag`` begin("trigger")/end pair."""
43
+
44
+ trigger: str
45
+ end: str
46
+
47
+
48
+class CohereTagStyle(NamedTuple):
49
+ """The structural tags style for a given model architecture."""
50
+
51
+ json: CohereTagRegistry
52
+ tools: CohereTagRegistry
53
+
54
+
55
+class CohereNormalizedTool(TypedDict):
56
+ """A tool definition normalized to the shape ``collect_tool_schema`` expects.
57
+
58
+ ``parameters`` is a JSON Schema object (possibly empty) describing the tool's
59
+ call signature.
60
+ """
61
+
62
+ name: str
63
+ parameters: dict[str, Any]
64
+
65
+
66
+COMMAND_A_TOOLS_TAG = CohereTagRegistry(
67
+ trigger="<|START_ACTION|>", end="<|END_ACTION|>"
68
+)
69
+COMMAND_A_JSON_TAG = CohereTagRegistry(
70
+ trigger="<|START_RESPONSE|>", end="<|END_RESPONSE|>"
71
+)
72
+
73
+MODEL_TO_TAG_STYLE: dict[str, CohereTagStyle] = {
74
+ "Cohere2ForCausalLM": CohereTagStyle(
75
+ json=COMMAND_A_JSON_TAG, tools=COMMAND_A_TOOLS_TAG
76
+ ),
77
+ "Cohere2VisionForConditionalGeneration": CohereTagStyle(
78
+ json=COMMAND_A_JSON_TAG, tools=COMMAND_A_TOOLS_TAG
79
+ ),
80
+}
81
+
82
+
83
+def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
84
+ """Build an xgrammar EBNF grammar that matches a JSON array of tool calls.
85
+
86
+ The grammar shape is architecture-independent; callers are responsible for
87
+ wrapping it in the correct structural tag (see ``CohereTagStyle.tools``).
88
+ """
89
+ tool_dictionary: dict[str, str] = {}
90
+ for tool in tool_schema:
91
+ tool_name = tool["name"]
92
+ tool_parameters = json.dumps(tool["parameters"])
93
+ json_schema = f"""{{
94
+ "type": "object",
95
+ "properties": {{
96
+ "tool_call_id": {{
97
+ "type": "string",
98
+ "pattern": "^[0-9]+$"
99
+ }},
100
+ "tool_name": {{
101
+ "type": "string",
102
+ "const": "{tool_name}"
103
+ }},
104
+ "parameters": {tool_parameters}
105
+ }}
106
+ }}"""
107
+ tool_grammar = str(xgr.Grammar.from_json_schema(json_schema))
108
+ for match in re.findall(r"\b(\w+)\s*::=", tool_grammar):
109
+ tool_grammar = re.sub(
110
+ rf"\b{re.escape(match)}\b", tool_name + match, tool_grammar
111
+ )
112
+ tool_dictionary[tool_name] = f"{tool_name} ::= {tool_name}root\n{tool_grammar}"
113
+ # Emitted grammar shape:
114
+ # root ::= tools
115
+ # tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws
116
+ # ws ::= (" " | "\t" | "\n")*
117
+ # tool ::= <tool_a> | <tool_b> | ... (one alternative per input)
118
+ # <tool_x> ::= <tool_x>root (per-tool xgrammar rules)
119
+ # <tool_x>root ::= ... (from xgr.Grammar.from_json_schema)
120
+ tool_alternatives = "tool ::= " + " | ".join(tool_dictionary.keys())
121
+ tool_rules = "\n ".join(tool_dictionary.values())
122
+ grammar = f"""root ::= tools
123
+ tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws
124
+ ws ::= (" " | "\\t" | "\\n")*
125
+ {tool_alternatives}
126
+ {tool_rules}
127
+ """
128
+ return grammar
129
+
130
+
131
+def _tool_definitions_to_schema_list(
132
+ tools: str | list[Any],
133
+) -> list[CohereNormalizedTool]:
134
+ """
135
+ Build the list of ``CohereNormalizedTool`` dicts expected by
136
+ ``collect_tool_schema``.
137
+
138
+ Accepts:
139
+ - JSON string
140
+ - list of dicts with top-level ``name`` / ``parameters``
141
+ - list of Chat Completions-style ``{"type": "function", "function": {...}}``
142
+ - list of Pydantic models with ``model_dump()``
143
+ """
144
+ if isinstance(tools, str):
145
+ try:
146
+ parsed = json.loads(tools)
147
+ except json.JSONDecodeError:
148
+ return []
149
+ if not isinstance(parsed, list):
150
+ return []
151
+ else:
152
+ parsed = list(tools)
153
+
154
+ out: list[CohereNormalizedTool] = []
155
+ for raw in parsed:
156
+ t = raw.model_dump() if hasattr(raw, "model_dump") else raw
157
+ if not isinstance(t, dict):
158
+ continue
159
+ # Unwrap Chat Completions' ``{"type": "function", "function": {...}}``
160
+ # shape; otherwise take the dict as-is.
161
+ if t.get("type") == "function" and isinstance(t.get("function"), dict):
162
+ t = t["function"]
163
+ name = t.get("name")
164
+ if not isinstance(name, str):
165
+ continue
166
+ params = t.get("parameters")
167
+ out.append(
168
+ CohereNormalizedTool(
169
+ name=name,
170
+ parameters=params if isinstance(params, dict) else {},
171
+ )
172
+ )
173
+ return out
174
+
175
+
176
+def _has_effective_tools(
177
+ tools: str | list[Any] | None,
178
+) -> TypeGuard[str | list[Any]]:
179
+ """
180
+ True when ``tools`` contains at least one tool definition to convert.
181
+
182
+ ``ResponsesRequest`` defaults ``tools`` to ``[]``; ``ChatCompletionRequest``
183
+ uses ``None``. Both mean "no tools" here. Strings (e.g. a JSON blob) are
184
+ treated as effective only when non-blank.
185
+ """
186
+ if tools is None:
187
+ return False
188
+ if isinstance(tools, str):
189
+ return bool(tools.strip())
190
+ return len(tools) > 0
191
+
192
+
193
+# Builder: produces vLLM response_format in xgrammar's canonical format.
194
+# See xgrammar docs: type "structural_tag" with "format" = triggered_tags
195
+# and tag content type = json_schema | grammar.
196
+def convert_schema_to_structural_tags(
197
+ schema: dict | None = None,
198
+ tools: str | list[Any] | None = None,
199
+ model_architecture: str | None = None,
200
+) -> str | None:
201
+ """
202
+ Returns a response_format string accepted by xgrammar's structural tag format.
203
+ Uses the canonical shape: {"type": "structural_tag", "format": {...}} with
204
+ format.type "triggered_tags" and tag content type "json_schema" or "grammar".
205
+
206
+ Callers that are not on an engine path (e.g. the reasoning parser) must pass
207
+ ``model_architecture`` explicitly.
208
+ """
209
+ if model_architecture is None or model_architecture not in MODEL_TO_TAG_STYLE:
210
+ return None
211
+ style = MODEL_TO_TAG_STYLE[model_architecture]
212
+
213
+ tags: list[dict] = []
214
+
215
+ def _add_tag(tag: CohereTagRegistry, content: dict) -> None:
216
+ tags.append({"begin": tag.trigger, "content": content, "end": tag.end})
217
+
218
+ if schema is not None:
219
+ # Add the JSON-schema tag both for schema-only requests and for the
220
+ # "tools plus JSON mode" case (North use case: follow the schema when
221
+ # the model decides not to call any tool).
222
+ _add_tag(style.json, {"type": "json_schema", "json_schema": schema})
223
+
224
+ if _has_effective_tools(tools):
225
+ # ``tools`` may be a JSON string (poseidon / RESPONSE_FORMAT_TOOL_DEFINITIONS)
226
+ # or a list (Chat Completions ``request.tools`` as Pydantic models or dicts).
227
+ tool_schema_list = _tool_definitions_to_schema_list(tools)
228
+ if not tool_schema_list:
229
+ raise ValueError(
230
+ "No valid tool definitions could be parsed from the request for "
231
+ "structural tag conversion."
232
+ )
233
+ tool_grammar = collect_tool_schema(tool_schema_list)
234
+ _add_tag(style.tools, {"type": "grammar", "grammar": tool_grammar})
235
+
236
+ if not tags:
237
+ return None
238
+ return json.dumps(
239
+ {
240
+ "type": "structural_tag",
241
+ "format": {
242
+ "type": "triggered_tags",
243
+ "triggers": [t["begin"] for t in tags],
244
+ "tags": tags,
245
+ },
246
+ }
247
+ )
248
+
249
+
250
+def _response_format_type(
251
+ response_format: AnyResponseFormat | dict | None,
252
+) -> str | None:
253
+ if response_format is None:
254
+ return None
255
+ if isinstance(response_format, dict):
256
+ t = response_format.get("type")
257
+ return t if isinstance(t, str) else None
258
+ return response_format.type
259
+
260
+
261
+def _maybe_parse_json_dict(value: Any) -> dict | None:
262
+ """If value is a JSON string, parse to dict; otherwise require dict."""
263
+ if isinstance(value, dict):
264
+ return value
265
+ if isinstance(value, str):
266
+ try:
267
+ parsed = json.loads(value)
268
+ except (TypeError, json.JSONDecodeError):
269
+ return None
270
+ return parsed if isinstance(parsed, dict) else None
271
+ return None
272
+
273
+
274
+def _unwrap_nested_schema(candidate: Any) -> dict | None:
275
+ """Return ``candidate`` as a dict, unwrapping a nested ``schema`` if present.
276
+
277
+ Returns ``None`` if ``candidate`` is not (and cannot be parsed into) a dict.
278
+ """
279
+ cand = _maybe_parse_json_dict(candidate)
280
+ if not isinstance(cand, dict):
281
+ return None
282
+ nested = cand.get("schema")
283
+ return nested if isinstance(nested, dict) else cand
284
+
285
+
286
+def _schema_from_json_schema_field(js_wr: Any) -> dict | None:
287
+ """
288
+ Extract the JSON Schema object from Chat Completions ``json_schema`` payload.
289
+
290
+ Accepts:
291
+ - ``JsonSchemaResponseFormat`` (Pydantic) with ``schema`` / ``json_schema`` field
292
+ - dict in OpenAI shape ``{"name": ..., "schema": {...}}``
293
+ - dict with ``json_schema`` key holding either the schema or a nested wrapper
294
+ - dict that is already a JSON Schema document (some clients omit the wrapper)
295
+ - JSON strings for any of the above
296
+ """
297
+ if js_wr is None:
298
+ return None
299
+
300
+ parsed_wr = _maybe_parse_json_dict(js_wr)
301
+ if parsed_wr is not None:
302
+ js_wr = parsed_wr
303
+
304
+ if hasattr(js_wr, "model_dump"):
305
+ for by_alias in (True, False):
306
+ try:
307
+ data = js_wr.model_dump(by_alias=by_alias, exclude_none=False)
308
+ except TypeError:
309
+ data = js_wr.model_dump(by_alias=by_alias)
310
+ out = _unwrap_nested_schema(data.get("schema") or data.get("json_schema"))
311
+ if out is not None:
312
+ return out
313
+ inner_attr = getattr(js_wr, "json_schema", None)
314
+ return inner_attr if isinstance(inner_attr, dict) else None
315
+
316
+ if isinstance(js_wr, dict):
317
+ for key in ("schema", "json_schema"):
318
+ out = _unwrap_nested_schema(js_wr.get(key))
319
+ if out is not None:
320
+ return out
321
+ return js_wr
322
+
323
+ return None
324
+
325
+
326
+def _schema_dict_from_chat_response_format(
327
+ rf: AnyResponseFormat | dict | None,
328
+) -> dict | None:
329
+ """JSON schema dict from Chat Completions ``request.response_format`` only."""
330
+ if rf is None:
331
+ return None
332
+ rf_type = _response_format_type(rf)
333
+ if rf_type == "json_object":
334
+ return {"type": "object"}
335
+ if rf_type != "json_schema":
336
+ return None
337
+ js_wr = (
338
+ rf.get("json_schema")
339
+ if isinstance(rf, dict)
340
+ else getattr(rf, "json_schema", None)
341
+ )
342
+ return _schema_from_json_schema_field(js_wr)
343
+
344
+
345
+def _schema_dict_from_structured_outputs(
346
+ so: StructuredOutputsParams | None,
347
+) -> dict | None:
348
+ """Schema dict from ``structured_outputs`` (``json`` / ``json_object``).
349
+
350
+ Same unwrapping as ``json_schema``. ``json`` is expected to be ``str`` or
351
+ ``dict`` (enforced by ``StructuredOutputsParams`` / request models); other
352
+ types raise ``ValueError`` only if a caller bypasses that validation.
353
+ """
354
+ if so is None:
355
+ return None
356
+ if so.json_object:
357
+ return {"type": "object"}
358
+ raw: Any = so.json
359
+ if raw is None:
360
+ return None
361
+
362
+ if hasattr(raw, "model_dump"):
363
+ out = _schema_from_json_schema_field(raw)
364
+ if out is None:
365
+ raise ValueError(
366
+ "structured_outputs.json model has no extractable JSON Schema."
367
+ )
368
+ return out
369
+
370
+ if isinstance(raw, str):
371
+ if not raw.strip():
372
+ raise ValueError("structured_outputs.json cannot be empty.")
373
+ try:
374
+ raw = json.loads(raw)
375
+ except json.JSONDecodeError as e:
376
+ raise ValueError("structured_outputs.json must be valid JSON.") from e
377
+ if not isinstance(raw, dict):
378
+ raise ValueError("structured_outputs.json must decode to a JSON object.")
379
+
380
+ if isinstance(raw, Mapping):
381
+ body = raw if isinstance(raw, dict) else dict(raw)
382
+ return _schema_from_json_schema_field(body) or body
383
+
384
+ raise ValueError(
385
+ f"structured_outputs.json has unsupported type {type(raw).__name__}."
386
+ )
387
+
388
+
389
+class BaseCohereCommandReasoningParser(ReasoningParser):
390
+ def __init__(
391
+ self,
392
+ tokenizer: TokenizerLike,
393
+ *args,
394
+ streaming_opts: PyFilterOptions,
395
+ unary_opts: PyFilterOptions,
396
+ **kwargs,
397
+ ):
398
+ super().__init__(tokenizer, *args, **kwargs)
399
+ self.end_token_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
400
+ self.unary_opts = unary_opts
401
+ self.melody_unary = PyFilter(unary_opts)
402
+ self.melody_streaming = PyFilter(streaming_opts)
403
+
404
+ @property
405
+ def reasoning_start_str(self) -> str | None:
406
+ return "<|START_THINKING|>"
407
+
408
+ @property
409
+ def reasoning_end_str(self) -> str | None:
410
+ return "<|END_THINKING|>"
411
+
412
+ def extract_reasoning_streaming(
413
+ self,
414
+ previous_text: str,
415
+ current_text: str,
416
+ delta_text: str,
417
+ previous_token_ids: Sequence[int],
418
+ current_token_ids: Sequence[int],
419
+ delta_token_ids: Sequence[int],
420
+ ) -> DeltaMessage | None:
421
+ r = self.melody_streaming.write_decoded(delta_text)
422
+ if r.content is None and r.reasoning is None and not r.tool_calls:
423
+ return None
424
+ msg = DeltaMessage()
425
+ if r.content is not None:
426
+ msg.content = r.content
427
+ if r.reasoning is not None:
428
+ msg.reasoning = r.reasoning
429
+ if r.tool_calls:
430
+ msg.tool_calls = [
431
+ DeltaToolCall(
432
+ id=tc.id,
433
+ index=tc.index,
434
+ type="function",
435
+ function=DeltaFunctionCall(name=tc.name, arguments=tc.arguments),
436
+ )
437
+ for tc in r.tool_calls
438
+ ]
439
+ return msg
440
+
441
+ def extract_reasoning(
442
+ self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
443
+ ) -> tuple[str | None, str | None]:
444
+ result = self.melody_unary.process_full_text(model_output)
445
+ return result.reasoning, result.content
446
+
447
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
448
+ token_buf: list[int] = []
449
+ content_ids: list[int] = []
450
+ content_filter = PyFilter(self.unary_opts)
451
+ for t in input_ids:
452
+ token_buf.append(t)
453
+ s = self.model_tokenizer.decode(token_buf, skip_special_tokens=False)
454
+ if s.endswith(REPLACEMENT_CHAR):
455
+ continue
456
+ r = content_filter.write_decoded(s)
457
+ if r.content is not None:
458
+ content_ids.extend(token_buf)
459
+ token_buf = []
460
+ return content_ids
461
+
462
+ def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
463
+ return any(tid == self.end_token_id for tid in reversed(input_ids))
464
+
465
+ def prepare_structured_tag(
466
+ self, original_tag: str | None, tool_server: ToolServer | None
467
+ ) -> str | None:
468
+ # Responses API replaces ``structural_tag`` via the reasoning parser.
469
+ # Default ``ReasoningParser.prepare_structured_tag`` returns None, which
470
+ # would clear a Cohere tag produced in ``adjust_request`` and break
471
+ # ``StructuredOutputsParams`` validation. Preserve the existing tag.
472
+ return original_tag
473
+
474
+ def adjust_request(
475
+ self, request: ChatCompletionRequest | ResponsesRequest
476
+ ) -> ChatCompletionRequest | ResponsesRequest:
477
+ so = request.structured_outputs
478
+ if so is not None and so.structural_tag:
479
+ return request
480
+ # Schema: prefer ``response_format`` (OpenAI Chat Completions), then
481
+ # ``structured_outputs.json`` / ``json_object`` (vLLM direct). Tools stay
482
+ # on ``request.tools``.
483
+ rf = (
484
+ request.response_format
485
+ if isinstance(request, ChatCompletionRequest)
486
+ else None
487
+ )
488
+ if rf is not None and _response_format_type(rf) == "structural_tag":
489
+ return request
490
+ model_architecture = (
491
+ self._model_config.architecture if self._model_config is not None else None
492
+ )
493
+ tools = request.tools
494
+ # ``response_format`` wins if both it and ``structured_outputs`` supply JSON.
495
+ schema = _schema_dict_from_chat_response_format(rf)
496
+ if schema is None:
497
+ schema = _schema_dict_from_structured_outputs(so)
498
+ if schema is None and not _has_effective_tools(tools):
499
+ return request
500
+ if model_architecture is None:
501
+ return request
502
+ result = convert_schema_to_structural_tags(
503
+ schema=schema,
504
+ tools=tools,
505
+ model_architecture=model_architecture,
506
+ )
507
+ if result is None:
508
+ # Unsupported architectures are not in ``MODEL_TO_TAG_STYLE``; conversion
509
+ raise ValueError(
510
+ "Failed to build structural_tag guided decoding constraints from "
511
+ "this request's JSON schema and/or tools. The configured model "
512
+ f"architecture ({model_architecture!r}) does not support Cohere "
513
+ "command structural tags, or the schema cannot be expressed in "
514
+ "that format."
515
+ )
516
+ request.structured_outputs = StructuredOutputsParams(structural_tag=result)
517
+ # Folded JSON constraints into ``structural_tag``; drop ``response_format``
518
+ # when it was the source so ``to_sampling_params`` does not also set ``json`` /
519
+ # ``json_object`` (mutually exclusive in ``StructuredOutputsParams``).
520
+ if isinstance(request, ChatCompletionRequest) and rf is not None:
521
+ rf_type = _response_format_type(rf)
522
+ if rf_type in ("json_schema", "json_object"):
523
+ request.response_format = None
524
+ return request
525
+
526
+
527
+class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
528
+ def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
529
+ super().__init__(
530
+ tokenizer,
531
+ *args,
532
+ streaming_opts=PyFilterOptions().cmd3(),
533
+ unary_opts=PyFilterOptions().cmd3().no_tools(),
534
+ **kwargs,
535
+ )
536
+
537
+
538
+class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
539
+ def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
540
+ super().__init__(
541
+ tokenizer,
542
+ *args,
543
+ streaming_opts=PyFilterOptions().cmd4(),
544
+ unary_opts=PyFilterOptions().cmd4().no_tools(),
545
+ **kwargs,
546
+ )
vllm/tool_parsers/__init__.py
+8
@@ -38,6 +38,14 @@ _TOOL_PARSERS_TO_REGISTER = {
38
"deepseekv4_tool_parser",
39
"DeepSeekV4ToolParser",
40
),
41
+ "cohere_command3": (
42
+ "cohere_command_tool_parser",
43
+ "CohereCommand3ToolParser",
44
+ ),
45
+ "cohere_command4": (
46
+ "cohere_command_tool_parser",
47
+ "CohereCommand4ToolParser",
48
+ ),
49
"ernie45": (
50
"ernie45_tool_parser",
51
"Ernie45ToolParser",
vllm/tool_parsers/cohere_command_tool_parser.py
new
+127
@@ -0,0 +1,127 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Sequence
5
+
6
+try:
7
+ from cohere_melody import PyFilter, PyFilterOptions
8
+except ImportError as e:
9
+ raise ImportError(
10
+ "The Cohere tool parser requires the `cohere_melody` "
11
+ "package, which is not installed. Install it with:\n"
12
+ " pip install cohere_melody"
13
+ ) from e
14
+
15
+from vllm.entrypoints.openai.chat_completion.protocol import (
16
+ ChatCompletionRequest,
17
+)
18
+from vllm.entrypoints.openai.engine.protocol import (
19
+ DeltaFunctionCall,
20
+ DeltaMessage,
21
+ DeltaToolCall,
22
+ ExtractedToolCallInformation,
23
+ FunctionCall,
24
+ ToolCall,
25
+)
26
+from vllm.entrypoints.openai.responses.protocol import (
27
+ ResponsesRequest,
28
+)
29
+from vllm.tokenizers import TokenizerLike
30
+from vllm.tool_parsers import ToolParser
31
+from vllm.tool_parsers.utils import Tool
32
+
33
+
34
+class BaseCohereCommandToolParser(ToolParser):
35
+ def __init__(
36
+ self,
37
+ tokenizer: TokenizerLike,
38
+ streaming_opts: PyFilterOptions,
39
+ unary_opts: PyFilterOptions,
40
+ ):
41
+ super().__init__(tokenizer)
42
+ self.melody_streaming = PyFilter(streaming_opts)
43
+ self.melody_unary = PyFilter(unary_opts)
44
+
45
+ def adjust_request(
46
+ self, request: ChatCompletionRequest | ResponsesRequest
47
+ ) -> ChatCompletionRequest | ResponsesRequest:
48
+ request = super().adjust_request(request)
49
+ request.skip_special_tokens = False
50
+ return request
51
+
52
+ def extract_tool_calls_streaming(
53
+ self,
54
+ previous_text: str,
55
+ current_text: str,
56
+ delta_text: str,
57
+ previous_token_ids: Sequence[int],
58
+ current_token_ids: Sequence[int],
59
+ delta_token_ids: Sequence[int],
60
+ request: ChatCompletionRequest,
61
+ ) -> DeltaMessage | None:
62
+ r = self.melody_streaming.write_decoded(delta_text)
63
+ if r.content is not None:
64
+ return DeltaMessage(content=r.content)
65
+ if r.reasoning is not None:
66
+ return DeltaMessage(reasoning=r.reasoning)
67
+ if r.tool_calls:
68
+ return DeltaMessage(
69
+ tool_calls=[
70
+ DeltaToolCall(
71
+ id=tc.id,
72
+ index=tc.index,
73
+ type="function",
74
+ function=DeltaFunctionCall(
75
+ name=tc.name, arguments=tc.arguments
76
+ ),
77
+ )
78
+ for tc in r.tool_calls
79
+ ]
80
+ )
81
+ return None
82
+
83
+ def extract_tool_calls(
84
+ self,
85
+ model_output: str,
86
+ request: ChatCompletionRequest,
87
+ ) -> ExtractedToolCallInformation:
88
+ result = self.melody_unary.process_full_text(model_output)
89
+ tool_calls = [
90
+ ToolCall(
91
+ id=tc.id,
92
+ type="function",
93
+ function=FunctionCall(name=tc.name, arguments=tc.arguments),
94
+ )
95
+ for tc in result.tool_calls
96
+ ]
97
+ return ExtractedToolCallInformation(
98
+ tools_called=len(tool_calls) > 0,
99
+ tool_calls=tool_calls,
100
+ content=result.content,
101
+ )
102
+
103
+
104
+class CohereCommand3ToolParser(BaseCohereCommandToolParser):
105
+ def __init__(
106
+ self,
107
+ tokenizer: TokenizerLike,
108
+ tools: list[Tool] | None = None,
109
+ ):
110
+ super().__init__(
111
+ tokenizer,
112
+ streaming_opts=PyFilterOptions().cmd3(),
113
+ unary_opts=PyFilterOptions().cmd3(),
114
+ )
115
+
116
+
117
+class CohereCommand4ToolParser(BaseCohereCommandToolParser):
118
+ def __init__(
119
+ self,
120
+ tokenizer: TokenizerLike,
121
+ tools: list[Tool] | None = None,
122
+ ):
123
+ super().__init__(
124
+ tokenizer,
125
+ streaming_opts=PyFilterOptions().cmd4(),
126
+ unary_opts=PyFilterOptions().cmd4(),
127
+ )