Add Cohere ChatV2 render endpoint (#53219)
andrewbcohere committed
Aug 24, 2026 at 08:27 UTC
41ae917c120f7036210053e0d79e0fb353fbdd46
5 files changed
+380
-24
tests/entrypoints/cohere/test_api_router.py
+195
@@ -11,6 +11,9 @@ Covers:
11
* The router wiring: response shapes (JSON + SSE), error translation,
12
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
13
Implemented).
14
+* The render route (``POST /cohere/v2/chat/render``): that the v2 body
15
+ reaches :class:`ServingRender` as a converted ``ChatCompletionRequest``
16
+ and that the resulting ``GenerateRequest`` is returned verbatim.
17
"""
18
19
import json
@@ -29,7 +32,9 @@ from vllm.entrypoints.cohere.protocol import (
32
AssistantMessageResponse,
33
CohereChatV2Response,
34
)
35
+from vllm.entrypoints.cohere.serving import CohereServingChatV2
36
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
37
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
38
from vllm.entrypoints.serve.exception_handling.handlers.http import (
39
http_exception_handler,
40
)
@@ -40,6 +45,7 @@ from vllm.entrypoints.serve.exception_handling.handlers.vllm_error import (
45
vllm_error_handler,
46
)
47
from vllm.exceptions import VLLMError
48
+from vllm.sampling_params import SamplingParams
49
50
51
@pytest.fixture(autouse=True)
@@ -78,6 +84,49 @@ class _Handler:
84
return self.result
85
86
87
+class _RenderChatHandler:
88
+ """Stand-in for :class:`CohereServingChatV2` on the render route.
89
+
90
+ Only ``to_chat_completion_request`` is reachable from
91
+ ``/cohere/v2/chat/render``, and it delegates to the real conversion
92
+ classmethod (which needs no engine or tokenizer) so the tests
93
+ exercise the actual v2 -> ``ChatCompletionRequest`` translation
94
+ rather than a stub of it.
95
+ """
96
+
97
+ def to_chat_completion_request(self, request):
98
+ return CohereServingChatV2._convert_v2_to_chat_completion(request)
99
+
100
+
101
+class _RenderHandler:
102
+ """Minimal stand-in for :class:`ServingRender`.
103
+
104
+ Records the ``ChatCompletionRequest`` it was handed so tests can
105
+ assert on the conversion, and returns ``self.result`` - either a
106
+ :class:`GenerateRequest`, an :class:`ErrorResponse`, or an exception
107
+ to raise.
108
+ """
109
+
110
+ def __init__(self, result):
111
+ self.result = result
112
+ self.seen_request = None
113
+
114
+ async def render_chat_request(self, request):
115
+ self.seen_request = request
116
+ if isinstance(self.result, Exception):
117
+ raise self.result
118
+ return self.result
119
+
120
+
121
+def _generate_request() -> GenerateRequest:
122
+ return GenerateRequest(
123
+ request_id="chatcmpl-abc",
124
+ token_ids=[5, 6, 7],
125
+ sampling_params=SamplingParams(temperature=0.25),
126
+ model="m",
127
+ )
128
+
129
+
130
def _build_app(handler: _Handler | None) -> FastAPI:
131
app = FastAPI()
132
attach_router(app)
@@ -85,6 +134,14 @@ def _build_app(handler: _Handler | None) -> FastAPI:
134
return app
135
136
137
+def _build_render_app(chat_handler, render_handler) -> FastAPI:
138
+ app = FastAPI()
139
+ attach_router(app)
140
+ app.state.cohere_serving_chat_v2 = chat_handler
141
+ app.state.serving_render = render_handler
142
+ return app
143
+
144
+
145
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
146
"""Build a FastAPI app that mirrors the real vLLM setup by installing
147
validation, HTTP, and vLLM exception handlers. The
@@ -321,6 +378,144 @@ class TestEndpoint:
378
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
379
380
381
+# ----------------------------------------------------------------------
382
+# Render endpoint
383
+# ----------------------------------------------------------------------
384
+
385
+
386
+class TestRenderEndpoint:
387
+ """``POST /cohere/v2/chat/render`` is the Cohere counterpart to
388
+ ``POST /v1/chat/completions/render``: it converts the v2 body to a
389
+ ``ChatCompletionRequest`` and hands it to the shared
390
+ ``ServingRender``, returning the resulting ``GenerateRequest``
391
+ instead of running generation.
392
+
393
+ The route therefore spans *two* app-state handlers, and these tests
394
+ pin both halves: that the conversion preserves the Cohere-specific
395
+ request surface, and that the render handler's output (or error)
396
+ reaches the client unchanged.
397
+ """
398
+
399
+ def test_route_registered(self):
400
+ app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
401
+ paths = [getattr(r, "path", None) for r in app.routes]
402
+ assert "/cohere/v2/chat/render" in paths
403
+
404
+ def test_route_not_registered_when_flag_unset(self, monkeypatch):
405
+ monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
406
+ app = FastAPI()
407
+ attach_router(app)
408
+ paths = [getattr(r, "path", None) for r in app.routes]
409
+ assert "/cohere/v2/chat/render" not in paths
410
+
411
+ def test_returns_generate_request_json(self):
412
+ render_handler = _RenderHandler(_generate_request())
413
+ app = _build_render_app(_RenderChatHandler(), render_handler)
414
+ with TestClient(app) as client:
415
+ r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
416
+ assert r.status_code == HTTPStatus.OK
417
+ assert r.headers["content-type"].startswith("application/json")
418
+ body = r.json()
419
+ assert body["request_id"] == "chatcmpl-abc"
420
+ assert body["token_ids"] == [5, 6, 7]
421
+ assert body["sampling_params"]["temperature"] == 0.25
422
+ # The body must be a wire-valid GenerateRequest so callers can
423
+ # feed it straight back to /v1/generate.
424
+ assert GenerateRequest.model_validate(body).token_ids == [5, 6, 7]
425
+
426
+ def test_cohere_request_surface_survives_conversion(self):
427
+ """The point of routing through ``CohereServingChatV2`` rather
428
+ than asking callers to pre-convert: Cohere-only fields
429
+ (``documents``, ``safety_mode``, ``citation_options``) must land
430
+ in ``chat_template_kwargs`` so the rendered prompt matches what
431
+ ``/cohere/v2/chat`` would have produced.
432
+ """
433
+ render_handler = _RenderHandler(_generate_request())
434
+ app = _build_render_app(_RenderChatHandler(), render_handler)
435
+ body = {
436
+ **_minimal_request_body(),
437
+ "documents": [{"id": "doc-1", "data": {"text": "Paris is in France."}}],
438
+ "safety_mode": "STRICT",
439
+ "max_tokens": 32,
440
+ }
441
+ with TestClient(app) as client:
442
+ r = client.post("/cohere/v2/chat/render", json=body)
443
+ assert r.status_code == HTTPStatus.OK
444
+
445
+ seen = render_handler.seen_request
446
+ assert seen is not None
447
+ assert seen.model == "m"
448
+ assert seen.messages[0]["content"] == "hi"
449
+ # ``max_tokens`` matters because ServingRender feeds it to
450
+ # ``get_max_tokens`` when building sampling params.
451
+ assert seen.max_completion_tokens == 32
452
+ kwargs = seen.chat_template_kwargs or {}
453
+ assert kwargs["documents"][0]["id"] == "doc-1"
454
+ # The conversion lower-cases safety_mode for the renderer.
455
+ assert kwargs["safety_mode"] == "strict"
456
+
457
+ def test_501_when_chat_handler_missing(self):
458
+ app = _build_render_app(None, _RenderHandler(_generate_request()))
459
+ with TestClient(app) as client:
460
+ r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
461
+ assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
462
+ assert "does not support" in r.json()["message"]
463
+
464
+ def test_501_when_render_handler_missing(self):
465
+ # A server without ``serving_render`` in app state (e.g. a
466
+ # non-generate task set) can't render, even though the Cohere
467
+ # chat handler exists.
468
+ app = _build_render_app(_RenderChatHandler(), None)
469
+ with TestClient(app) as client:
470
+ r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
471
+ assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
472
+ assert "does not support" in r.json()["message"]
473
+
474
+ def test_error_response_translated_to_cohere_envelope(self):
475
+ err = ErrorResponse(
476
+ error=ErrorInfo(message="unknown model", type="NotFound", code=404)
477
+ )
478
+ app = _build_render_app(_RenderChatHandler(), _RenderHandler(err))
479
+ with TestClient(app) as client:
480
+ r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
481
+ assert r.status_code == HTTPStatus.NOT_FOUND
482
+ assert r.json() == {"message": "unknown model"}
483
+
484
+ def test_render_exception_returns_500_envelope(self):
485
+ app = _build_render_app(
486
+ _RenderChatHandler(), _RenderHandler(RuntimeError("tokenizer exploded"))
487
+ )
488
+ with TestClient(app) as client:
489
+ r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
490
+ assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
491
+ assert r.json() == {"message": "tokenizer exploded"}
492
+
493
+ def test_conversion_exception_returns_500_envelope(self):
494
+ """Conversion runs inside the same ``try`` as the render call, so
495
+ a bad v2 body that slips past Pydantic still yields the Cohere
496
+ error envelope rather than an unhandled 500 with vLLM's shape.
497
+ """
498
+
499
+ class _Boom:
500
+ def to_chat_completion_request(self, request):
501
+ raise ValueError("unconvertible message")
502
+
503
+ app = _build_render_app(_Boom(), _RenderHandler(_generate_request()))
504
+ with TestClient(app) as client:
505
+ r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
506
+ assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
507
+ assert r.json() == {"message": "unconvertible message"}
508
+
509
+ def test_invalid_body_returns_422(self):
510
+ app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
511
+ with TestClient(app) as client:
512
+ r = client.post(
513
+ "/cohere/v2/chat/render",
514
+ json={"messages": [{"role": "user", "content": "hi"}]},
515
+ )
516
+ assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
517
+
518
+
519
# ----------------------------------------------------------------------
520
# CohereErrorEnvelopeMiddleware
521
# ----------------------------------------------------------------------
tests/entrypoints/cohere/test_chat_v2.py
+83
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
3
+"""End-to-end integration tests for the Cohere Chat v2 endpoints
4
+(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
5
6
These tests spin up a real ``vllm serve`` process via
7
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
@@ -173,6 +174,87 @@ async def test_cohere_v2_chat_validation_error_returns_400(
174
assert resp.status_code == 400
175
176
177
+@pytest.mark.asyncio
178
+async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
179
+ """``/cohere/v2/chat/render`` tokenizes without running the model.
180
+
181
+ Beyond the ``GenerateRequest`` shape, this pins the cross-layer
182
+ contract that makes the endpoint useful: v2 sampling fields
183
+ (``temperature``, ``stop_sequences``) survive the v2 ->
184
+ ``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
185
+ render once and generate later without re-deriving them.
186
+ """
187
+ resp = await httpx_client.post(
188
+ "/cohere/v2/chat/render",
189
+ json={
190
+ "model": SERVED_MODEL_NAME,
191
+ "messages": [{"role": "user", "content": "Say hi."}],
192
+ "max_tokens": 16,
193
+ "temperature": 0.125,
194
+ "stop_sequences": ["END"],
195
+ },
196
+ )
197
+ assert resp.status_code == 200, resp.text
198
+ data = resp.json()
199
+
200
+ token_ids = data["token_ids"]
201
+ assert isinstance(token_ids, list) and len(token_ids) > 0
202
+ assert all(isinstance(t, int) and t >= 0 for t in token_ids)
203
+ assert data["request_id"]
204
+
205
+ sampling_params = data["sampling_params"]
206
+ assert sampling_params["temperature"] == 0.125
207
+ assert sampling_params["stop"] == ["END"]
208
+
209
+ # No generation happened, so the response carries none of the chat
210
+ # envelope the /cohere/v2/chat endpoint would return.
211
+ assert "message" not in data
212
+ assert "finish_reason" not in data
213
+
214
+
215
+@pytest.mark.asyncio
216
+async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
217
+ httpx_client: httpx.AsyncClient,
218
+):
219
+ """The rendered ids are the real templated prompt, not a debug dump.
220
+
221
+ Round-tripping them through ``/detokenize`` and finding the user
222
+ message back proves the endpoint applied the chat template and
223
+ tokenized it, which is what a caller splitting prompt construction
224
+ from generation depends on.
225
+ """
226
+ render_resp = await httpx_client.post(
227
+ "/cohere/v2/chat/render",
228
+ json={
229
+ "model": SERVED_MODEL_NAME,
230
+ "messages": [{"role": "user", "content": "What is the capital of France?"}],
231
+ "max_tokens": 8,
232
+ },
233
+ )
234
+ assert render_resp.status_code == 200, render_resp.text
235
+ token_ids = render_resp.json()["token_ids"]
236
+
237
+ detok_resp = await httpx_client.post(
238
+ "/detokenize",
239
+ json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
240
+ )
241
+ assert detok_resp.status_code == 200, detok_resp.text
242
+ prompt = detok_resp.json()["prompt"]
243
+ assert "What is the capital of France?" in prompt
244
+
245
+
246
+@pytest.mark.asyncio
247
+async def test_cohere_v2_chat_render_validation_error_returns_400(
248
+ httpx_client: httpx.AsyncClient,
249
+):
250
+ """Bad bodies get the Cohere error envelope, not vLLM's."""
251
+ resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
252
+ assert resp.status_code == 400
253
+ body = resp.json()
254
+ assert "error" not in body
255
+ assert body.get("message")
256
+
257
+
258
@pytest.mark.asyncio
259
async def test_cohere_v2_chat_documents_field_accepted(
260
httpx_client: httpx.AsyncClient,
vllm/entrypoints/cohere/api_router.py
+79
-15
@@ -1,24 +1,30 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
3
+"""FastAPI router for the Cohere Chat v2 API.
4
+
5
+Exposes two routes:
6
+
7
+* ``POST /cohere/v2/chat`` - the chat endpoint itself.
8
+* ``POST /cohere/v2/chat/render`` - tokenize a request without running
9
+ generation, mirroring ``POST /v1/chat/completions/render``.
10
11
The Cohere v2 protocol models are sourced from the official ``cohere``
12
Python SDK (``pip install cohere``). To keep that an *optional*
7
-dependency for vLLM, the SDK-dependent imports - and the route handler
8
-itself - are gated on a one-shot probe at module load. If the SDK isn't
9
-installed, :func:`attach_router` becomes a no-op (with an info log) and
10
-vLLM continues to boot normally.
13
+dependency for vLLM, the SDK-dependent imports - and the route handlers
14
+themselves - are gated on a one-shot probe at module load. If the SDK
15
+isn't installed, :func:`attach_router` becomes a no-op (with an info
16
+log) and vLLM continues to boot normally.
17
18
Even when the SDK is installed, :func:`attach_router` also requires
19
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
14
-the route. This keeps non-Cohere deployments that pull in the SDK for
20
+the routes. This keeps non-Cohere deployments that pull in the SDK for
21
unrelated reasons (e.g. test dependencies) from accidentally exposing
22
the api.
23
18
-Note: the handler must live at module scope (not inside
24
+Note: the handlers must live at module scope (not inside
25
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
26
the ``CohereChatV2Request`` body annotation against the module's
21
-globals. Defining it locally inside ``attach_router`` would hide the
27
+globals. Defining them locally inside ``attach_router`` would hide the
28
type from ``get_type_hints``, causing FastAPI to silently degrade the
29
body parameter into a query parameter and reject every request with
30
422.
@@ -61,12 +67,17 @@ if _SDK_AVAILABLE:
67
CohereError,
68
)
69
from vllm.entrypoints.cohere.serving import CohereServingChatV2
70
+ from vllm.entrypoints.scale_out.render.serving import ServingRender
71
+ from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
72
73
router = APIRouter()
74
75
def _serving(request: Request) -> CohereServingChatV2 | None:
76
return getattr(request.app.state, "cohere_serving_chat_v2", None)
77
78
+ def _serving_render(request: Request) -> ServingRender | None:
79
+ return getattr(request.app.state, "serving_render", None)
80
+
81
def _request_id(raw_request: Request | None) -> str | None:
82
"""Best-effort lookup of the active request id.
83
@@ -143,6 +154,59 @@ if _SDK_AVAILABLE:
154
case _:
155
return StreamingResponse(content=result, media_type="text/event-stream")
156
157
+ @router.post(
158
+ "/cohere/v2/chat/render",
159
+ dependencies=[Depends(validate_json_request)],
160
+ response_model=GenerateRequest,
161
+ responses={
162
+ HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
163
+ HTTPStatus.NOT_FOUND.value: {"model": CohereError},
164
+ HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
165
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
166
+ },
167
+ )
168
+ @with_cancellation
169
+ async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
170
+ """Tokenize a Cohere v2 chat request without running generation.
171
+
172
+ The Cohere counterpart to ``POST /v1/chat/completions/render``.
173
+ The v2 body goes through the same conversion
174
+ ``/cohere/v2/chat`` uses and is then handed to the shared
175
+ :class:`ServingRender`, so the returned ``GenerateRequest``
176
+ carries the prompt tokens and sampling params the chat endpoint
177
+ would have sent to the engine.
178
+ """
179
+ handler = _serving(raw_request)
180
+ render_handler = _serving_render(raw_request)
181
+ if handler is None or render_handler is None:
182
+ return JSONResponse(
183
+ status_code=HTTPStatus.NOT_IMPLEMENTED.value,
184
+ content=CohereError(
185
+ message=(
186
+ "The model does not support the Cohere v2 chat render API."
187
+ ),
188
+ id=_request_id(raw_request),
189
+ ).model_dump(exclude_none=True),
190
+ )
191
+
192
+ try:
193
+ chat_request = handler.to_chat_completion_request(request)
194
+ result = await render_handler.render_chat_request(chat_request)
195
+ except Exception as e: # noqa: BLE001 - report as 500 for parity
196
+ logger.exception("Error in /cohere/v2/chat/render: %s", e)
197
+ return JSONResponse(
198
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
199
+ content=CohereError(
200
+ message=sanitize_message(str(e)),
201
+ id=_request_id(raw_request),
202
+ ).model_dump(exclude_none=True),
203
+ )
204
+
205
+ if isinstance(result, ErrorResponse):
206
+ return _error_response(result, raw_request)
207
+
208
+ return JSONResponse(content=result.model_dump())
209
+
210
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
211
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
212
@@ -209,12 +273,12 @@ if _SDK_AVAILABLE:
273
274
275
def attach_router(app: FastAPI) -> None:
212
- """Register ``POST /cohere/v2/chat`` on ``app``.
276
+ """Register the ``/cohere/v2/chat`` routes on ``app``.
277
278
No-op when either:
279
280
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
217
- Cohere v2 endpoint is opt-in because it carries Cohere-specific
281
+ Cohere v2 endpoints are opt-in because they carry Cohere-specific
282
request/response semantics (grounding citations, tool_plan,
283
PLAN/THINKING_CONTENT blocks) that are only meaningful when
284
serving a Cohere Command-family model.
@@ -223,21 +287,21 @@ def attach_router(app: FastAPI) -> None:
287
288
The two skip paths log at different levels: an operator who set
289
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
226
- a WARNING (they explicitly asked for the endpoint and it's silently
227
- absent), whereas the default-off skip logs at debug.
290
+ a WARNING (they explicitly asked for the endpoints and they're
291
+ silently absent), whereas the default-off skip logs at debug.
292
"""
293
enabled = envs.VLLM_ENABLE_COHERE_API
294
if not enabled:
295
logger.debug(
232
- "VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
233
- "disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
296
+ "VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
297
+ "disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
298
)
299
return
300
if not _SDK_AVAILABLE:
301
logger.warning(
302
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
303
"installed; /cohere/v2/chat will not be exposed. Install "
240
- "with `pip install cohere` to enable the endpoint."
304
+ "with `pip install cohere` to enable the endpoints."
305
)
306
return
307
app.include_router(router)
vllm/entrypoints/cohere/serving.py
+21
-7
@@ -304,13 +304,7 @@ class CohereServingChatV2(OpenAIServingChat):
304
"Received Cohere v2 chat request %s", request.model_dump_json()
305
)
306
307
- chat_req = self._convert_v2_to_chat_completion(request)
308
- if logger.isEnabledFor(logging.DEBUG):
309
- logger.debug(
310
- "Converted Cohere v2 -> ChatCompletion: %s",
311
- chat_req.model_dump_json(),
312
- )
313
-
307
+ chat_req = self.to_chat_completion_request(request)
308
generator = await self.create_chat_completion(chat_req, raw_request)
309
310
match generator:
@@ -321,6 +315,26 @@ class CohereServingChatV2(OpenAIServingChat):
315
case _:
316
return self._chat_completion_stream_to_v2(generator, request)
317
318
+ def to_chat_completion_request(
319
+ self, request: CohereChatV2Request
320
+ ) -> ChatCompletionRequest:
321
+ """Convert a Cohere v2 request into its ``ChatCompletionRequest``.
322
+
323
+ Exposed for callers that need the converted request without
324
+ running generation - notably the render endpoint
325
+ (``POST /cohere/v2/chat/render``), which hands the result to
326
+ :meth:`vllm.entrypoints.scale_out.render.serving.ServingRender.render_chat_request`
327
+ so that Cohere requests tokenize through exactly the same path
328
+ as ``/v1/chat/completions/render``.
329
+ """
330
+ chat_req = self._convert_v2_to_chat_completion(request)
331
+ if logger.isEnabledFor(logging.DEBUG):
332
+ logger.debug(
333
+ "Converted Cohere v2 -> ChatCompletion: %s",
334
+ chat_req.model_dump_json(),
335
+ )
336
+ return chat_req
337
+
338
def _engine_chat_template_kwargs(
339
self, chat_template_kwargs: dict[str, Any]
340
) -> dict[str, Any]:
vllm/envs.py
+2
-1
@@ -1868,7 +1868,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
1868
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
1869
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
1870
),
1871
- # If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``.
1871
+ # If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
1872
+ # and its render endpoint at ``POST /cohere/v2/chat/render``.
1873
# Default off
1874
"VLLM_ENABLE_COHERE_API": lambda: bool(
1875
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))