[Bugfix][Frontend] Report Cohere stop sequences correctly (#51556)
Signed-off-by: cherry77-cloud <1615405@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com>
cherry77-cloud committed
Aug 11, 2026 at 12:49 UTC
65b7662d3fcb773afaf751ab29ac6960a0cf011d
3 files changed
+63
-5
tests/entrypoints/cohere/test_serving_conversion.py
+22
-1
@@ -75,6 +75,7 @@ def _build_chat_completion_response(
75
reasoning: str | None = None,
76
tool_calls: list[dict[str, Any]] | None = None,
77
finish_reason: str | None = "stop",
78
+ stop_reason: int | str | None = None,
79
citations: list[Any] | None = None,
80
usage: dict[str, Any] | None = None,
81
kv_transfer_params: dict[str, Any] | None = None,
@@ -93,7 +94,14 @@ def _build_chat_completion_response(
94
object="chat.completion",
95
created=0,
96
model="m",
96
- choices=[{"index": 0, "message": message, "finish_reason": finish_reason}],
97
+ choices=[
98
+ {
99
+ "index": 0,
100
+ "message": message,
101
+ "finish_reason": finish_reason,
102
+ "stop_reason": stop_reason,
103
+ }
104
+ ],
105
# ``usage`` is a required field on ChatCompletionResponse, but the
106
# production code defensively handles ``None`` -> no usage block;
107
# we round-trip that behavior by post-setting the attribute below.
@@ -137,6 +145,11 @@ class TestMapFinishReason:
145
def test_unknown_reason_defaults_to_complete(self):
146
assert _map_finish_reason("not_a_real_reason") == "COMPLETE"
147
148
+ def test_only_string_stop_reasons_are_stop_sequences(self):
149
+ assert _map_finish_reason("stop", "<END>") == "STOP_SEQUENCE"
150
+ assert _map_finish_reason("stop", 42) == "COMPLETE"
151
+ assert _map_finish_reason("stop", None) == "COMPLETE"
152
+
153
def test_finish_reason_map_is_complete(self):
154
# Sanity check that the lookup table covers all documented states.
155
assert set(_FINISH_REASON_MAP) == {
@@ -1488,6 +1501,14 @@ class TestChatCompletionToV2:
1501
assert v2.message.tool_plan is None
1502
assert v2.usage is None
1503
1504
+ def test_stop_sequence_finish_reason(self):
1505
+ serving = _serving()
1506
+ resp = _build_chat_completion_response(
1507
+ finish_reason="stop", stop_reason="<END>"
1508
+ )
1509
+ v2 = serving._chat_completion_to_v2(resp, _make_request())
1510
+ assert v2.finish_reason == "STOP_SEQUENCE"
1511
+
1512
def test_reasoning_model_keeps_thinking_with_tool_calls(self):
1513
# Reasoning Command models: ``thinking`` block stays in
1514
# ``message.content`` and ``tool_plan`` is left unset, even when
tests/entrypoints/cohere/test_serving_streaming.py
+24
-1
@@ -81,6 +81,7 @@ def _make_chunk(
81
reasoning: str | None = None,
82
tool_calls: list[dict[str, Any]] | None = None,
83
finish_reason: str | None = None,
84
+ stop_reason: int | str | None = None,
85
usage: dict[str, Any] | None = None,
86
omit_choices: bool = False,
87
citations: list[Any] | None = None,
@@ -107,7 +108,12 @@ def _make_chunk(
108
}
109
if not omit_choices:
110
payload["choices"] = [
110
- {"index": 0, "delta": delta, "finish_reason": finish_reason}
111
+ {
112
+ "index": 0,
113
+ "delta": delta,
114
+ "finish_reason": finish_reason,
115
+ "stop_reason": stop_reason,
116
+ }
117
]
118
else:
119
payload["choices"] = []
@@ -652,6 +658,23 @@ class TestChatCompletionStreamToV2:
658
assert end_payload["delta"]["finish_reason"] == "COMPLETE"
659
assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4
660
661
+ @pytest.mark.asyncio
662
+ async def test_stop_sequence_finish_reason(self):
663
+ serving = _serving()
664
+ frames = await _drain(
665
+ serving,
666
+ [
667
+ _make_chunk(role="assistant"),
668
+ _make_chunk(
669
+ finish_reason="stop",
670
+ stop_reason="<END>",
671
+ ),
672
+ _make_chunk(omit_choices=True),
673
+ ],
674
+ )
675
+ end_payload = _parse_event(frames[-2])
676
+ assert end_payload["delta"]["finish_reason"] == "STOP_SEQUENCE"
677
+
678
@pytest.mark.asyncio
679
async def test_text_then_tool_call_closes_text_first(self):
680
serving = _serving(is_reasoning_model=True)
vllm/entrypoints/cohere/serving.py
+17
-3
@@ -155,7 +155,13 @@ _FINISH_REASON_MAP: dict[str | None, CohereFinishReason] = {
155
}
156
157
158
-def _map_finish_reason(reason: str | None) -> CohereFinishReason:
158
+def _map_finish_reason(
159
+ reason: str | None, stop_reason: int | str | None = None
160
+) -> CohereFinishReason:
161
+ # vLLM reports both EOS and matched stop strings as finish_reason="stop".
162
+ # The latter is distinguished by the matched string in stop_reason.
163
+ if reason == "stop" and isinstance(stop_reason, str):
164
+ return "STOP_SEQUENCE"
165
return _FINISH_REASON_MAP.get(reason, "COMPLETE")
166
167
@@ -1006,7 +1012,10 @@ class CohereServingChatV2(OpenAIServingChat):
1012
1013
return CohereChatV2Response(
1014
id=response.id or f"chat_{int(time.time() * 1000)}",
1009
- finish_reason=_map_finish_reason(choice.finish_reason),
1015
+ finish_reason=_map_finish_reason(
1016
+ choice.finish_reason,
1017
+ choice.stop_reason,
1018
+ ),
1019
message=assistant_msg,
1020
usage=usage,
1021
kv_transfer_params=response.kv_transfer_params,
@@ -1328,6 +1337,7 @@ class CohereServingChatV2(OpenAIServingChat):
1337
yield self._build_message_end_event(
1338
chunk_id=chunk.id,
1339
finish_reason=state.finish_reason,
1340
+ stop_reason=state.stop_reason,
1341
usage_chunk=chunk,
1342
)
1343
state.ended = True
@@ -1336,6 +1346,7 @@ class CohereServingChatV2(OpenAIServingChat):
1346
choice = chunk.choices[0]
1347
if choice.finish_reason is not None:
1348
state.finish_reason = choice.finish_reason
1349
+ state.stop_reason = choice.stop_reason
1350
1351
delta = choice.delta
1352
@@ -1391,6 +1402,7 @@ class CohereServingChatV2(OpenAIServingChat):
1402
yield self._build_message_end_event(
1403
chunk_id=state.last_chunk_id,
1404
finish_reason=state.finish_reason,
1405
+ stop_reason=state.stop_reason,
1406
usage_chunk=None,
1407
)
1408
state.ended = True
@@ -1594,10 +1606,11 @@ class CohereServingChatV2(OpenAIServingChat):
1606
self,
1607
chunk_id: str,
1608
finish_reason: str | None,
1609
+ stop_reason: int | str | None = None,
1610
usage_chunk: ChatCompletionStreamResponse | None = None,
1611
) -> str:
1612
delta: dict[str, Any] = {
1600
- "finish_reason": _map_finish_reason(finish_reason),
1613
+ "finish_reason": _map_finish_reason(finish_reason, stop_reason),
1614
}
1615
if usage_chunk is not None and usage_chunk.usage is not None:
1616
prompt = usage_chunk.usage.prompt_tokens
@@ -1659,6 +1672,7 @@ class _StreamState:
1672
self.started: bool = False
1673
self.ended: bool = False
1674
self.finish_reason: str | None = None
1675
+ self.stop_reason: int | str | None = None
1676
self.last_chunk_id: str = ""
1677
self.active_block: ContentBlockType | None = None
1678
self.active_block_index: int | None = None