[Frontend] Handle Disable Thinking in /v1/messages (#58613)
Signed-off-by: jryberg <johan.ryberg@security.ntt> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: jryberg <johan.ryberg@security.ntt> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Robert Shaw committed
Sep 25, 2026 at 00:52 UTC
5a2ed2e7ee1063810f3ffe4a4c8781219e3eddb4
6 files changed
+324
-3
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+174
-1
@@ -15,8 +15,9 @@ Also covers cache usage computation in ``_build_anthropic_usage``.
15
import json
16
from argparse import Namespace
17
from http import HTTPStatus
18
+from types import SimpleNamespace
19
from typing import Annotated
19
-from unittest.mock import MagicMock
20
+from unittest.mock import AsyncMock, MagicMock
21
22
import pytest
23
from fastapi import FastAPI
@@ -1727,3 +1728,175 @@ class TestClientErrorResponses:
1728
1729
assert response.status_code == HTTPStatus.BAD_REQUEST
1730
assert response.json()["error"]["type"] == "BadRequestError"
1731
+
1732
+
1733
+# ======================================================================
1734
+# thinking configuration pass-through
1735
+# ======================================================================
1736
+
1737
+
1738
+class TestThinkingConfig:
1739
+ def test_absent_thinking_leaves_reasoning_untouched(self):
1740
+ """Requests without `thinking` must convert exactly as before."""
1741
+ request = _make_request([{"role": "user", "content": "Hello"}])
1742
+
1743
+ result = _convert(request)
1744
+ assert result.reasoning_effort is None
1745
+ assert result.thinking_token_budget is None
1746
+ assert result.include_reasoning is True
1747
+
1748
+ def test_disabled_clears_reasoning_effort(self):
1749
+ """`disabled` maps to reasoning_effort="none", which is what clears
1750
+ enable_thinking for templates that honor it."""
1751
+ request = _make_request(
1752
+ [{"role": "user", "content": "Hello"}],
1753
+ thinking={"type": "disabled"},
1754
+ )
1755
+
1756
+ result = _convert(request)
1757
+ assert result.reasoning_effort == "none"
1758
+
1759
+ def test_disabled_overrides_output_config_effort(self):
1760
+ """`thinking` is applied after output_config so an explicit opt-out wins
1761
+ over an inherited effort ceiling."""
1762
+ request = _make_request(
1763
+ [{"role": "user", "content": "Hello"}],
1764
+ output_config={"effort": "high"},
1765
+ thinking={"type": "disabled"},
1766
+ )
1767
+
1768
+ result = _convert(request)
1769
+ assert result.reasoning_effort == "none"
1770
+
1771
+ def test_enabled_sets_thinking_token_budget(self):
1772
+ request = AnthropicMessagesRequest(
1773
+ model="test-model",
1774
+ max_tokens=4096,
1775
+ messages=[{"role": "user", "content": "Hello"}],
1776
+ thinking={"type": "enabled", "budget_tokens": 2048},
1777
+ )
1778
+
1779
+ result = _convert(request)
1780
+ assert result.thinking_token_budget == 2048
1781
+ assert result.reasoning_effort is None
1782
+
1783
+ @pytest.mark.parametrize(
1784
+ "thinking",
1785
+ [
1786
+ pytest.param({"budget_tokens": 2048}, id="missing-type"),
1787
+ pytest.param({"type": "enabled"}, id="enabled-missing-budget"),
1788
+ pytest.param(
1789
+ {"type": "enabled", "budget_tokens": 1023}, id="budget-below-1024"
1790
+ ),
1791
+ pytest.param(
1792
+ {"type": "enabled", "budget_tokens": 4096}, id="budget-not-below-max"
1793
+ ),
1794
+ pytest.param({"type": "adaptive", "display": "full"}, id="bad-display"),
1795
+ ],
1796
+ )
1797
+ def test_rejects_invalid_thinking(self, thinking):
1798
+ """Mirror the Anthropic API's BetaThinkingConfigParam constraints."""
1799
+ with pytest.raises(ValidationError):
1800
+ AnthropicMessagesRequest(
1801
+ model="test-model",
1802
+ max_tokens=4096,
1803
+ messages=[{"role": "user", "content": "Hello"}],
1804
+ thinking=thinking,
1805
+ )
1806
+
1807
+ def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self):
1808
+ """`adaptive` lets the model choose depth, so only the ceiling from
1809
+ output_config.effort should survive."""
1810
+ request = _make_request(
1811
+ [{"role": "user", "content": "Hello"}],
1812
+ output_config={"effort": "low"},
1813
+ thinking={"type": "adaptive"},
1814
+ )
1815
+
1816
+ result = _convert(request)
1817
+ assert result.reasoning_effort == "low"
1818
+ assert result.thinking_token_budget is None
1819
+
1820
+ def test_disabled_uses_configured_effort(self):
1821
+ """Models that always think (GLM-5.3) or reject "none" (Harmony) are
1822
+ served with a low effort instead."""
1823
+ request = _make_request(
1824
+ [{"role": "user", "content": "Hello"}],
1825
+ thinking={"type": "disabled"},
1826
+ )
1827
+
1828
+ result = _convert(request, disabled_thinking_effort="low")
1829
+ assert result.reasoning_effort == "low"
1830
+
1831
+ @pytest.mark.parametrize("display", ["omitted", "summarized", "updates"])
1832
+ def test_display_keeps_reasoning_included(self, display):
1833
+ """Suppressing reasoning would mark it ended for structured outputs and
1834
+ drop it from multi-turn history, so `display` is ignored."""
1835
+ request = _make_request(
1836
+ [{"role": "user", "content": "Hello"}],
1837
+ thinking={"type": "adaptive", "display": display},
1838
+ )
1839
+
1840
+ result = _convert(request)
1841
+ assert result.include_reasoning is True
1842
+
1843
+ def test_claude_code_payload(self):
1844
+ """The combination Claude Code sends on every request: an effort ceiling
1845
+ plus adaptive thinking with reasoning display omitted."""
1846
+ request = _make_request(
1847
+ [{"role": "user", "content": "Hello"}],
1848
+ output_config={"effort": "high"},
1849
+ thinking={"type": "adaptive", "display": "omitted"},
1850
+ )
1851
+
1852
+ result = _convert(request)
1853
+ assert result.reasoning_effort == "high"
1854
+ assert result.include_reasoning is True
1855
+ assert result.thinking_token_budget is None
1856
+
1857
+
1858
+class TestProbeDisabledThinkingEffort:
1859
+ """``auto`` falls back to ``low`` when ``none`` cannot turn thinking off."""
1860
+
1861
+ @staticmethod
1862
+ async def _probe(render):
1863
+ obj = MagicMock(spec=AnthropicServingMessages)
1864
+ obj.online_renderer = MagicMock()
1865
+ obj.online_renderer.render_chat = AsyncMock(
1866
+ side_effect=lambda req: ([], [render(req.reasoning_effort)])
1867
+ )
1868
+ obj._extract_prompt_components = lambda engine_input: SimpleNamespace(
1869
+ token_ids=engine_input, text=None
1870
+ )
1871
+ obj._render_probe_prompt = (
1872
+ AnthropicServingMessages._render_probe_prompt.__get__(obj)
1873
+ )
1874
+ return await AnthropicServingMessages._probe_disabled_thinking_effort(obj)
1875
+
1876
+ @staticmethod
1877
+ def _reject_none(effort):
1878
+ """Harmony (gpt-oss) raises on ``none``."""
1879
+ if effort == "none":
1880
+ raise ValueError(f"unsupported {effort=}")
1881
+ return [1]
1882
+
1883
+ @staticmethod
1884
+ def _none_as_max(effort):
1885
+ """GLM-5.3 treats efforts other than low/high as max."""
1886
+ return [1, {"low": 0, "high": 1}.get(effort, 2)]
1887
+
1888
+ @pytest.mark.asyncio
1889
+ async def test_template_honors_none(self):
1890
+ assert await self._probe(lambda effort: [1, int(effort == "none")]) == "none"
1891
+
1892
+ @pytest.mark.asyncio
1893
+ async def test_template_ignores_effort(self):
1894
+ assert await self._probe(lambda effort: [1]) == "low"
1895
+
1896
+ @pytest.mark.asyncio
1897
+ async def test_template_renders_none_as_thinking_effort(self):
1898
+ assert await self._probe(self._none_as_max) == "low"
1899
+
1900
+ @pytest.mark.asyncio
1901
+ async def test_renderer_rejects_none(self):
1902
+ assert await self._probe(self._reject_none) == "low"
tests/entrypoints/anthropic/test_protocol_exports.py
+2
@@ -19,6 +19,7 @@ from vllm.entrypoints.anthropic.protocol import (
19
AnthropicMessagesResponse,
20
AnthropicOutputConfig,
21
AnthropicStreamEvent,
22
+ AnthropicThinkingConfig,
23
AnthropicUsage,
24
)
25
@@ -35,6 +36,7 @@ SERVING_PROTOCOL_EXPORTS = (
36
AnthropicMessagesResponse,
37
AnthropicOutputConfig,
38
AnthropicStreamEvent,
39
+ AnthropicThinkingConfig,
40
AnthropicUsage,
41
)
42
vllm/entrypoints/anthropic/protocol.py
+51
-1
@@ -111,13 +111,53 @@ class AnthropicJsonOutputFormat(BaseModel):
111
type: Literal["json_schema"] = "json_schema"
112
113
114
+AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"]
115
+# reasoning_effort sent to the chat template for thinking.type=disabled.
116
+AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort]
117
+AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort]
118
+AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"]
119
+
120
+
121
class AnthropicOutputConfig(BaseModel):
122
"""Configuration options for the model's output, such as the output format."""
123
117
- effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
124
+ effort: AnthropicEffort | None = None
125
format: AnthropicJsonOutputFormat | None = None
126
127
128
+class AnthropicThinkingConfigEnabled(BaseModel):
129
+ """Extended thinking with a fixed token budget.
130
+
131
+ ``display`` is accepted but ignored: reasoning is always returned.
132
+ """
133
+
134
+ type: Literal["enabled"]
135
+ budget_tokens: int = Field(ge=1024)
136
+ display: AnthropicThinkingDisplay | None = None
137
+
138
+
139
+class AnthropicThinkingConfigDisabled(BaseModel):
140
+ type: Literal["disabled"]
141
+
142
+
143
+class AnthropicThinkingConfigAdaptive(BaseModel):
144
+ """Extended thinking whose depth the model chooses.
145
+
146
+ ``display`` is accepted but ignored: reasoning is always returned.
147
+ """
148
+
149
+ type: Literal["adaptive"]
150
+ display: AnthropicThinkingDisplay | None = None
151
+
152
+
153
+AnthropicThinkingConfig = Annotated[
154
+ AnthropicThinkingConfigEnabled
155
+ | AnthropicThinkingConfigDisabled
156
+ | AnthropicThinkingConfigAdaptive,
157
+ Field(discriminator="type"),
158
+]
159
+
160
+
161
class AnthropicMessagesRequest(BaseModel):
162
"""Anthropic Messages API request."""
163
@@ -126,6 +166,7 @@ class AnthropicMessagesRequest(BaseModel):
166
max_tokens: int
167
metadata: dict[str, Any] | None = None
168
output_config: AnthropicOutputConfig | None = None
169
+ thinking: AnthropicThinkingConfig | None = None
170
stop_sequences: (
171
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
172
) = None
@@ -190,6 +231,15 @@ class AnthropicMessagesRequest(BaseModel):
231
raise ValueError("max_tokens must be positive")
232
return v
233
234
+ @model_validator(mode="after")
235
+ def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
236
+ if (
237
+ isinstance(self.thinking, AnthropicThinkingConfigEnabled)
238
+ and self.thinking.budget_tokens >= self.max_tokens
239
+ ):
240
+ raise ValueError("thinking.budget_tokens must be less than max_tokens")
241
+ return self
242
+
243
244
class AnthropicDelta(BaseModel):
245
"""Delta for streaming responses."""
vllm/entrypoints/anthropic/serving.py
+86
-1
@@ -10,7 +10,7 @@ import logging
10
import time
11
import uuid
12
from collections.abc import AsyncGenerator
13
-from typing import Any
13
+from typing import Any, get_args
14
15
import jinja2
16
from fastapi import Request
@@ -22,11 +22,15 @@ from vllm.entrypoints.anthropic.protocol import (
22
AnthropicCountTokensRequest,
23
AnthropicCountTokensResponse,
24
AnthropicDelta,
25
+ AnthropicDisabledThinkingEffort,
26
+ AnthropicDisabledThinkingEffortOption,
27
+ AnthropicEffort,
28
AnthropicError,
29
AnthropicMessagesRequest,
30
AnthropicMessagesResponse,
31
AnthropicOutputConfig,
32
AnthropicStreamEvent,
33
+ AnthropicThinkingConfig,
34
AnthropicUsage,
35
)
36
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
@@ -116,6 +120,7 @@ class AnthropicServingMessages(OpenAIServingChat):
120
enable_prompt_tokens_details: bool = False,
121
enable_force_include_usage: bool = False,
122
default_chat_template_kwargs: dict[str, Any] | None = None,
123
+ disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto",
124
):
125
super().__init__(
126
engine_client=engine_client,
@@ -140,6 +145,53 @@ class AnthropicServingMessages(OpenAIServingChat):
145
"tool_calls": "tool_use",
146
}
147
self._merge_inline_system = self._detect_merge_inline_system(chat_template)
148
+ # Resolved lazily from the renderer when "auto".
149
+ self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
150
+ None if disabled_thinking_effort == "auto" else disabled_thinking_effort
151
+ )
152
+
153
+ async def _get_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort:
154
+ if self._disabled_thinking_effort is None:
155
+ self._disabled_thinking_effort = (
156
+ await self._probe_disabled_thinking_effort()
157
+ )
158
+ logger.info(
159
+ "Anthropic thinking.type=disabled maps to reasoning_effort=%r",
160
+ self._disabled_thinking_effort,
161
+ )
162
+ return self._disabled_thinking_effort
163
+
164
+ async def _probe_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort:
165
+ """Use ``low`` if the renderer rejects or ignores ``none``.
166
+
167
+ ``none`` is ignored when it renders the same prompt as a thinking
168
+ effort, e.g. GLM-5.3 treats unknown efforts as ``max``.
169
+ """
170
+ none_prompt = await self._render_probe_prompt("none")
171
+ if none_prompt is None:
172
+ return "low"
173
+ for effort in get_args(AnthropicEffort):
174
+ if await self._render_probe_prompt(effort) == none_prompt:
175
+ return "low"
176
+ return "none"
177
+
178
+ async def _render_probe_prompt(
179
+ self, effort: AnthropicDisabledThinkingEffort
180
+ ) -> tuple[list[int] | None, str | None] | None:
181
+ request = ChatCompletionRequest(
182
+ messages=[{"role": "user", "content": "Hi"}],
183
+ reasoning_effort=effort,
184
+ )
185
+ try:
186
+ result = await self.online_renderer.render_chat(request)
187
+ except Exception:
188
+ logger.debug("Rendering reasoning_effort=%r failed", effort, exc_info=True)
189
+ return None
190
+ if isinstance(result, ErrorResponse):
191
+ return None
192
+ _, (engine_input,) = result
193
+ components = self._extract_prompt_components(engine_input)
194
+ return components.token_ids, components.text
195
196
@staticmethod
197
def _detect_merge_inline_system(chat_template: str | None) -> bool:
@@ -197,6 +249,7 @@ class AnthropicServingMessages(OpenAIServingChat):
249
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
250
*,
251
merge_inline_system: bool = False,
252
+ disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none",
253
) -> ChatCompletionRequest:
254
"""Convert Anthropic message format to OpenAI format."""
255
openai_messages: list[dict[str, Any]] = []
@@ -214,6 +267,7 @@ class AnthropicServingMessages(OpenAIServingChat):
267
req = cls._build_base_request(anthropic_request, openai_messages)
268
cls._handle_streaming_options(req, anthropic_request)
269
cls._handle_output_config(req, anthropic_request)
270
+ cls._handle_thinking(req, anthropic_request, disabled_thinking_effort)
271
cls._convert_tool_choice(anthropic_request, req)
272
cls._convert_tools(anthropic_request, req)
273
return req
@@ -494,6 +548,33 @@ class AnthropicServingMessages(OpenAIServingChat):
548
chat_template_kwargs=anthropic_request.chat_template_kwargs,
549
)
550
551
+ @classmethod
552
+ def _handle_thinking(
553
+ cls,
554
+ req: ChatCompletionRequest,
555
+ anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
556
+ disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none",
557
+ ) -> None:
558
+ """Handle extended-thinking configuration.
559
+
560
+ ``display`` is intentionally ignored: suppressing reasoning would mark
561
+ it ended for structured outputs and drop it from multi-turn history.
562
+ """
563
+ if isinstance(anthropic_request, AnthropicCountTokensRequest):
564
+ return
565
+ thinking: AnthropicThinkingConfig | None = anthropic_request.thinking
566
+ if thinking is None:
567
+ return
568
+
569
+ if thinking.type == "disabled":
570
+ # "none" clears enable_thinking for templates that honor it; models
571
+ # that cannot disable thinking are configured with a low effort.
572
+ req.reasoning_effort = disabled_thinking_effort
573
+ elif thinking.type == "enabled":
574
+ req.thinking_token_budget = thinking.budget_tokens
575
+ # "adaptive" pins nothing: the model chooses depth beneath the ceiling
576
+ # already set from output_config.effort.
577
+
578
@classmethod
579
def _handle_output_config(
580
cls,
@@ -602,9 +683,13 @@ class AnthropicServingMessages(OpenAIServingChat):
683
"""
684
if logger.isEnabledFor(logging.DEBUG):
685
logger.debug("Received messages request %s", request.model_dump_json())
686
+ disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none"
687
+ if request.thinking is not None and request.thinking.type == "disabled":
688
+ disabled_thinking_effort = await self._get_disabled_thinking_effort()
689
chat_req = self.to_chat_completion_request(
690
request,
691
merge_inline_system=self._merge_inline_system,
692
+ disabled_thinking_effort=disabled_thinking_effort,
693
)
694
if logger.isEnabledFor(logging.DEBUG):
695
logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json())
vllm/entrypoints/generate/api_router.py
+1
@@ -185,6 +185,7 @@ async def init_generate_state(
185
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
186
enable_force_include_usage=args.enable_force_include_usage,
187
default_chat_template_kwargs=default_chat_template_kwargs,
188
+ disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
189
)
190
if "generate" in supported_tasks
191
else None
vllm/entrypoints/launchers/cli_args.py
+10
@@ -14,6 +14,9 @@ from typing import Any, Literal
14
import vllm.envs as envs
15
from vllm.config import config
16
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
17
+from vllm.entrypoints.anthropic.protocol import (
18
+ AnthropicDisabledThinkingEffortOption,
19
+)
20
from vllm.entrypoints.chat_utils import (
21
ChatTemplateContentFormatOption,
22
validate_chat_template,
@@ -181,6 +184,13 @@ class BaseFrontendArgs:
184
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
185
explicit request-level ``chat_template_kwargs.cohere_format`` takes
186
priority."""
187
+ anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto"
188
+ """Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for
189
+ requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking
190
+ off for models that support it; ``low`` suits models that always think
191
+ (e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses
192
+ ``low`` when the renderer rejects ``none`` or renders it the same as a
193
+ thinking effort, and ``none`` otherwise."""
194
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
195
"""If set to True, log the stack trace of error responses"""
196
tokens_only: bool = False