A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 14:45 UTC
[Feature] Add structured output and effort support to Anthropic Messages API (#42396)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
May 28, 2026 at 20:06 UTC
d692b89c2c599314565f9c1a073e5ec655919302
3 files changed
+78
tests/entrypoints/anthropic/test_messages.py
+38
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import json
5
+
6
import anthropic
7
import pytest
8
import pytest_asyncio
@@ -153,3 +155,39 @@ async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
155
156
async for chunk in resp:
157
print(chunk.model_dump_json())
158
+
159
+
160
+@pytest.mark.asyncio
161
+async def test_anthropic_structured_output(client: anthropic.AsyncAnthropic):
162
+ response = await client.messages.create(
163
+ model="claude-3-7-sonnet-latest",
164
+ max_tokens=1024,
165
+ messages=[
166
+ {
167
+ "role": "user",
168
+ "content": "Extract the key information from this email:"
169
+ "John Smith (john@example.com) is interested in our "
170
+ "Enterprise plan and wants to schedule a demo for next Tuesday at 2pm.",
171
+ }
172
+ ],
173
+ output_config={
174
+ "format": {
175
+ "type": "json_schema",
176
+ "schema": {
177
+ "type": "object",
178
+ "properties": {
179
+ "name": {"type": "string"},
180
+ "email": {"type": "string"},
181
+ "plan_interest": {"type": "string"},
182
+ "demo_requested": {"type": "boolean"},
183
+ },
184
+ "required": ["name", "email", "plan_interest", "demo_requested"],
185
+ "additionalProperties": False,
186
+ },
187
+ }
188
+ },
189
+ )
190
+ print(response.content[0].text)
191
+ json_obj = json.loads(response.content[0].text)
192
+ for key in ["name", "email", "plan_interest", "demo_requested"]:
193
+ assert key in json_obj, f"Missing key in output: {key}"
vllm/entrypoints/anthropic/protocol.py
+15
@@ -100,6 +100,20 @@ class AnthropicToolChoice(BaseModel):
100
return self
101
102
103
+class AnthropicJsonOutputFormat(BaseModel):
104
+ """JSON output format configuration"""
105
+
106
+ json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
107
+ type: Literal["json_schema"] = "json_schema"
108
+
109
+
110
+class AnthropicOutputConfig(BaseModel):
111
+ """Configuration options for the model's output, such as the output format."""
112
+
113
+ effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
114
+ format: AnthropicJsonOutputFormat | None = None
115
+
116
+
117
class AnthropicMessagesRequest(BaseModel):
118
"""Anthropic Messages API request"""
119
@@ -107,6 +121,7 @@ class AnthropicMessagesRequest(BaseModel):
121
messages: list[AnthropicMessage]
122
max_tokens: int
123
metadata: dict[str, Any] | None = None
124
+ output_config: AnthropicOutputConfig | None = None
125
stop_sequences: list[str] | None = None
126
stream: bool | None = False
127
system: str | list[AnthropicContentBlock] | None = None
vllm/entrypoints/anthropic/serving.py
+25
@@ -24,6 +24,7 @@ from vllm.entrypoints.anthropic.protocol import (
24
AnthropicError,
25
AnthropicMessagesRequest,
26
AnthropicMessagesResponse,
27
+ AnthropicOutputConfig,
28
AnthropicStreamEvent,
29
AnthropicUsage,
30
)
@@ -39,6 +40,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
40
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
41
from vllm.entrypoints.openai.engine.protocol import (
42
ErrorResponse,
43
+ JsonSchemaResponseFormat,
44
+ ResponseFormat,
45
StreamOptions,
46
)
47
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -128,6 +131,7 @@ class AnthropicServingMessages(OpenAIServingChat):
131
cls._convert_messages(anthropic_request.messages, openai_messages)
132
req = cls._build_base_request(anthropic_request, openai_messages)
133
cls._handle_streaming_options(req, anthropic_request)
134
+ cls._handle_output_config(req, anthropic_request)
135
cls._convert_tool_choice(anthropic_request, req)
136
cls._convert_tools(anthropic_request, req)
137
return req
@@ -359,6 +363,27 @@ class AnthropicServingMessages(OpenAIServingChat):
363
chat_template_kwargs=anthropic_request.chat_template_kwargs,
364
)
365
366
+ @classmethod
367
+ def _handle_output_config(
368
+ cls,
369
+ req: ChatCompletionRequest,
370
+ anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
371
+ ) -> None:
372
+ """Handle output configuration such as output format and effort"""
373
+ if isinstance(anthropic_request, AnthropicCountTokensRequest):
374
+ return
375
+ output_config: AnthropicOutputConfig | None = anthropic_request.output_config
376
+ if output_config and output_config.format and output_config.format.json_schema:
377
+ req.response_format = ResponseFormat(
378
+ type=output_config.format.type,
379
+ json_schema=JsonSchemaResponseFormat(
380
+ schema=output_config.format.json_schema,
381
+ name=output_config.format.type,
382
+ ),
383
+ )
384
+ if output_config and output_config.effort is not None:
385
+ req.reasoning_effort = output_config.effort
386
+
387
@classmethod
388
def _handle_streaming_options(
389
cls,