| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Pydantic models for Anthropic API protocol.""" |
| 4 | |
| 5 | import time |
| 6 | from typing import Annotated, Any, Literal |
| 7 | |
| 8 | from pydantic import BaseModel, Field, field_validator, model_validator |
| 9 | |
| 10 | import vllm.envs as envs |
| 11 | |
| 12 | |
| 13 | class AnthropicError(BaseModel): |
| 14 | """Error structure for Anthropic API.""" |
| 15 | |
| 16 | type: str |
| 17 | message: str |
| 18 | |
| 19 | |
| 20 | class AnthropicErrorResponse(BaseModel): |
| 21 | """Error response structure for Anthropic API.""" |
| 22 | |
| 23 | type: Literal["error"] = "error" |
| 24 | error: AnthropicError |
| 25 | |
| 26 | |
| 27 | class AnthropicUsage(BaseModel): |
| 28 | """Token usage information.""" |
| 29 | |
| 30 | input_tokens: int |
| 31 | output_tokens: int |
| 32 | cache_creation_input_tokens: int | None = None |
| 33 | cache_read_input_tokens: int | None = None |
| 34 | |
| 35 | |
| 36 | class AnthropicContentBlock(BaseModel): |
| 37 | """Content block in message.""" |
| 38 | |
| 39 | type: Literal[ |
| 40 | "text", |
| 41 | "image", |
| 42 | "tool_use", |
| 43 | "tool_result", |
| 44 | "tool_reference", |
| 45 | "thinking", |
| 46 | "redacted_thinking", |
| 47 | "tool_addition", |
| 48 | "tool_removal", |
| 49 | ] |
| 50 | text: str | None = None |
| 51 | # For image content |
| 52 | source: dict[str, Any] | None = None |
| 53 | # For tool use/result |
| 54 | id: str | None = None |
| 55 | tool_use_id: str | None = None |
| 56 | name: str | None = None |
| 57 | input: dict[str, Any] | None = None |
| 58 | content: str | list[dict[str, Any]] | None = None |
| 59 | is_error: bool | None = None |
| 60 | # For tool_reference content |
| 61 | tool_name: str | None = None |
| 62 | # For thinking content |
| 63 | thinking: str | None = None |
| 64 | signature: str | None = None |
| 65 | # For redacted thinking content (safety-filtered by the API) |
| 66 | data: str | None = None |
| 67 | # For tool_addition/tool_removal content |
| 68 | tool: "AnthropicToolChange | None" = None |
| 69 | |
| 70 | @model_validator(mode="after") |
| 71 | def validate_tool_change(self) -> "AnthropicContentBlock": |
| 72 | if self.type not in ("tool_addition", "tool_removal"): |
| 73 | return self |
| 74 | if self.tool is None: |
| 75 | raise ValueError(f"tool is required for {self.type} blocks") |
| 76 | if self.type == "tool_removal" and self.tool.type != "tool_reference": |
| 77 | raise ValueError("tool_removal only accepts a tool_reference") |
| 78 | return self |
| 79 | |
| 80 | |
| 81 | def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None: |
| 82 | if content is None or isinstance(content, str): |
| 83 | return |
| 84 | for block in content: |
| 85 | if block.type in ("tool_addition", "tool_removal"): |
| 86 | raise ValueError( |
| 87 | f'{block.type} blocks are only allowed in messages with role "system"' |
| 88 | ) |
| 89 | |
| 90 | |
| 91 | class AnthropicMessage(BaseModel): |
| 92 | """Message structure.""" |
| 93 | |
| 94 | role: Literal["user", "assistant", "system"] |
| 95 | content: str | list[AnthropicContentBlock] |
| 96 | |
| 97 | @model_validator(mode="after") |
| 98 | def validate_tool_change_role(self) -> "AnthropicMessage": |
| 99 | if self.role != "system": |
| 100 | _reject_tool_changes(self.content) |
| 101 | return self |
| 102 | |
| 103 | |
| 104 | class AnthropicTool(BaseModel): |
| 105 | """Tool definition.""" |
| 106 | |
| 107 | name: str |
| 108 | description: str | None = None |
| 109 | input_schema: dict[str, Any] |
| 110 | strict: bool | None = None |
| 111 | defer_loading: bool | None = None |
| 112 | |
| 113 | @field_validator("input_schema") |
| 114 | @classmethod |
| 115 | def validate_input_schema(cls, v): |
| 116 | if not isinstance(v, dict): |
| 117 | raise ValueError("input_schema must be a dictionary") |
| 118 | if "type" not in v: |
| 119 | v["type"] = "object" # Default to object type |
| 120 | return v |
| 121 | |
| 122 | |
| 123 | class AnthropicToolChangeReference(BaseModel): |
| 124 | """A tool named by a tool_addition or tool_removal block.""" |
| 125 | |
| 126 | type: Literal["tool_reference"] |
| 127 | name: str |
| 128 | |
| 129 | |
| 130 | class AnthropicToolChangeDefinition(BaseModel): |
| 131 | """A tool defined by value in a tool_addition block.""" |
| 132 | |
| 133 | type: Literal["tool_definition"] |
| 134 | definition: AnthropicTool |
| 135 | |
| 136 | |
| 137 | AnthropicToolChange = Annotated[ |
| 138 | AnthropicToolChangeReference | AnthropicToolChangeDefinition, |
| 139 | Field(discriminator="type"), |
| 140 | ] |
| 141 | |
| 142 | |
| 143 | class AnthropicToolChoice(BaseModel): |
| 144 | """Tool Choice definition.""" |
| 145 | |
| 146 | type: Literal["auto", "any", "tool", "none"] |
| 147 | name: str | None = None |
| 148 | disable_parallel_tool_use: bool | None = None |
| 149 | |
| 150 | @model_validator(mode="after") |
| 151 | def validate_name_required_for_tool(self) -> "AnthropicToolChoice": |
| 152 | if self.type == "tool" and not self.name: |
| 153 | raise ValueError("tool_choice.name is required when type is 'tool'") |
| 154 | return self |
| 155 | |
| 156 | |
| 157 | class AnthropicJsonOutputFormat(BaseModel): |
| 158 | """JSON output format configuration.""" |
| 159 | |
| 160 | json_schema: dict[str, Any] | None = Field(default=None, alias="schema") |
| 161 | type: Literal["json_schema"] = "json_schema" |
| 162 | |
| 163 | |
| 164 | AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"] |
| 165 | # reasoning_effort sent to the chat template for thinking.type=disabled. |
| 166 | AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort] |
| 167 | AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort] |
| 168 | AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"] |
| 169 | |
| 170 | |
| 171 | class AnthropicOutputConfig(BaseModel): |
| 172 | """Configuration options for the model's output, such as the output format.""" |
| 173 | |
| 174 | effort: AnthropicEffort | None = None |
| 175 | format: AnthropicJsonOutputFormat | None = None |
| 176 | |
| 177 | |
| 178 | class AnthropicThinkingConfigEnabled(BaseModel): |
| 179 | """Extended thinking with a fixed token budget. |
| 180 | |
| 181 | ``display`` is accepted but ignored: reasoning is always returned. |
| 182 | """ |
| 183 | |
| 184 | type: Literal["enabled"] |
| 185 | budget_tokens: int = Field(ge=1024) |
| 186 | display: AnthropicThinkingDisplay | None = None |
| 187 | |
| 188 | |
| 189 | class AnthropicThinkingConfigDisabled(BaseModel): |
| 190 | type: Literal["disabled"] |
| 191 | |
| 192 | |
| 193 | class AnthropicThinkingConfigAdaptive(BaseModel): |
| 194 | """Extended thinking whose depth the model chooses. |
| 195 | |
| 196 | ``display`` is accepted but ignored: reasoning is always returned. |
| 197 | """ |
| 198 | |
| 199 | type: Literal["adaptive"] |
| 200 | display: AnthropicThinkingDisplay | None = None |
| 201 | |
| 202 | |
| 203 | AnthropicThinkingConfig = Annotated[ |
| 204 | AnthropicThinkingConfigEnabled |
| 205 | | AnthropicThinkingConfigDisabled |
| 206 | | AnthropicThinkingConfigAdaptive, |
| 207 | Field(discriminator="type"), |
| 208 | ] |
| 209 | |
| 210 | |
| 211 | class AnthropicMessagesRequest(BaseModel): |
| 212 | """Anthropic Messages API request.""" |
| 213 | |
| 214 | model: str |
| 215 | messages: list[AnthropicMessage] |
| 216 | max_tokens: int |
| 217 | metadata: dict[str, Any] | None = None |
| 218 | output_config: AnthropicOutputConfig | None = None |
| 219 | thinking: AnthropicThinkingConfig | None = None |
| 220 | stop_sequences: ( |
| 221 | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None |
| 222 | ) = None |
| 223 | stream: bool | None = False |
| 224 | system: str | list[AnthropicContentBlock] | None = None |
| 225 | temperature: float | None = None |
| 226 | tool_choice: AnthropicToolChoice | None = None |
| 227 | tools: list[AnthropicTool] | None = None |
| 228 | top_k: int | None = None |
| 229 | top_p: float | None = None |
| 230 | |
| 231 | # vLLM-specific fields that are not in Anthropic spec |
| 232 | cache_salt: str | None = Field( |
| 233 | default=None, |
| 234 | min_length=1, |
| 235 | max_length=1024, |
| 236 | description=( |
| 237 | "If specified, the prefix cache will be salted with the provided " |
| 238 | "string to prevent an attacker to guess prompts in multi-user " |
| 239 | "environments. The salt should be random, protected from " |
| 240 | "access by 3rd parties, and long enough to be " |
| 241 | "unpredictable (e.g., 43 characters base64-encoded, corresponding " |
| 242 | "to 256 bit)." |
| 243 | ), |
| 244 | ) |
| 245 | kv_transfer_params: dict[str, Any] | None = Field( |
| 246 | default=None, |
| 247 | description="KVTransfer parameters used for disaggregated serving.", |
| 248 | ) |
| 249 | ec_transfer_params: dict[str, Any] | None = Field( |
| 250 | default=None, |
| 251 | description=( |
| 252 | "ECTransfer parameters used for encoder-cache disaggregated serving." |
| 253 | ), |
| 254 | ) |
| 255 | vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field( |
| 256 | default=None, |
| 257 | description=( |
| 258 | "Additional request parameters with (list of) string or " |
| 259 | "numeric values, used by custom extensions." |
| 260 | ), |
| 261 | ) |
| 262 | chat_template_kwargs: dict[str, Any] | None = Field( |
| 263 | default=None, |
| 264 | description=( |
| 265 | "Additional keyword args to pass to the chat template renderer. " |
| 266 | "Will be accessible by the template." |
| 267 | ), |
| 268 | ) |
| 269 | |
| 270 | @field_validator("model") |
| 271 | @classmethod |
| 272 | def validate_model(cls, v): |
| 273 | if not v: |
| 274 | raise ValueError("Model is required") |
| 275 | return v |
| 276 | |
| 277 | @field_validator("max_tokens") |
| 278 | @classmethod |
| 279 | def validate_max_tokens(cls, v): |
| 280 | if v <= 0: |
| 281 | raise ValueError("max_tokens must be positive") |
| 282 | return v |
| 283 | |
| 284 | @field_validator("system") |
| 285 | @classmethod |
| 286 | def validate_system(cls, v): |
| 287 | _reject_tool_changes(v) |
| 288 | return v |
| 289 | |
| 290 | @model_validator(mode="after") |
| 291 | def validate_thinking_budget(self) -> "AnthropicMessagesRequest": |
| 292 | # P/D prefill legs are sent with max_tokens=1 and never decode; the |
| 293 | # decode leg carries the client's max_tokens and is still checked. |
| 294 | if self.kv_transfer_params and self.kv_transfer_params.get("do_remote_decode"): |
| 295 | return self |
| 296 | if ( |
| 297 | isinstance(self.thinking, AnthropicThinkingConfigEnabled) |
| 298 | and self.thinking.budget_tokens >= self.max_tokens |
| 299 | ): |
| 300 | raise ValueError("thinking.budget_tokens must be less than max_tokens") |
| 301 | return self |
| 302 | |
| 303 | |
| 304 | class AnthropicDelta(BaseModel): |
| 305 | """Delta for streaming responses.""" |
| 306 | |
| 307 | type: ( |
| 308 | Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"] |
| 309 | | None |
| 310 | ) = None |
| 311 | text: str | None = None |
| 312 | thinking: str | None = None |
| 313 | partial_json: str | None = None |
| 314 | signature: str | None = None |
| 315 | |
| 316 | # Message delta |
| 317 | stop_reason: ( |
| 318 | Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None |
| 319 | ) = None |
| 320 | stop_sequence: str | None = None |
| 321 | |
| 322 | |
| 323 | class AnthropicStreamEvent(BaseModel): |
| 324 | """Streaming event.""" |
| 325 | |
| 326 | type: Literal[ |
| 327 | "message_start", |
| 328 | "message_delta", |
| 329 | "message_stop", |
| 330 | "content_block_start", |
| 331 | "content_block_delta", |
| 332 | "content_block_stop", |
| 333 | "ping", |
| 334 | "error", |
| 335 | ] |
| 336 | message: "AnthropicMessagesResponse | None" = None |
| 337 | delta: AnthropicDelta | None = None |
| 338 | content_block: AnthropicContentBlock | None = None |
| 339 | index: int | None = None |
| 340 | error: AnthropicError | None = None |
| 341 | usage: AnthropicUsage | None = None |
| 342 | |
| 343 | |
| 344 | class AnthropicMessagesResponse(BaseModel): |
| 345 | """Anthropic Messages API response.""" |
| 346 | |
| 347 | id: str |
| 348 | type: Literal["message"] = "message" |
| 349 | role: Literal["assistant"] = "assistant" |
| 350 | content: list[AnthropicContentBlock] |
| 351 | model: str |
| 352 | stop_reason: ( |
| 353 | Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None |
| 354 | ) = None |
| 355 | stop_sequence: str | None = None |
| 356 | usage: AnthropicUsage | None = None |
| 357 | |
| 358 | # vLLM-specific fields that are not in Anthropic spec |
| 359 | kv_transfer_params: dict[str, Any] | None = Field( |
| 360 | default=None, description="KVTransfer parameters." |
| 361 | ) |
| 362 | ec_transfer_params: dict[str, Any] | None = Field( |
| 363 | default=None, description="ECTransfer parameters." |
| 364 | ) |
| 365 | |
| 366 | def model_post_init(self, __context): |
| 367 | if not self.id: |
| 368 | self.id = f"msg_{int(time.time() * 1000)}" |
| 369 | |
| 370 | |
| 371 | class AnthropicContextManagement(BaseModel): |
| 372 | """Context management information for token counting.""" |
| 373 | |
| 374 | original_input_tokens: int |
| 375 | |
| 376 | |
| 377 | class AnthropicCountTokensRequest(BaseModel): |
| 378 | """Anthropic messages.count_tokens request.""" |
| 379 | |
| 380 | model: str |
| 381 | messages: list[AnthropicMessage] |
| 382 | system: str | list[AnthropicContentBlock] | None = None |
| 383 | tool_choice: AnthropicToolChoice | None = None |
| 384 | tools: list[AnthropicTool] | None = None |
| 385 | |
| 386 | # vLLM-specific fields that are not in Anthropic spec |
| 387 | chat_template_kwargs: dict[str, Any] | None = Field( |
| 388 | default=None, |
| 389 | description=( |
| 390 | "Additional keyword args to pass to the chat template renderer. " |
| 391 | "Will be accessible by the template." |
| 392 | ), |
| 393 | ) |
| 394 | |
| 395 | @field_validator("model") |
| 396 | @classmethod |
| 397 | def validate_model(cls, v): |
| 398 | if not v: |
| 399 | raise ValueError("Model is required") |
| 400 | return v |
| 401 | |
| 402 | @field_validator("system") |
| 403 | @classmethod |
| 404 | def validate_system(cls, v): |
| 405 | _reject_tool_changes(v) |
| 406 | return v |
| 407 | |
| 408 | |
| 409 | class AnthropicCountTokensResponse(BaseModel): |
| 410 | """Anthropic messages.count_tokens response.""" |
| 411 | |
| 412 | input_tokens: int |
| 413 | context_management: AnthropicContextManagement | None = None |