| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for Anthropic-to-OpenAI request conversion. |
| 4 | |
| 5 | Tests the image source handling and tool_result content parsing in |
| 6 | AnthropicServingMessages.to_chat_completion_request(). |
| 7 | |
| 8 | Also covers extended-thinking edge cases such as ``redacted_thinking`` |
| 9 | blocks echoed back by Anthropic clients, and streaming conversion in |
| 10 | ``message_stream_converter``. |
| 11 | |
| 12 | Also covers cache usage computation in ``_build_anthropic_usage``. |
| 13 | """ |
| 14 | |
| 15 | import json |
| 16 | from argparse import Namespace |
| 17 | from http import HTTPStatus |
| 18 | from types import SimpleNamespace |
| 19 | from typing import Annotated |
| 20 | from unittest.mock import AsyncMock, MagicMock |
| 21 | |
| 22 | import pytest |
| 23 | from fastapi import FastAPI |
| 24 | from fastapi.exceptions import RequestValidationError |
| 25 | from fastapi.testclient import TestClient |
| 26 | from pydantic import BaseModel, Field, ValidationError |
| 27 | |
| 28 | from vllm.entrypoints.anthropic.api_router import attach_router |
| 29 | from vllm.entrypoints.anthropic.protocol import ( |
| 30 | AnthropicCountTokensRequest, |
| 31 | AnthropicMessagesRequest, |
| 32 | ) |
| 33 | from vllm.entrypoints.anthropic.serving import ( |
| 34 | AnthropicServingMessages, |
| 35 | _build_anthropic_usage, |
| 36 | ) |
| 37 | from vllm.entrypoints.generate.base.protocol import ( |
| 38 | DeltaFunctionCall, |
| 39 | DeltaMessage, |
| 40 | DeltaToolCall, |
| 41 | FunctionCall, |
| 42 | ToolCall, |
| 43 | ) |
| 44 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 45 | ChatCompletionResponse, |
| 46 | ChatCompletionResponseChoice, |
| 47 | ChatCompletionResponseStreamChoice, |
| 48 | ChatCompletionStreamResponse, |
| 49 | ChatMessage, |
| 50 | ) |
| 51 | from vllm.entrypoints.serve.engine.protocol import PromptTokenUsageInfo, UsageInfo |
| 52 | from vllm.entrypoints.serve.exception_handling.handlers.validation import ( |
| 53 | validation_exception_handler, |
| 54 | ) |
| 55 | from vllm.exceptions import VLLMValidationError |
| 56 | |
| 57 | _convert = AnthropicServingMessages.to_chat_completion_request |
| 58 | _img_url = AnthropicServingMessages._convert_image_source_to_url |
| 59 | |
| 60 | |
| 61 | def _make_request( |
| 62 | messages: list[dict], |
| 63 | **kwargs, |
| 64 | ) -> AnthropicMessagesRequest: |
| 65 | return AnthropicMessagesRequest( |
| 66 | model="test-model", |
| 67 | max_tokens=128, |
| 68 | messages=messages, |
| 69 | **kwargs, |
| 70 | ) |
| 71 | |
| 72 | |
| 73 | # ====================================================================== |
| 74 | # _convert_image_source_to_url |
| 75 | # ====================================================================== |
| 76 | |
| 77 | |
| 78 | class TestConvertImageSourceToUrl: |
| 79 | def test_base64_source(self): |
| 80 | source = { |
| 81 | "type": "base64", |
| 82 | "media_type": "image/jpeg", |
| 83 | "data": "iVBORw0KGgo=", |
| 84 | } |
| 85 | assert _img_url(source) == "data:image/jpeg;base64,iVBORw0KGgo=" |
| 86 | |
| 87 | def test_base64_png(self): |
| 88 | source = { |
| 89 | "type": "base64", |
| 90 | "media_type": "image/png", |
| 91 | "data": "AAAA", |
| 92 | } |
| 93 | assert _img_url(source) == "data:image/png;base64,AAAA" |
| 94 | |
| 95 | def test_url_source(self): |
| 96 | source = { |
| 97 | "type": "url", |
| 98 | "url": "https://example.com/image.jpg", |
| 99 | } |
| 100 | assert _img_url(source) == "https://example.com/image.jpg" |
| 101 | |
| 102 | def test_missing_type_defaults_to_base64(self): |
| 103 | """When 'type' is absent, treat as base64.""" |
| 104 | source = { |
| 105 | "media_type": "image/webp", |
| 106 | "data": "UklGR", |
| 107 | } |
| 108 | assert _img_url(source) == "data:image/webp;base64,UklGR" |
| 109 | |
| 110 | def test_missing_media_type_defaults_to_jpeg(self): |
| 111 | source = {"type": "base64", "data": "abc123"} |
| 112 | assert _img_url(source) == "data:image/jpeg;base64,abc123" |
| 113 | |
| 114 | def test_url_source_missing_url_returns_empty(self): |
| 115 | source = {"type": "url"} |
| 116 | assert _img_url(source) == "" |
| 117 | |
| 118 | def test_empty_source_returns_data_uri_shell(self): |
| 119 | source: dict = {} |
| 120 | assert _img_url(source) == "data:image/jpeg;base64," |
| 121 | |
| 122 | |
| 123 | # ====================================================================== |
| 124 | # Image blocks inside user messages |
| 125 | # ====================================================================== |
| 126 | |
| 127 | |
| 128 | class TestImageContentBlocks: |
| 129 | def test_base64_image_in_user_message(self): |
| 130 | request = _make_request( |
| 131 | [ |
| 132 | { |
| 133 | "role": "user", |
| 134 | "content": [ |
| 135 | {"type": "text", "text": "Describe this image"}, |
| 136 | { |
| 137 | "type": "image", |
| 138 | "source": { |
| 139 | "type": "base64", |
| 140 | "media_type": "image/jpeg", |
| 141 | "data": "iVBORw0KGgo=", |
| 142 | }, |
| 143 | }, |
| 144 | ], |
| 145 | } |
| 146 | ] |
| 147 | ) |
| 148 | |
| 149 | result = _convert(request) |
| 150 | user_msg = result.messages[0] |
| 151 | assert user_msg["role"] == "user" |
| 152 | |
| 153 | parts = user_msg["content"] |
| 154 | assert len(parts) == 2 |
| 155 | assert parts[0] == {"type": "text", "text": "Describe this image"} |
| 156 | assert parts[1] == { |
| 157 | "type": "image_url", |
| 158 | "image_url": {"url": "data:image/jpeg;base64,iVBORw0KGgo="}, |
| 159 | } |
| 160 | |
| 161 | def test_url_image_in_user_message(self): |
| 162 | request = _make_request( |
| 163 | [ |
| 164 | { |
| 165 | "role": "user", |
| 166 | "content": [ |
| 167 | {"type": "text", "text": "What is this?"}, |
| 168 | { |
| 169 | "type": "image", |
| 170 | "source": { |
| 171 | "type": "url", |
| 172 | "url": "https://example.com/cat.png", |
| 173 | }, |
| 174 | }, |
| 175 | ], |
| 176 | } |
| 177 | ] |
| 178 | ) |
| 179 | |
| 180 | result = _convert(request) |
| 181 | parts = result.messages[0]["content"] |
| 182 | assert parts[1] == { |
| 183 | "type": "image_url", |
| 184 | "image_url": {"url": "https://example.com/cat.png"}, |
| 185 | } |
| 186 | |
| 187 | |
| 188 | # ====================================================================== |
| 189 | # vllm_xargs pass-through |
| 190 | # ====================================================================== |
| 191 | |
| 192 | |
| 193 | class TestVllmXargs: |
| 194 | def test_vllm_xargs_passed_through(self): |
| 195 | request = _make_request( |
| 196 | [{"role": "user", "content": "Hello"}], |
| 197 | vllm_xargs={ |
| 198 | "kv_cache_report_mode": "full", |
| 199 | "existing_extension": 7, |
| 200 | }, |
| 201 | ) |
| 202 | |
| 203 | result = _convert(request) |
| 204 | |
| 205 | assert result.vllm_xargs == { |
| 206 | "kv_cache_report_mode": "full", |
| 207 | "existing_extension": 7, |
| 208 | } |
| 209 | |
| 210 | def test_vllm_xargs_reaches_sampling_params_with_kv_transfer(self): |
| 211 | kv_transfer_params = { |
| 212 | "do_remote_decode": True, |
| 213 | "do_remote_prefill": False, |
| 214 | } |
| 215 | request = _make_request( |
| 216 | [{"role": "user", "content": "Hello"}], |
| 217 | vllm_xargs={ |
| 218 | "kv_cache_report_mode": "full", |
| 219 | "existing_extension": "kept", |
| 220 | }, |
| 221 | kv_transfer_params=kv_transfer_params, |
| 222 | ) |
| 223 | |
| 224 | converted = _convert(request) |
| 225 | sampling_params = converted.to_sampling_params( |
| 226 | max_tokens=converted.max_completion_tokens or 0, |
| 227 | default_sampling_params={}, |
| 228 | ) |
| 229 | |
| 230 | assert converted.kv_transfer_params == kv_transfer_params |
| 231 | assert sampling_params.extra_args == { |
| 232 | "kv_cache_report_mode": "full", |
| 233 | "existing_extension": "kept", |
| 234 | "kv_transfer_params": kv_transfer_params, |
| 235 | } |
| 236 | |
| 237 | |
| 238 | # ====================================================================== |
| 239 | # tool_result content handling |
| 240 | # ====================================================================== |
| 241 | |
| 242 | |
| 243 | class TestToolResultContent: |
| 244 | def _make_tool_result_request( |
| 245 | self, tool_result_content |
| 246 | ) -> AnthropicMessagesRequest: |
| 247 | """Build a request with assistant tool_use followed by user |
| 248 | tool_result.""" |
| 249 | return _make_request( |
| 250 | [ |
| 251 | { |
| 252 | "role": "assistant", |
| 253 | "content": [ |
| 254 | { |
| 255 | "type": "tool_use", |
| 256 | "id": "call_001", |
| 257 | "name": "read_file", |
| 258 | "input": {"path": "/tmp/img.png"}, |
| 259 | } |
| 260 | ], |
| 261 | }, |
| 262 | { |
| 263 | "role": "user", |
| 264 | "content": [ |
| 265 | { |
| 266 | "type": "tool_result", |
| 267 | "tool_use_id": "call_001", |
| 268 | "content": tool_result_content, |
| 269 | } |
| 270 | ], |
| 271 | }, |
| 272 | ] |
| 273 | ) |
| 274 | |
| 275 | def test_tool_result_string_content(self): |
| 276 | request = self._make_tool_result_request("file contents here") |
| 277 | result = _convert(request) |
| 278 | |
| 279 | tool_msg = [m for m in result.messages if m["role"] == "tool"] |
| 280 | assert len(tool_msg) == 1 |
| 281 | assert tool_msg[0]["content"] == "file contents here" |
| 282 | assert tool_msg[0]["tool_call_id"] == "call_001" |
| 283 | |
| 284 | def test_tool_result_text_blocks(self): |
| 285 | request = self._make_tool_result_request( |
| 286 | [ |
| 287 | {"type": "text", "text": "line 1"}, |
| 288 | {"type": "text", "text": "line 2"}, |
| 289 | ] |
| 290 | ) |
| 291 | result = _convert(request) |
| 292 | |
| 293 | tool_msg = [m for m in result.messages if m["role"] == "tool"] |
| 294 | assert len(tool_msg) == 1 |
| 295 | assert tool_msg[0]["content"] == "line 1\nline 2" |
| 296 | |
| 297 | def test_tool_result_with_image(self): |
| 298 | """Image in tool_result should produce a follow-up user message.""" |
| 299 | request = self._make_tool_result_request( |
| 300 | [ |
| 301 | { |
| 302 | "type": "image", |
| 303 | "source": { |
| 304 | "type": "base64", |
| 305 | "media_type": "image/png", |
| 306 | "data": "AAAA", |
| 307 | }, |
| 308 | } |
| 309 | ] |
| 310 | ) |
| 311 | result = _convert(request) |
| 312 | |
| 313 | tool_msg = [m for m in result.messages if m["role"] == "tool"] |
| 314 | assert len(tool_msg) == 1 |
| 315 | assert tool_msg[0]["content"] == "" |
| 316 | |
| 317 | # The image should be injected as a follow-up user message |
| 318 | follow_up = [ |
| 319 | m |
| 320 | for m in result.messages |
| 321 | if m["role"] == "user" and isinstance(m.get("content"), list) |
| 322 | ] |
| 323 | assert len(follow_up) == 1 |
| 324 | img_parts = follow_up[0]["content"] |
| 325 | assert len(img_parts) == 1 |
| 326 | assert img_parts[0] == { |
| 327 | "type": "image_url", |
| 328 | "image_url": {"url": "data:image/png;base64,AAAA"}, |
| 329 | } |
| 330 | |
| 331 | def test_tool_result_with_text_and_image(self): |
| 332 | """Mixed text+image tool_result: text in tool msg, image in user |
| 333 | msg.""" |
| 334 | request = self._make_tool_result_request( |
| 335 | [ |
| 336 | {"type": "text", "text": "Here is the screenshot"}, |
| 337 | { |
| 338 | "type": "image", |
| 339 | "source": { |
| 340 | "type": "base64", |
| 341 | "media_type": "image/jpeg", |
| 342 | "data": "QUFB", |
| 343 | }, |
| 344 | }, |
| 345 | ] |
| 346 | ) |
| 347 | result = _convert(request) |
| 348 | |
| 349 | tool_msg = [m for m in result.messages if m["role"] == "tool"] |
| 350 | assert len(tool_msg) == 1 |
| 351 | assert tool_msg[0]["content"] == "Here is the screenshot" |
| 352 | |
| 353 | follow_up = [ |
| 354 | m |
| 355 | for m in result.messages |
| 356 | if m["role"] == "user" and isinstance(m.get("content"), list) |
| 357 | ] |
| 358 | assert len(follow_up) == 1 |
| 359 | content = follow_up[0]["content"] |
| 360 | assert isinstance(content, list) |
| 361 | image = content[0] |
| 362 | assert image["type"] == "image_url" |
| 363 | assert image["image_url"]["url"] == "data:image/jpeg;base64,QUFB" |
| 364 | |
| 365 | def test_tool_result_with_multiple_images(self): |
| 366 | request = self._make_tool_result_request( |
| 367 | [ |
| 368 | { |
| 369 | "type": "image", |
| 370 | "source": { |
| 371 | "type": "base64", |
| 372 | "media_type": "image/png", |
| 373 | "data": "IMG1", |
| 374 | }, |
| 375 | }, |
| 376 | { |
| 377 | "type": "image", |
| 378 | "source": { |
| 379 | "type": "url", |
| 380 | "url": "https://example.com/img2.jpg", |
| 381 | }, |
| 382 | }, |
| 383 | ] |
| 384 | ) |
| 385 | result = _convert(request) |
| 386 | |
| 387 | follow_up = [ |
| 388 | m |
| 389 | for m in result.messages |
| 390 | if m["role"] == "user" and isinstance(m.get("content"), list) |
| 391 | ] |
| 392 | assert len(follow_up) == 1 |
| 393 | content = follow_up[0]["content"] |
| 394 | assert isinstance(content, list) |
| 395 | urls = [] |
| 396 | for part in content: |
| 397 | assert part["type"] == "image_url" |
| 398 | urls.append(part["image_url"]["url"]) |
| 399 | assert urls == [ |
| 400 | "data:image/png;base64,IMG1", |
| 401 | "https://example.com/img2.jpg", |
| 402 | ] |
| 403 | |
| 404 | def test_tool_result_none_content(self): |
| 405 | request = self._make_tool_result_request(None) |
| 406 | result = _convert(request) |
| 407 | |
| 408 | tool_msg = [m for m in result.messages if m["role"] == "tool"] |
| 409 | assert len(tool_msg) == 1 |
| 410 | assert tool_msg[0]["content"] == "" |
| 411 | |
| 412 | def test_tool_result_no_follow_up_when_no_images(self): |
| 413 | """Ensure no extra user message is added when there are no images.""" |
| 414 | request = self._make_tool_result_request( |
| 415 | [ |
| 416 | {"type": "text", "text": "just text"}, |
| 417 | ] |
| 418 | ) |
| 419 | result = _convert(request) |
| 420 | |
| 421 | user_follow_ups = [ |
| 422 | m |
| 423 | for m in result.messages |
| 424 | if m["role"] == "user" and isinstance(m.get("content"), list) |
| 425 | ] |
| 426 | assert len(user_follow_ups) == 0 |
| 427 | |
| 428 | |
| 429 | # ====================================================================== |
| 430 | # Attribution header stripping |
| 431 | # ====================================================================== |
| 432 | |
| 433 | |
| 434 | class TestAttributionHeaderStripping: |
| 435 | def test_billing_header_stripped_from_system(self): |
| 436 | """Claude Code's x-anthropic-billing-header block should be |
| 437 | stripped to preserve prefix caching.""" |
| 438 | request = _make_request( |
| 439 | [{"role": "user", "content": "Hello"}], |
| 440 | system=[ |
| 441 | {"type": "text", "text": "You are a helpful assistant."}, |
| 442 | { |
| 443 | "type": "text", |
| 444 | "text": "x-anthropic-billing-header: " |
| 445 | "cc_version=2.1.37.abc; cc_entrypoint=cli;", |
| 446 | }, |
| 447 | ], |
| 448 | ) |
| 449 | result = _convert(request) |
| 450 | system_msg = result.messages[0] |
| 451 | assert system_msg["role"] == "system" |
| 452 | assert system_msg["content"] == "You are a helpful assistant." |
| 453 | |
| 454 | def test_system_without_billing_header_unchanged(self): |
| 455 | """Normal system blocks should pass through unchanged.""" |
| 456 | request = _make_request( |
| 457 | [{"role": "user", "content": "Hello"}], |
| 458 | system=[ |
| 459 | {"type": "text", "text": "You are a helpful assistant."}, |
| 460 | {"type": "text", "text": " Be concise."}, |
| 461 | ], |
| 462 | ) |
| 463 | result = _convert(request) |
| 464 | system_msg = result.messages[0] |
| 465 | assert system_msg["content"] == "You are a helpful assistant. Be concise." |
| 466 | |
| 467 | def test_system_string_unchanged(self): |
| 468 | """String system prompts should pass through unchanged.""" |
| 469 | request = _make_request( |
| 470 | [{"role": "user", "content": "Hello"}], |
| 471 | system="You are a helpful assistant.", |
| 472 | ) |
| 473 | result = _convert(request) |
| 474 | system_msg = result.messages[0] |
| 475 | assert system_msg["content"] == "You are a helpful assistant." |
| 476 | |
| 477 | |
| 478 | # ====================================================================== |
| 479 | # Thinking block conversion (Anthropic → OpenAI) |
| 480 | # ====================================================================== |
| 481 | |
| 482 | |
| 483 | class TestThinkingBlockConversion: |
| 484 | """Verify that thinking blocks in assistant messages are correctly |
| 485 | moved to the ``reasoning`` field and stripped from ``content`` during |
| 486 | the Anthropic→OpenAI conversion. |
| 487 | |
| 488 | This is the Anthropic-endpoint path: the client echoes back the full |
| 489 | assistant message (including thinking blocks emitted by vllm) in |
| 490 | subsequent requests. |
| 491 | """ |
| 492 | |
| 493 | def test_thinking_plus_text_in_assistant_message(self): |
| 494 | """Thinking + text → reasoning field + plain-string content.""" |
| 495 | request = _make_request( |
| 496 | [ |
| 497 | {"role": "user", "content": "Write me some code."}, |
| 498 | { |
| 499 | "role": "assistant", |
| 500 | "content": [ |
| 501 | { |
| 502 | "type": "thinking", |
| 503 | "thinking": "I should write a simple example.", |
| 504 | "signature": "sig_abc123", |
| 505 | }, |
| 506 | {"type": "text", "text": "Sure! Here is the code."}, |
| 507 | ], |
| 508 | }, |
| 509 | {"role": "user", "content": "Can you fix the bug?"}, |
| 510 | ] |
| 511 | ) |
| 512 | result = _convert(request) |
| 513 | |
| 514 | # Find the assistant message in the converted output. |
| 515 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 516 | assert len(asst_msgs) == 1 |
| 517 | asst = asst_msgs[0] |
| 518 | |
| 519 | # Thinking content must be in reasoning, NOT in content. |
| 520 | assert asst.get("reasoning") == "I should write a simple example." |
| 521 | assert asst.get("content") == "Sure! Here is the code." |
| 522 | |
| 523 | def test_thinking_only_in_assistant_message(self): |
| 524 | """Assistant message with only a thinking block (no visible text). |
| 525 | |
| 526 | This can happen when the model emits reasoning but no final answer |
| 527 | yet (e.g. a mid-turn reasoning step). Content should be None. |
| 528 | """ |
| 529 | request = _make_request( |
| 530 | [ |
| 531 | {"role": "user", "content": "Hello"}, |
| 532 | { |
| 533 | "role": "assistant", |
| 534 | "content": [ |
| 535 | { |
| 536 | "type": "thinking", |
| 537 | "thinking": "Just thinking...", |
| 538 | "signature": "sig_xyz", |
| 539 | } |
| 540 | ], |
| 541 | }, |
| 542 | {"role": "user", "content": "Go on."}, |
| 543 | ] |
| 544 | ) |
| 545 | result = _convert(request) |
| 546 | |
| 547 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 548 | assert len(asst_msgs) == 1 |
| 549 | asst = asst_msgs[0] |
| 550 | |
| 551 | assert asst.get("reasoning") == "Just thinking..." |
| 552 | # No visible text → content should be absent or None. |
| 553 | assert asst.get("content") is None |
| 554 | |
| 555 | def test_thinking_plus_tool_use_in_assistant_message(self): |
| 556 | """Thinking + tool_use: reasoning field set, tool_calls populated.""" |
| 557 | request = _make_request( |
| 558 | [ |
| 559 | {"role": "user", "content": "What is 2+2?"}, |
| 560 | { |
| 561 | "role": "assistant", |
| 562 | "content": [ |
| 563 | { |
| 564 | "type": "thinking", |
| 565 | "thinking": "I need to call the calculator.", |
| 566 | "signature": "sig_tool", |
| 567 | }, |
| 568 | { |
| 569 | "type": "tool_use", |
| 570 | "id": "call_001", |
| 571 | "name": "calculator", |
| 572 | "input": {"expression": "2+2"}, |
| 573 | }, |
| 574 | ], |
| 575 | }, |
| 576 | { |
| 577 | "role": "user", |
| 578 | "content": [ |
| 579 | { |
| 580 | "type": "tool_result", |
| 581 | "tool_use_id": "call_001", |
| 582 | "content": "4", |
| 583 | } |
| 584 | ], |
| 585 | }, |
| 586 | ] |
| 587 | ) |
| 588 | result = _convert(request) |
| 589 | |
| 590 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 591 | assert len(asst_msgs) == 1 |
| 592 | asst = asst_msgs[0] |
| 593 | |
| 594 | assert asst.get("reasoning") == "I need to call the calculator." |
| 595 | tool_calls = asst.get("tool_calls") |
| 596 | assert tool_calls is not None |
| 597 | assert len(tool_calls) == 1 |
| 598 | assert tool_calls[0]["function"]["name"] == "calculator" |
| 599 | # No text content alongside reasoning + tool_use. |
| 600 | assert asst.get("content") is None |
| 601 | |
| 602 | def test_multiple_thinking_blocks_concatenated(self): |
| 603 | """Multiple thinking blocks should be joined in order.""" |
| 604 | request = _make_request( |
| 605 | [ |
| 606 | {"role": "user", "content": "Think hard."}, |
| 607 | { |
| 608 | "role": "assistant", |
| 609 | "content": [ |
| 610 | { |
| 611 | "type": "thinking", |
| 612 | "thinking": "First thought. ", |
| 613 | "signature": "s1", |
| 614 | }, |
| 615 | { |
| 616 | "type": "thinking", |
| 617 | "thinking": "Second thought.", |
| 618 | "signature": "s2", |
| 619 | }, |
| 620 | {"type": "text", "text": "Done."}, |
| 621 | ], |
| 622 | }, |
| 623 | ] |
| 624 | ) |
| 625 | result = _convert(request) |
| 626 | |
| 627 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 628 | assert len(asst_msgs) == 1 |
| 629 | asst = asst_msgs[0] |
| 630 | |
| 631 | assert asst.get("reasoning") == "First thought. Second thought." |
| 632 | assert asst.get("content") == "Done." |
| 633 | |
| 634 | def test_no_thinking_blocks_unchanged(self): |
| 635 | """Messages without thinking blocks must not be modified.""" |
| 636 | request = _make_request( |
| 637 | [ |
| 638 | {"role": "user", "content": "Hi"}, |
| 639 | {"role": "assistant", "content": "Hello!"}, |
| 640 | ] |
| 641 | ) |
| 642 | result = _convert(request) |
| 643 | |
| 644 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 645 | assert len(asst_msgs) == 1 |
| 646 | asst = asst_msgs[0] |
| 647 | |
| 648 | assert asst.get("content") == "Hello!" |
| 649 | assert "reasoning" not in asst |
| 650 | |
| 651 | def test_multi_turn_with_thinking_blocks(self): |
| 652 | """Full multi-turn conversation: previous assistant messages that |
| 653 | include thinking blocks must all be converted without a 400 error. |
| 654 | |
| 655 | This is the primary regression scenario from the bug report: |
| 656 | upgrading vllm from v0.15.1 → v0.17.0 introduced thinking-block |
| 657 | support in responses, but echoing those responses back in subsequent |
| 658 | requests caused a Pydantic validation failure. |
| 659 | """ |
| 660 | request = _make_request( |
| 661 | [ |
| 662 | {"role": "user", "content": "Turn 1 question"}, |
| 663 | { |
| 664 | "role": "assistant", |
| 665 | "content": [ |
| 666 | { |
| 667 | "type": "thinking", |
| 668 | "thinking": "Reasoning for turn 1.", |
| 669 | "signature": "s_t1", |
| 670 | }, |
| 671 | {"type": "text", "text": "Answer for turn 1."}, |
| 672 | ], |
| 673 | }, |
| 674 | {"role": "user", "content": "Turn 2 question"}, |
| 675 | { |
| 676 | "role": "assistant", |
| 677 | "content": [ |
| 678 | { |
| 679 | "type": "thinking", |
| 680 | "thinking": "Reasoning for turn 2.", |
| 681 | "signature": "s_t2", |
| 682 | }, |
| 683 | {"type": "text", "text": "Answer for turn 2."}, |
| 684 | ], |
| 685 | }, |
| 686 | {"role": "user", "content": "Turn 3 question"}, |
| 687 | ] |
| 688 | ) |
| 689 | # Must not raise a ValidationError / 400. |
| 690 | result = _convert(request) |
| 691 | |
| 692 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 693 | assert len(asst_msgs) == 2 |
| 694 | |
| 695 | assert asst_msgs[0].get("reasoning") == "Reasoning for turn 1." |
| 696 | assert asst_msgs[0].get("content") == "Answer for turn 1." |
| 697 | assert asst_msgs[1].get("reasoning") == "Reasoning for turn 2." |
| 698 | assert asst_msgs[1].get("content") == "Answer for turn 2." |
| 699 | |
| 700 | def test_redacted_thinking_block_is_accepted(self): |
| 701 | """Anthropic clients may echo back redacted thinking blocks. |
| 702 | |
| 703 | vLLM should accept these blocks (to avoid 400 validation errors) |
| 704 | and ignore them when constructing the OpenAI-format prompt. |
| 705 | """ |
| 706 | request = _make_request( |
| 707 | [ |
| 708 | {"role": "user", "content": "Hello"}, |
| 709 | { |
| 710 | "role": "assistant", |
| 711 | "content": [ |
| 712 | { |
| 713 | "type": "thinking", |
| 714 | "thinking": "Thinking...", |
| 715 | "signature": "sig_think", |
| 716 | }, |
| 717 | { |
| 718 | "type": "redacted_thinking", |
| 719 | "data": "BASE64_OR_OTHER_OPAQUE_DATA", |
| 720 | }, |
| 721 | {"type": "text", "text": "Hi!"}, |
| 722 | ], |
| 723 | }, |
| 724 | {"role": "user", "content": "Continue"}, |
| 725 | ] |
| 726 | ) |
| 727 | result = _convert(request) |
| 728 | |
| 729 | asst_msgs = [m for m in result.messages if m.get("role") == "assistant"] |
| 730 | assert len(asst_msgs) == 1 |
| 731 | asst = asst_msgs[0] |
| 732 | |
| 733 | # Redacted thinking is ignored, normal thinking still becomes reasoning. |
| 734 | assert asst.get("reasoning") == "Thinking..." |
| 735 | assert asst.get("content") == "Hi!" |
| 736 | |
| 737 | |
| 738 | # ====================================================================== |
| 739 | # Cache usage computation |
| 740 | # ====================================================================== |
| 741 | |
| 742 | |
| 743 | class TestBuildAnthropicUsage: |
| 744 | """Tests for _build_anthropic_usage helper. |
| 745 | |
| 746 | Anthropic defines: total_input = input_tokens + cache_read + cache_creation |
| 747 | vLLM's prompt_tokens is the total. |
| 748 | """ |
| 749 | |
| 750 | def test_cache_hit(self): |
| 751 | """When cache is hit, input_tokens excludes cached tokens.""" |
| 752 | usage = UsageInfo( |
| 753 | prompt_tokens=100, |
| 754 | completion_tokens=10, |
| 755 | prompt_tokens_details=PromptTokenUsageInfo( |
| 756 | cached_tokens=80, created_cache_tokens=10 |
| 757 | ), |
| 758 | ) |
| 759 | result = _build_anthropic_usage(usage) |
| 760 | assert result.input_tokens == 10 # 100 - 80 - 10 |
| 761 | assert result.output_tokens == 10 |
| 762 | assert result.cache_read_input_tokens == 80 |
| 763 | assert result.cache_creation_input_tokens == 10 |
| 764 | |
| 765 | def test_zero_cached_tokens(self): |
| 766 | """Zero cached tokens should still set cache_creation to 0.""" |
| 767 | usage = UsageInfo( |
| 768 | prompt_tokens=100, |
| 769 | completion_tokens=10, |
| 770 | prompt_tokens_details=PromptTokenUsageInfo( |
| 771 | cached_tokens=0, created_cache_tokens=0 |
| 772 | ), |
| 773 | ) |
| 774 | result = _build_anthropic_usage(usage) |
| 775 | assert result.input_tokens == 100 # 100 - 0 - 0 |
| 776 | assert result.cache_read_input_tokens == 0 |
| 777 | assert result.cache_creation_input_tokens == 0 |
| 778 | |
| 779 | def test_all_tokens_cached(self): |
| 780 | """When all tokens are cached, input_tokens should be 0.""" |
| 781 | usage = UsageInfo( |
| 782 | prompt_tokens=100, |
| 783 | completion_tokens=10, |
| 784 | prompt_tokens_details=PromptTokenUsageInfo( |
| 785 | cached_tokens=100, created_cache_tokens=0 |
| 786 | ), |
| 787 | ) |
| 788 | result = _build_anthropic_usage(usage) |
| 789 | assert result.input_tokens == 0 |
| 790 | assert result.cache_read_input_tokens == 100 |
| 791 | assert result.cache_creation_input_tokens == 0 |
| 792 | |
| 793 | def test_no_prompt_tokens_details(self): |
| 794 | """UsageInfo without prompt_tokens_details returns no cache info.""" |
| 795 | usage = UsageInfo(prompt_tokens=100, completion_tokens=10) |
| 796 | result = _build_anthropic_usage(usage) |
| 797 | assert result.input_tokens == 100 |
| 798 | assert result.cache_read_input_tokens is None |
| 799 | assert result.cache_creation_input_tokens is None |
| 800 | |
| 801 | |
| 802 | class TestInlineSystemMessageInMessagesArray: |
| 803 | """Verify that ``role: system`` messages embedded inside the ``messages`` |
| 804 | array are preserved in their original position. |
| 805 | |
| 806 | Unlike the previous approach that merged all system messages into a single |
| 807 | leading system message (breaking prefix caching), this preserves the |
| 808 | conversation structure so KV-cache hits remain intact. |
| 809 | """ |
| 810 | |
| 811 | def test_inline_system_merged_with_top_level_system(self): |
| 812 | """Full integration: inline system + top-level system + user message.""" |
| 813 | request = _make_request( |
| 814 | [ |
| 815 | { |
| 816 | "role": "user", |
| 817 | "content": [ |
| 818 | { |
| 819 | "type": "text", |
| 820 | "text": "<system-reminder>\n.....\n</system-reminder>\n\n", |
| 821 | }, |
| 822 | { |
| 823 | "type": "text", |
| 824 | "text": "help?", |
| 825 | "cache_control": {"type": "ephemeral"}, |
| 826 | }, |
| 827 | ], |
| 828 | }, |
| 829 | { |
| 830 | "role": "system", |
| 831 | "content": ".....", |
| 832 | }, |
| 833 | ], |
| 834 | system=[ |
| 835 | { |
| 836 | "type": "text", |
| 837 | "text": "x-anthropic-billing-header: " |
| 838 | "cc_version=2.1.160.bca; cc_entrypoint=cli; cch=d1d48;", |
| 839 | }, |
| 840 | { |
| 841 | "type": "text", |
| 842 | "text": "You are Claude Code, Anthropic's official CLI for Claude.", |
| 843 | "cache_control": {"type": "ephemeral"}, |
| 844 | }, |
| 845 | { |
| 846 | "type": "text", |
| 847 | "text": "....", |
| 848 | "cache_control": {"type": "ephemeral"}, |
| 849 | }, |
| 850 | ], |
| 851 | tools=[], |
| 852 | ) |
| 853 | |
| 854 | result = _convert(request) |
| 855 | |
| 856 | # First message: top-level system prompt (billing header stripped). |
| 857 | assert result.messages[0]["role"] == "system" |
| 858 | assert ( |
| 859 | result.messages[0]["content"] |
| 860 | == "You are Claude Code, Anthropic's official CLI for Claude." |
| 861 | "...." |
| 862 | ) |
| 863 | |
| 864 | # Second message: user message, content preserved at original position. |
| 865 | assert result.messages[1]["role"] == "user" |
| 866 | user_content = result.messages[1]["content"] |
| 867 | assert len(user_content) == 2 |
| 868 | assert user_content[0] == { |
| 869 | "type": "text", |
| 870 | "text": "<system-reminder>\n.....\n</system-reminder>\n\n", |
| 871 | } |
| 872 | assert user_content[1] == { |
| 873 | "type": "text", |
| 874 | "text": "help?", |
| 875 | } |
| 876 | |
| 877 | # Third message: inline system stays in original position |
| 878 | # (after user, not merged into leading system). |
| 879 | assert result.messages[2]["role"] == "system" |
| 880 | assert result.messages[2]["content"] == "....." |
| 881 | |
| 882 | def test_inline_system_string_only(self): |
| 883 | """Only an inline system string, no top-level system.""" |
| 884 | request = _make_request( |
| 885 | [ |
| 886 | {"role": "user", "content": "Hello"}, |
| 887 | {"role": "system", "content": "Be concise."}, |
| 888 | ] |
| 889 | ) |
| 890 | result = _convert(request) |
| 891 | |
| 892 | # Inline system stays in its original position. |
| 893 | assert result.messages[0]["role"] == "user" |
| 894 | assert result.messages[0]["content"] == "Hello" |
| 895 | assert result.messages[1]["role"] == "system" |
| 896 | assert result.messages[1]["content"] == "Be concise." |
| 897 | |
| 898 | def test_inline_system_list_content(self): |
| 899 | """Inline system with list content blocks.""" |
| 900 | request = _make_request( |
| 901 | [ |
| 902 | {"role": "user", "content": "Hi"}, |
| 903 | { |
| 904 | "role": "system", |
| 905 | "content": [ |
| 906 | {"type": "text", "text": "Part one. "}, |
| 907 | {"type": "text", "text": "Part two."}, |
| 908 | ], |
| 909 | }, |
| 910 | ] |
| 911 | ) |
| 912 | result = _convert(request) |
| 913 | |
| 914 | # Inline system stays in its original position; |
| 915 | # text blocks are concatenated (same as top-level system). |
| 916 | assert result.messages[0]["role"] == "user" |
| 917 | assert result.messages[0]["content"] == "Hi" |
| 918 | assert result.messages[1]["role"] == "system" |
| 919 | assert result.messages[1]["content"] == "Part one. Part two." |
| 920 | |
| 921 | def test_multiple_inline_system_messages(self): |
| 922 | """Multiple inline system messages each stay in their position.""" |
| 923 | request = _make_request( |
| 924 | [ |
| 925 | {"role": "system", "content": "First system."}, |
| 926 | {"role": "user", "content": "Hello"}, |
| 927 | {"role": "system", "content": "Second system."}, |
| 928 | ] |
| 929 | ) |
| 930 | result = _convert(request) |
| 931 | |
| 932 | # Each system message stays in its original position. |
| 933 | assert result.messages[0]["role"] == "system" |
| 934 | assert result.messages[0]["content"] == "First system." |
| 935 | assert result.messages[1]["role"] == "user" |
| 936 | assert result.messages[1]["content"] == "Hello" |
| 937 | assert result.messages[2]["role"] == "system" |
| 938 | assert result.messages[2]["content"] == "Second system." |
| 939 | |
| 940 | def test_inline_system_with_top_level_string(self): |
| 941 | """Top-level system is a string, inline system is also present.""" |
| 942 | request = _make_request( |
| 943 | [ |
| 944 | {"role": "user", "content": "Hello"}, |
| 945 | {"role": "system", "content": "Inline hint."}, |
| 946 | ], |
| 947 | system="Top-level prompt.", |
| 948 | ) |
| 949 | result = _convert(request) |
| 950 | |
| 951 | # Top-level system goes first; inline system stays in position. |
| 952 | assert result.messages[0]["role"] == "system" |
| 953 | assert result.messages[0]["content"] == "Top-level prompt." |
| 954 | assert result.messages[1]["role"] == "user" |
| 955 | assert result.messages[1]["content"] == "Hello" |
| 956 | assert result.messages[2]["role"] == "system" |
| 957 | assert result.messages[2]["content"] == "Inline hint." |
| 958 | |
| 959 | def test_inline_system_billing_header_stripped(self): |
| 960 | """Inline system that is only a billing header is omitted.""" |
| 961 | request = _make_request( |
| 962 | [ |
| 963 | {"role": "user", "content": "Hello"}, |
| 964 | { |
| 965 | "role": "system", |
| 966 | "content": "x-anthropic-billing-header: cc_version=2.1.160", |
| 967 | }, |
| 968 | {"role": "assistant", "content": "Hi there"}, |
| 969 | ] |
| 970 | ) |
| 971 | result = _convert(request) |
| 972 | |
| 973 | # Billing-header-only system message should be dropped entirely. |
| 974 | assert len(result.messages) == 2 |
| 975 | assert result.messages[0]["role"] == "user" |
| 976 | assert result.messages[1]["role"] == "assistant" |
| 977 | |
| 978 | def test_inline_system_billing_header_mixed_with_content(self): |
| 979 | """Inline system with billing header block + real content.""" |
| 980 | request = _make_request( |
| 981 | [ |
| 982 | {"role": "user", "content": "Hello"}, |
| 983 | { |
| 984 | "role": "system", |
| 985 | "content": [ |
| 986 | { |
| 987 | "type": "text", |
| 988 | "text": "x-anthropic-billing-header: " |
| 989 | "cc_version=2.1.160.bca; cch=d1d48;", |
| 990 | }, |
| 991 | {"type": "text", "text": "Real system content."}, |
| 992 | ], |
| 993 | }, |
| 994 | ] |
| 995 | ) |
| 996 | result = _convert(request) |
| 997 | |
| 998 | # Billing header stripped, real content preserved in position. |
| 999 | assert len(result.messages) == 2 |
| 1000 | assert result.messages[0]["role"] == "user" |
| 1001 | assert result.messages[0]["content"] == "Hello" |
| 1002 | assert result.messages[1]["role"] == "system" |
| 1003 | assert result.messages[1]["content"] == "Real system content." |
| 1004 | |
| 1005 | |
| 1006 | # ====================================================================== |
| 1007 | # Streaming conversion: message_stream_converter |
| 1008 | # ====================================================================== |
| 1009 | |
| 1010 | |
| 1011 | def _make_stream_converter(): |
| 1012 | obj = MagicMock(spec=AnthropicServingMessages) |
| 1013 | obj.message_stream_converter = ( |
| 1014 | AnthropicServingMessages.message_stream_converter.__get__(obj) |
| 1015 | ) |
| 1016 | return obj |
| 1017 | |
| 1018 | |
| 1019 | def _parse_sse_events(raw_events: list[str]) -> list[tuple[str, dict]]: |
| 1020 | results = [] |
| 1021 | for raw in raw_events: |
| 1022 | headers = dict( |
| 1023 | line.split(": ", 1) for line in raw.strip().split("\n") if ": " in line |
| 1024 | ) |
| 1025 | if "event" in headers and "data" in headers: |
| 1026 | results.append((headers["event"], json.loads(headers["data"]))) |
| 1027 | return results |
| 1028 | |
| 1029 | |
| 1030 | def _make_stream_chunk( |
| 1031 | *, |
| 1032 | delta: DeltaMessage | None = None, |
| 1033 | finish_reason: str | None = None, |
| 1034 | stop_reason: int | str | None = None, |
| 1035 | choices: list[ChatCompletionResponseStreamChoice] | None = None, |
| 1036 | usage: UsageInfo | None = None, |
| 1037 | ) -> str: |
| 1038 | if choices is None: |
| 1039 | choices = [ |
| 1040 | ChatCompletionResponseStreamChoice( |
| 1041 | index=0, |
| 1042 | delta=delta or DeltaMessage(), |
| 1043 | finish_reason=finish_reason, |
| 1044 | stop_reason=stop_reason, |
| 1045 | ) |
| 1046 | ] |
| 1047 | chunk = ChatCompletionStreamResponse( |
| 1048 | id="chatcmpl-test", |
| 1049 | created=0, |
| 1050 | model="test-model", |
| 1051 | choices=choices, |
| 1052 | usage=usage, |
| 1053 | ) |
| 1054 | return f"data: {chunk.model_dump_json()}" |
| 1055 | |
| 1056 | |
| 1057 | def _tc(*, args, id=None, name=None): |
| 1058 | return DeltaToolCall( |
| 1059 | index=0, |
| 1060 | id=id, |
| 1061 | function=DeltaFunctionCall(name=name, arguments=args), |
| 1062 | ) |
| 1063 | |
| 1064 | |
| 1065 | class TestMessageStreamConverterToolUseContentBuffering: |
| 1066 | """Regression test for tool_use arguments being silently dropped. |
| 1067 | |
| 1068 | With speculative decoding or multi-token prediction, a single delta |
| 1069 | can carry both the final tool_call argument fragment and trailing |
| 1070 | content. |
| 1071 | """ |
| 1072 | |
| 1073 | @pytest.mark.asyncio |
| 1074 | async def test_tool_use_args_not_dropped_when_content_in_same_chunk( |
| 1075 | self, |
| 1076 | ): |
| 1077 | async def sse_input(): |
| 1078 | yield _make_stream_chunk( |
| 1079 | delta=DeltaMessage(role="assistant"), |
| 1080 | usage=UsageInfo(prompt_tokens=10, total_tokens=10), |
| 1081 | ) |
| 1082 | yield _make_stream_chunk( |
| 1083 | delta=DeltaMessage( |
| 1084 | tool_calls=[ |
| 1085 | _tc(id="call_abc123", name="read_file", args=""), |
| 1086 | ] |
| 1087 | ) |
| 1088 | ) |
| 1089 | yield _make_stream_chunk( |
| 1090 | delta=DeltaMessage( |
| 1091 | tool_calls=[ |
| 1092 | _tc(args='{"path":"/tmp/f"'), |
| 1093 | ] |
| 1094 | ) |
| 1095 | ) |
| 1096 | # BUG TRIGGER: final tool_call args and trailing content in |
| 1097 | # one delta, as happens with spec decoding / multi-token |
| 1098 | # prediction where multiple tokens land in a single chunk. |
| 1099 | yield _make_stream_chunk( |
| 1100 | delta=DeltaMessage( |
| 1101 | content="\nOkay", |
| 1102 | tool_calls=[_tc(args="}")], |
| 1103 | ) |
| 1104 | ) |
| 1105 | yield _make_stream_chunk(finish_reason="tool_calls") |
| 1106 | yield _make_stream_chunk( |
| 1107 | choices=[], |
| 1108 | usage=UsageInfo( |
| 1109 | prompt_tokens=10, |
| 1110 | total_tokens=30, |
| 1111 | completion_tokens=20, |
| 1112 | ), |
| 1113 | ) |
| 1114 | yield "data: [DONE]" |
| 1115 | |
| 1116 | converter = _make_stream_converter() |
| 1117 | output = [] |
| 1118 | async for event in converter.message_stream_converter(sse_input()): |
| 1119 | output.append(event) |
| 1120 | |
| 1121 | events = _parse_sse_events(output) |
| 1122 | |
| 1123 | assert events[0][0] == "message_start" |
| 1124 | |
| 1125 | arg_fragments = [ |
| 1126 | data["delta"]["partial_json"] |
| 1127 | for _, data in events |
| 1128 | if data.get("delta", {}).get("type") == "input_json_delta" |
| 1129 | ] |
| 1130 | full_args = "".join(arg_fragments) |
| 1131 | assert full_args == '{"path":"/tmp/f"}' |
| 1132 | |
| 1133 | text_deltas = [ |
| 1134 | data["delta"]["text"] |
| 1135 | for _, data in events |
| 1136 | if data.get("delta", {}).get("type") == "text_delta" |
| 1137 | ] |
| 1138 | assert text_deltas == ["\nOkay"] |
| 1139 | |
| 1140 | block_starts = [ |
| 1141 | (data["content_block"]["type"], data.get("index")) |
| 1142 | for ev_type, data in events |
| 1143 | if ev_type == "content_block_start" |
| 1144 | ] |
| 1145 | assert block_starts[0] == ("tool_use", 0) |
| 1146 | assert block_starts[1] == ("text", 1) |
| 1147 | |
| 1148 | msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"] |
| 1149 | assert msg_deltas[0]["delta"]["stop_reason"] == "tool_use" |
| 1150 | |
| 1151 | assert events[-1][0] == "message_stop" |
| 1152 | |
| 1153 | @pytest.mark.asyncio |
| 1154 | async def test_buffered_content_flushed_on_done_without_usage_chunk(self): |
| 1155 | """Content buffered during tool_use must be emitted even if the |
| 1156 | stream jumps straight from finish_reason to [DONE], skipping the |
| 1157 | empty-choices usage chunk.""" |
| 1158 | |
| 1159 | async def sse_input(): |
| 1160 | yield _make_stream_chunk( |
| 1161 | delta=DeltaMessage(role="assistant"), |
| 1162 | usage=UsageInfo(prompt_tokens=10, total_tokens=10), |
| 1163 | ) |
| 1164 | yield _make_stream_chunk( |
| 1165 | delta=DeltaMessage( |
| 1166 | tool_calls=[ |
| 1167 | _tc(id="call_xyz", name="get_weather", args=""), |
| 1168 | ] |
| 1169 | ) |
| 1170 | ) |
| 1171 | yield _make_stream_chunk( |
| 1172 | delta=DeltaMessage( |
| 1173 | tool_calls=[_tc(args='{"city":"NYC"}')], |
| 1174 | ) |
| 1175 | ) |
| 1176 | yield _make_stream_chunk( |
| 1177 | delta=DeltaMessage(content="\nDone"), |
| 1178 | finish_reason="tool_calls", |
| 1179 | ) |
| 1180 | # No empty-choices usage chunk — go straight to [DONE]. |
| 1181 | yield "data: [DONE]" |
| 1182 | |
| 1183 | converter = _make_stream_converter() |
| 1184 | output = [] |
| 1185 | async for event in converter.message_stream_converter(sse_input()): |
| 1186 | output.append(event) |
| 1187 | |
| 1188 | events = _parse_sse_events(output) |
| 1189 | |
| 1190 | text_deltas = [ |
| 1191 | data["delta"]["text"] |
| 1192 | for _, data in events |
| 1193 | if data.get("delta", {}).get("type") == "text_delta" |
| 1194 | ] |
| 1195 | assert text_deltas == ["\nDone"] |
| 1196 | |
| 1197 | block_starts = [ |
| 1198 | data["content_block"]["type"] |
| 1199 | for ev_type, data in events |
| 1200 | if ev_type == "content_block_start" |
| 1201 | ] |
| 1202 | assert "tool_use" in block_starts |
| 1203 | assert "text" in block_starts |
| 1204 | |
| 1205 | assert events[-1][0] == "message_stop" |
| 1206 | |
| 1207 | |
| 1208 | class TestMessageStartIncludesTypeAndRole: |
| 1209 | """Regression test for issue #45367: the streaming message_start event is |
| 1210 | serialized with exclude_unset=True, which silently dropped the |
| 1211 | default-valued ``type``/``role`` fields of the nested message object. |
| 1212 | Strict Anthropic SDK clients (e.g. Claude Code) validate |
| 1213 | ``message_start.message.type``/``role`` and reject the whole stream when |
| 1214 | they are missing. |
| 1215 | """ |
| 1216 | |
| 1217 | @pytest.mark.asyncio |
| 1218 | async def test_message_start_contains_message_type_and_role(self): |
| 1219 | async def sse_input(): |
| 1220 | yield _make_stream_chunk( |
| 1221 | delta=DeltaMessage(content="Hello"), |
| 1222 | usage=UsageInfo( |
| 1223 | prompt_tokens=20, |
| 1224 | total_tokens=20, |
| 1225 | completion_tokens=0, |
| 1226 | ), |
| 1227 | ) |
| 1228 | yield _make_stream_chunk(finish_reason="stop") |
| 1229 | yield "data: [DONE]" |
| 1230 | |
| 1231 | converter = _make_stream_converter() |
| 1232 | output = [] |
| 1233 | async for event in converter.message_stream_converter(sse_input()): |
| 1234 | output.append(event) |
| 1235 | |
| 1236 | events = _parse_sse_events(output) |
| 1237 | |
| 1238 | assert events[0][0] == "message_start" |
| 1239 | message = events[0][1]["message"] |
| 1240 | assert message["type"] == "message" |
| 1241 | assert message["role"] == "assistant" |
| 1242 | |
| 1243 | |
| 1244 | class TestStreamingCacheUsageSemantics: |
| 1245 | """Locks in the documented streaming behavior of cache usage fields. |
| 1246 | |
| 1247 | vLLM's OpenAI chat completion streaming only attaches |
| 1248 | ``prompt_tokens_details`` to the terminal usage chunk. The Anthropic layer |
| 1249 | mirrors that contract: cache fields are omitted on ``message_start`` (key |
| 1250 | absence signals "unknown") and populated on ``message_delta`` (the final |
| 1251 | cumulative count). This is intentionally consistent with vLLM's OpenAI |
| 1252 | behavior, even though Anthropic's upstream API populates cache fields on |
| 1253 | ``message_start``; closing that gap requires plumbing cache info into the |
| 1254 | first chunk at the OpenAI layer, which is out of scope here. |
| 1255 | """ |
| 1256 | |
| 1257 | @pytest.mark.asyncio |
| 1258 | async def test_streaming_cache_fields_absent_then_populated(self): |
| 1259 | """First chunk lacks prompt_tokens_details (vLLM contract); |
| 1260 | message_start omits cache fields. The final chunk carries |
| 1261 | prompt_tokens_details, so message_delta carries resolved values.""" |
| 1262 | |
| 1263 | async def sse_input(): |
| 1264 | yield _make_stream_chunk( |
| 1265 | delta=DeltaMessage(role="assistant", content="hi"), |
| 1266 | usage=UsageInfo(prompt_tokens=100, total_tokens=100), |
| 1267 | ) |
| 1268 | yield _make_stream_chunk(finish_reason="stop") |
| 1269 | yield _make_stream_chunk( |
| 1270 | choices=[], |
| 1271 | usage=UsageInfo( |
| 1272 | prompt_tokens=100, |
| 1273 | completion_tokens=5, |
| 1274 | total_tokens=105, |
| 1275 | prompt_tokens_details=PromptTokenUsageInfo( |
| 1276 | cached_tokens=80, created_cache_tokens=10 |
| 1277 | ), |
| 1278 | ), |
| 1279 | ) |
| 1280 | yield "data: [DONE]" |
| 1281 | |
| 1282 | converter = _make_stream_converter() |
| 1283 | output = [] |
| 1284 | async for event in converter.message_stream_converter(sse_input()): |
| 1285 | output.append(event) |
| 1286 | events = _parse_sse_events(output) |
| 1287 | |
| 1288 | # message_start: cache fields unknown → omitted from JSON entirely. |
| 1289 | start_usage = events[0][1]["message"]["usage"] |
| 1290 | assert events[0][0] == "message_start" |
| 1291 | assert start_usage["input_tokens"] == 100 |
| 1292 | assert "cache_read_input_tokens" not in start_usage |
| 1293 | assert "cache_creation_input_tokens" not in start_usage |
| 1294 | |
| 1295 | # message_delta: authoritative usage with cache fields populated. |
| 1296 | delta_usage = next( |
| 1297 | data["usage"] for ev, data in events if ev == "message_delta" |
| 1298 | ) |
| 1299 | assert delta_usage["input_tokens"] == 10 # 100 - 80 - 10 |
| 1300 | assert delta_usage["cache_read_input_tokens"] == 80 |
| 1301 | assert delta_usage["cache_creation_input_tokens"] == 10 |
| 1302 | |
| 1303 | @pytest.mark.asyncio |
| 1304 | async def test_streaming_no_cache_hit(self): |
| 1305 | """When the final chunk reports cached_tokens=0, message_delta carries |
| 1306 | cache fields = 0 (cache miss); message_start still omits them.""" |
| 1307 | |
| 1308 | async def sse_input(): |
| 1309 | yield _make_stream_chunk( |
| 1310 | delta=DeltaMessage(role="assistant"), |
| 1311 | usage=UsageInfo(prompt_tokens=50, total_tokens=50), |
| 1312 | ) |
| 1313 | yield _make_stream_chunk(finish_reason="stop") |
| 1314 | yield _make_stream_chunk( |
| 1315 | choices=[], |
| 1316 | usage=UsageInfo( |
| 1317 | prompt_tokens=50, |
| 1318 | completion_tokens=5, |
| 1319 | total_tokens=55, |
| 1320 | prompt_tokens_details=PromptTokenUsageInfo( |
| 1321 | cached_tokens=0, created_cache_tokens=0 |
| 1322 | ), |
| 1323 | ), |
| 1324 | ) |
| 1325 | yield "data: [DONE]" |
| 1326 | |
| 1327 | converter = _make_stream_converter() |
| 1328 | output = [] |
| 1329 | async for event in converter.message_stream_converter(sse_input()): |
| 1330 | output.append(event) |
| 1331 | events = _parse_sse_events(output) |
| 1332 | |
| 1333 | start_usage = events[0][1]["message"]["usage"] |
| 1334 | delta_usage = next( |
| 1335 | data["usage"] for ev, data in events if ev == "message_delta" |
| 1336 | ) |
| 1337 | assert start_usage["input_tokens"] == 50 |
| 1338 | assert "cache_read_input_tokens" not in start_usage |
| 1339 | assert "cache_creation_input_tokens" not in start_usage |
| 1340 | assert delta_usage["input_tokens"] == 50 # 50 - 0 - 0 |
| 1341 | assert delta_usage["cache_read_input_tokens"] == 0 |
| 1342 | assert delta_usage["cache_creation_input_tokens"] == 0 |
| 1343 | |
| 1344 | @pytest.mark.asyncio |
| 1345 | async def test_streaming_no_prompt_tokens_details_at_all(self): |
| 1346 | """If --enable-prompt-tokens-details is off, no chunk carries cache |
| 1347 | info; both message_start and message_delta omit cache fields.""" |
| 1348 | |
| 1349 | async def sse_input(): |
| 1350 | yield _make_stream_chunk( |
| 1351 | delta=DeltaMessage(role="assistant"), |
| 1352 | usage=UsageInfo(prompt_tokens=30, total_tokens=30), |
| 1353 | ) |
| 1354 | yield _make_stream_chunk(finish_reason="stop") |
| 1355 | yield _make_stream_chunk( |
| 1356 | choices=[], |
| 1357 | usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32), |
| 1358 | ) |
| 1359 | yield "data: [DONE]" |
| 1360 | |
| 1361 | converter = _make_stream_converter() |
| 1362 | output = [] |
| 1363 | async for event in converter.message_stream_converter(sse_input()): |
| 1364 | output.append(event) |
| 1365 | events = _parse_sse_events(output) |
| 1366 | |
| 1367 | start_usage = events[0][1]["message"]["usage"] |
| 1368 | delta_usage = next( |
| 1369 | data["usage"] for ev, data in events if ev == "message_delta" |
| 1370 | ) |
| 1371 | assert "cache_read_input_tokens" not in start_usage |
| 1372 | assert "cache_creation_input_tokens" not in start_usage |
| 1373 | assert "cache_read_input_tokens" not in delta_usage |
| 1374 | assert "cache_creation_input_tokens" not in delta_usage |
| 1375 | |
| 1376 | |
| 1377 | # ====================================================================== |
| 1378 | # Auto-detection of system-first template requirement |
| 1379 | # ====================================================================== |
| 1380 | |
| 1381 | |
| 1382 | Q35_TEMPLATE = ( |
| 1383 | "{%- for message in messages %}" |
| 1384 | "{%- if message.role == 'system' %}" |
| 1385 | "{%- if not loop.first %}" |
| 1386 | "{{- raise_exception('System message must be at the beginning.') }}" |
| 1387 | "{%- endif %}" |
| 1388 | "{%- endif %}" |
| 1389 | "{%- endfor %}" |
| 1390 | ) |
| 1391 | |
| 1392 | PERMISSIVE_TEMPLATE = ( |
| 1393 | "{%- for message in messages %}{{- message.role }}: {{ message.content }}\n" |
| 1394 | "{%- endfor %}" |
| 1395 | ) |
| 1396 | |
| 1397 | |
| 1398 | class TestDetectMergeInlineSystem: |
| 1399 | """Verify _detect_merge_inline_system auto-detection. |
| 1400 | |
| 1401 | Tests three scenarios: |
| 1402 | 1. Template with system-first guard (e.g. Qwen) → merge needed |
| 1403 | 2. Template without restrictions → no merge, cache-friendly |
| 1404 | 3. No template provided → safe default: merge |
| 1405 | """ |
| 1406 | |
| 1407 | def test_qwen_template_requires_merge(self): |
| 1408 | """Template with loop.first guard rejects mid-conversation system.""" |
| 1409 | assert ( |
| 1410 | AnthropicServingMessages._detect_merge_inline_system(Q35_TEMPLATE) is True |
| 1411 | ) |
| 1412 | |
| 1413 | def test_no_restriction_no_merge(self): |
| 1414 | """Template without restriction accepts mid-conversation system.""" |
| 1415 | assert ( |
| 1416 | AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE) |
| 1417 | is False |
| 1418 | ) |
| 1419 | |
| 1420 | def test_no_template_defaults_merge(self): |
| 1421 | """No chat_template → conservative default: merge.""" |
| 1422 | assert AnthropicServingMessages._detect_merge_inline_system(None) is True |
| 1423 | |
| 1424 | @pytest.mark.parametrize( |
| 1425 | ("resolved", "expected"), |
| 1426 | [(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)], |
| 1427 | ) |
| 1428 | def test_probes_resolved_template_without_cli_override( |
| 1429 | self, monkeypatch, resolved, expected |
| 1430 | ): |
| 1431 | """Without --chat-template, probe the tokenizer's template (#58727). |
| 1432 | |
| 1433 | Merging defeats prefix caching, so it is warned about at startup. |
| 1434 | """ |
| 1435 | import vllm.entrypoints.anthropic.serving as serving_mod |
| 1436 | from vllm.renderers.hf import HfRenderer |
| 1437 | |
| 1438 | monkeypatch.setattr( |
| 1439 | serving_mod, "resolve_chat_template", lambda *a, **kw: resolved |
| 1440 | ) |
| 1441 | mock_logger = MagicMock() |
| 1442 | monkeypatch.setattr(serving_mod, "logger", mock_logger) |
| 1443 | renderer = MagicMock(spec=HfRenderer) |
| 1444 | renderer.tokenizer = MagicMock() |
| 1445 | online_renderer = SimpleNamespace( |
| 1446 | renderer=renderer, |
| 1447 | chat_template=None, |
| 1448 | model_config=None, |
| 1449 | ) |
| 1450 | assert ( |
| 1451 | AnthropicServingMessages._should_merge_inline_system(online_renderer) |
| 1452 | is expected |
| 1453 | ) |
| 1454 | assert mock_logger.warning_once.called is expected |
| 1455 | |
| 1456 | |
| 1457 | # ====================================================================== |
| 1458 | # Full (non-streaming) response conversion: messages_full_converter |
| 1459 | # ====================================================================== |
| 1460 | |
| 1461 | |
| 1462 | def _make_full_converter(): |
| 1463 | obj = MagicMock(spec=AnthropicServingMessages) |
| 1464 | obj.messages_full_converter = ( |
| 1465 | AnthropicServingMessages.messages_full_converter.__get__(obj) |
| 1466 | ) |
| 1467 | return obj |
| 1468 | |
| 1469 | |
| 1470 | class TestMessagesFullConverter: |
| 1471 | def test_empty_completion_emits_one_text_block(self): |
| 1472 | """An empty completion still yields exactly one (empty) text block.""" |
| 1473 | generator = ChatCompletionResponse( |
| 1474 | id="chatcmpl-empty", |
| 1475 | model="test-model", |
| 1476 | choices=[ |
| 1477 | ChatCompletionResponseChoice( |
| 1478 | index=0, |
| 1479 | message=ChatMessage(role="assistant", content=None), |
| 1480 | finish_reason="stop", |
| 1481 | ) |
| 1482 | ], |
| 1483 | usage=UsageInfo(prompt_tokens=10, completion_tokens=0, total_tokens=10), |
| 1484 | ) |
| 1485 | |
| 1486 | result = _make_full_converter().messages_full_converter(generator) |
| 1487 | |
| 1488 | assert len(result.content) == 1 |
| 1489 | assert result.content[0].type == "text" |
| 1490 | assert result.content[0].text == "" |
| 1491 | |
| 1492 | |
| 1493 | # ====================================================================== |
| 1494 | # cache_salt pass-through (Issue #46688) |
| 1495 | # ====================================================================== |
| 1496 | |
| 1497 | |
| 1498 | class TestCacheSalt: |
| 1499 | def test_cache_salt_passed_through(self): |
| 1500 | """cache_salt on the Anthropic request reaches the converted |
| 1501 | ChatCompletionRequest so prefix-cache isolation works via /v1/messages.""" |
| 1502 | request = _make_request( |
| 1503 | [{"role": "user", "content": "Hello"}], |
| 1504 | cache_salt="tenant-abc-secret-salt", |
| 1505 | ) |
| 1506 | result = _convert(request) |
| 1507 | assert result.cache_salt == "tenant-abc-secret-salt" |
| 1508 | |
| 1509 | def test_cache_salt_defaults_to_none(self): |
| 1510 | """Omitting cache_salt leaves it unset (unchanged default behavior).""" |
| 1511 | request = _make_request([{"role": "user", "content": "Hello"}]) |
| 1512 | result = _convert(request) |
| 1513 | assert result.cache_salt is None |
| 1514 | |
| 1515 | @staticmethod |
| 1516 | def _make_api_app(): |
| 1517 | app = FastAPI() |
| 1518 | attach_router(app) |
| 1519 | app.state.args = Namespace(log_error_stack=False) |
| 1520 | app.exception_handler(RequestValidationError)(validation_exception_handler) |
| 1521 | |
| 1522 | handler = MagicMock(spec=AnthropicServingMessages) |
| 1523 | handler.create_messages.side_effect = AssertionError( |
| 1524 | "invalid requests must not reach the serving handler" |
| 1525 | ) |
| 1526 | app.state.anthropic_serving_messages = handler |
| 1527 | return app, handler |
| 1528 | |
| 1529 | def test_cache_salt_openapi_requires_non_empty_string(self): |
| 1530 | app, _ = self._make_api_app() |
| 1531 | field_schema = app.openapi()["components"]["schemas"][ |
| 1532 | "AnthropicMessagesRequest" |
| 1533 | ]["properties"]["cache_salt"] |
| 1534 | string_schema = next( |
| 1535 | option for option in field_schema["anyOf"] if option.get("type") == "string" |
| 1536 | ) |
| 1537 | |
| 1538 | assert string_schema["minLength"] == 1 |
| 1539 | |
| 1540 | def test_empty_cache_salt_returns_bad_request(self): |
| 1541 | app, handler = self._make_api_app() |
| 1542 | with TestClient(app, raise_server_exceptions=False) as client: |
| 1543 | response = client.post( |
| 1544 | "/v1/messages", |
| 1545 | json={ |
| 1546 | "model": "test-model", |
| 1547 | "max_tokens": 1, |
| 1548 | "messages": [{"role": "user", "content": "Hello"}], |
| 1549 | "cache_salt": "", |
| 1550 | }, |
| 1551 | ) |
| 1552 | |
| 1553 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 1554 | handler.create_messages.assert_not_awaited() |
| 1555 | |
| 1556 | |
| 1557 | class TestStopSequenceReason: |
| 1558 | """When generation stops because a configured stop string matched, the |
| 1559 | Anthropic Messages API must report ``stop_reason="stop_sequence"`` and echo |
| 1560 | the matched string in ``stop_sequence``. vLLM surfaces the matched string in |
| 1561 | the OpenAI choice's ``stop_reason`` field (a str) while ``finish_reason`` |
| 1562 | stays ``"stop"``. A natural EOS (stop_reason None) or a stop token id (int) |
| 1563 | must still map to ``end_turn``. |
| 1564 | """ |
| 1565 | |
| 1566 | def test_non_streaming_stop_string_maps_to_stop_sequence(self): |
| 1567 | converter = _make_full_converter() |
| 1568 | response = ChatCompletionResponse( |
| 1569 | id="chatcmpl-test", |
| 1570 | model="test-model", |
| 1571 | choices=[ |
| 1572 | ChatCompletionResponseChoice( |
| 1573 | index=0, |
| 1574 | message=ChatMessage(role="assistant", content="hello"), |
| 1575 | finish_reason="stop", |
| 1576 | stop_reason="</tool>", |
| 1577 | ) |
| 1578 | ], |
| 1579 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1580 | ) |
| 1581 | |
| 1582 | result = converter.messages_full_converter(response) |
| 1583 | |
| 1584 | assert result.stop_reason == "stop_sequence" |
| 1585 | assert result.stop_sequence == "</tool>" |
| 1586 | |
| 1587 | def test_non_streaming_natural_eos_maps_to_end_turn(self): |
| 1588 | converter = _make_full_converter() |
| 1589 | response = ChatCompletionResponse( |
| 1590 | id="chatcmpl-test", |
| 1591 | model="test-model", |
| 1592 | choices=[ |
| 1593 | ChatCompletionResponseChoice( |
| 1594 | index=0, |
| 1595 | message=ChatMessage(role="assistant", content="hello"), |
| 1596 | finish_reason="stop", |
| 1597 | stop_reason=None, |
| 1598 | ) |
| 1599 | ], |
| 1600 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1601 | ) |
| 1602 | |
| 1603 | result = converter.messages_full_converter(response) |
| 1604 | |
| 1605 | assert result.stop_reason == "end_turn" |
| 1606 | assert result.stop_sequence is None |
| 1607 | |
| 1608 | def test_non_streaming_stop_token_id_maps_to_end_turn(self): |
| 1609 | converter = _make_full_converter() |
| 1610 | response = ChatCompletionResponse( |
| 1611 | id="chatcmpl-test", |
| 1612 | model="test-model", |
| 1613 | choices=[ |
| 1614 | ChatCompletionResponseChoice( |
| 1615 | index=0, |
| 1616 | message=ChatMessage(role="assistant", content="hello"), |
| 1617 | finish_reason="stop", |
| 1618 | stop_reason=128009, |
| 1619 | ) |
| 1620 | ], |
| 1621 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1622 | ) |
| 1623 | |
| 1624 | result = converter.messages_full_converter(response) |
| 1625 | |
| 1626 | assert result.stop_reason == "end_turn" |
| 1627 | assert result.stop_sequence is None |
| 1628 | |
| 1629 | @pytest.mark.asyncio |
| 1630 | async def test_streaming_stop_string_maps_to_stop_sequence(self): |
| 1631 | async def sse_input(): |
| 1632 | yield _make_stream_chunk( |
| 1633 | delta=DeltaMessage(content="hi"), |
| 1634 | usage=UsageInfo(prompt_tokens=5, total_tokens=5, completion_tokens=0), |
| 1635 | ) |
| 1636 | yield _make_stream_chunk(finish_reason="stop", stop_reason="</tool>") |
| 1637 | yield _make_stream_chunk( |
| 1638 | choices=[], |
| 1639 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1640 | ) |
| 1641 | yield "data: [DONE]" |
| 1642 | |
| 1643 | converter = _make_stream_converter() |
| 1644 | output = [] |
| 1645 | async for event in converter.message_stream_converter(sse_input()): |
| 1646 | output.append(event) |
| 1647 | |
| 1648 | events = _parse_sse_events(output) |
| 1649 | msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"] |
| 1650 | assert msg_deltas[0]["delta"]["stop_reason"] == "stop_sequence" |
| 1651 | assert msg_deltas[0]["delta"]["stop_sequence"] == "</tool>" |
| 1652 | |
| 1653 | @pytest.mark.asyncio |
| 1654 | async def test_streaming_no_stop_string_emits_explicit_null_stop_sequence(self): |
| 1655 | """exclude_unset=True drops stop_sequence unless it is set explicitly.""" |
| 1656 | |
| 1657 | async def sse_input(): |
| 1658 | yield _make_stream_chunk(delta=DeltaMessage(role="assistant")) |
| 1659 | yield _make_stream_chunk(delta=DeltaMessage(content="hi")) |
| 1660 | yield _make_stream_chunk(finish_reason="stop") |
| 1661 | yield _make_stream_chunk( |
| 1662 | choices=[], |
| 1663 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1664 | ) |
| 1665 | yield "data: [DONE]" |
| 1666 | |
| 1667 | converter = _make_stream_converter() |
| 1668 | output = [] |
| 1669 | async for event in converter.message_stream_converter(sse_input()): |
| 1670 | output.append(event) |
| 1671 | |
| 1672 | events = _parse_sse_events(output) |
| 1673 | msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"] |
| 1674 | assert len(msg_deltas) == 1 |
| 1675 | assert msg_deltas[0]["delta"]["stop_reason"] == "end_turn" |
| 1676 | assert "stop_sequence" in msg_deltas[0]["delta"] |
| 1677 | assert msg_deltas[0]["delta"]["stop_sequence"] is None |
| 1678 | |
| 1679 | |
| 1680 | class TestToolUseStopReason: |
| 1681 | """A response carrying a tool_use block reports ``stop_reason="tool_use"`` |
| 1682 | even when the chat layer says ``finish_reason="stop"``, as it does for a |
| 1683 | forced (named) tool. A call cut short by max_tokens or a stop string does not. |
| 1684 | """ |
| 1685 | |
| 1686 | @pytest.mark.parametrize( |
| 1687 | ("finish_reason", "stop_reason", "expected"), |
| 1688 | [ |
| 1689 | ("stop", None, "tool_use"), |
| 1690 | ("tool_calls", "</x>", "tool_use"), |
| 1691 | ], |
| 1692 | ) |
| 1693 | def test_non_streaming(self, finish_reason, stop_reason, expected): |
| 1694 | tool_call = ToolCall( |
| 1695 | function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}') |
| 1696 | ) |
| 1697 | response = ChatCompletionResponse( |
| 1698 | id="chatcmpl-test", |
| 1699 | model="test-model", |
| 1700 | choices=[ |
| 1701 | ChatCompletionResponseChoice( |
| 1702 | index=0, |
| 1703 | message=ChatMessage(role="assistant", tool_calls=[tool_call]), |
| 1704 | finish_reason=finish_reason, |
| 1705 | stop_reason=stop_reason, |
| 1706 | ) |
| 1707 | ], |
| 1708 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1709 | ) |
| 1710 | |
| 1711 | result = _make_full_converter().messages_full_converter(response) |
| 1712 | |
| 1713 | assert result.content[0].type == "tool_use" |
| 1714 | assert result.stop_reason == expected |
| 1715 | |
| 1716 | @pytest.mark.asyncio |
| 1717 | @pytest.mark.parametrize( |
| 1718 | ("args", "finish_reason", "stop_reason", "expected"), |
| 1719 | [ |
| 1720 | ('{"city": "Paris"}', "stop", None, "tool_use"), |
| 1721 | ('{"city": "Pa', "length", None, "max_tokens"), |
| 1722 | ('{"city": "Paris"}', "tool_calls", "</x>", "tool_use"), |
| 1723 | ('{"city": "Pa', "tool_calls", "</x>", "stop_sequence"), |
| 1724 | ], |
| 1725 | ) |
| 1726 | async def test_streaming(self, args, finish_reason, stop_reason, expected): |
| 1727 | async def sse_input(): |
| 1728 | yield _make_stream_chunk(delta=DeltaMessage(role="assistant")) |
| 1729 | yield _make_stream_chunk( |
| 1730 | delta=DeltaMessage( |
| 1731 | tool_calls=[_tc(id="call_1", name="get_weather", args=args)] |
| 1732 | ) |
| 1733 | ) |
| 1734 | yield _make_stream_chunk( |
| 1735 | finish_reason=finish_reason, stop_reason=stop_reason |
| 1736 | ) |
| 1737 | yield _make_stream_chunk( |
| 1738 | choices=[], |
| 1739 | usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), |
| 1740 | ) |
| 1741 | yield "data: [DONE]" |
| 1742 | |
| 1743 | converter = _make_stream_converter() |
| 1744 | events = _parse_sse_events( |
| 1745 | [e async for e in converter.message_stream_converter(sse_input())] |
| 1746 | ) |
| 1747 | |
| 1748 | msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"] |
| 1749 | assert msg_deltas[0]["delta"]["stop_reason"] == expected |
| 1750 | |
| 1751 | |
| 1752 | # ====================================================================== |
| 1753 | # Client-caused errors are 4xx, not 500 (Issue #52088) |
| 1754 | # ====================================================================== |
| 1755 | |
| 1756 | |
| 1757 | class TestClientErrorResponses: |
| 1758 | @staticmethod |
| 1759 | def _make_api_app(handler: MagicMock): |
| 1760 | app = FastAPI() |
| 1761 | attach_router(app) |
| 1762 | app.state.args = Namespace(log_error_stack=False) |
| 1763 | app.exception_handler(RequestValidationError)(validation_exception_handler) |
| 1764 | app.state.anthropic_serving_messages = handler |
| 1765 | return app |
| 1766 | |
| 1767 | @staticmethod |
| 1768 | def _request_body() -> dict: |
| 1769 | return { |
| 1770 | "model": "test-model", |
| 1771 | "max_tokens": 1, |
| 1772 | "messages": [{"role": "user", "content": "Hello"}], |
| 1773 | } |
| 1774 | |
| 1775 | @staticmethod |
| 1776 | def _conversion_error() -> ValidationError: |
| 1777 | """A real pydantic ValidationError like the one ChatCompletionRequest |
| 1778 | construction raises when Anthropic input violates the OpenAI schema.""" |
| 1779 | |
| 1780 | class _StubRequest(BaseModel): |
| 1781 | stop: Annotated[list[str], Field(max_length=4)] | None = None |
| 1782 | |
| 1783 | with pytest.raises(ValidationError) as exc_info: |
| 1784 | _StubRequest(stop=["a"] * 6) |
| 1785 | return exc_info.value |
| 1786 | |
| 1787 | def test_validation_error_returns_bad_request(self): |
| 1788 | """A pydantic ValidationError during Anthropic->OpenAI conversion is |
| 1789 | surfaced as a 400 BadRequestError, not a 500.""" |
| 1790 | handler = MagicMock(spec=AnthropicServingMessages) |
| 1791 | handler.create_messages.side_effect = self._conversion_error() |
| 1792 | |
| 1793 | app = self._make_api_app(handler) |
| 1794 | with TestClient(app, raise_server_exceptions=False) as client: |
| 1795 | response = client.post("/v1/messages", json=self._request_body()) |
| 1796 | |
| 1797 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 1798 | body = response.json() |
| 1799 | assert body["type"] == "error" |
| 1800 | assert body["error"]["type"] == "BadRequestError" |
| 1801 | assert "at most 4 items" in body["error"]["message"] |
| 1802 | |
| 1803 | def test_vllm_client_error_returns_bad_request(self): |
| 1804 | """VLLMClientError raised by the serving layer maps to 400.""" |
| 1805 | handler = MagicMock(spec=AnthropicServingMessages) |
| 1806 | handler.create_messages.side_effect = VLLMValidationError( |
| 1807 | "Invalid value for stop", parameter="stop" |
| 1808 | ) |
| 1809 | |
| 1810 | app = self._make_api_app(handler) |
| 1811 | with TestClient(app, raise_server_exceptions=False) as client: |
| 1812 | response = client.post("/v1/messages", json=self._request_body()) |
| 1813 | |
| 1814 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 1815 | assert response.json()["error"]["type"] == "BadRequestError" |
| 1816 | |
| 1817 | def test_generic_error_still_returns_internal_server_error(self): |
| 1818 | """Non-client errors keep the existing 500 behaviour.""" |
| 1819 | handler = MagicMock(spec=AnthropicServingMessages) |
| 1820 | handler.create_messages.side_effect = RuntimeError("boom") |
| 1821 | |
| 1822 | app = self._make_api_app(handler) |
| 1823 | with TestClient(app, raise_server_exceptions=False) as client: |
| 1824 | response = client.post("/v1/messages", json=self._request_body()) |
| 1825 | |
| 1826 | assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR |
| 1827 | assert response.json()["error"]["type"] == "InternalServerError" |
| 1828 | |
| 1829 | def test_count_tokens_validation_error_returns_bad_request(self): |
| 1830 | """The count_tokens route maps conversion errors to 400 as well.""" |
| 1831 | handler = MagicMock(spec=AnthropicServingMessages) |
| 1832 | handler.count_tokens.side_effect = self._conversion_error() |
| 1833 | |
| 1834 | app = self._make_api_app(handler) |
| 1835 | with TestClient(app, raise_server_exceptions=False) as client: |
| 1836 | response = client.post( |
| 1837 | "/v1/messages/count_tokens", json=self._request_body() |
| 1838 | ) |
| 1839 | |
| 1840 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 1841 | assert response.json()["error"]["type"] == "BadRequestError" |
| 1842 | |
| 1843 | |
| 1844 | # ====================================================================== |
| 1845 | # thinking configuration pass-through |
| 1846 | # ====================================================================== |
| 1847 | |
| 1848 | |
| 1849 | class TestThinkingConfig: |
| 1850 | def test_absent_thinking_leaves_reasoning_untouched(self): |
| 1851 | """Requests without `thinking` must convert exactly as before.""" |
| 1852 | request = _make_request([{"role": "user", "content": "Hello"}]) |
| 1853 | |
| 1854 | result = _convert(request) |
| 1855 | assert result.reasoning_effort is None |
| 1856 | assert result.thinking_token_budget is None |
| 1857 | assert result.include_reasoning is True |
| 1858 | |
| 1859 | def test_disabled_clears_reasoning_effort(self): |
| 1860 | """`disabled` maps to reasoning_effort="none", which is what clears |
| 1861 | enable_thinking for templates that honor it.""" |
| 1862 | request = _make_request( |
| 1863 | [{"role": "user", "content": "Hello"}], |
| 1864 | thinking={"type": "disabled"}, |
| 1865 | ) |
| 1866 | |
| 1867 | result = _convert(request) |
| 1868 | assert result.reasoning_effort == "none" |
| 1869 | |
| 1870 | def test_disabled_overrides_output_config_effort(self): |
| 1871 | """`thinking` is applied after output_config so an explicit opt-out wins |
| 1872 | over an inherited effort ceiling.""" |
| 1873 | request = _make_request( |
| 1874 | [{"role": "user", "content": "Hello"}], |
| 1875 | output_config={"effort": "high"}, |
| 1876 | thinking={"type": "disabled"}, |
| 1877 | ) |
| 1878 | |
| 1879 | result = _convert(request) |
| 1880 | assert result.reasoning_effort == "none" |
| 1881 | |
| 1882 | def test_enabled_sets_thinking_token_budget(self): |
| 1883 | request = AnthropicMessagesRequest( |
| 1884 | model="test-model", |
| 1885 | max_tokens=4096, |
| 1886 | messages=[{"role": "user", "content": "Hello"}], |
| 1887 | thinking={"type": "enabled", "budget_tokens": 2048}, |
| 1888 | ) |
| 1889 | |
| 1890 | result = _convert(request) |
| 1891 | assert result.thinking_token_budget == 2048 |
| 1892 | assert result.reasoning_effort is None |
| 1893 | |
| 1894 | @pytest.mark.parametrize( |
| 1895 | "thinking", |
| 1896 | [ |
| 1897 | pytest.param({"budget_tokens": 2048}, id="missing-type"), |
| 1898 | pytest.param({"type": "enabled"}, id="enabled-missing-budget"), |
| 1899 | pytest.param( |
| 1900 | {"type": "enabled", "budget_tokens": 1023}, id="budget-below-1024" |
| 1901 | ), |
| 1902 | pytest.param( |
| 1903 | {"type": "enabled", "budget_tokens": 4096}, id="budget-not-below-max" |
| 1904 | ), |
| 1905 | pytest.param({"type": "adaptive", "display": "full"}, id="bad-display"), |
| 1906 | ], |
| 1907 | ) |
| 1908 | def test_rejects_invalid_thinking(self, thinking): |
| 1909 | """Mirror the Anthropic API's BetaThinkingConfigParam constraints.""" |
| 1910 | with pytest.raises(ValidationError): |
| 1911 | AnthropicMessagesRequest( |
| 1912 | model="test-model", |
| 1913 | max_tokens=4096, |
| 1914 | messages=[{"role": "user", "content": "Hello"}], |
| 1915 | thinking=thinking, |
| 1916 | ) |
| 1917 | |
| 1918 | def test_pd_prefill_leg_skips_budget_check(self): |
| 1919 | """P/D sidecars resend the request to the prefill node with |
| 1920 | max_tokens=1; the budget is enforced on the decode leg instead.""" |
| 1921 | thinking = {"type": "enabled", "budget_tokens": 1024} |
| 1922 | request = AnthropicMessagesRequest( |
| 1923 | model="test-model", |
| 1924 | max_tokens=1, |
| 1925 | messages=[{"role": "user", "content": "Hello"}], |
| 1926 | thinking=thinking, |
| 1927 | kv_transfer_params={"do_remote_decode": True}, |
| 1928 | ) |
| 1929 | |
| 1930 | assert _convert(request).thinking_token_budget == 1024 |
| 1931 | with pytest.raises(ValidationError): |
| 1932 | AnthropicMessagesRequest( |
| 1933 | model="test-model", |
| 1934 | max_tokens=1, |
| 1935 | messages=[{"role": "user", "content": "Hello"}], |
| 1936 | thinking=thinking, |
| 1937 | kv_transfer_params={"do_remote_decode": False}, |
| 1938 | ) |
| 1939 | |
| 1940 | def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self): |
| 1941 | """`adaptive` lets the model choose depth, so only the ceiling from |
| 1942 | output_config.effort should survive.""" |
| 1943 | request = _make_request( |
| 1944 | [{"role": "user", "content": "Hello"}], |
| 1945 | output_config={"effort": "low"}, |
| 1946 | thinking={"type": "adaptive"}, |
| 1947 | ) |
| 1948 | |
| 1949 | result = _convert(request) |
| 1950 | assert result.reasoning_effort == "low" |
| 1951 | assert result.thinking_token_budget is None |
| 1952 | |
| 1953 | def test_disabled_uses_configured_effort(self): |
| 1954 | """Models that always think (GLM-5.3) or reject "none" (Harmony) are |
| 1955 | served with a low effort instead.""" |
| 1956 | request = _make_request( |
| 1957 | [{"role": "user", "content": "Hello"}], |
| 1958 | thinking={"type": "disabled"}, |
| 1959 | ) |
| 1960 | |
| 1961 | result = _convert(request, disabled_thinking_effort="low") |
| 1962 | assert result.reasoning_effort == "low" |
| 1963 | |
| 1964 | @pytest.mark.parametrize("display", ["omitted", "summarized", "updates"]) |
| 1965 | def test_display_keeps_reasoning_included(self, display): |
| 1966 | """Suppressing reasoning would mark it ended for structured outputs and |
| 1967 | drop it from multi-turn history, so `display` is ignored.""" |
| 1968 | request = _make_request( |
| 1969 | [{"role": "user", "content": "Hello"}], |
| 1970 | thinking={"type": "adaptive", "display": display}, |
| 1971 | ) |
| 1972 | |
| 1973 | result = _convert(request) |
| 1974 | assert result.include_reasoning is True |
| 1975 | |
| 1976 | def test_claude_code_payload(self): |
| 1977 | """The combination Claude Code sends on every request: an effort ceiling |
| 1978 | plus adaptive thinking with reasoning display omitted.""" |
| 1979 | request = _make_request( |
| 1980 | [{"role": "user", "content": "Hello"}], |
| 1981 | output_config={"effort": "high"}, |
| 1982 | thinking={"type": "adaptive", "display": "omitted"}, |
| 1983 | ) |
| 1984 | |
| 1985 | result = _convert(request) |
| 1986 | assert result.reasoning_effort == "high" |
| 1987 | assert result.include_reasoning is True |
| 1988 | assert result.thinking_token_budget is None |
| 1989 | |
| 1990 | |
| 1991 | class TestProbeDisabledThinkingEffort: |
| 1992 | """``auto`` falls back to ``low`` when ``none`` cannot turn thinking off.""" |
| 1993 | |
| 1994 | @staticmethod |
| 1995 | async def _probe(render): |
| 1996 | obj = MagicMock(spec=AnthropicServingMessages) |
| 1997 | obj.online_renderer = MagicMock() |
| 1998 | obj.online_renderer.render_chat = AsyncMock( |
| 1999 | side_effect=lambda req: ([], [render(req.reasoning_effort)]) |
| 2000 | ) |
| 2001 | obj._extract_prompt_components = lambda engine_input: SimpleNamespace( |
| 2002 | token_ids=engine_input, text=None |
| 2003 | ) |
| 2004 | obj._render_probe_prompt = ( |
| 2005 | AnthropicServingMessages._render_probe_prompt.__get__(obj) |
| 2006 | ) |
| 2007 | return await AnthropicServingMessages._probe_disabled_thinking_effort(obj) |
| 2008 | |
| 2009 | @staticmethod |
| 2010 | def _reject_none(effort): |
| 2011 | """Harmony (gpt-oss) raises on ``none``.""" |
| 2012 | if effort == "none": |
| 2013 | raise ValueError(f"unsupported {effort=}") |
| 2014 | return [1] |
| 2015 | |
| 2016 | @staticmethod |
| 2017 | def _none_as_max(effort): |
| 2018 | """GLM-5.3 treats efforts other than low/high as max.""" |
| 2019 | return [1, {"low": 0, "high": 1}.get(effort, 2)] |
| 2020 | |
| 2021 | @pytest.mark.asyncio |
| 2022 | async def test_template_honors_none(self): |
| 2023 | assert await self._probe(lambda effort: [1, int(effort == "none")]) == "none" |
| 2024 | |
| 2025 | @pytest.mark.asyncio |
| 2026 | async def test_template_ignores_effort(self): |
| 2027 | assert await self._probe(lambda effort: [1]) == "low" |
| 2028 | |
| 2029 | @pytest.mark.asyncio |
| 2030 | async def test_template_renders_none_as_thinking_effort(self): |
| 2031 | assert await self._probe(self._none_as_max) == "low" |
| 2032 | |
| 2033 | @pytest.mark.asyncio |
| 2034 | async def test_renderer_rejects_none(self): |
| 2035 | assert await self._probe(self._reject_none) == "low" |
| 2036 | |
| 2037 | |
| 2038 | class TestMidConversationToolChanges: |
| 2039 | """``tool_addition``/``tool_removal`` blocks decide which tools the chat |
| 2040 | template is given, since it has no other way to see them.""" |
| 2041 | |
| 2042 | @staticmethod |
| 2043 | def _convert_tools(*changes: dict) -> dict[str, bool | None]: |
| 2044 | request = _make_request( |
| 2045 | [ |
| 2046 | {"role": "user", "content": "Hello"}, |
| 2047 | { |
| 2048 | "role": "system", |
| 2049 | "content": [{"type": "text", "text": "Tools changed."}, *changes], |
| 2050 | }, |
| 2051 | ], |
| 2052 | tools=[ |
| 2053 | {"name": "bash", "input_schema": {}}, |
| 2054 | {"name": "search", "input_schema": {}, "defer_loading": True}, |
| 2055 | {"name": "fetch", "input_schema": {}, "defer_loading": True}, |
| 2056 | ], |
| 2057 | ) |
| 2058 | result = _convert(request) |
| 2059 | assert result.messages[-1] == {"role": "system", "content": "Tools changed."} |
| 2060 | assert result.tools is not None |
| 2061 | return {t.function.name: t.function.defer_loading for t in result.tools} |
| 2062 | |
| 2063 | @staticmethod |
| 2064 | def _change(block_type: str, name: str) -> dict: |
| 2065 | return {"type": block_type, "tool": {"type": "tool_reference", "name": name}} |
| 2066 | |
| 2067 | def test_addition_loads_deferred_tool(self): |
| 2068 | tools = self._convert_tools(self._change("tool_addition", "search")) |
| 2069 | assert tools == {"bash": None, "search": None, "fetch": True} |
| 2070 | |
| 2071 | def test_removal_withdraws_tool_until_added_again(self): |
| 2072 | removal = self._change("tool_removal", "bash") |
| 2073 | assert "bash" not in self._convert_tools(removal) |
| 2074 | readded = self._convert_tools(removal, self._change("tool_addition", "bash")) |
| 2075 | assert list(readded) == ["bash", "search", "fetch"] |
| 2076 | |
| 2077 | def test_addition_defines_tool_by_value(self): |
| 2078 | definition = {"name": "db_query", "input_schema": {}} |
| 2079 | tools = self._convert_tools( |
| 2080 | { |
| 2081 | "type": "tool_addition", |
| 2082 | "tool": {"type": "tool_definition", "definition": definition}, |
| 2083 | } |
| 2084 | ) |
| 2085 | assert list(tools) == ["bash", "search", "fetch", "db_query"] |
| 2086 | |
| 2087 | def test_unknown_tool_reference_is_rejected(self): |
| 2088 | with pytest.raises(ValueError, match="not declared in tools"): |
| 2089 | self._convert_tools(self._change("tool_addition", "missing")) |
| 2090 | |
| 2091 | def test_removal_by_value_is_rejected(self): |
| 2092 | definition = {"name": "bash", "input_schema": {}} |
| 2093 | with pytest.raises(ValidationError, match="only accepts a tool_reference"): |
| 2094 | self._convert_tools( |
| 2095 | { |
| 2096 | "type": "tool_removal", |
| 2097 | "tool": {"type": "tool_definition", "definition": definition}, |
| 2098 | } |
| 2099 | ) |
| 2100 | |
| 2101 | @pytest.mark.parametrize( |
| 2102 | "request_cls", [AnthropicMessagesRequest, AnthropicCountTokensRequest] |
| 2103 | ) |
| 2104 | @pytest.mark.parametrize("in_top_level_system", [False, True]) |
| 2105 | def test_tool_change_outside_system_message_is_rejected( |
| 2106 | self, request_cls, in_top_level_system |
| 2107 | ): |
| 2108 | change = self._change("tool_addition", "search") |
| 2109 | if in_top_level_system: |
| 2110 | fields = { |
| 2111 | "messages": [{"role": "user", "content": "Hi"}], |
| 2112 | "system": [change], |
| 2113 | } |
| 2114 | else: |
| 2115 | fields = {"messages": [{"role": "user", "content": [change]}]} |
| 2116 | with pytest.raises( |
| 2117 | ValidationError, match='only allowed in messages with role "system"' |
| 2118 | ): |
| 2119 | request_cls(model="test-model", max_tokens=128, **fields) |