| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from argparse import Namespace |
| 5 | from dataclasses import dataclass, field |
| 6 | from http import HTTPStatus |
| 7 | from typing import Any |
| 8 | from unittest.mock import AsyncMock, MagicMock |
| 9 | |
| 10 | import pytest |
| 11 | from fastapi import FastAPI |
| 12 | from fastapi.testclient import TestClient |
| 13 | |
| 14 | from vllm.config.multimodal import MultiModalConfig |
| 15 | from vllm.entrypoints.openai.models.protocol import BaseModelPath |
| 16 | from vllm.entrypoints.openai.models.serving import OpenAIServingModels |
| 17 | from vllm.entrypoints.serve.exception_handling.register import init_exception_handler |
| 18 | from vllm.entrypoints.serve.tokenize.api_router import attach_router |
| 19 | from vllm.entrypoints.serve.tokenize.protocol import ( |
| 20 | TokenizeChatRequest, |
| 21 | TokenizeCompletionRequest, |
| 22 | TokenizeResponse, |
| 23 | ) |
| 24 | from vllm.entrypoints.serve.tokenize.serving import ServingTokenization |
| 25 | from vllm.exceptions import VLLMNotFoundError, VLLMValidationError |
| 26 | from vllm.renderers.online_renderer import OnlineRenderer |
| 27 | from vllm.v1.engine.async_llm import AsyncLLM |
| 28 | |
| 29 | MODEL_NAME = "openai-community/gpt2" |
| 30 | BASE_MODEL_PATHS = [ |
| 31 | BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME), |
| 32 | ] |
| 33 | |
| 34 | |
| 35 | @dataclass |
| 36 | class MockHFConfig: |
| 37 | model_type: str = "any" |
| 38 | |
| 39 | |
| 40 | @dataclass |
| 41 | class MockModelConfig: |
| 42 | task = "generate" |
| 43 | runner_type = "generate" |
| 44 | model = MODEL_NAME |
| 45 | tokenizer = MODEL_NAME |
| 46 | trust_remote_code = False |
| 47 | tokenizer_mode = "auto" |
| 48 | max_model_len = 100 |
| 49 | tokenizer_revision = None |
| 50 | multimodal_config = MultiModalConfig() |
| 51 | hf_config = MockHFConfig() |
| 52 | hf_text_config = MockHFConfig() |
| 53 | logits_processors: list[str] | None = None |
| 54 | diff_sampling_param: dict | None = None |
| 55 | allowed_local_media_path: str = "" |
| 56 | allowed_media_domains: list[str] | None = None |
| 57 | encoder_config = None |
| 58 | generation_config: str = "auto" |
| 59 | media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict) |
| 60 | skip_tokenizer_init: bool = False |
| 61 | is_encoder_decoder: bool = False |
| 62 | is_multimodal_model: bool = False |
| 63 | supports_multimodal_inputs: bool = False |
| 64 | renderer_num_workers: int = 1 |
| 65 | |
| 66 | def get_diff_sampling_param(self): |
| 67 | return self.diff_sampling_param or {} |
| 68 | |
| 69 | |
| 70 | def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization: |
| 71 | models = OpenAIServingModels( |
| 72 | engine_client=engine, |
| 73 | base_model_paths=BASE_MODEL_PATHS, |
| 74 | ) |
| 75 | online_renderer = OnlineRenderer( |
| 76 | model_config=engine.model_config, |
| 77 | renderer=engine.renderer, |
| 78 | request_logger=None, |
| 79 | chat_template=None, |
| 80 | chat_template_content_format="auto", |
| 81 | ) |
| 82 | return ServingTokenization( |
| 83 | models, |
| 84 | online_renderer=online_renderer, |
| 85 | chat_template=None, |
| 86 | chat_template_content_format="auto", |
| 87 | ) |
| 88 | |
| 89 | |
| 90 | @pytest.mark.asyncio |
| 91 | async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path(): |
| 92 | mock_engine = MagicMock(spec=AsyncLLM) |
| 93 | mock_engine.errored = False |
| 94 | mock_engine.model_config = MockModelConfig() |
| 95 | mock_engine.input_processor = MagicMock() |
| 96 | mock_engine.renderer = MagicMock() |
| 97 | |
| 98 | serving = _build_serving_tokenization(mock_engine) |
| 99 | serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign] |
| 100 | return_value=( |
| 101 | [{"role": "user", "content": "Test"}], |
| 102 | [{"prompt_token_ids": [1, 2, 3]}], |
| 103 | ) |
| 104 | ) |
| 105 | |
| 106 | request = TokenizeChatRequest( |
| 107 | model=MODEL_NAME, |
| 108 | messages=[{"role": "user", "content": "Test prompt"}], |
| 109 | ) |
| 110 | |
| 111 | response = await serving.create_tokenize(request, MagicMock(headers={})) |
| 112 | |
| 113 | assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3] |
| 114 | assert ( |
| 115 | serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"] |
| 116 | is True |
| 117 | ) |
| 118 | |
| 119 | |
| 120 | @pytest.mark.asyncio |
| 121 | async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path(): |
| 122 | mock_engine = MagicMock(spec=AsyncLLM) |
| 123 | mock_engine.errored = False |
| 124 | mock_engine.model_config = MockModelConfig() |
| 125 | mock_engine.input_processor = MagicMock() |
| 126 | mock_engine.renderer = MagicMock() |
| 127 | |
| 128 | serving = _build_serving_tokenization(mock_engine) |
| 129 | serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign] |
| 130 | return_value=[{"prompt_token_ids": [1, 2, 3]}] |
| 131 | ) |
| 132 | |
| 133 | request = TokenizeCompletionRequest( |
| 134 | model=MODEL_NAME, |
| 135 | prompt="Test prompt", |
| 136 | ) |
| 137 | |
| 138 | response = await serving.create_tokenize(request, MagicMock(headers={})) |
| 139 | |
| 140 | assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3] |
| 141 | assert ( |
| 142 | serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"] |
| 143 | is True |
| 144 | ) |
| 145 | |
| 146 | |
| 147 | class TestDetokenizeClientErrorResponses: |
| 148 | """Client-caused errors from /detokenize are 4xx, not 500. |
| 149 | |
| 150 | /detokenize used to wrap ``create_detokenize`` in a blanket |
| 151 | ``except Exception`` that reported every failure as 500, while its |
| 152 | sibling /tokenize let exceptions propagate to the global exception |
| 153 | handlers, which map client-caused errors to 4xx and keep genuine |
| 154 | server errors at 500. These tests pin the shared contract for both |
| 155 | endpoints. See #52246 for the same fix in the Anthropic entrypoint. |
| 156 | """ |
| 157 | |
| 158 | @staticmethod |
| 159 | def _make_api_app(handler: MagicMock) -> FastAPI: |
| 160 | app = FastAPI() |
| 161 | app.state.args = Namespace(log_error_stack=False) |
| 162 | app.state.serving_tokenization = handler |
| 163 | attach_router(app) |
| 164 | init_exception_handler(app) |
| 165 | return app |
| 166 | |
| 167 | def _post(self, handler: MagicMock, path: str = "/detokenize"): |
| 168 | app = self._make_api_app(handler) |
| 169 | body: dict[str, Any] = {"model": "test-model"} |
| 170 | if path == "/detokenize": |
| 171 | body["tokens"] = [1, 2, 3] |
| 172 | else: |
| 173 | body["prompt"] = "Hello" |
| 174 | with TestClient(app, raise_server_exceptions=False) as client: |
| 175 | return client.post(path, json=body) |
| 176 | |
| 177 | def test_vllm_validation_error_returns_bad_request(self): |
| 178 | handler = MagicMock(spec=ServingTokenization) |
| 179 | handler.create_detokenize.side_effect = VLLMValidationError( |
| 180 | "invalid token ids", parameter="tokens" |
| 181 | ) |
| 182 | |
| 183 | response = self._post(handler) |
| 184 | |
| 185 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 186 | error = response.json()["error"] |
| 187 | assert error["type"] == "BadRequestError" |
| 188 | assert error["param"] == "tokens" |
| 189 | |
| 190 | def test_vllm_not_found_error_returns_not_found(self): |
| 191 | handler = MagicMock(spec=ServingTokenization) |
| 192 | handler.create_detokenize.side_effect = VLLMNotFoundError( |
| 193 | "LoRA adapter nonexistent-lora not found" |
| 194 | ) |
| 195 | |
| 196 | response = self._post(handler) |
| 197 | |
| 198 | assert response.status_code == HTTPStatus.NOT_FOUND |
| 199 | assert response.json()["error"]["type"] == "NotFoundError" |
| 200 | |
| 201 | def test_value_error_returns_bad_request(self): |
| 202 | handler = MagicMock(spec=ServingTokenization) |
| 203 | handler.create_detokenize.side_effect = ValueError("bad input") |
| 204 | |
| 205 | response = self._post(handler) |
| 206 | |
| 207 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 208 | assert response.json()["error"]["type"] == "BadRequestError" |
| 209 | |
| 210 | def test_overflow_error_returns_bad_request(self): |
| 211 | """Fast tokenizers raise OverflowError for out-of-range token ids. |
| 212 | |
| 213 | This used to be special-cased into a RequestValidationError; the |
| 214 | global OverflowError handler keeps the same 400 status code. |
| 215 | """ |
| 216 | handler = MagicMock(spec=ServingTokenization) |
| 217 | handler.create_detokenize.side_effect = OverflowError( |
| 218 | "out of range integral type conversion attempted" |
| 219 | ) |
| 220 | |
| 221 | response = self._post(handler) |
| 222 | |
| 223 | assert response.status_code == HTTPStatus.BAD_REQUEST |
| 224 | assert response.json()["error"]["type"] == "BadRequestError" |
| 225 | |
| 226 | def test_server_error_still_returns_internal_server_error(self): |
| 227 | """Genuine server bugs keep the existing 500 behaviour.""" |
| 228 | handler = MagicMock(spec=ServingTokenization) |
| 229 | handler.create_detokenize.side_effect = RuntimeError("boom") |
| 230 | |
| 231 | response = self._post(handler) |
| 232 | |
| 233 | assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR |
| 234 | assert response.json()["error"]["type"] == "InternalServerError" |
| 235 | |
| 236 | def test_tokenize_and_detokenize_agree_on_client_errors(self): |
| 237 | """The two sibling endpoints report the same client error alike.""" |
| 238 | statuses = {} |
| 239 | for path, method in ( |
| 240 | ("/tokenize", "create_tokenize"), |
| 241 | ("/detokenize", "create_detokenize"), |
| 242 | ): |
| 243 | handler = MagicMock(spec=ServingTokenization) |
| 244 | getattr(handler, method).side_effect = VLLMValidationError( |
| 245 | "invalid input", parameter="model" |
| 246 | ) |
| 247 | statuses[path] = self._post(handler, path=path).status_code |
| 248 | |
| 249 | assert statuses["/tokenize"] == HTTPStatus.BAD_REQUEST |
| 250 | assert statuses["/detokenize"] == HTTPStatus.BAD_REQUEST |