| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | from http import HTTPStatus |
| 6 | from unittest.mock import AsyncMock, Mock |
| 7 | |
| 8 | import openai |
| 9 | import pytest |
| 10 | import pytest_asyncio |
| 11 | import requests |
| 12 | from fastapi import Request |
| 13 | |
| 14 | from tests.utils import RemoteOpenAIServer |
| 15 | from vllm import envs |
| 16 | from vllm.v1.engine.exceptions import EngineDeadError |
| 17 | from vllm.version import __version__ as VLLM_VERSION |
| 18 | |
| 19 | MODEL_NAME = "Qwen/Qwen3-0.6B" |
| 20 | |
| 21 | |
| 22 | @pytest.fixture(scope="module") |
| 23 | def server_args(request: pytest.FixtureRequest) -> list[str]: |
| 24 | """Provide extra arguments to the server via indirect parametrization. |
| 25 | |
| 26 | Usage: |
| 27 | |
| 28 | >>> @pytest.mark.parametrize( |
| 29 | >>> "server_args", |
| 30 | >>> [ |
| 31 | >>> ["--max-model-len", "10100"], |
| 32 | >>> [ |
| 33 | >>> "--model=NousResearch/Hermes-3-Llama-3.1-70B", |
| 34 | >>> "--enable-auto-tool-choice", |
| 35 | >>> ], |
| 36 | >>> ], |
| 37 | >>> indirect=True, |
| 38 | >>> ) |
| 39 | >>> def test_foo(server, client): |
| 40 | >>> ... |
| 41 | |
| 42 | This will run `test_foo` twice with servers with: |
| 43 | - `--max-model-len 10100` |
| 44 | - `--model=NousResearch/Hermes-3-Llama-3.1-70B --enable-auto-tool-choice`. |
| 45 | |
| 46 | """ |
| 47 | if not hasattr(request, "param"): |
| 48 | return [] |
| 49 | |
| 50 | val = request.param |
| 51 | |
| 52 | if isinstance(val, str): |
| 53 | return [val] |
| 54 | |
| 55 | return request.param |
| 56 | |
| 57 | |
| 58 | @pytest.fixture(scope="module") |
| 59 | def server(server_args): |
| 60 | args = [ |
| 61 | # use half precision for speed and memory savings in CI environment |
| 62 | "--dtype", |
| 63 | "bfloat16", |
| 64 | "--max-model-len", |
| 65 | "8192", |
| 66 | "--enforce-eager", |
| 67 | "--max-num-seqs", |
| 68 | "128", |
| 69 | *server_args, |
| 70 | ] |
| 71 | |
| 72 | with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: |
| 73 | yield remote_server |
| 74 | |
| 75 | |
| 76 | @pytest_asyncio.fixture |
| 77 | async def client(server): |
| 78 | async with server.get_async_client() as async_client: |
| 79 | yield async_client |
| 80 | |
| 81 | |
| 82 | @pytest.mark.asyncio |
| 83 | async def test_show_version(server: RemoteOpenAIServer): |
| 84 | response = requests.get(server.url_for("version")) |
| 85 | response.raise_for_status() |
| 86 | |
| 87 | payload = response.json() |
| 88 | assert payload["version"] == VLLM_VERSION |
| 89 | if envs.VLLM_USE_RUST_FRONTEND: |
| 90 | assert payload["rust_frontend_version"] == VLLM_VERSION |
| 91 | |
| 92 | |
| 93 | @pytest.mark.asyncio |
| 94 | async def test_check_health(server: RemoteOpenAIServer): |
| 95 | response = requests.get(server.url_for("health")) |
| 96 | |
| 97 | assert response.status_code == HTTPStatus.OK |
| 98 | |
| 99 | |
| 100 | @pytest.mark.parametrize( |
| 101 | "server_args", |
| 102 | [ |
| 103 | pytest.param(["--max-model-len", "10100"]), |
| 104 | ], |
| 105 | indirect=True, |
| 106 | ) |
| 107 | @pytest.mark.asyncio |
| 108 | async def test_request_cancellation(server: RemoteOpenAIServer): |
| 109 | # clunky test: send an ungodly amount of load in with short timeouts |
| 110 | # then ensure that it still responds quickly afterwards |
| 111 | |
| 112 | chat_input = [{"role": "user", "content": "Write a long story"}] |
| 113 | client = server.get_async_client(timeout=0.5) |
| 114 | tasks = [] |
| 115 | # Request about 2 million tokens |
| 116 | for _ in range(200): |
| 117 | task = asyncio.create_task( |
| 118 | client.chat.completions.create( |
| 119 | messages=chat_input, |
| 120 | model=MODEL_NAME, |
| 121 | max_tokens=10000, |
| 122 | extra_body={"min_tokens": 10000}, |
| 123 | temperature=0.0, |
| 124 | ) |
| 125 | ) |
| 126 | tasks.append(task) |
| 127 | |
| 128 | done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) |
| 129 | |
| 130 | # Make sure all requests were sent to the server and timed out |
| 131 | # (We don't want to hide other errors like 400s that would invalidate this |
| 132 | # test) |
| 133 | assert len(pending) == 0 |
| 134 | for d in done: |
| 135 | with pytest.raises(openai.APITimeoutError): |
| 136 | d.result() |
| 137 | |
| 138 | # If the server had not cancelled all the other requests, then it would not |
| 139 | # be able to respond to this one within the timeout |
| 140 | client = server.get_async_client(timeout=5) |
| 141 | response = await client.chat.completions.create( |
| 142 | messages=chat_input, model=MODEL_NAME, max_tokens=10, temperature=0.0 |
| 143 | ) |
| 144 | |
| 145 | assert len(response.choices) == 1 |
| 146 | |
| 147 | |
| 148 | @pytest.mark.asyncio |
| 149 | async def test_request_wrong_content_type(server: RemoteOpenAIServer): |
| 150 | chat_input = [{"role": "user", "content": "Write a long story"}] |
| 151 | client = server.get_async_client() |
| 152 | |
| 153 | with pytest.raises(openai.APIStatusError): |
| 154 | await client.chat.completions.create( |
| 155 | messages=chat_input, |
| 156 | model=MODEL_NAME, |
| 157 | max_tokens=10000, |
| 158 | extra_headers={"Content-Type": "application/x-www-form-urlencoded"}, |
| 159 | ) |
| 160 | |
| 161 | |
| 162 | @pytest.mark.parametrize( |
| 163 | "server_args", |
| 164 | [pytest.param(["--enable-server-load-tracking"], id="enable-server-load-tracking")], |
| 165 | indirect=True, |
| 166 | ) |
| 167 | @pytest.mark.asyncio |
| 168 | async def test_server_load(server: RemoteOpenAIServer): |
| 169 | # Check initial server load |
| 170 | response = requests.get(server.url_for("load")) |
| 171 | assert response.status_code == HTTPStatus.OK |
| 172 | assert response.json().get("server_load") == 0 |
| 173 | |
| 174 | def make_long_completion_request(): |
| 175 | return requests.post( |
| 176 | server.url_for("v1/completions"), |
| 177 | headers={"Content-Type": "application/json"}, |
| 178 | json={ |
| 179 | "prompt": "Give me a long story", |
| 180 | "max_tokens": 1000, |
| 181 | "temperature": 0, |
| 182 | }, |
| 183 | ) |
| 184 | |
| 185 | # Start the completion request in a background thread. |
| 186 | completion_future = asyncio.create_task( |
| 187 | asyncio.to_thread(make_long_completion_request) |
| 188 | ) |
| 189 | |
| 190 | # Give a short delay to ensure the request has started. |
| 191 | await asyncio.sleep(0.1) |
| 192 | |
| 193 | # Check server load while the completion request is running. |
| 194 | response = requests.get(server.url_for("load")) |
| 195 | assert response.status_code == HTTPStatus.OK |
| 196 | assert response.json().get("server_load") == 1 |
| 197 | |
| 198 | # Wait for the completion request to finish. |
| 199 | await completion_future |
| 200 | await asyncio.sleep(0.1) |
| 201 | |
| 202 | # Check server load after the completion request has finished. |
| 203 | response = requests.get(server.url_for("load")) |
| 204 | assert response.status_code == HTTPStatus.OK |
| 205 | assert response.json().get("server_load") == 0 |
| 206 | |
| 207 | |
| 208 | @pytest.mark.asyncio |
| 209 | async def test_health_check_engine_dead_error(): |
| 210 | # Import the health function directly to test it in isolation |
| 211 | from vllm.entrypoints.serve.instrumentator.health import health |
| 212 | |
| 213 | # Create a mock request that simulates what FastAPI would provide |
| 214 | mock_request = Mock(spec=Request) |
| 215 | mock_app_state = Mock() |
| 216 | mock_engine_client = AsyncMock() |
| 217 | mock_engine_client.check_health.side_effect = EngineDeadError() |
| 218 | mock_app_state.engine_client = mock_engine_client |
| 219 | mock_request.app.state = mock_app_state |
| 220 | |
| 221 | # Test the health function directly with our mocked request |
| 222 | # This simulates what would happen if the engine dies |
| 223 | response = await health(mock_request) |
| 224 | |
| 225 | # Assert that it returns 503 Service Unavailable |
| 226 | assert response.status_code == 503 |