@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d01d0b4646
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 1, 23:45 UTC

[Frontend] Consolidate online serving utils. (#44479)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

wang.yuqi committed Jun 4, 2026 at 14:49 UTC d01d0b46468f08e6b7643ae0574899f4f31fd15a
81 files changed +466 -435
.buildkite/test-amd.yaml
+4 -5
@@ -1299,12 +1299,11 @@ steps:
1299 source_file_dependencies:
1300 - vllm/
1301 - tests/entrypoints/llm
1302 - - tests/entrypoints/offline_mode
1302 commands:
1303 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1305 - - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
1306 - - pytest -v -s entrypoints/llm/test_generate.py
1307 - - pytest -v -s entrypoints/offline_mode
1304 + - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py --ignore=entrypoints/llm/offline_mode
1305 + - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
1306 + - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
1307
1308 - label: Entrypoints Integration (Pooling) # TBD
1309 timeout_in_minutes: 180
@@ -1346,7 +1345,7 @@ steps:
1345 - vllm/platforms/rocm.py
1346 commands:
1347 - pytest -v -s entrypoints/openai/tool_parsers
1349 - - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/offline_mode --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
1348 + - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
1349
1350 - label: OpenAI API correctness # TBD
1351 timeout_in_minutes: 180
.buildkite/test_areas/entrypoints.yaml
+3 -4
@@ -11,7 +11,7 @@ steps:
11 - tests/entrypoints/
12 commands:
13 - pytest -v -s entrypoints/openai/tool_parsers
14 - - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/offline_mode --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
14 + - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
15
16 - label: Entrypoints Integration (LLM)
17 key: entrypoints-integration-llm
@@ -20,12 +20,11 @@ steps:
20 source_file_dependencies:
21 - vllm/
22 - tests/entrypoints/llm
23 - - tests/entrypoints/offline_mode
23 commands:
24 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
26 - - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
25 + - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py --ignore=entrypoints/llm/offline_mode
26 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
28 - - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
27 + - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
28 mirror:
29 amd:
30 device: mi325_1
.github/CODEOWNERS
+2 -1
@@ -34,10 +34,11 @@
34 /vllm/entrypoints/speech_to_text/realtime @njhill
35 /vllm/entrypoints/speech_to_text @NickLucche
36 /vllm/entrypoints/pooling @noooop
37 -/vllm/entrypoints/sagemaker @DarkLight1337
37 +/vllm/entrypoints/serve/sagemaker @DarkLight1337
38 /vllm/entrypoints/serve @njhill
39 /vllm/entrypoints/*.py @njhill
40 /vllm/entrypoints/chat_utils.py @DarkLight1337
41 +/vllm/entrypoints/offline_utils.py @DarkLight1337
42 /vllm/entrypoints/llm.py @DarkLight1337
43
44 # Rust Frontend
tests/entrypoints/llm/offline_mode/__init__.py renamed
tests/entrypoints/llm/offline_mode/test_offline_mode.py renamed
tests/entrypoints/serve/sagemaker/__init__.py renamed
tests/entrypoints/serve/sagemaker/conftest.py renamed
+1 -1
@@ -6,7 +6,7 @@
6 import pytest
7 import pytest_asyncio
8
9 -from ...utils import RemoteOpenAIServer
9 +from tests.utils import RemoteOpenAIServer
10
11 # Model name constants used across tests
12 MODEL_NAME_SMOLLM = "HuggingFaceTB/SmolLM2-135M-Instruct"
tests/entrypoints/serve/sagemaker/test_sagemaker_handler_overrides.py renamed
+2 -1
@@ -22,7 +22,8 @@ import tempfile
22 import pytest
23 import requests
24
25 -from ...utils import RemoteOpenAIServer
25 +from tests.utils import RemoteOpenAIServer
26 +
27 from .conftest import (
28 MODEL_NAME_SMOLLM,
29 )
tests/entrypoints/serve/sagemaker/test_sagemaker_lora_adapters.py renamed
+2 -1
@@ -4,7 +4,8 @@ import openai # use the official async_client for correctness check
4 import pytest
5 import requests
6
7 -from ...utils import RemoteOpenAIServer
7 +from tests.utils import RemoteOpenAIServer
8 +
9 from .conftest import MODEL_NAME_SMOLLM
10
11
tests/entrypoints/serve/sagemaker/test_sagemaker_middleware_integration.py renamed
+2 -1
@@ -12,7 +12,8 @@ import tempfile
12 import pytest
13 import requests
14
15 -from ...utils import RemoteOpenAIServer
15 +from tests.utils import RemoteOpenAIServer
16 +
17 from .conftest import (
18 MODEL_NAME_SMOLLM,
19 )
tests/entrypoints/serve/sagemaker/test_sagemaker_stateful_sessions.py renamed
+2 -1
@@ -6,7 +6,8 @@ import openai # use the official client for correctness check
6 import pytest
7 import requests
8
9 -from ...utils import RemoteOpenAIServer
9 +from tests.utils import RemoteOpenAIServer
10 +
11 from .conftest import (
12 HEADER_SAGEMAKER_CLOSED_SESSION_ID,
13 HEADER_SAGEMAKER_NEW_SESSION_ID,
tests/entrypoints/serve/utils/__init__.py
tests/entrypoints/serve/utils/test_api_utils.py renamed
+1 -1
@@ -4,7 +4,7 @@
4 import pytest
5
6 from vllm.entrypoints.openai.engine.protocol import StreamOptions
7 -from vllm.entrypoints.utils import (
7 +from vllm.entrypoints.serve.utils.api_utils import (
8 get_max_tokens,
9 sanitize_message,
10 should_include_usage,
tests/entrypoints/serve/utils/test_fingerprint.py renamed
+1 -1
@@ -6,7 +6,7 @@ from types import SimpleNamespace
6
7 import pytest
8
9 -from vllm.entrypoints.openai import fingerprint as fp
9 +from vllm.entrypoints.serve.utils import fingerprint as fp
10
11
12 def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
tests/entrypoints/serve/utils/test_request_logger.py new
+248
@@ -0,0 +1,248 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock, patch
5 +
6 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
7 +
8 +
9 +def test_request_logger_log_outputs():
10 + """Test the new log_outputs functionality."""
11 + # Create a mock logger to capture log calls
12 + mock_logger = MagicMock()
13 +
14 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
15 + request_logger = RequestLogger(max_log_len=None)
16 +
17 + # Test basic output logging
18 + request_logger.log_outputs(
19 + request_id="test-123",
20 + outputs="Hello, world!",
21 + output_token_ids=[1, 2, 3, 4],
22 + finish_reason="stop",
23 + is_streaming=False,
24 + delta=False,
25 + )
26 +
27 + mock_logger.info.assert_called_once()
28 + call_args = mock_logger.info.call_args.args
29 + assert "Generated response %s%s" in call_args[0]
30 + assert call_args[1] == "test-123"
31 + assert call_args[3] == "Hello, world!"
32 + assert call_args[4] == [1, 2, 3, 4]
33 + assert call_args[5] == "stop"
34 +
35 +
36 +def test_request_logger_log_outputs_streaming_delta():
37 + """Test log_outputs with streaming delta mode."""
38 + mock_logger = MagicMock()
39 +
40 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
41 + request_logger = RequestLogger(max_log_len=None)
42 +
43 + # Test streaming delta logging
44 + request_logger.log_outputs(
45 + request_id="test-456",
46 + outputs="Hello",
47 + output_token_ids=[1],
48 + finish_reason=None,
49 + is_streaming=True,
50 + delta=True,
51 + )
52 +
53 + mock_logger.info.assert_called_once()
54 + call_args = mock_logger.info.call_args.args
55 + assert "Generated response %s%s" in call_args[0]
56 + assert call_args[1] == "test-456"
57 + assert call_args[2] == " (streaming delta)"
58 + assert call_args[3] == "Hello"
59 + assert call_args[4] == [1]
60 + assert call_args[5] is None
61 +
62 +
63 +def test_request_logger_log_outputs_streaming_complete():
64 + """Test log_outputs with streaming complete mode."""
65 + mock_logger = MagicMock()
66 +
67 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
68 + request_logger = RequestLogger(max_log_len=None)
69 +
70 + # Test streaming complete logging
71 + request_logger.log_outputs(
72 + request_id="test-789",
73 + outputs="Complete response",
74 + output_token_ids=[1, 2, 3],
75 + finish_reason="length",
76 + is_streaming=True,
77 + delta=False,
78 + )
79 +
80 + mock_logger.info.assert_called_once()
81 + call_args = mock_logger.info.call_args.args
82 + assert "Generated response %s%s" in call_args[0]
83 + assert call_args[1] == "test-789"
84 + assert call_args[2] == " (streaming complete)"
85 + assert call_args[3] == "Complete response"
86 + assert call_args[4] == [1, 2, 3]
87 + assert call_args[5] == "length"
88 +
89 +
90 +def test_request_logger_log_outputs_with_truncation():
91 + """Test log_outputs respects max_log_len setting."""
92 + mock_logger = MagicMock()
93 +
94 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
95 + # Set max_log_len to 10
96 + request_logger = RequestLogger(max_log_len=10)
97 +
98 + # Test output truncation
99 + long_output = "This is a very long output that should be truncated"
100 + long_token_ids = list(range(20)) # 20 tokens
101 +
102 + request_logger.log_outputs(
103 + request_id="test-truncate",
104 + outputs=long_output,
105 + output_token_ids=long_token_ids,
106 + finish_reason="stop",
107 + is_streaming=False,
108 + delta=False,
109 + )
110 +
111 + mock_logger.info.assert_called_once()
112 + call_args = mock_logger.info.call_args
113 +
114 + # Check that output was truncated to first 10 characters
115 + logged_output = call_args[0][3]
116 + assert logged_output == "This is a "
117 + assert len(logged_output) == 10
118 +
119 + # Check that token IDs were truncated to first 10 tokens
120 + logged_token_ids = call_args[0][4]
121 + assert logged_token_ids == list(range(10))
122 + assert len(logged_token_ids) == 10
123 +
124 +
125 +def test_request_logger_log_outputs_none_values():
126 + """Test log_outputs handles None values correctly."""
127 + mock_logger = MagicMock()
128 +
129 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
130 + request_logger = RequestLogger(max_log_len=None)
131 +
132 + # Test with None output_token_ids
133 + request_logger.log_outputs(
134 + request_id="test-none",
135 + outputs="Test output",
136 + output_token_ids=None,
137 + finish_reason="stop",
138 + is_streaming=False,
139 + delta=False,
140 + )
141 +
142 + mock_logger.info.assert_called_once()
143 + call_args = mock_logger.info.call_args.args
144 + assert "Generated response %s%s" in call_args[0]
145 + assert call_args[1] == "test-none"
146 + assert call_args[3] == "Test output"
147 + assert call_args[4] is None
148 + assert call_args[5] == "stop"
149 +
150 +
151 +def test_request_logger_log_outputs_empty_output():
152 + """Test log_outputs handles empty output correctly."""
153 + mock_logger = MagicMock()
154 +
155 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
156 + request_logger = RequestLogger(max_log_len=5)
157 +
158 + # Test with empty output
159 + request_logger.log_outputs(
160 + request_id="test-empty",
161 + outputs="",
162 + output_token_ids=[],
163 + finish_reason="stop",
164 + is_streaming=False,
165 + delta=False,
166 + )
167 +
168 + mock_logger.info.assert_called_once()
169 + call_args = mock_logger.info.call_args.args
170 + assert "Generated response %s%s" in call_args[0]
171 + assert call_args[1] == "test-empty"
172 + assert call_args[3] == ""
173 + assert call_args[4] == []
174 + assert call_args[5] == "stop"
175 +
176 +
177 +def test_request_logger_log_outputs_integration():
178 + """Test that log_outputs can be called alongside log_inputs."""
179 + mock_logger = MagicMock()
180 +
181 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
182 + request_logger = RequestLogger(max_log_len=None)
183 +
184 + # Test that both methods can be called without interference
185 + request_logger.log_inputs(
186 + request_id="test-integration",
187 + prompt="Test prompt",
188 + prompt_token_ids=[1, 2, 3],
189 + prompt_embeds=None,
190 + params=None,
191 + lora_request=None,
192 + )
193 +
194 + request_logger.log_outputs(
195 + request_id="test-integration",
196 + outputs="Test output",
197 + output_token_ids=[4, 5, 6],
198 + finish_reason="stop",
199 + is_streaming=False,
200 + delta=False,
201 + )
202 +
203 + # Should have been called twice - once for inputs, once for outputs
204 + assert mock_logger.info.call_count == 2
205 +
206 + # Check that the calls were made with correct patterns
207 + input_call = mock_logger.info.call_args_list[0][0]
208 + output_call = mock_logger.info.call_args_list[1][0]
209 +
210 + assert "Received request %s" in input_call[0]
211 + assert input_call[1] == "test-integration"
212 +
213 + assert "Generated response %s%s" in output_call[0]
214 + assert output_call[1] == "test-integration"
215 +
216 +
217 +def test_streaming_complete_logs_full_text_content():
218 + """Test that streaming complete logging includes
219 + full accumulated text, not just token count."""
220 + mock_logger = MagicMock()
221 +
222 + with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
223 + request_logger = RequestLogger(max_log_len=None)
224 +
225 + # Test with actual content instead of token count format
226 + full_response = "This is a complete response from streaming"
227 + request_logger.log_outputs(
228 + request_id="test-streaming-full-text",
229 + outputs=full_response,
230 + output_token_ids=None,
231 + finish_reason="streaming_complete",
232 + is_streaming=True,
233 + delta=False,
234 + )
235 +
236 + mock_logger.info.assert_called_once()
237 + call_args = mock_logger.info.call_args.args
238 +
239 + # Verify the logged output is the full text, not a token count format
240 + logged_output = call_args[3]
241 + assert logged_output == full_response
242 + assert "tokens>" not in logged_output
243 + assert "streaming_complete" not in logged_output
244 +
245 + # Verify other parameters
246 + assert call_args[1] == "test-streaming-full-text"
247 + assert call_args[2] == " (streaming complete)"
248 + assert call_args[5] == "streaming_complete"
tests/entrypoints/serve/utils/test_ssl_cert_refresher.py renamed
+1 -1
@@ -7,7 +7,7 @@ from ssl import SSLContext
7
8 import pytest
9
10 -from vllm.entrypoints.ssl import SSLCertRefresher
10 +from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
11
12
13 class MockSSLContext(SSLContext):
tests/test_logger.py
+1 -244
@@ -10,12 +10,11 @@ from dataclasses import dataclass
10 from json.decoder import JSONDecodeError
11 from tempfile import NamedTemporaryFile
12 from typing import Any
13 -from unittest.mock import MagicMock, patch
13 +from unittest.mock import patch
14 from uuid import uuid4
15
16 import pytest
17
18 -from vllm.entrypoints.logger import RequestLogger
18 from vllm.logger import (
19 _DATE_FORMAT,
20 _FORMAT,
@@ -269,248 +268,6 @@ def test_prepare_object_to_dump():
268 assert prepare_object_to_dump(CustomClass(1, "b")) == "CustomClass(a=1, b='b')"
269
270
272 -def test_request_logger_log_outputs():
273 - """Test the new log_outputs functionality."""
274 - # Create a mock logger to capture log calls
275 - mock_logger = MagicMock()
276 -
277 - with patch("vllm.entrypoints.logger.logger", mock_logger):
278 - request_logger = RequestLogger(max_log_len=None)
279 -
280 - # Test basic output logging
281 - request_logger.log_outputs(
282 - request_id="test-123",
283 - outputs="Hello, world!",
284 - output_token_ids=[1, 2, 3, 4],
285 - finish_reason="stop",
286 - is_streaming=False,
287 - delta=False,
288 - )
289 -
290 - mock_logger.info.assert_called_once()
291 - call_args = mock_logger.info.call_args.args
292 - assert "Generated response %s%s" in call_args[0]
293 - assert call_args[1] == "test-123"
294 - assert call_args[3] == "Hello, world!"
295 - assert call_args[4] == [1, 2, 3, 4]
296 - assert call_args[5] == "stop"
297 -
298 -
299 -def test_request_logger_log_outputs_streaming_delta():
300 - """Test log_outputs with streaming delta mode."""
301 - mock_logger = MagicMock()
302 -
303 - with patch("vllm.entrypoints.logger.logger", mock_logger):
304 - request_logger = RequestLogger(max_log_len=None)
305 -
306 - # Test streaming delta logging
307 - request_logger.log_outputs(
308 - request_id="test-456",
309 - outputs="Hello",
310 - output_token_ids=[1],
311 - finish_reason=None,
312 - is_streaming=True,
313 - delta=True,
314 - )
315 -
316 - mock_logger.info.assert_called_once()
317 - call_args = mock_logger.info.call_args.args
318 - assert "Generated response %s%s" in call_args[0]
319 - assert call_args[1] == "test-456"
320 - assert call_args[2] == " (streaming delta)"
321 - assert call_args[3] == "Hello"
322 - assert call_args[4] == [1]
323 - assert call_args[5] is None
324 -
325 -
326 -def test_request_logger_log_outputs_streaming_complete():
327 - """Test log_outputs with streaming complete mode."""
328 - mock_logger = MagicMock()
329 -
330 - with patch("vllm.entrypoints.logger.logger", mock_logger):
331 - request_logger = RequestLogger(max_log_len=None)
332 -
333 - # Test streaming complete logging
334 - request_logger.log_outputs(
335 - request_id="test-789",
336 - outputs="Complete response",
337 - output_token_ids=[1, 2, 3],
338 - finish_reason="length",
339 - is_streaming=True,
340 - delta=False,
341 - )
342 -
343 - mock_logger.info.assert_called_once()
344 - call_args = mock_logger.info.call_args.args
345 - assert "Generated response %s%s" in call_args[0]
346 - assert call_args[1] == "test-789"
347 - assert call_args[2] == " (streaming complete)"
348 - assert call_args[3] == "Complete response"
349 - assert call_args[4] == [1, 2, 3]
350 - assert call_args[5] == "length"
351 -
352 -
353 -def test_request_logger_log_outputs_with_truncation():
354 - """Test log_outputs respects max_log_len setting."""
355 - mock_logger = MagicMock()
356 -
357 - with patch("vllm.entrypoints.logger.logger", mock_logger):
358 - # Set max_log_len to 10
359 - request_logger = RequestLogger(max_log_len=10)
360 -
361 - # Test output truncation
362 - long_output = "This is a very long output that should be truncated"
363 - long_token_ids = list(range(20)) # 20 tokens
364 -
365 - request_logger.log_outputs(
366 - request_id="test-truncate",
367 - outputs=long_output,
368 - output_token_ids=long_token_ids,
369 - finish_reason="stop",
370 - is_streaming=False,
371 - delta=False,
372 - )
373 -
374 - mock_logger.info.assert_called_once()
375 - call_args = mock_logger.info.call_args
376 -
377 - # Check that output was truncated to first 10 characters
378 - logged_output = call_args[0][3]
379 - assert logged_output == "This is a "
380 - assert len(logged_output) == 10
381 -
382 - # Check that token IDs were truncated to first 10 tokens
383 - logged_token_ids = call_args[0][4]
384 - assert logged_token_ids == list(range(10))
385 - assert len(logged_token_ids) == 10
386 -
387 -
388 -def test_request_logger_log_outputs_none_values():
389 - """Test log_outputs handles None values correctly."""
390 - mock_logger = MagicMock()
391 -
392 - with patch("vllm.entrypoints.logger.logger", mock_logger):
393 - request_logger = RequestLogger(max_log_len=None)
394 -
395 - # Test with None output_token_ids
396 - request_logger.log_outputs(
397 - request_id="test-none",
398 - outputs="Test output",
399 - output_token_ids=None,
400 - finish_reason="stop",
401 - is_streaming=False,
402 - delta=False,
403 - )
404 -
405 - mock_logger.info.assert_called_once()
406 - call_args = mock_logger.info.call_args.args
407 - assert "Generated response %s%s" in call_args[0]
408 - assert call_args[1] == "test-none"
409 - assert call_args[3] == "Test output"
410 - assert call_args[4] is None
411 - assert call_args[5] == "stop"
412 -
413 -
414 -def test_request_logger_log_outputs_empty_output():
415 - """Test log_outputs handles empty output correctly."""
416 - mock_logger = MagicMock()
417 -
418 - with patch("vllm.entrypoints.logger.logger", mock_logger):
419 - request_logger = RequestLogger(max_log_len=5)
420 -
421 - # Test with empty output
422 - request_logger.log_outputs(
423 - request_id="test-empty",
424 - outputs="",
425 - output_token_ids=[],
426 - finish_reason="stop",
427 - is_streaming=False,
428 - delta=False,
429 - )
430 -
431 - mock_logger.info.assert_called_once()
432 - call_args = mock_logger.info.call_args.args
433 - assert "Generated response %s%s" in call_args[0]
434 - assert call_args[1] == "test-empty"
435 - assert call_args[3] == ""
436 - assert call_args[4] == []
437 - assert call_args[5] == "stop"
438 -
439 -
440 -def test_request_logger_log_outputs_integration():
441 - """Test that log_outputs can be called alongside log_inputs."""
442 - mock_logger = MagicMock()
443 -
444 - with patch("vllm.entrypoints.logger.logger", mock_logger):
445 - request_logger = RequestLogger(max_log_len=None)
446 -
447 - # Test that both methods can be called without interference
448 - request_logger.log_inputs(
449 - request_id="test-integration",
450 - prompt="Test prompt",
451 - prompt_token_ids=[1, 2, 3],
452 - prompt_embeds=None,
453 - params=None,
454 - lora_request=None,
455 - )
456 -
457 - request_logger.log_outputs(
458 - request_id="test-integration",
459 - outputs="Test output",
460 - output_token_ids=[4, 5, 6],
461 - finish_reason="stop",
462 - is_streaming=False,
463 - delta=False,
464 - )
465 -
466 - # Should have been called twice - once for inputs, once for outputs
467 - assert mock_logger.info.call_count == 2
468 -
469 - # Check that the calls were made with correct patterns
470 - input_call = mock_logger.info.call_args_list[0][0]
471 - output_call = mock_logger.info.call_args_list[1][0]
472 -
473 - assert "Received request %s" in input_call[0]
474 - assert input_call[1] == "test-integration"
475 -
476 - assert "Generated response %s%s" in output_call[0]
477 - assert output_call[1] == "test-integration"
478 -
479 -
480 -def test_streaming_complete_logs_full_text_content():
481 - """Test that streaming complete logging includes
482 - full accumulated text, not just token count."""
483 - mock_logger = MagicMock()
484 -
485 - with patch("vllm.entrypoints.logger.logger", mock_logger):
486 - request_logger = RequestLogger(max_log_len=None)
487 -
488 - # Test with actual content instead of token count format
489 - full_response = "This is a complete response from streaming"
490 - request_logger.log_outputs(
491 - request_id="test-streaming-full-text",
492 - outputs=full_response,
493 - output_token_ids=None,
494 - finish_reason="streaming_complete",
495 - is_streaming=True,
496 - delta=False,
497 - )
498 -
499 - mock_logger.info.assert_called_once()
500 - call_args = mock_logger.info.call_args.args
501 -
502 - # Verify the logged output is the full text, not a token count format
503 - logged_output = call_args[3]
504 - assert logged_output == full_response
505 - assert "tokens>" not in logged_output
506 - assert "streaming_complete" not in logged_output
507 -
508 - # Verify other parameters
509 - assert call_args[1] == "test-streaming-full-text"
510 - assert call_args[2] == " (streaming complete)"
511 - assert call_args[5] == "streaming_complete"
512 -
513 -
271 # Add vllm prefix to make sure logs go through the vllm logger
272 test_logger = init_logger("vllm.test_logger")
273
vllm/benchmarks/sweep/cli.py
+1 -1
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import argparse
4
5 -from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
5 +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
6
7 from .plot import SweepPlotArgs
8 from .plot import main as plot_main
vllm/entrypoints/anthropic/api_router.py
+2 -2
@@ -17,9 +17,9 @@ from vllm.entrypoints.anthropic.protocol import (
17 )
18 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
19 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
20 -from vllm.entrypoints.openai.utils import validate_json_request
21 -from vllm.entrypoints.utils import (
20 +from vllm.entrypoints.serve.utils.api_utils import (
21 load_aware_call,
22 + validate_json_request,
23 with_cancellation,
24 )
25 from vllm.logger import init_logger
vllm/entrypoints/anthropic/serving.py
+1 -1
@@ -29,7 +29,6 @@ from vllm.entrypoints.anthropic.protocol import (
29 AnthropicUsage,
30 )
31 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
32 -from vllm.entrypoints.logger import RequestLogger
32 from vllm.entrypoints.openai.chat_completion.protocol import (
33 ChatCompletionNamedToolChoiceParam,
34 ChatCompletionRequest,
@@ -45,6 +44,7 @@ from vllm.entrypoints.openai.engine.protocol import (
44 StreamOptions,
45 )
46 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
47 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
48
49 if TYPE_CHECKING:
50 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
vllm/entrypoints/api_server.py
+1 -1
@@ -22,7 +22,7 @@ import vllm.envs as envs
22 from vllm.engine.arg_utils import AsyncEngineArgs
23 from vllm.engine.async_llm_engine import AsyncLLMEngine
24 from vllm.entrypoints.launcher import serve_http
25 -from vllm.entrypoints.utils import with_cancellation
25 +from vllm.entrypoints.serve.utils.api_utils import with_cancellation
26 from vllm.logger import init_logger
27 from vllm.sampling_params import SamplingParams
28 from vllm.usage.usage_lib import UsageContext
vllm/entrypoints/cli/benchmark/main.py
+1 -1
@@ -7,7 +7,7 @@ import typing
7
8 from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
9 from vllm.entrypoints.cli.types import CLISubcommand
10 -from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
10 +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
11
12 if typing.TYPE_CHECKING:
13 from vllm.utils.argparse_utils import FlexibleArgumentParser
vllm/entrypoints/cli/launch.py
+1 -1
@@ -18,7 +18,7 @@ from vllm.entrypoints.openai.cli_args import (
18 make_arg_parser,
19 validate_parsed_serve_args,
20 )
21 -from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
21 +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
22 from vllm.logger import init_logger
23 from vllm.utils.argparse_utils import FlexibleArgumentParser
24
vllm/entrypoints/cli/main.py
+4 -1
@@ -21,7 +21,10 @@ def main():
21 import vllm.entrypoints.cli.openai
22 import vllm.entrypoints.cli.run_batch
23 import vllm.entrypoints.cli.serve
24 - from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup
24 + from vllm.entrypoints.serve.utils.api_utils import (
25 + VLLM_SUBCMD_PARSER_EPILOG,
26 + cli_env_setup,
27 + )
28 from vllm.utils.argparse_utils import FlexibleArgumentParser
29
30 CMD_MODULES = [
vllm/entrypoints/cli/run_batch.py
+1 -1
@@ -7,7 +7,7 @@ import importlib.metadata
7 import typing
8
9 from vllm.entrypoints.cli.types import CLISubcommand
10 -from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
10 +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
11 from vllm.logger import init_logger
12
13 if typing.TYPE_CHECKING:
vllm/entrypoints/cli/serve.py
+1 -1
@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se
15 from vllm.entrypoints.openai.dp_supervisor import (
16 run_dp_supervisor,
17 )
18 -from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
18 +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
19 from vllm.logger import init_logger
20 from vllm.usage.usage_lib import UsageContext
21 from vllm.utils.argparse_utils import FlexibleArgumentParser
vllm/entrypoints/generate/api_router.py
+2 -2
@@ -10,7 +10,7 @@ if TYPE_CHECKING:
10 from starlette.datastructures import State
11
12 from vllm.engine.protocol import EngineClient
13 - from vllm.entrypoints.logger import RequestLogger
13 + from vllm.entrypoints.serve.utils.request_logger import RequestLogger
14 from vllm.tasks import SupportedTask
15 else:
16 RequestLogger = object
@@ -65,9 +65,9 @@ async def init_generate_state(
65 )
66 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
67 from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
68 - from vllm.entrypoints.openai.fingerprint import set_default_fingerprint_mode
68 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
69 from vllm.entrypoints.serve.disagg.serving import ServingTokens
70 + from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
71
72 # Applied before any serving class is constructed so that each one picks
73 # up the chosen mode on its first cache miss.
vllm/entrypoints/generate/factories.py
+1 -1
@@ -6,7 +6,7 @@ from vllm.config import ModelConfig
6 from vllm.tasks import SupportedTask
7
8 if TYPE_CHECKING:
9 - from vllm.entrypoints.sagemaker.api_router import (
9 + from vllm.entrypoints.serve.sagemaker.api_router import (
10 EndpointFn,
11 GetHandlerFn,
12 RequestType,
vllm/entrypoints/generate/generative_scoring/api_router.py
+5 -2
@@ -10,8 +10,11 @@ from vllm.entrypoints.generate.generative_scoring.serving import (
10 ServingGenerativeScoring,
11 )
12 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
13 -from vllm.entrypoints.openai.utils import validate_json_request
14 -from vllm.entrypoints.utils import load_aware_call, with_cancellation
13 +from vllm.entrypoints.serve.utils.api_utils import (
14 + load_aware_call,
15 + validate_json_request,
16 + with_cancellation,
17 +)
18 from vllm.logger import init_logger
19
20 router = APIRouter()
vllm/entrypoints/generate/generative_scoring/serving.py
+1 -1
@@ -18,7 +18,6 @@ from fastapi import Request
18 from pydantic import Field
19
20 from vllm.engine.protocol import EngineClient
21 -from vllm.entrypoints.logger import RequestLogger
21 from vllm.entrypoints.openai.engine.protocol import (
22 ErrorResponse,
23 OpenAIBaseModel,
@@ -26,6 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (
25 )
26 from vllm.entrypoints.openai.engine.serving import OpenAIServing
27 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
28 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
29 from vllm.inputs import EngineInput, tokens_input
30 from vllm.logger import init_logger
31 from vllm.outputs import RequestOutput
vllm/entrypoints/grpc_server.py
+1 -1
@@ -43,7 +43,7 @@ import uvloop
43
44 from vllm import envs
45 from vllm.engine.arg_utils import AsyncEngineArgs
46 -from vllm.entrypoints.utils import log_version_and_model
46 +from vllm.entrypoints.serve.utils.api_utils import log_version_and_model
47 from vllm.logger import init_logger
48 from vllm.usage.usage_lib import UsageContext
49 from vllm.utils.argparse_utils import FlexibleArgumentParser
vllm/entrypoints/launcher.py
+2 -2
@@ -12,11 +12,11 @@ from fastapi import FastAPI
12
13 from vllm import envs
14 from vllm.engine.protocol import EngineClient
15 -from vllm.entrypoints.constants import (
15 +from vllm.entrypoints.serve.utils.constants import (
16 H11_MAX_HEADER_COUNT_DEFAULT,
17 H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
18 )
19 -from vllm.entrypoints.ssl import SSLCertRefresher
19 +from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
20 from vllm.logger import init_logger
21 from vllm.utils.network_utils import find_process_using_port
22
vllm/entrypoints/llm.py
+1 -1
@@ -40,7 +40,7 @@ from vllm.entrypoints.chat_utils import (
40 )
41 from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
42 from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
43 -from vllm.entrypoints.utils import log_non_default_args
43 +from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
44 from vllm.inputs import PromptType
45 from vllm.logger import init_logger
46 from vllm.lora.request import LoRARequest
vllm/entrypoints/openai/api_server.py
+15 -15
@@ -27,12 +27,22 @@ from vllm.engine.arg_utils import AsyncEngineArgs
27 from vllm.engine.protocol import EngineClient
28 from vllm.entrypoints.chat_utils import load_chat_template
29 from vllm.entrypoints.launcher import serve_http
30 -from vllm.entrypoints.logger import RequestLogger
30 from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
31 from vllm.entrypoints.openai.engine.protocol import GenerationError
32 from vllm.entrypoints.openai.models.protocol import BaseModelPath
33 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
35 -from vllm.entrypoints.openai.server_utils import (
34 +from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
35 +from vllm.entrypoints.serve.render.serving import OpenAIServingRender
36 +from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
37 +from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
38 +from vllm.entrypoints.serve.utils.api_utils import (
39 + cli_env_setup,
40 + log_non_default_args,
41 + log_version_and_model,
42 + process_lora_modules,
43 +)
44 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
45 +from vllm.entrypoints.serve.utils.server_utils import (
46 engine_error_handler,
47 exception_handler,
48 generation_error_handler,
@@ -42,16 +52,6 @@ from vllm.entrypoints.openai.server_utils import (
52 log_response,
53 validation_exception_handler,
54 )
45 -from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
46 -from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
47 -from vllm.entrypoints.serve.render.serving import OpenAIServingRender
48 -from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
49 -from vllm.entrypoints.utils import (
50 - cli_env_setup,
51 - log_non_default_args,
52 - log_version_and_model,
53 - process_lora_modules,
54 -)
55 from vllm.logger import init_logger
56 from vllm.reasoning import ReasoningParserManager
57 from vllm.tasks import POOLING_TASKS, SupportedTask
@@ -187,7 +187,7 @@ def build_app(
187
188 register_models_api_router(app)
189
190 - from vllm.entrypoints.sagemaker.api_router import (
190 + from vllm.entrypoints.serve.sagemaker.api_router import (
191 attach_router as register_sagemaker_api_router,
192 )
193
@@ -254,12 +254,12 @@ def build_app(
254
255 # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
256 if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
257 - from vllm.entrypoints.openai.server_utils import AuthenticationMiddleware
257 + from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
258
259 app.add_middleware(AuthenticationMiddleware, tokens=tokens)
260
261 if args.enable_request_id_headers:
262 - from vllm.entrypoints.openai.server_utils import XRequestIdMiddleware
262 + from vllm.entrypoints.serve.utils.server_utils import XRequestIdMiddleware
263
264 app.add_middleware(XRequestIdMiddleware)
265
vllm/entrypoints/openai/chat_completion/api_router.py
+3 -3
@@ -15,12 +15,12 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
15 )
16 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
17 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
18 -from vllm.entrypoints.openai.orca_metrics import metrics_header
19 -from vllm.entrypoints.openai.utils import validate_json_request
20 -from vllm.entrypoints.utils import (
18 +from vllm.entrypoints.serve.utils.api_utils import (
19 load_aware_call,
20 + validate_json_request,
21 with_cancellation,
22 )
23 +from vllm.entrypoints.serve.utils.orca_metrics import metrics_header
24 from vllm.logger import init_logger
25
26 logger = init_logger(__name__)
vllm/entrypoints/openai/chat_completion/batch_serving.py
+1 -1
@@ -21,7 +21,7 @@ from vllm.entrypoints.openai.engine.protocol import (
21 RequestResponseMetadata,
22 UsageInfo,
23 )
24 -from vllm.entrypoints.utils import get_max_tokens
24 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
25 from vllm.inputs import EngineInput
26 from vllm.logger import init_logger
27 from vllm.outputs import RequestOutput
vllm/entrypoints/openai/chat_completion/serving.py
+5 -3
@@ -21,7 +21,6 @@ from vllm.entrypoints.chat_utils import (
21 get_tool_call_id_type,
22 make_tool_call_id,
23 )
24 -from vllm.entrypoints.logger import RequestLogger
24 from vllm.entrypoints.openai.chat_completion.protocol import (
25 ChatCompletionLogProb,
26 ChatCompletionLogProbs,
@@ -57,8 +56,11 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
56 get_streamable_parser_for_assistant,
57 parse_chat_output,
58 )
60 -from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
61 -from vllm.entrypoints.utils import get_max_tokens, should_include_usage
59 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
60 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
61 +from vllm.entrypoints.serve.utils.tool_calls_utils import (
62 + maybe_filter_parallel_tool_calls,
63 +)
64 from vllm.inputs import EngineInput
65 from vllm.logger import init_logger
66 from vllm.logprobs import Logprob
vllm/entrypoints/openai/cli_args.py
+2 -2
@@ -20,11 +20,11 @@ from vllm.entrypoints.chat_utils import (
20 ChatTemplateContentFormatOption,
21 validate_chat_template,
22 )
23 -from vllm.entrypoints.constants import (
23 +from vllm.entrypoints.openai.models.protocol import LoRAModulePath
24 +from vllm.entrypoints.serve.utils.constants import (
25 H11_MAX_HEADER_COUNT_DEFAULT,
26 H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
27 )
27 -from vllm.entrypoints.openai.models.protocol import LoRAModulePath
28 from vllm.logger import init_logger
29 from vllm.tool_parsers import ToolParserManager
30 from vllm.utils.argparse_utils import FlexibleArgumentParser
vllm/entrypoints/openai/completion/api_router.py
+3 -3
@@ -13,12 +13,12 @@ from vllm.entrypoints.openai.completion.protocol import (
13 )
14 from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
15 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
16 -from vllm.entrypoints.openai.orca_metrics import metrics_header
17 -from vllm.entrypoints.openai.utils import validate_json_request
18 -from vllm.entrypoints.utils import (
16 +from vllm.entrypoints.serve.utils.api_utils import (
17 load_aware_call,
18 + validate_json_request,
19 with_cancellation,
20 )
21 +from vllm.entrypoints.serve.utils.orca_metrics import metrics_header
22 from vllm.logger import init_logger
23
24 logger = init_logger(__name__)
vllm/entrypoints/openai/completion/serving.py
+2 -2
@@ -13,7 +13,6 @@ import pybase64 as base64
13 from fastapi import Request
14
15 from vllm.engine.protocol import EngineClient
16 -from vllm.entrypoints.logger import RequestLogger
16 from vllm.entrypoints.openai.completion.protocol import (
17 CompletionLogProbs,
18 CompletionRequest,
@@ -34,7 +33,8 @@ from vllm.entrypoints.openai.engine.serving import (
33 clamp_prompt_logprobs,
34 )
35 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
37 -from vllm.entrypoints.utils import get_max_tokens, should_include_usage
36 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
37 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
38 from vllm.exceptions import VLLMValidationError
39 from vllm.inputs import EngineInput
40 from vllm.logger import init_logger
vllm/entrypoints/openai/engine/serving.py
+3 -3
@@ -16,7 +16,6 @@ from vllm.config import ModelConfig
16 from vllm.engine.protocol import EngineClient
17 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
18 from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
19 -from vllm.entrypoints.logger import RequestLogger
19 from vllm.entrypoints.openai.chat_completion.protocol import (
20 BatchChatCompletionRequest,
21 ChatCompletionRequest,
@@ -39,12 +38,13 @@ from vllm.entrypoints.serve.tokenize.protocol import (
38 TokenizeCompletionRequest,
39 TokenizeResponse,
40 )
41 +from vllm.entrypoints.serve.utils.error_response import create_error_response
42 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
43 from vllm.entrypoints.speech_to_text.transcription.protocol import (
44 TranscriptionRequest,
45 TranscriptionResponse,
46 )
47 from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
47 -from vllm.entrypoints.utils import create_error_response
48 from vllm.inputs import EngineInput, PromptType
49 from vllm.logger import init_logger
50 from vllm.logprobs import Logprob, PromptLogprobs
@@ -153,7 +153,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
153 # Computed once at startup (cached by ``vllm_config`` identity) and
154 # stamped on non-streaming responses. Streaming chunks deliberately
155 # omit it to avoid per-chunk overhead.
156 - from vllm.entrypoints.openai.fingerprint import get_system_fingerprint
156 + from vllm.entrypoints.serve.utils.fingerprint import get_system_fingerprint
157
158 try:
159 self.system_fingerprint: str | None = get_system_fingerprint(
vllm/entrypoints/openai/models/serving.py
+1 -1
@@ -18,7 +18,7 @@ from vllm.entrypoints.serve.lora.protocol import (
18 LoadLoRAAdapterRequest,
19 UnloadLoRAAdapterRequest,
20 )
21 -from vllm.entrypoints.utils import create_error_response
21 +from vllm.entrypoints.serve.utils.error_response import create_error_response
22 from vllm.exceptions import LoRAAdapterNotFoundError
23 from vllm.logger import init_logger
24 from vllm.lora.request import LoRARequest
vllm/entrypoints/openai/parser/responses_parser.py
+1 -1
@@ -12,11 +12,11 @@ from openai.types.responses.response_output_message import ResponseOutputMessage
12 from openai.types.responses.response_output_text import ResponseOutputText
13
14 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
15 -from vllm.entrypoints.constants import MCP_PREFIX
15 from vllm.entrypoints.openai.responses.protocol import (
16 ResponseInputOutputItem,
17 ResponsesRequest,
18 )
19 +from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
20 from vllm.outputs import CompletionOutput
21 from vllm.parser.abstract_parser import Parser
22 from vllm.tokenizers import TokenizerLike
vllm/entrypoints/openai/responses/api_router.py
+2 -2
@@ -15,9 +15,9 @@ from vllm.entrypoints.openai.responses.protocol import (
15 StreamingResponsesResponse,
16 )
17 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
18 -from vllm.entrypoints.openai.utils import validate_json_request
19 -from vllm.entrypoints.utils import (
18 +from vllm.entrypoints.serve.utils.api_utils import (
19 load_aware_call,
20 + validate_json_request,
21 with_cancellation,
22 )
23 from vllm.logger import init_logger
vllm/entrypoints/openai/responses/context.py
+1 -1
@@ -20,7 +20,6 @@ from vllm import envs
20 from vllm.entrypoints.chat_utils import (
21 ChatTemplateContentFormatOption,
22 )
23 -from vllm.entrypoints.constants import MCP_PREFIX
23 from vllm.entrypoints.mcp.tool import Tool
24 from vllm.entrypoints.mcp.tool_server import ToolServer
25 from vllm.entrypoints.openai.engine.protocol import (
@@ -40,6 +39,7 @@ from vllm.entrypoints.openai.responses.protocol import (
39 ResponsesRequest,
40 )
41 from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
42 +from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
43 from vllm.outputs import RequestOutput
44 from vllm.parser.abstract_parser import Parser
45 from vllm.tokenizers import TokenizerLike
vllm/entrypoints/openai/responses/serving.py
+2 -2
@@ -32,7 +32,6 @@ from vllm.entrypoints.chat_utils import (
32 ChatTemplateContentFormatOption,
33 get_tool_call_id_type,
34 )
35 -from vllm.entrypoints.logger import RequestLogger
35 from vllm.entrypoints.mcp.tool_server import ToolServer
36 from vllm.entrypoints.openai.engine.protocol import (
37 DeltaMessage,
@@ -93,7 +92,8 @@ from vllm.entrypoints.openai.responses.utils import (
92 extract_tool_types,
93 )
94 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
96 -from vllm.entrypoints.utils import get_max_tokens
95 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
96 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
97 from vllm.exceptions import VLLMValidationError
98 from vllm.inputs import EngineInput, tokens_input
99 from vllm.logger import init_logger
vllm/entrypoints/openai/run_batch.py
+1 -1
@@ -51,6 +51,7 @@ from vllm.entrypoints.pooling.scoring.protocol import (
51 ScoreRequest,
52 ScoreResponse,
53 )
54 +from vllm.entrypoints.serve.utils.error_response import create_error_response
55 from vllm.entrypoints.speech_to_text.transcription.protocol import (
56 TranscriptionRequest,
57 TranscriptionResponse,
@@ -61,7 +62,6 @@ from vllm.entrypoints.speech_to_text.translation.protocol import (
62 TranslationResponse,
63 TranslationResponseVerbose,
64 )
64 -from vllm.entrypoints.utils import create_error_response
65 from vllm.exceptions import VLLMValidationError
66 from vllm.logger import init_logger
67 from vllm.reasoning import ReasoningParserManager
vllm/entrypoints/pooling/base/serving.py
+1 -1
@@ -16,9 +16,9 @@ from vllm import PoolingParams, PoolingRequestOutput, envs
16 from vllm.config import VllmConfig
17 from vllm.engine.protocol import EngineClient
18 from vllm.entrypoints.chat_utils import ChatTemplateConfig
19 -from vllm.entrypoints.logger import RequestLogger
19 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
20 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
21 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
22 from vllm.exceptions import VLLMNotFoundError
23 from vllm.inputs import EngineInput
24 from vllm.lora.request import LoRARequest
vllm/entrypoints/pooling/classify/api_router.py
+2 -2
@@ -4,9 +4,9 @@
4 from fastapi import APIRouter, Depends, Request
5 from fastapi.responses import Response
6
7 -from vllm.entrypoints.openai.utils import validate_json_request
8 -from vllm.entrypoints.utils import (
7 +from vllm.entrypoints.serve.utils.api_utils import (
8 load_aware_call,
9 + validate_json_request,
10 with_cancellation,
11 )
12
vllm/entrypoints/pooling/embed/api_router.py
+5 -2
@@ -6,8 +6,11 @@ from http import HTTPStatus
6 from fastapi import APIRouter, Depends, Request
7
8 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
9 -from vllm.entrypoints.openai.utils import validate_json_request
10 -from vllm.entrypoints.utils import load_aware_call, with_cancellation
9 +from vllm.entrypoints.serve.utils.api_utils import (
10 + load_aware_call,
11 + validate_json_request,
12 + with_cancellation,
13 +)
14
15 from .protocol import CohereEmbedRequest, EmbeddingRequest
16 from .serving import ServingEmbedding
vllm/entrypoints/pooling/factories.py
+2 -2
@@ -21,12 +21,12 @@ if TYPE_CHECKING:
21 from starlette.datastructures import State
22
23 from vllm.engine.protocol import EngineClient
24 - from vllm.entrypoints.logger import RequestLogger
25 - from vllm.entrypoints.sagemaker.api_router import (
24 + from vllm.entrypoints.serve.sagemaker.api_router import (
25 EndpointFn,
26 GetHandlerFn,
27 RequestType,
28 )
29 + from vllm.entrypoints.serve.utils.request_logger import RequestLogger
30
31 else:
32 RequestLogger = object
vllm/entrypoints/pooling/pooling/api_router.py
+5 -2
@@ -5,8 +5,11 @@ from http import HTTPStatus
5 from fastapi import APIRouter, Depends, Request
6
7 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
8 -from vllm.entrypoints.openai.utils import validate_json_request
9 -from vllm.entrypoints.utils import load_aware_call, with_cancellation
8 +from vllm.entrypoints.serve.utils.api_utils import (
9 + load_aware_call,
10 + validate_json_request,
11 + with_cancellation,
12 +)
13
14 from .protocol import PoolingRequest
15 from .serving import ServingPooling
vllm/entrypoints/pooling/scoring/api_router.py
+5 -2
@@ -5,8 +5,11 @@ from http import HTTPStatus
5 from fastapi import APIRouter, Depends, Request
6
7 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
8 -from vllm.entrypoints.openai.utils import validate_json_request
9 -from vllm.entrypoints.utils import load_aware_call, with_cancellation
8 +from vllm.entrypoints.serve.utils.api_utils import (
9 + load_aware_call,
10 + validate_json_request,
11 + with_cancellation,
12 +)
13 from vllm.logger import init_logger
14
15 from .protocol import RerankRequest, ScoreRequest
vllm/entrypoints/serve/disagg/api_router.py
+2 -2
@@ -13,7 +13,6 @@ from vllm.engine.protocol import EngineClient
13 from vllm.entrypoints.openai.engine.protocol import (
14 ErrorResponse,
15 )
16 -from vllm.entrypoints.openai.utils import validate_json_request
16 from vllm.entrypoints.serve.disagg.protocol import (
17 GenerateRequest,
18 GenerateResponse,
@@ -22,8 +21,9 @@ from vllm.entrypoints.serve.disagg.serving import (
21 ServingTokens,
22 )
23 from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
25 -from vllm.entrypoints.utils import (
24 +from vllm.entrypoints.serve.utils.api_utils import (
25 load_aware_call,
26 + validate_json_request,
27 with_cancellation,
28 )
29 from vllm.logger import init_logger
vllm/entrypoints/serve/disagg/serving.py
+2 -2
@@ -14,7 +14,6 @@ import pybase64 as base64
14 from fastapi import Request
15
16 from vllm.engine.protocol import EngineClient
17 -from vllm.entrypoints.logger import RequestLogger
17 from vllm.entrypoints.openai.chat_completion.protocol import (
18 ChatCompletionLogProb,
19 ChatCompletionLogProbs,
@@ -38,7 +37,8 @@ from vllm.entrypoints.serve.disagg.protocol import (
37 GenerateStreamResponse,
38 )
39 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
41 -from vllm.entrypoints.utils import get_max_tokens, should_include_usage
40 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
41 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
42 from vllm.inputs import EngineInput, mm_input
43 from vllm.logger import init_logger
44 from vllm.logprobs import Logprob
vllm/entrypoints/serve/elastic_ep/api_router.py
+1 -1
@@ -12,11 +12,11 @@ from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.openai.engine.protocol import (
13 ErrorResponse,
14 )
15 -from vllm.entrypoints.openai.utils import validate_json_request
15 from vllm.entrypoints.serve.elastic_ep.middleware import (
16 get_scaling_elastic_ep,
17 set_scaling_elastic_ep,
18 )
19 +from vllm.entrypoints.serve.utils.api_utils import validate_json_request
20 from vllm.logger import init_logger
21
22 logger = init_logger(__name__)
vllm/entrypoints/serve/lora/api_router.py
+1 -1
@@ -12,11 +12,11 @@ from vllm.entrypoints.openai.engine.protocol import (
12 )
13 from vllm.entrypoints.openai.models.api_router import models
14 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15 -from vllm.entrypoints.openai.utils import validate_json_request
15 from vllm.entrypoints.serve.lora.protocol import (
16 LoadLoRAAdapterRequest,
17 UnloadLoRAAdapterRequest,
18 )
19 +from vllm.entrypoints.serve.utils.api_utils import validate_json_request
20 from vllm.logger import init_logger
21
22 logger = init_logger(__name__)
vllm/entrypoints/serve/render/api_router.py
+1 -1
@@ -8,9 +8,9 @@ from fastapi.responses import JSONResponse
8 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
10 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11 -from vllm.entrypoints.openai.utils import validate_json_request
11 from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
12 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
13 +from vllm.entrypoints.serve.utils.api_utils import validate_json_request
14 from vllm.logger import init_logger
15
16 logger = init_logger(__name__)
vllm/entrypoints/serve/render/serving.py
+3 -5
@@ -11,7 +11,6 @@ from vllm.entrypoints.chat_utils import (
11 ChatTemplateContentFormatOption,
12 ConversationMessage,
13 )
14 -from vllm.entrypoints.logger import RequestLogger
14 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
15 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
16 from vllm.entrypoints.openai.engine.protocol import (
@@ -31,10 +30,9 @@ from vllm.entrypoints.serve.disagg.protocol import (
30 MultiModalFeatures,
31 PlaceholderRangeInfo,
32 )
34 -from vllm.entrypoints.utils import (
35 - create_error_response,
36 - get_max_tokens,
37 -)
33 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
34 +from vllm.entrypoints.serve.utils.error_response import create_error_response
35 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
36 from vllm.inputs import (
37 EngineInput,
38 MultiModalHashes,
vllm/entrypoints/serve/sagemaker/__init__.py renamed
vllm/entrypoints/serve/sagemaker/api_router.py renamed
+1 -1
@@ -14,11 +14,11 @@ from vllm.config import ModelConfig
14 from vllm.entrypoints.generate.factories import get_generate_invocation_types
15 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
16 from vllm.entrypoints.openai.engine.serving import OpenAIServing
17 -from vllm.entrypoints.openai.utils import validate_json_request
17 from vllm.entrypoints.pooling.base.serving import PoolingServingBase
18 from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
19 from vllm.entrypoints.serve.instrumentator.basic import base
20 from vllm.entrypoints.serve.instrumentator.health import health
21 +from vllm.entrypoints.serve.utils.api_utils import validate_json_request
22 from vllm.tasks import SupportedTask
23
24 # TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
vllm/entrypoints/serve/tokenize/api_router.py
+2 -2
@@ -12,7 +12,6 @@ from typing_extensions import assert_never
12 from vllm.entrypoints.openai.engine.protocol import (
13 ErrorResponse,
14 )
15 -from vllm.entrypoints.openai.utils import validate_json_request
15 from vllm.entrypoints.serve.tokenize.protocol import (
16 DetokenizeRequest,
17 DetokenizeResponse,
@@ -20,7 +19,8 @@ from vllm.entrypoints.serve.tokenize.protocol import (
19 TokenizeResponse,
20 )
21 from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
23 -from vllm.entrypoints.utils import (
22 +from vllm.entrypoints.serve.utils.api_utils import (
23 + validate_json_request,
24 with_cancellation,
25 )
26 from vllm.logger import init_logger
vllm/entrypoints/serve/tokenize/serving.py
+1 -1
@@ -7,7 +7,6 @@ from fastapi import Request
7
8 from vllm.engine.protocol import EngineClient
9 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
10 -from vllm.entrypoints.logger import RequestLogger
10 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11 from vllm.entrypoints.openai.engine.serving import OpenAIServing
12 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -20,6 +19,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
19 TokenizeResponse,
20 TokenizerInfoResponse,
21 )
22 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
23 from vllm.inputs import TokensPrompt, tokens_input
24 from vllm.logger import init_logger
25 from vllm.tokenizers import TokenizerLike
vllm/entrypoints/serve/utils/__init__.py
vllm/entrypoints/serve/utils/api_utils.py renamed
+9 -64
@@ -6,24 +6,19 @@ import dataclasses
6 import functools
7 import os
8 from argparse import Namespace
9 -from http import HTTPStatus
9 from logging import Logger
10 from string import Template
11 from typing import Any
12
13 import regex as re
14 from fastapi import Request
15 +from fastapi.exceptions import RequestValidationError
16 from fastapi.responses import JSONResponse, StreamingResponse
17 from starlette.background import BackgroundTask, BackgroundTasks
18
19 from vllm import envs
20 from vllm.engine.arg_utils import EngineArgs
21 -from vllm.entrypoints.openai.engine.protocol import (
22 - ErrorInfo,
23 - ErrorResponse,
24 - GenerationError,
25 - StreamOptions,
26 -)
21 +from vllm.entrypoints.openai.engine.protocol import StreamOptions
22 from vllm.entrypoints.openai.models.protocol import LoRAModulePath
23 from vllm.logger import current_formatter_type, init_logger
24 from vllm.platforms import current_platform
@@ -279,7 +274,7 @@ def log_non_default_args(args: Namespace | EngineArgs):
274
275
276 def should_include_usage(
282 - stream_options: "StreamOptions | None", enable_force_include_usage: bool
277 + stream_options: StreamOptions | None, enable_force_include_usage: bool
278 ) -> tuple[bool, bool]:
279 if enable_force_include_usage:
280 return True, True
@@ -344,60 +339,10 @@ def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
339 lgr.info(message, version, model_name)
340
341
347 -def create_error_response(
348 - message: str | Exception,
349 - err_type: str = "BadRequestError",
350 - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
351 - param: str | None = None,
352 -) -> ErrorResponse:
353 - exc: Exception | None = None
354 -
355 - if isinstance(message, Exception):
356 - exc = message
357 - logger.debug(
358 - "create_error_response called with %s: %s", type(exc).__name__, exc
359 - )
360 -
361 - from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
362 -
363 - if isinstance(exc, VLLMValidationError):
364 - err_type = "BadRequestError"
365 - status_code = HTTPStatus.BAD_REQUEST
366 - param = exc.parameter
367 - elif isinstance(exc, VLLMNotFoundError):
368 - err_type = "NotFoundError"
369 - status_code = HTTPStatus.NOT_FOUND
370 - param = None
371 - elif isinstance(exc, (ValueError, TypeError, OverflowError)):
372 - # Common validation errors from user input
373 - err_type = "BadRequestError"
374 - status_code = HTTPStatus.BAD_REQUEST
375 - param = None
376 - elif isinstance(exc, NotImplementedError):
377 - err_type = "NotImplementedError"
378 - status_code = HTTPStatus.NOT_IMPLEMENTED
379 - param = None
380 - elif isinstance(exc, GenerationError):
381 - err_type = "InternalServerError"
382 - status_code = exc.status_code
383 - param = None
384 - elif any(cls.__name__ == "TemplateError" for cls in type(exc).__mro__):
385 - # jinja2.TemplateError and its subclasses (avoid importing jinja2)
386 - err_type = "BadRequestError"
387 - status_code = HTTPStatus.BAD_REQUEST
388 - param = None
389 - else:
390 - err_type = "InternalServerError"
391 - status_code = HTTPStatus.INTERNAL_SERVER_ERROR
392 - param = None
393 -
394 - message = str(exc)
395 -
396 - return ErrorResponse(
397 - error=ErrorInfo(
398 - message=sanitize_message(message),
399 - type=err_type,
400 - code=status_code.value,
401 - param=param,
342 +async def validate_json_request(raw_request: Request):
343 + content_type = raw_request.headers.get("content-type", "").lower()
344 + media_type = content_type.split(";", maxsplit=1)[0]
345 + if media_type != "application/json":
346 + raise RequestValidationError(
347 + errors=["Unsupported Media Type: Only 'application/json' is allowed"]
348 )
403 - )
vllm/entrypoints/serve/utils/constants.py renamed
vllm/entrypoints/serve/utils/error_response.py new
+72
@@ -0,0 +1,72 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from http import HTTPStatus
4 +
5 +from vllm.entrypoints.openai.engine.protocol import (
6 + ErrorInfo,
7 + ErrorResponse,
8 + GenerationError,
9 +)
10 +from vllm.entrypoints.serve.utils.api_utils import sanitize_message
11 +from vllm.logger import init_logger
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +def create_error_response(
17 + message: str | Exception,
18 + err_type: str = "BadRequestError",
19 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
20 + param: str | None = None,
21 +) -> ErrorResponse:
22 + exc: Exception | None = None
23 +
24 + if isinstance(message, Exception):
25 + exc = message
26 + logger.debug(
27 + "create_error_response called with %s: %s", type(exc).__name__, exc
28 + )
29 +
30 + from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
31 +
32 + if isinstance(exc, VLLMValidationError):
33 + err_type = "BadRequestError"
34 + status_code = HTTPStatus.BAD_REQUEST
35 + param = exc.parameter
36 + elif isinstance(exc, VLLMNotFoundError):
37 + err_type = "NotFoundError"
38 + status_code = HTTPStatus.NOT_FOUND
39 + param = None
40 + elif isinstance(exc, (ValueError, TypeError, OverflowError)):
41 + # Common validation errors from user input
42 + err_type = "BadRequestError"
43 + status_code = HTTPStatus.BAD_REQUEST
44 + param = None
45 + elif isinstance(exc, NotImplementedError):
46 + err_type = "NotImplementedError"
47 + status_code = HTTPStatus.NOT_IMPLEMENTED
48 + param = None
49 + elif isinstance(exc, GenerationError):
50 + err_type = "InternalServerError"
51 + status_code = exc.status_code
52 + param = None
53 + elif any(cls.__name__ == "TemplateError" for cls in type(exc).__mro__):
54 + # jinja2.TemplateError and its subclasses (avoid importing jinja2)
55 + err_type = "BadRequestError"
56 + status_code = HTTPStatus.BAD_REQUEST
57 + param = None
58 + else:
59 + err_type = "InternalServerError"
60 + status_code = HTTPStatus.INTERNAL_SERVER_ERROR
61 + param = None
62 +
63 + message = str(exc)
64 +
65 + return ErrorResponse(
66 + error=ErrorInfo(
67 + message=sanitize_message(message),
68 + type=err_type,
69 + code=status_code.value,
70 + param=param,
71 + )
72 + )
vllm/entrypoints/serve/utils/fingerprint.py renamed
vllm/entrypoints/serve/utils/orca_metrics.py renamed
vllm/entrypoints/serve/utils/request_logger.py renamed
vllm/entrypoints/serve/utils/server_utils.py renamed
+4 -1
@@ -26,7 +26,10 @@ from vllm.entrypoints.openai.engine.protocol import (
26 ErrorResponse,
27 GenerationError,
28 )
29 -from vllm.entrypoints.utils import create_error_response, sanitize_message
29 +from vllm.entrypoints.serve.utils.error_response import (
30 + create_error_response,
31 + sanitize_message,
32 +)
33 from vllm.exceptions import VLLMValidationError
34 from vllm.logger import init_logger
35 from vllm.utils.gc_utils import freeze_gc_heap
vllm/entrypoints/serve/utils/ssl.py renamed
vllm/entrypoints/serve/utils/tool_calls_utils.py renamed
-12
@@ -2,9 +2,6 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from typing import TypeVar
4
5 -from fastapi import Request
6 -from fastapi.exceptions import RequestValidationError
7 -
5 from vllm.entrypoints.openai.chat_completion.protocol import (
6 ChatCompletionRequest,
7 ChatCompletionResponseChoice,
@@ -38,12 +35,3 @@ def maybe_filter_parallel_tool_calls(
35 ]
36
37 return choice
41 -
42 -
43 -async def validate_json_request(raw_request: Request):
44 - content_type = raw_request.headers.get("content-type", "").lower()
45 - media_type = content_type.split(";", maxsplit=1)[0]
46 - if media_type != "application/json":
47 - raise RequestValidationError(
48 - errors=["Unsupported Media Type: Only 'application/json' is allowed"]
49 - )
vllm/entrypoints/speech_to_text/base/serving.py
+2 -2
@@ -15,7 +15,6 @@ from transformers import PreTrainedTokenizerBase
15
16 import vllm.envs as envs
17 from vllm.engine.protocol import EngineClient
18 -from vllm.entrypoints.logger import RequestLogger
18 from vllm.entrypoints.openai.engine.protocol import (
19 DeltaMessage,
20 ErrorResponse,
@@ -24,7 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import (
23 )
24 from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
25 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27 -from vllm.entrypoints.utils import get_max_tokens
26 +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
27 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
28 from vllm.exceptions import VLLMValidationError
29 from vllm.inputs import EncoderDecoderInput, EngineInput
30 from vllm.logger import init_logger
vllm/entrypoints/speech_to_text/factories.py
+1 -1
@@ -12,7 +12,7 @@ if TYPE_CHECKING:
12 from starlette.datastructures import State
13
14 from vllm.engine.protocol import EngineClient
15 - from vllm.entrypoints.logger import RequestLogger
15 + from vllm.entrypoints.serve.utils.request_logger import RequestLogger
16 from vllm.tasks import SupportedTask
17 else:
18 RequestLogger = object
vllm/entrypoints/speech_to_text/realtime/serving.py
+1 -1
@@ -9,9 +9,9 @@ from typing import Literal, cast
9 import numpy as np
10
11 from vllm.engine.protocol import EngineClient, StreamingInput
12 -from vllm.entrypoints.logger import RequestLogger
12 from vllm.entrypoints.openai.engine.serving import OpenAIServing
13 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
14 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
15 from vllm.inputs import PromptType
16 from vllm.logger import init_logger
17 from vllm.model_executor.models.interfaces import SupportsRealtime
vllm/entrypoints/speech_to_text/transcription/api_router.py
+1 -1
@@ -9,7 +9,7 @@ from fastapi import APIRouter, Form, Request
9 from fastapi.responses import JSONResponse, StreamingResponse
10
11 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12 -from vllm.entrypoints.utils import (
12 +from vllm.entrypoints.serve.utils.api_utils import (
13 load_aware_call,
14 with_cancellation,
15 )
vllm/entrypoints/speech_to_text/transcription/serving.py
+1 -1
@@ -5,12 +5,12 @@ from collections.abc import AsyncGenerator
5 from fastapi import Request
6
7 from vllm.engine.protocol import EngineClient
8 -from vllm.entrypoints.logger import RequestLogger
8 from vllm.entrypoints.openai.engine.protocol import (
9 ErrorResponse,
10 RequestResponseMetadata,
11 )
12 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
13 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
14 from vllm.logger import init_logger
15 from vllm.outputs import RequestOutput
16
vllm/entrypoints/speech_to_text/translation/api_router.py
+1 -1
@@ -9,7 +9,7 @@ from fastapi import APIRouter, Form, Request
9 from fastapi.responses import JSONResponse, StreamingResponse
10
11 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12 -from vllm.entrypoints.utils import (
12 +from vllm.entrypoints.serve.utils.api_utils import (
13 load_aware_call,
14 with_cancellation,
15 )
vllm/entrypoints/speech_to_text/translation/serving.py
+1 -1
@@ -5,12 +5,12 @@ from collections.abc import AsyncGenerator
5 from fastapi import Request
6
7 from vllm.engine.protocol import EngineClient
8 -from vllm.entrypoints.logger import RequestLogger
8 from vllm.entrypoints.openai.engine.protocol import (
9 ErrorResponse,
10 RequestResponseMetadata,
11 )
12 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
13 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
14 from vllm.logger import init_logger
15 from vllm.outputs import RequestOutput
16
vllm/v1/utils.py
+1 -1
@@ -359,7 +359,7 @@ class RustFrontendProcessManager:
359 ]
360 if stats_update_address is not None:
361 cmd.extend(["--coordinator-address", stats_update_address])
362 - from vllm.entrypoints.utils import jsonify_non_default_args
362 + from vllm.entrypoints.serve.utils.api_utils import jsonify_non_default_args
363
364 args_json = json.dumps(
365 jsonify_non_default_args(args, exclude={"api_server_count"}),