[Frontend] Refine the entrypoint class's inheritance hierarchy. (#47498)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jul 3, 2026 at 18:50 UTC
a14f57a3ac0f011a50b62adc410cef93de4d7854
23 files changed
+90
-191
tests/entrypoints/generate/generative_scoring/test_generative_scoring.py
-1
@@ -78,7 +78,6 @@ def _create_mock_engine():
78
mock_engine.model_config = MockModelConfig()
79
mock_engine.input_processor = MagicMock()
80
81
- # renderer is accessed by OpenAIServing.__init__ and serving.py
81
mock_renderer = MagicMock()
82
mock_renderer.tokenizer = get_tokenizer(MODEL_NAME)
83
mock_engine.renderer = mock_renderer
tests/entrypoints/openai/responses/test_errors.py
+5
-5
@@ -7,20 +7,20 @@ from unittest.mock import MagicMock
7
import pytest
8
9
import vllm.envs as envs
10
-from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
10
+from vllm.entrypoints.generate.base.serving import GenerateBaseServing, GenerationError
11
from vllm.envs import disable_envs_cache
12
13
14
@pytest.mark.asyncio
15
async def test_raise_if_error_raises_generation_error():
16
"""test _raise_if_error raises GenerationError"""
17
- # create a minimal OpenAIServing instance
17
+ # create a minimal GenerateBaseServing instance
18
mock_engine = MagicMock()
19
mock_engine.model_config = MagicMock()
20
mock_engine.model_config.max_model_len = 100
21
mock_models = MagicMock()
22
23
- serving = OpenAIServing(
23
+ serving = GenerateBaseServing(
24
engine_client=mock_engine,
25
models=mock_models,
26
request_logger=None,
@@ -47,7 +47,7 @@ async def test_convert_generation_error_to_streaming_response():
47
mock_engine.model_config.max_model_len = 100
48
mock_models = MagicMock()
49
50
- serving = OpenAIServing(
50
+ serving = GenerateBaseServing(
51
engine_client=mock_engine,
52
models=mock_models,
53
request_logger=None,
@@ -77,7 +77,7 @@ def test_is_model_supported_skip_name_validation_env(
77
mock_models = MagicMock()
78
mock_models.is_base_model.return_value = False
79
80
- serving = OpenAIServing(
80
+ serving = GenerateBaseServing(
81
engine_client=mock_engine,
82
models=mock_models,
83
request_logger=None,
tests/entrypoints/openai/test_return_tokens_as_ids.py
+2
-2
@@ -127,13 +127,13 @@ def test_responses_api_logprobs_with_return_tokens_as_token_ids():
127
"""Test that return_tokens_as_token_ids works in Responses API logprobs."""
128
from unittest.mock import MagicMock
129
130
- from vllm.entrypoints.openai.engine.serving import OpenAIServing
130
+ from vllm.entrypoints.generate.base.serving import GenerateBaseServing
131
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
132
from vllm.logprobs import Logprob as SampleLogprob
133
134
serving = MagicMock(spec=OpenAIServingResponses)
135
serving.return_tokens_as_token_ids = True
136
- serving._get_decoded_token = OpenAIServing._get_decoded_token
136
+ serving._get_decoded_token = GenerateBaseServing._get_decoded_token
137
138
tokenizer = MagicMock()
139
tokenizer.decode = lambda token_id: "decoded"
tests/entrypoints/serve/lora/test_serving_models.py
+4
-4
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
14
)
15
from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
-from vllm.entrypoints.pooling.base.serving import PoolingServingBase
17
+from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
18
from vllm.entrypoints.pooling.typing import PoolingServeContext
19
from vllm.entrypoints.serve.lora.protocol import (
20
LoadLoRAAdapterRequest,
@@ -136,7 +136,7 @@ async def test_unload_lora_adapter_not_found():
136
assert response.error.code == HTTPStatus.NOT_FOUND
137
138
139
-class _ConcretePoolingServing(PoolingServingBase):
139
+class _ConcretePoolingServing(PoolingBaseServing):
140
"""Minimal concrete subclass used only in these unit tests."""
141
142
request_id_prefix = "test"
@@ -178,7 +178,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
178
serving = _make_pooling_serving(lora_name)
179
ctx = _make_pooling_ctx(lora_name)
180
181
- serving._maybe_get_adapters(ctx)
181
+ ctx.lora_request = serving._maybe_get_adapters(ctx.request)
182
183
assert ctx.lora_request is not None
184
assert ctx.lora_request.lora_name == lora_name
@@ -190,4 +190,4 @@ def test_pooling_maybe_get_adapters_unknown_model_raises():
190
ctx = _make_pooling_ctx("unknown-model")
191
192
with pytest.raises(VLLMNotFoundError):
193
- serving._maybe_get_adapters(ctx)
193
+ serving._maybe_get_adapters(ctx.request)
tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py
+4
-4
@@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock
7
8
import pytest
9
10
-from vllm.entrypoints.speech_to_text.base.serving import OpenAISpeechToText
10
+from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
11
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse
12
13
@@ -43,7 +43,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
43
is_tracing_enabled=AsyncMock(return_value=False),
44
)
45
46
- server = OpenAISpeechToText.__new__(OpenAISpeechToText)
46
+ server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
47
server.engine_client = engine_client
48
server.task_type = "transcribe"
49
server.models = SimpleNamespace(model_name=lambda: "audio")
@@ -112,7 +112,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
112
{"prompt": "chunk-1"},
113
{"prompt": "chunk-2"},
114
]
115
- server = OpenAISpeechToText.__new__(OpenAISpeechToText)
115
+ server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
116
server.engine_client = engine_client
117
server.task_type = "transcribe"
118
server.models = SimpleNamespace(model_name=lambda: "audio")
@@ -170,7 +170,7 @@ async def test_language_detection_cancel_aborts_engine_request():
170
abort=AsyncMock(),
171
)
172
173
- server = OpenAISpeechToText.__new__(OpenAISpeechToText)
173
+ server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
174
server.engine_client = engine_client
175
server.asr_config = SimpleNamespace()
176
server.tokenizer = Mock()
tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py
+4
-2
@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (
25
)
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
from vllm.entrypoints.speech_to_text.base.serving import (
28
- OpenAISpeechToText,
28
+ SpeechToTextBaseServing,
29
asr_inter_chunk_separator,
30
)
31
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
@@ -234,7 +234,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language():
234
"vllm.model_executor.model_loader.get_model_cls",
235
return_value=_StubTranscriptionModel,
236
),
237
- patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock),
237
+ patch.object(
238
+ SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock
239
+ ),
240
):
241
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
242
vllm/entrypoints/generate/base/__init__.py
vllm/entrypoints/generate/base/serving.py
renamed
+1
-1
@@ -53,7 +53,7 @@ class ServeContext(Generic[RequestT]):
53
model_config = ConfigDict(arbitrary_types_allowed=True)
54
55
56
-class OpenAIServing(BaseServing, BeamSearchOnlineMixin):
56
+class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
57
request_id_prefix: ClassVar[str] = """
58
A short string prepended to every request’s ID.
59
"""
vllm/entrypoints/generate/generative_scoring/serving.py
+5
-3
@@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import (
23
OpenAIBaseModel,
24
UsageInfo,
25
)
26
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
+from vllm.entrypoints.serve.engine.serving import BaseServing
28
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
29
from vllm.inputs import EngineInput, tokens_input
30
from vllm.logger import init_logger
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
142
# ============================================================================
143
144
145
-class ServingGenerativeScoring(OpenAIServing):
145
+class ServingGenerativeScoring(BaseServing):
146
"""Serving class for generative scoring computation.
147
148
This class handles computing the probability of specified token IDs
@@ -164,10 +164,12 @@ class ServingGenerativeScoring(OpenAIServing):
164
request_logger: RequestLogger | None,
165
) -> None:
166
super().__init__(
167
- engine_client=engine_client,
167
models=models,
168
+ model_config=engine_client.model_config,
169
request_logger=request_logger,
170
)
171
+ self.engine_client = engine_client
172
+ self.renderer = engine_client.renderer
173
174
async def create_generative_scoring(
175
self,
vllm/entrypoints/openai/chat_completion/serving.py
+8
-8
@@ -19,6 +19,12 @@ from vllm.entrypoints.chat_utils import (
19
ConversationMessage,
20
make_tool_call_id,
21
)
22
+from vllm.entrypoints.generate.base.serving import (
23
+ GenerateBaseServing,
24
+ GenerationError,
25
+ clamp_prompt_logprobs,
26
+ format_token_id_placeholder,
27
+)
28
from vllm.entrypoints.openai.chat_completion.protocol import (
29
ChatCompletionLogProb,
30
ChatCompletionLogProbs,
@@ -40,12 +46,6 @@ from vllm.entrypoints.openai.engine.protocol import (
46
ToolCall,
47
UsageInfo,
48
)
43
-from vllm.entrypoints.openai.engine.serving import (
44
- GenerationError,
45
- OpenAIServing,
46
- clamp_prompt_logprobs,
47
- format_token_id_placeholder,
48
-)
49
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
50
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
51
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -76,7 +76,7 @@ def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
76
counted in ``usage.prompt_tokens``.
77
"""
78
mm_placeholders = cast(
79
- "MultiModalPlaceholders | None", engine_input.get("mm_placeholders")
79
+ MultiModalPlaceholders | None, engine_input.get("mm_placeholders")
80
)
81
return {
82
modality: sum(p.length for p in ranges)
@@ -101,7 +101,7 @@ def _make_prompt_tokens_details(
101
)
102
103
104
-class OpenAIServingChat(OpenAIServing):
104
+class OpenAIServingChat(GenerateBaseServing):
105
def __init__(
106
self,
107
engine_client: EngineClient,
vllm/entrypoints/openai/completion/serving.py
+7
-7
@@ -13,6 +13,12 @@ import pybase64 as base64
13
from fastapi import Request
14
15
from vllm.engine.protocol import EngineClient
16
+from vllm.entrypoints.generate.base.serving import (
17
+ GenerateBaseServing,
18
+ GenerationError,
19
+ clamp_prompt_logprobs,
20
+ format_token_id_placeholder,
21
+)
22
from vllm.entrypoints.openai.completion.protocol import (
23
CompletionLogProbs,
24
CompletionRequest,
@@ -27,12 +33,6 @@ from vllm.entrypoints.openai.engine.protocol import (
33
RequestResponseMetadata,
34
UsageInfo,
35
)
30
-from vllm.entrypoints.openai.engine.serving import (
31
- GenerationError,
32
- OpenAIServing,
33
- clamp_prompt_logprobs,
34
- format_token_id_placeholder,
35
-)
36
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
37
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
38
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -50,7 +50,7 @@ from vllm.utils.collection_utils import as_list
50
logger = init_logger(__name__)
51
52
53
-class OpenAIServingCompletion(OpenAIServing):
53
+class OpenAIServingCompletion(GenerateBaseServing):
54
def __init__(
55
self,
56
engine_client: EngineClient,
vllm/entrypoints/openai/responses/serving.py
+5
-5
@@ -31,16 +31,16 @@ from vllm.entrypoints.chat_utils import (
31
ChatCompletionMessageParam,
32
ChatTemplateContentFormatOption,
33
)
34
+from vllm.entrypoints.generate.base.serving import (
35
+ GenerateBaseServing,
36
+ GenerationError,
37
+)
38
from vllm.entrypoints.mcp.tool_server import ToolServer
39
from vllm.entrypoints.openai.engine.protocol import (
40
DeltaMessage,
41
ErrorResponse,
42
RequestResponseMetadata,
43
)
40
-from vllm.entrypoints.openai.engine.serving import (
41
- GenerationError,
42
- OpenAIServing,
43
-)
44
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
45
from vllm.entrypoints.openai.parser.harmony_utils import (
46
build_harmony_preamble,
@@ -147,7 +147,7 @@ def _extract_allowed_tools_from_mcp_requests(
147
return allowed_tools_map
148
149
150
-class OpenAIServingResponses(OpenAIServing):
150
+class OpenAIServingResponses(GenerateBaseServing):
151
def __init__(
152
self,
153
engine_client: EngineClient,
vllm/entrypoints/pooling/base/serving.py
+13
-123
@@ -12,31 +12,29 @@ from fastapi import Request
12
from fastapi.responses import Response
13
from starlette.datastructures import Headers
14
15
-from vllm import PoolingParams, PoolingRequestOutput, envs
15
+from vllm import PoolingRequestOutput, envs
16
from vllm.config import VllmConfig
17
from vllm.engine.protocol import EngineClient
18
from vllm.entrypoints.chat_utils import ChatTemplateConfig
19
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
20
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
21
+from vllm.entrypoints.serve.engine.serving import BaseServing
22
+from vllm.entrypoints.serve.engine.typing import AnyRequest
23
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
22
-from vllm.exceptions import VLLMNotFoundError
23
-from vllm.inputs import EngineInput
24
from vllm.lora.request import LoRARequest
25
from vllm.renderers.base import BaseRenderer
26
-from vllm.renderers.inputs.preprocess import extract_prompt_components
26
from vllm.tracing import (
27
contains_trace_headers,
28
extract_trace_headers,
29
log_tracing_disabled_warning,
30
)
32
-from vllm.utils import random_uuid
31
from vllm.utils.async_utils import make_async, merge_async_iterators
32
33
from ..typing import AnyPoolingRequest, PoolingServeContext
34
from .io_processor import PoolingIOProcessor
35
36
39
-class PoolingServingBase(ABC):
37
+class PoolingBaseServing(ABC, BaseServing):
38
request_id_prefix: ClassVar[str]
39
40
def __init__(
@@ -49,13 +47,16 @@ class PoolingServingBase(ABC):
47
return_tokens_as_token_ids: bool = False,
48
log_error_stack: bool = False,
49
):
50
+ super().__init__(
51
+ models=models,
52
+ model_config=models.model_config,
53
+ request_logger=request_logger,
54
+ )
55
+
56
self.engine_client = engine_client
53
- self.models = models
54
- self.model_config = models.model_config
57
self.renderer = engine_client.renderer
58
self.vllm_config = engine_client.vllm_config
59
self.max_model_len = self.model_config.max_model_len
58
- self.request_logger = request_logger
60
self.return_tokens_as_token_ids = return_tokens_as_token_ids
61
self.log_error_stack = log_error_stack
62
self.chat_template_config = chat_template_config
@@ -118,7 +119,7 @@ class PoolingServingBase(ABC):
119
)
120
121
self._validate_request(ctx)
121
- self._maybe_get_adapters(ctx)
122
+ ctx.lora_request = self._maybe_get_adapters(ctx.request)
123
return ctx
124
125
async def _prepare_generators(
@@ -207,26 +208,9 @@ class PoolingServingBase(ABC):
208
) -> Response:
209
raise NotImplementedError
210
210
- @staticmethod
211
- def _base_request_id(
212
- raw_request: Request | None, default: str | None = None
213
- ) -> str | None:
214
- """Pulls the request id to use from a header, if provided"""
215
- if raw_request is not None and (
216
- (req_id := raw_request.headers.get("X-Request-Id")) is not None
217
- ):
218
- return req_id
219
-
220
- return random_uuid() if default is None else default
221
-
222
- def _is_model_supported(self, model_name: str | None) -> bool:
223
- if not model_name:
224
- return True
225
- return self.models.is_base_model(model_name)
226
-
211
async def _check_model(
212
self,
229
- request: AnyPoolingRequest,
213
+ request: AnyRequest | AnyPoolingRequest,
214
) -> ErrorResponse | None:
215
if self._is_model_supported(request.model):
216
return None
@@ -275,102 +259,8 @@ class PoolingServingBase(ABC):
259
260
return None
261
278
- def _maybe_get_adapters(
279
- self,
280
- ctx: PoolingServeContext,
281
- supports_default_mm_loras: bool = False,
282
- ):
283
- request = ctx.request
284
- if request.model in self.models.lora_requests:
285
- ctx.lora_request = self.models.lora_requests[request.model]
286
- return None
287
-
288
- # Currently only support default modality specific loras
289
- # if we have exactly one lora matched on the request.
290
- if supports_default_mm_loras:
291
- default_mm_lora = self._get_active_default_mm_loras(request)
292
- if default_mm_lora is not None:
293
- ctx.lora_request = default_mm_lora
294
-
295
- if self._is_model_supported(request.model):
296
- return None
297
-
298
- # if _check_model has been called earlier, this will be unreachable
299
- raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
300
-
301
- def _get_active_default_mm_loras(
302
- self, request: AnyPoolingRequest
303
- ) -> LoRARequest | None:
304
- """Determine if there are any active default multimodal loras."""
305
- # TODO: Currently this is only enabled for chat completions
306
- # to be better aligned with only being enabled for .generate
307
- # when run offline. It would be nice to support additional
308
- # tasks types in the future.
309
- message_types = self._get_message_types(request)
310
- default_mm_loras = set()
311
-
312
- for lora in self.models.lora_requests.values():
313
- # Best effort match for default multimodal lora adapters;
314
- # There is probably a better way to do this, but currently
315
- # this matches against the set of 'types' in any content lists
316
- # up until '_', e.g., to match audio_url -> audio
317
- if lora.lora_name in message_types:
318
- default_mm_loras.add(lora)
319
-
320
- # Currently only support default modality specific loras if
321
- # we have exactly one lora matched on the request.
322
- if len(default_mm_loras) == 1:
323
- return default_mm_loras.pop()
324
- return None
325
-
326
- def _get_message_types(self, request: AnyPoolingRequest) -> set[str]:
327
- """Retrieve the set of types from message content dicts up
328
- until `_`; we use this to match potential multimodal data
329
- with default per modality loras.
330
- """
331
- message_types: set[str] = set()
332
-
333
- if not hasattr(request, "messages"):
334
- return message_types
335
-
336
- messages = request.messages
337
- if messages is None or isinstance(messages, (str, bytes)):
338
- return message_types
339
-
340
- for message in messages:
341
- if (
342
- isinstance(message, dict)
343
- and "content" in message
344
- and isinstance(message["content"], list)
345
- ):
346
- for content_dict in message["content"]:
347
- if "type" in content_dict:
348
- message_types.add(content_dict["type"].split("_")[0])
349
- return message_types
350
-
351
- def _log_inputs(
352
- self,
353
- request_id: str,
354
- inputs: EngineInput,
355
- params: PoolingParams,
356
- lora_request: LoRARequest | None,
357
- ) -> None:
358
- if self.request_logger is None:
359
- return
360
-
361
- components = extract_prompt_components(self.model_config, inputs)
362
-
363
- self.request_logger.log_inputs(
364
- request_id,
365
- components.text,
366
- components.token_ids,
367
- components.embeds,
368
- params=params,
369
- lora_request=lora_request,
370
- )
371
-
262
373
-class PoolingServing(PoolingServingBase, ABC):
263
+class PoolingServing(PoolingBaseServing, ABC):
264
def __init__(self, *args, **kwargs):
265
super().__init__(*args, **kwargs)
266
vllm/entrypoints/pooling/pooling/serving.py
+2
-2
@@ -9,7 +9,7 @@ from vllm.tasks import SupportedTask
9
from vllm.utils.serial_utils import EmbedDType, Endianness
10
11
from ..base.io_processor import PoolingIOProcessor
12
-from ..base.serving import PoolingServingBase
12
+from ..base.serving import PoolingBaseServing
13
from ..factories import init_pooling_io_processors
14
from ..typing import AnyPoolingRequest, PoolingServeContext
15
from ..utils import (
@@ -30,7 +30,7 @@ from .protocol import (
30
logger = init_logger(__name__)
31
32
33
-class ServingPooling(PoolingServingBase):
33
+class ServingPooling(PoolingBaseServing):
34
request_id_prefix = "pooling"
35
36
def __init__(
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+5
-2
@@ -14,6 +14,10 @@ import pybase64 as base64
14
from fastapi import Request
15
16
from vllm.engine.protocol import EngineClient
17
+from vllm.entrypoints.generate.base.serving import (
18
+ GenerateBaseServing,
19
+ clamp_prompt_logprobs,
20
+)
21
from vllm.entrypoints.openai.chat_completion.protocol import (
22
ChatCompletionLogProb,
23
ChatCompletionLogProbs,
@@ -26,7 +30,6 @@ from vllm.entrypoints.openai.engine.protocol import (
30
RequestResponseMetadata,
31
UsageInfo,
32
)
29
-from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
33
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
34
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
35
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -55,7 +58,7 @@ from .protocol import (
58
logger = init_logger(__name__)
59
60
58
-class ServingTokens(OpenAIServing):
61
+class ServingTokens(GenerateBaseServing):
62
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
63
64
def __init__(
vllm/entrypoints/serve/engine/serving.py
+9
-5
@@ -11,9 +11,11 @@ from vllm.entrypoints.openai.models.serving import (
11
OpenAIModelRegistry,
12
OpenAIServingModels,
13
)
14
+from vllm.entrypoints.pooling.typing import AnyPoolingRequest
15
from vllm.entrypoints.serve.engine.typing import AnyRequest
16
from vllm.entrypoints.serve.utils.error_response import create_error_response
17
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
18
+from vllm.exceptions import VLLMNotFoundError
19
from vllm.inputs import EngineInput
20
from vllm.lora.request import LoRARequest
21
from vllm.renderers.inputs.preprocess import (
@@ -37,7 +39,7 @@ class BaseServing:
39
40
async def _check_model(
41
self,
40
- request: AnyRequest,
42
+ request: AnyRequest | AnyPoolingRequest,
43
) -> ErrorResponse | None:
44
error_response = None
45
@@ -123,7 +125,7 @@ class BaseServing:
125
126
return random_uuid() if default is None else default
127
126
- def _get_message_types(self, request: AnyRequest) -> set[str]:
128
+ def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
129
"""Retrieve the set of types from message content dicts up
130
until `_`; we use this to match potential multimodal data
131
with default per modality loras.
@@ -148,7 +150,9 @@ class BaseServing:
150
message_types.add(content_dict["type"].split("_")[0])
151
return message_types
152
151
- def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
153
+ def _get_active_default_mm_loras(
154
+ self, request: AnyRequest | AnyPoolingRequest
155
+ ) -> LoRARequest | None:
156
"""Determine if there are any active default multimodal loras."""
157
# TODO: Currently this is only enabled for chat completions
158
# to be better aligned with only being enabled for .generate
@@ -173,7 +177,7 @@ class BaseServing:
177
178
def _maybe_get_adapters(
179
self,
176
- request: AnyRequest,
180
+ request: AnyRequest | AnyPoolingRequest,
181
supports_default_mm_loras: bool = False,
182
) -> LoRARequest | None:
183
if request.model in self.models.lora_requests:
@@ -190,4 +194,4 @@ class BaseServing:
194
return None
195
196
# if _check_model has been called earlier, this will be unreachable
193
- raise ValueError(f"The model `{request.model}` does not exist.")
197
+ raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
vllm/entrypoints/serve/sagemaker/api_router.py
+2
-3
@@ -13,9 +13,8 @@ from fastapi.responses import JSONResponse, Response
13
from vllm.config import ModelConfig
14
from vllm.entrypoints.generate.factories import get_generate_invocation_types
15
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
16
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
17
-from vllm.entrypoints.pooling.base.serving import PoolingServingBase
16
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
17
+from vllm.entrypoints.serve.engine.serving import BaseServing
18
from vllm.entrypoints.serve.instrumentator.basic import base
19
from vllm.entrypoints.serve.instrumentator.health import health
20
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
@@ -24,7 +23,7 @@ from vllm.tasks import SupportedTask
23
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
24
# (requires typing_extensions >= 4.13)
25
RequestType = Any
27
-GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServingBase | None]
26
+GetHandlerFn = Callable[[Request], BaseServing | None]
27
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
28
29
vllm/entrypoints/speech_to_text/base/serving.py
+3
-3
@@ -16,13 +16,13 @@ from transformers import PreTrainedTokenizerBase
16
17
import vllm.envs as envs
18
from vllm.engine.protocol import EngineClient
19
+from vllm.entrypoints.generate.base.serving import GenerateBaseServing
20
from vllm.entrypoints.openai.engine.protocol import (
21
DeltaMessage,
22
ErrorResponse,
23
RequestResponseMetadata,
24
UsageInfo,
25
)
25
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
28
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
@@ -87,7 +87,7 @@ def asr_inter_chunk_separator(
87
return "" if language and language.lower() in no_space_languages else " "
88
89
90
-class OpenAISpeechToText(OpenAIServing):
90
+class SpeechToTextBaseServing(GenerateBaseServing):
91
"""Base class for speech-to-text operations like transcription and
92
translation."""
93
@@ -475,7 +475,7 @@ class OpenAISpeechToText(OpenAIServing):
475
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
476
477
input_len = (
478
- OpenAISpeechToText._get_decoder_prompt_len(engine_inputs)
478
+ SpeechToTextBaseServing._get_decoder_prompt_len(engine_inputs)
479
if request.use_beam_search
480
else 0
481
)
vllm/entrypoints/speech_to_text/realtime/serving.py
+2
-2
@@ -9,7 +9,7 @@ from typing import Literal, cast
9
import numpy as np
10
11
from vllm.engine.protocol import EngineClient, StreamingInput
12
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
12
+from vllm.entrypoints.generate.base.serving import GenerateBaseServing
13
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
14
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
15
from vllm.inputs import PromptType
@@ -20,7 +20,7 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt
20
logger = init_logger(__name__)
21
22
23
-class OpenAIServingRealtime(OpenAIServing):
23
+class OpenAIServingRealtime(GenerateBaseServing):
24
"""Realtime audio transcription service via WebSocket streaming.
25
26
Provides streaming audio-to-text transcription by transforming audio chunks
vllm/entrypoints/speech_to_text/transcription/serving.py
+2
-2
@@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
14
from vllm.logger import init_logger
15
from vllm.outputs import RequestOutput
16
17
-from ..base.serving import OpenAISpeechToText
17
+from ..base.serving import SpeechToTextBaseServing
18
from .protocol import (
19
TranscriptionRequest,
20
TranscriptionResponse,
@@ -26,7 +26,7 @@ from .protocol import (
26
logger = init_logger(__name__)
27
28
29
-class OpenAIServingTranscription(OpenAISpeechToText):
29
+class OpenAIServingTranscription(SpeechToTextBaseServing):
30
"""Handles transcription requests."""
31
32
def __init__(
vllm/entrypoints/speech_to_text/translation/serving.py
+2
-2
@@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
14
from vllm.logger import init_logger
15
from vllm.outputs import RequestOutput
16
17
-from ..base.serving import OpenAISpeechToText
17
+from ..base.serving import SpeechToTextBaseServing
18
from .protocol import (
19
TranslationRequest,
20
TranslationResponse,
@@ -26,7 +26,7 @@ from .protocol import (
26
logger = init_logger(__name__)
27
28
29
-class OpenAIServingTranslation(OpenAISpeechToText):
29
+class OpenAIServingTranslation(SpeechToTextBaseServing):
30
"""Handles translation requests."""
31
32
def __init__(
vllm/renderers/online_derenderer.py
+1
-1
@@ -4,6 +4,7 @@ from typing import Any
4
5
from vllm.config import ModelConfig
6
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
7
+from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder
8
from vllm.entrypoints.openai.chat_completion.protocol import (
9
ChatCompletionLogProbs,
10
ChatCompletionRequest,
@@ -15,7 +16,6 @@ from vllm.entrypoints.openai.completion.protocol import (
16
CompletionResponseChoice,
17
)
18
from vllm.entrypoints.openai.engine.protocol import ToolCall
18
-from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
19
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
20
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
21
from vllm.logger import init_logger
vllm/renderers/online_renderer.py
+4
-4
@@ -264,7 +264,7 @@ class OnlineRenderer:
264
chat_template_kwargs: dict[str, Any] | None,
265
trust_request_chat_template: bool,
266
) -> ErrorResponse | None:
267
- """Copied from OpenAIServing._validate_chat_template."""
267
+ """Copied from GenerateBaseServing._validate_chat_template."""
268
if not trust_request_chat_template and (
269
request_chat_template is not None
270
or (
@@ -287,7 +287,7 @@ class OnlineRenderer:
287
*,
288
skip_mm_cache: bool = False,
289
) -> list[EngineInput]:
290
- """Copied from OpenAIServing._preprocess_completion."""
290
+ """Copied from GenerateBaseServing._preprocess_completion."""
291
prompts = list[SingletonPrompt | bytes]()
292
if prompt_embeds is not None: # embeds take higher priority
293
prompts.extend(prompt_to_seq(prompt_embeds))
@@ -302,7 +302,7 @@ class OnlineRenderer:
302
*,
303
skip_mm_cache: bool = False,
304
) -> list[EngineInput]:
305
- """Copied from OpenAIServing._preprocess_cmpl."""
305
+ """Copied from GenerateBaseServing._preprocess_cmpl."""
306
renderer = self.renderer
307
model_config = self.model_config
308
@@ -339,7 +339,7 @@ class OnlineRenderer:
339
*,
340
skip_mm_cache: bool = False,
341
) -> tuple[list[ConversationMessage], list[EngineInput]]:
342
- """Copied from OpenAIServing._preprocess_chat."""
342
+ """Copied from GenerateBaseServing._preprocess_chat."""
343
renderer = self.renderer
344
mm_config = self.model_config.multimodal_config
345