[Frontend] Refactor ServingTokenization entrypoint. (#46022)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jun 22, 2026 at 14:27 UTC
1eb2cc961e997dbea323872f75e8fee7346dc640
15 files changed
+322
-302
tests/entrypoints/serve/tokenize/test_serving_tokenization.py
+3
-4
@@ -15,7 +15,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
15
TokenizeChatRequest,
16
TokenizeCompletionRequest,
17
)
18
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
18
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
19
from vllm.v1.engine.async_llm import AsyncLLM
20
21
MODEL_NAME = "openai-community/gpt2"
@@ -58,7 +58,7 @@ class MockModelConfig:
58
return self.diff_sampling_param or {}
59
60
61
-def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
61
+def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
62
models = OpenAIServingModels(
63
engine_client=engine,
64
base_model_paths=BASE_MODEL_PATHS,
@@ -71,8 +71,7 @@ def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
71
chat_template=None,
72
chat_template_content_format="auto",
73
)
74
- return OpenAIServingTokenization(
75
- engine,
74
+ return ServingTokenization(
75
models,
76
openai_serving_render=serving_render,
77
request_logger=None,
vllm/entrypoints/anthropic/api_router.py
+2
-2
@@ -61,7 +61,7 @@ def translate_error_response(response: ErrorResponse) -> JSONResponse:
61
async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
62
handler = messages(raw_request)
63
if handler is None:
64
- base_server = raw_request.app.state.openai_serving_tokenization
64
+ base_server = raw_request.app.state.serving_tokenization
65
error = base_server.create_error_response(
66
NotImplementedError("The model does not support Messages API")
67
)
@@ -107,7 +107,7 @@ async def create_messages(request: AnthropicMessagesRequest, raw_request: Reques
107
async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
108
handler = messages(raw_request)
109
if handler is None:
110
- base_server = raw_request.app.state.openai_serving_tokenization
110
+ base_server = raw_request.app.state.serving_tokenization
111
error = base_server.create_error_response(
112
NotImplementedError("The model does not support Messages API")
113
)
vllm/entrypoints/openai/api_server.py
+11
-6
@@ -34,7 +34,7 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
34
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
35
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
36
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
37
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
37
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
38
from vllm.entrypoints.serve.utils.api_utils import (
39
cli_env_setup,
40
log_non_default_args,
@@ -376,8 +376,7 @@ async def init_app_state(
376
log_error_stack=args.log_error_stack,
377
)
378
379
- state.openai_serving_tokenization = OpenAIServingTokenization(
380
- engine_client,
379
+ state.serving_tokenization = ServingTokenization(
380
state.openai_serving_models,
381
state.openai_serving_render,
382
request_logger=request_logger,
@@ -461,9 +460,15 @@ async def init_render_app_state(
460
)
461
462
state.openai_serving_models = model_registry
464
-
465
- # Expose tokenization via the render handler (no engine required).
466
- state.openai_serving_tokenization = state.openai_serving_render
463
+ state.serving_tokenization = ServingTokenization(
464
+ model_registry,
465
+ state.openai_serving_render,
466
+ request_logger=request_logger,
467
+ chat_template=resolved_chat_template,
468
+ chat_template_content_format=args.chat_template_content_format,
469
+ default_chat_template_kwargs=args.default_chat_template_kwargs,
470
+ trust_request_chat_template=args.trust_request_chat_template,
471
+ )
472
473
state.vllm_config = vllm_config
474
# Disable stats logging — there is no engine to poll.
vllm/entrypoints/openai/engine/serving.py
+10
-265
@@ -5,25 +5,19 @@ import time
5
from collections.abc import Awaitable, Mapping
6
from dataclasses import dataclass, field
7
from http import HTTPStatus
8
-from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
8
+from typing import ClassVar, Generic, TypeVar
9
10
from fastapi import Request
11
from pydantic import ConfigDict
12
from starlette.datastructures import Headers
13
14
-import vllm.envs as envs
15
-from vllm.config import ModelConfig
14
from vllm.engine.protocol import EngineClient
17
-from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
15
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
16
from vllm.entrypoints.openai.chat_completion.protocol import (
20
- BatchChatCompletionRequest,
17
ChatCompletionRequest,
22
- ChatCompletionResponse,
18
)
19
from vllm.entrypoints.openai.completion.protocol import (
20
CompletionRequest,
26
- CompletionResponse,
21
)
22
from vllm.entrypoints.openai.engine.protocol import (
23
ErrorResponse,
@@ -31,81 +25,22 @@ from vllm.entrypoints.openai.engine.protocol import (
25
)
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
34
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
35
-from vllm.entrypoints.serve.tokenize.protocol import (
36
- DetokenizeRequest,
37
- TokenizeChatRequest,
38
- TokenizeCompletionRequest,
39
- TokenizeResponse,
40
-)
41
-from vllm.entrypoints.serve.utils.error_response import create_error_response
28
+from vllm.entrypoints.serve.engine.serving import BaseServing
29
+from vllm.entrypoints.serve.engine.typing import AnyRequest
30
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
43
-from vllm.entrypoints.speech_to_text.transcription.protocol import (
44
- TranscriptionRequest,
45
- TranscriptionResponse,
46
-)
47
-from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
48
-from vllm.inputs import EngineInput, PromptType
31
+from vllm.inputs import EngineInput
32
from vllm.logger import init_logger
33
from vllm.logprobs import Logprob, PromptLogprobs
34
from vllm.lora.request import LoRARequest
52
-from vllm.renderers import ChatParams, TokenizeParams
53
-from vllm.renderers.inputs.preprocess import (
54
- extract_prompt_components,
55
- extract_prompt_len,
56
-)
57
-from vllm.sampling_params import BeamSearchParams, SamplingParams
35
from vllm.tokenizers import TokenizerLike
36
from vllm.tracing import (
37
contains_trace_headers,
38
extract_trace_headers,
39
log_tracing_disabled_warning,
40
)
64
-from vllm.utils import random_uuid
41
42
logger = init_logger(__name__)
43
68
-
69
-class RendererRequest(Protocol):
70
- def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
71
- raise NotImplementedError
72
-
73
-
74
-class RendererChatRequest(RendererRequest, Protocol):
75
- def build_chat_params(
76
- self,
77
- default_template: str | None,
78
- default_template_content_format: ChatTemplateContentFormatOption,
79
- ) -> ChatParams:
80
- raise NotImplementedError
81
-
82
-
83
-CompletionLikeRequest: TypeAlias = (
84
- CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
85
-)
86
-
87
-ChatLikeRequest: TypeAlias = (
88
- ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
89
-)
90
-
91
-SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
92
-
93
-AnyRequest: TypeAlias = (
94
- CompletionLikeRequest
95
- | ChatLikeRequest
96
- | SpeechToTextRequest
97
- | ResponsesRequest
98
- | GenerateRequest
99
-)
100
-
101
-AnyResponse: TypeAlias = (
102
- CompletionResponse
103
- | ChatCompletionResponse
104
- | TranscriptionResponse
105
- | TokenizeResponse
106
- | GenerateResponse
107
-)
108
-
44
RequestT = TypeVar("RequestT", bound=AnyRequest)
45
_T = TypeVar("_T")
46
@@ -122,7 +57,7 @@ class ServeContext(Generic[RequestT]):
57
model_config = ConfigDict(arbitrary_types_allowed=True)
58
59
125
-class OpenAIServing(BeamSearchOnlineMixin):
60
+class OpenAIServing(BaseServing, BeamSearchOnlineMixin):
61
request_id_prefix: ClassVar[str] = """
62
A short string prepended to every request’s ID.
63
"""
@@ -135,15 +70,14 @@ class OpenAIServing(BeamSearchOnlineMixin):
70
request_logger: RequestLogger | None,
71
return_tokens_as_token_ids: bool = False,
72
):
138
- super().__init__()
73
+ super().__init__(
74
+ models=models,
75
+ model_config=engine_client.model_config,
76
+ request_logger=request_logger,
77
+ )
78
79
self.engine_client = engine_client
141
- self.models = models
142
-
143
- self.request_logger = request_logger
80
self.return_tokens_as_token_ids = return_tokens_as_token_ids
145
-
146
- self.model_config = engine_client.model_config
81
self.renderer = engine_client.renderer
82
self.input_processor = engine_client.input_processor
83
vllm_config = getattr(engine_client, "vllm_config", None)
@@ -163,15 +97,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
97
# Never fail server startup over the fingerprint.
98
self.system_fingerprint = None
99
166
- @staticmethod
167
- def create_error_response(
168
- message: str | Exception,
169
- err_type: str = "BadRequestError",
170
- status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
171
- param: str | None = None,
172
- ) -> ErrorResponse:
173
- return create_error_response(message, err_type, status_code, param)
174
-
100
def create_streaming_error_response(
101
self,
102
message: str | Exception,
@@ -208,167 +133,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
133
status_code=e.status_code,
134
)
135
211
- async def _check_model(
212
- self,
213
- request: AnyRequest,
214
- ) -> ErrorResponse | None:
215
- error_response = None
216
-
217
- if self._is_model_supported(request.model):
218
- return None
219
- if request.model in self.models.lora_requests:
220
- return None
221
- if (
222
- envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
223
- and request.model
224
- and (load_result := await self.models.resolve_lora(request.model))
225
- ):
226
- if isinstance(load_result, LoRARequest):
227
- return None
228
- if (
229
- isinstance(load_result, ErrorResponse)
230
- and load_result.error.code == HTTPStatus.BAD_REQUEST.value
231
- ):
232
- error_response = load_result
233
-
234
- return error_response or self.create_error_response(
235
- message=f"The model `{request.model}` does not exist.",
236
- err_type="NotFoundError",
237
- status_code=HTTPStatus.NOT_FOUND,
238
- param="model",
239
- )
240
-
241
- def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
242
- """Determine if there are any active default multimodal loras."""
243
- # TODO: Currently this is only enabled for chat completions
244
- # to be better aligned with only being enabled for .generate
245
- # when run offline. It would be nice to support additional
246
- # tasks types in the future.
247
- message_types = self._get_message_types(request)
248
- default_mm_loras = set()
249
-
250
- for lora in self.models.lora_requests.values():
251
- # Best effort match for default multimodal lora adapters;
252
- # There is probably a better way to do this, but currently
253
- # this matches against the set of 'types' in any content lists
254
- # up until '_', e.g., to match audio_url -> audio
255
- if lora.lora_name in message_types:
256
- default_mm_loras.add(lora)
257
-
258
- # Currently only support default modality specific loras if
259
- # we have exactly one lora matched on the request.
260
- if len(default_mm_loras) == 1:
261
- return default_mm_loras.pop()
262
- return None
263
-
264
- def _maybe_get_adapters(
265
- self,
266
- request: AnyRequest,
267
- supports_default_mm_loras: bool = False,
268
- ) -> LoRARequest | None:
269
- if request.model in self.models.lora_requests:
270
- return self.models.lora_requests[request.model]
271
-
272
- # Currently only support default modality specific loras
273
- # if we have exactly one lora matched on the request.
274
- if supports_default_mm_loras:
275
- default_mm_lora = self._get_active_default_mm_loras(request)
276
- if default_mm_lora is not None:
277
- return default_mm_lora
278
-
279
- if self._is_model_supported(request.model):
280
- return None
281
-
282
- # if _check_model has been called earlier, this will be unreachable
283
- raise ValueError(f"The model `{request.model}` does not exist.")
284
-
285
- def _get_message_types(self, request: AnyRequest) -> set[str]:
286
- """Retrieve the set of types from message content dicts up
287
- until `_`; we use this to match potential multimodal data
288
- with default per modality loras.
289
- """
290
- message_types: set[str] = set()
291
-
292
- if not hasattr(request, "messages"):
293
- return message_types
294
-
295
- messages = request.messages
296
- if messages is None or isinstance(messages, (str, bytes)):
297
- return message_types
298
-
299
- for message in messages:
300
- if (
301
- isinstance(message, dict)
302
- and "content" in message
303
- and isinstance(message["content"], list)
304
- ):
305
- for content_dict in message["content"]:
306
- if "type" in content_dict:
307
- message_types.add(content_dict["type"].split("_")[0])
308
- return message_types
309
-
310
- def _validate_chat_template(
311
- self,
312
- request_chat_template: str | None,
313
- chat_template_kwargs: dict[str, Any] | None,
314
- trust_request_chat_template: bool,
315
- ) -> ErrorResponse | None:
316
- if not trust_request_chat_template and (
317
- request_chat_template is not None
318
- or (
319
- chat_template_kwargs
320
- and chat_template_kwargs.get("chat_template") is not None
321
- )
322
- ):
323
- return self.create_error_response(
324
- "Chat template is passed with request, but "
325
- "--trust-request-chat-template is not set. "
326
- "Refused request with untrusted chat template."
327
- )
328
- return None
329
-
330
- @staticmethod
331
- def _prepare_extra_chat_template_kwargs(
332
- request_chat_template_kwargs: dict[str, Any] | None = None,
333
- default_chat_template_kwargs: dict[str, Any] | None = None,
334
- ) -> dict[str, Any]:
335
- """Helper to merge server-default and request-specific chat template kwargs."""
336
- request_chat_template_kwargs = request_chat_template_kwargs or {}
337
- if default_chat_template_kwargs is None:
338
- return request_chat_template_kwargs
339
- # Apply server defaults first, then request kwargs override.
340
- return default_chat_template_kwargs | request_chat_template_kwargs
341
-
342
- def _extract_prompt_components(self, prompt: PromptType | EngineInput):
343
- return extract_prompt_components(self.model_config, prompt)
344
-
345
- def _extract_prompt_text(self, prompt: PromptType | EngineInput):
346
- return self._extract_prompt_components(prompt).text
347
-
348
- def _extract_prompt_len(self, prompt: EngineInput):
349
- return extract_prompt_len(self.model_config, prompt)
350
-
351
- def _log_inputs(
352
- self,
353
- request_id: str,
354
- inputs: PromptType | EngineInput,
355
- params: SamplingParams | BeamSearchParams | None,
356
- lora_request: LoRARequest | None,
357
- ) -> None:
358
- if self.request_logger is None:
359
- return
360
-
361
- components = self._extract_prompt_components(inputs)
362
-
363
- self.request_logger.log_inputs(
364
- request_id,
365
- components.text,
366
- components.token_ids,
367
- components.embeds,
368
- params=params,
369
- lora_request=lora_request,
370
- )
371
-
136
async def _get_trace_headers(
137
self,
138
headers: Headers,
@@ -383,18 +147,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
147
148
return None
149
386
- @staticmethod
387
- def _base_request_id(
388
- raw_request: Request | None, default: str | None = None
389
- ) -> str | None:
390
- """Pulls the request id to use from a header, if provided"""
391
- if raw_request is not None and (
392
- (req_id := raw_request.headers.get("X-Request-Id")) is not None
393
- ):
394
- return req_id
395
-
396
- return random_uuid() if default is None else default
397
-
150
@staticmethod
151
def _get_data_parallel_rank(raw_request: Request | None) -> int | None:
152
"""Pulls the data parallel rank from a header, if provided"""
@@ -464,13 +216,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
216
217
return tokenizer.decode([token_id])
218
467
- def _is_model_supported(self, model_name: str | None) -> bool:
468
- if not model_name:
469
- return True
470
- if envs.VLLM_SKIP_MODEL_NAME_VALIDATION:
471
- return True
472
- return self.models.is_base_model(model_name)
473
-
219
220
def format_token_id_placeholder(token_id: int) -> str:
221
return f"token_id:{token_id}"
vllm/entrypoints/openai/models/serving.py
+7
@@ -42,6 +42,10 @@ class OpenAIModelRegistry:
42
) -> None:
43
self.model_config = model_config
44
self.base_model_paths = base_model_paths
45
+ self.lora_requests: dict[str, LoRARequest] = {}
46
+
47
+ def model_name(self, lora_request: LoRARequest | None = None) -> str:
48
+ return self.base_model_paths[0].name
49
50
def is_base_model(self, model_name: str) -> bool:
51
return any(model.name == model_name for model in self.base_model_paths)
@@ -72,6 +76,9 @@ class OpenAIModelRegistry:
76
]
77
)
78
79
+ async def resolve_lora(self, lora_name: str):
80
+ raise RuntimeError("The OpenAIModelRegistry has no LoRA support.")
81
+
82
83
class OpenAIServingModels:
84
"""Shared instance to hold data about the loaded base model(s) and adapters.
vllm/entrypoints/pooling/base/io_processor.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (
12
ChatTemplateContentFormatOption,
13
ConversationMessage,
14
)
15
-from vllm.entrypoints.openai.engine.serving import RendererChatRequest, RendererRequest
15
+from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
16
from vllm.inputs import EngineInput, SingletonPrompt
17
from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs
18
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
vllm/entrypoints/pooling/base/serving.py
+2
-2
@@ -52,7 +52,7 @@ class PoolingServingBase(ABC):
52
self.engine_client = engine_client
53
self.models = models
54
self.model_config = models.model_config
55
- self.renderer = models.renderer
55
+ self.renderer = engine_client.renderer
56
self.vllm_config = engine_client.vllm_config
57
self.max_model_len = self.model_config.max_model_len
58
self.request_logger = request_logger
@@ -61,7 +61,7 @@ class PoolingServingBase(ABC):
61
self.chat_template_config = chat_template_config
62
63
# Shared thread pool executor for preprocessing and postprocessing.
64
- self._executor: Executor = models.renderer._executor
64
+ self._executor: Executor = self.renderer._executor
65
self._preprocessing_async = make_async(
66
self._preprocessing, executor=self._executor
67
)
vllm/entrypoints/serve/disagg/api_router.py
+3
-3
@@ -20,7 +20,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
20
from vllm.entrypoints.serve.disagg.serving import (
21
ServingTokens,
22
)
23
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
23
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
24
from vllm.entrypoints.serve.utils.api_utils import (
25
load_aware_call,
26
validate_json_request,
@@ -31,8 +31,8 @@ from vllm.logger import init_logger
31
logger = init_logger(__name__)
32
33
34
-def tokenization(request: Request) -> OpenAIServingTokenization:
35
- return request.app.state.openai_serving_tokenization
34
+def tokenization(request: Request) -> ServingTokenization:
35
+ return request.app.state.serving_tokenization
36
37
38
def generate_tokens(request: Request) -> ServingTokens | None:
vllm/entrypoints/serve/engine/__init__.py
vllm/entrypoints/serve/engine/serving.py
new
+193
@@ -0,0 +1,193 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from http import HTTPStatus
4
+
5
+from fastapi import Request
6
+
7
+from vllm import PromptType, SamplingParams, envs
8
+from vllm.config import ModelConfig
9
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
+from vllm.entrypoints.openai.models.serving import (
11
+ OpenAIModelRegistry,
12
+ OpenAIServingModels,
13
+)
14
+from vllm.entrypoints.serve.engine.typing import AnyRequest
15
+from vllm.entrypoints.serve.utils.error_response import create_error_response
16
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
17
+from vllm.inputs import EngineInput
18
+from vllm.lora.request import LoRARequest
19
+from vllm.renderers.inputs.preprocess import (
20
+ extract_prompt_components,
21
+ extract_prompt_len,
22
+)
23
+from vllm.sampling_params import BeamSearchParams
24
+from vllm.utils import random_uuid
25
+
26
+
27
+class BaseServing:
28
+ def __init__(
29
+ self,
30
+ models: OpenAIServingModels | OpenAIModelRegistry,
31
+ model_config: ModelConfig,
32
+ request_logger: RequestLogger | None = None,
33
+ ):
34
+ self.models = models
35
+ self.model_config = model_config
36
+ self.request_logger = request_logger
37
+
38
+ async def _check_model(
39
+ self,
40
+ request: AnyRequest,
41
+ ) -> ErrorResponse | None:
42
+ error_response = None
43
+
44
+ if self._is_model_supported(request.model):
45
+ return None
46
+ if request.model in self.models.lora_requests:
47
+ return None
48
+ if (
49
+ envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
50
+ and request.model
51
+ and (load_result := await self.models.resolve_lora(request.model))
52
+ ):
53
+ if isinstance(load_result, LoRARequest):
54
+ return None
55
+ if (
56
+ isinstance(load_result, ErrorResponse)
57
+ and load_result.error.code == HTTPStatus.BAD_REQUEST.value
58
+ ):
59
+ error_response = load_result
60
+
61
+ return error_response or self.create_error_response(
62
+ message=f"The model `{request.model}` does not exist.",
63
+ err_type="NotFoundError",
64
+ status_code=HTTPStatus.NOT_FOUND,
65
+ param="model",
66
+ )
67
+
68
+ def _is_model_supported(self, model_name: str | None) -> bool:
69
+ if not model_name:
70
+ return True
71
+ if envs.VLLM_SKIP_MODEL_NAME_VALIDATION:
72
+ return True
73
+ return self.models.is_base_model(model_name)
74
+
75
+ @staticmethod
76
+ def create_error_response(
77
+ message: str | Exception,
78
+ err_type: str = "BadRequestError",
79
+ status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
80
+ param: str | None = None,
81
+ ) -> ErrorResponse:
82
+ return create_error_response(message, err_type, status_code, param)
83
+
84
+ def _extract_prompt_components(self, prompt: PromptType | EngineInput):
85
+ return extract_prompt_components(self.model_config, prompt)
86
+
87
+ def _extract_prompt_text(self, prompt: PromptType | EngineInput):
88
+ return self._extract_prompt_components(prompt).text
89
+
90
+ def _extract_prompt_len(self, prompt: EngineInput):
91
+ return extract_prompt_len(self.model_config, prompt)
92
+
93
+ def _log_inputs(
94
+ self,
95
+ request_id: str,
96
+ inputs: PromptType | EngineInput,
97
+ params: SamplingParams | BeamSearchParams | None,
98
+ lora_request: LoRARequest | None,
99
+ ) -> None:
100
+ if self.request_logger is None:
101
+ return
102
+
103
+ components = self._extract_prompt_components(inputs)
104
+
105
+ self.request_logger.log_inputs(
106
+ request_id,
107
+ components.text,
108
+ components.token_ids,
109
+ components.embeds,
110
+ params=params,
111
+ lora_request=lora_request,
112
+ )
113
+
114
+ @staticmethod
115
+ def _base_request_id(
116
+ raw_request: Request | None, default: str | None = None
117
+ ) -> str | None:
118
+ """Pulls the request id to use from a header, if provided"""
119
+ if raw_request is not None and (
120
+ (req_id := raw_request.headers.get("X-Request-Id")) is not None
121
+ ):
122
+ return req_id
123
+
124
+ return random_uuid() if default is None else default
125
+
126
+ def _get_message_types(self, request: AnyRequest) -> set[str]:
127
+ """Retrieve the set of types from message content dicts up
128
+ until `_`; we use this to match potential multimodal data
129
+ with default per modality loras.
130
+ """
131
+ message_types: set[str] = set()
132
+
133
+ if not hasattr(request, "messages"):
134
+ return message_types
135
+
136
+ messages = request.messages
137
+ if messages is None or isinstance(messages, (str, bytes)):
138
+ return message_types
139
+
140
+ for message in messages:
141
+ if (
142
+ isinstance(message, dict)
143
+ and "content" in message
144
+ and isinstance(message["content"], list)
145
+ ):
146
+ for content_dict in message["content"]:
147
+ if "type" in content_dict:
148
+ message_types.add(content_dict["type"].split("_")[0])
149
+ return message_types
150
+
151
+ def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
152
+ """Determine if there are any active default multimodal loras."""
153
+ # TODO: Currently this is only enabled for chat completions
154
+ # to be better aligned with only being enabled for .generate
155
+ # when run offline. It would be nice to support additional
156
+ # tasks types in the future.
157
+ message_types = self._get_message_types(request)
158
+ default_mm_loras = set()
159
+
160
+ for lora in self.models.lora_requests.values():
161
+ # Best effort match for default multimodal lora adapters;
162
+ # There is probably a better way to do this, but currently
163
+ # this matches against the set of 'types' in any content lists
164
+ # up until '_', e.g., to match audio_url -> audio
165
+ if lora.lora_name in message_types:
166
+ default_mm_loras.add(lora)
167
+
168
+ # Currently only support default modality specific loras if
169
+ # we have exactly one lora matched on the request.
170
+ if len(default_mm_loras) == 1:
171
+ return default_mm_loras.pop()
172
+ return None
173
+
174
+ def _maybe_get_adapters(
175
+ self,
176
+ request: AnyRequest,
177
+ supports_default_mm_loras: bool = False,
178
+ ) -> LoRARequest | None:
179
+ if request.model in self.models.lora_requests:
180
+ return self.models.lora_requests[request.model]
181
+
182
+ # Currently only support default modality specific loras
183
+ # if we have exactly one lora matched on the request.
184
+ if supports_default_mm_loras:
185
+ default_mm_lora = self._get_active_default_mm_loras(request)
186
+ if default_mm_lora is not None:
187
+ return default_mm_lora
188
+
189
+ if self._is_model_supported(request.model):
190
+ return None
191
+
192
+ # if _check_model has been called earlier, this will be unreachable
193
+ raise ValueError(f"The model `{request.model}` does not exist.")
vllm/entrypoints/serve/engine/typing.py
new
+71
@@ -0,0 +1,71 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import Protocol, TypeAlias
5
+
6
+from vllm.config import ModelConfig
7
+from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
8
+from vllm.entrypoints.openai.chat_completion.protocol import (
9
+ BatchChatCompletionRequest,
10
+ ChatCompletionRequest,
11
+ ChatCompletionResponse,
12
+)
13
+from vllm.entrypoints.openai.completion.protocol import (
14
+ CompletionRequest,
15
+ CompletionResponse,
16
+)
17
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
18
+from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
19
+from vllm.entrypoints.serve.tokenize.protocol import (
20
+ DetokenizeRequest,
21
+ TokenizeChatRequest,
22
+ TokenizeCompletionRequest,
23
+ TokenizeResponse,
24
+)
25
+from vllm.entrypoints.speech_to_text.transcription.protocol import (
26
+ TranscriptionRequest,
27
+ TranscriptionResponse,
28
+)
29
+from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
30
+from vllm.renderers import ChatParams, TokenizeParams
31
+
32
+
33
+class RendererRequest(Protocol):
34
+ def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
35
+ raise NotImplementedError
36
+
37
+
38
+class RendererChatRequest(RendererRequest, Protocol):
39
+ def build_chat_params(
40
+ self,
41
+ default_template: str | None,
42
+ default_template_content_format: ChatTemplateContentFormatOption,
43
+ ) -> ChatParams:
44
+ raise NotImplementedError
45
+
46
+
47
+CompletionLikeRequest: TypeAlias = (
48
+ CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
49
+)
50
+
51
+ChatLikeRequest: TypeAlias = (
52
+ ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
53
+)
54
+
55
+SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
56
+
57
+AnyRequest: TypeAlias = (
58
+ CompletionLikeRequest
59
+ | ChatLikeRequest
60
+ | SpeechToTextRequest
61
+ | ResponsesRequest
62
+ | GenerateRequest
63
+)
64
+
65
+AnyResponse: TypeAlias = (
66
+ CompletionResponse
67
+ | ChatCompletionResponse
68
+ | TranscriptionResponse
69
+ | TokenizeResponse
70
+ | GenerateResponse
71
+)
vllm/entrypoints/serve/instrumentator/basic.py
+4
-5
@@ -5,8 +5,7 @@ from fastapi import APIRouter, Request
5
from fastapi.responses import JSONResponse
6
7
from vllm.engine.protocol import EngineClient
8
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
9
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
8
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
9
from vllm.logger import init_logger
10
from vllm.version import __version__ as VLLM_VERSION
11
@@ -15,13 +14,13 @@ router = APIRouter()
14
logger = init_logger(__name__)
15
16
18
-def base(request: Request) -> OpenAIServing:
17
+def base(request: Request) -> ServingTokenization:
18
# Reuse the existing instance
19
return tokenization(request)
20
21
23
-def tokenization(request: Request) -> OpenAIServingTokenization:
24
- return request.app.state.openai_serving_tokenization
22
+def tokenization(request: Request) -> ServingTokenization:
23
+ return request.app.state.serving_tokenization
24
25
26
def engine_client(request: Request) -> EngineClient:
vllm/entrypoints/serve/tokenize/api_router.py
+4
-6
@@ -9,16 +9,14 @@ from fastapi.exceptions import RequestValidationError
9
from fastapi.responses import JSONResponse
10
from typing_extensions import assert_never
11
12
-from vllm.entrypoints.openai.engine.protocol import (
13
- ErrorResponse,
14
-)
12
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
13
from vllm.entrypoints.serve.tokenize.protocol import (
14
DetokenizeRequest,
15
DetokenizeResponse,
16
TokenizeRequest,
17
TokenizeResponse,
18
)
21
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
19
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
20
from vllm.entrypoints.serve.utils.api_utils import (
21
validate_json_request,
22
with_cancellation,
@@ -28,8 +26,8 @@ from vllm.logger import init_logger
26
logger = init_logger(__name__)
27
28
31
-def tokenization(request: Request) -> OpenAIServingTokenization:
32
- return request.app.state.openai_serving_tokenization
29
+def tokenization(request: Request) -> ServingTokenization:
30
+ return request.app.state.serving_tokenization
31
32
33
router = APIRouter()
vllm/entrypoints/serve/tokenize/serving.py
+9
-7
@@ -5,11 +5,13 @@ from typing import Any, Final
5
6
from fastapi import Request
7
8
-from vllm.engine.protocol import EngineClient
8
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
9
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
12
-from vllm.entrypoints.openai.models.serving import OpenAIServingModels
10
+from vllm.entrypoints.openai.models.serving import (
11
+ OpenAIModelRegistry,
12
+ OpenAIServingModels,
13
+)
14
+from vllm.entrypoints.serve.engine.serving import BaseServing
15
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
16
from vllm.entrypoints.serve.tokenize.protocol import (
17
DetokenizeRequest,
@@ -27,11 +29,10 @@ from vllm.tokenizers import TokenizerLike
29
logger = init_logger(__name__)
30
31
30
-class OpenAIServingTokenization(OpenAIServing):
32
+class ServingTokenization(BaseServing):
33
def __init__(
34
self,
33
- engine_client: EngineClient,
34
- models: OpenAIServingModels,
35
+ models: OpenAIServingModels | OpenAIModelRegistry,
36
openai_serving_render: OpenAIServingRender,
37
*,
38
request_logger: RequestLogger | None,
@@ -41,11 +42,12 @@ class OpenAIServingTokenization(OpenAIServing):
42
trust_request_chat_template: bool = False,
43
) -> None:
44
super().__init__(
44
- engine_client=engine_client,
45
models=models,
46
+ model_config=openai_serving_render.model_config,
47
request_logger=request_logger,
48
)
49
50
+ self.renderer = openai_serving_render.renderer
51
self.openai_serving_render = openai_serving_render
52
self.chat_template = chat_template
53
self.chat_template_content_format: Final = chat_template_content_format
vllm/entrypoints/speech_to_text/base/serving.py
+2
-1
@@ -22,8 +22,9 @@ from vllm.entrypoints.openai.engine.protocol import (
22
RequestResponseMetadata,
23
UsageInfo,
24
)
25
-from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
25
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
+from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
28
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
29
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
30
from vllm.exceptions import VLLMValidationError