@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dcb31196da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 12:30 UTC

[Chore] Remove redundant `RequestPrompt` (#30612)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 14, 2025 at 17:22 UTC dcb31196dae923e06da81eae02de1de662a97d2b
12 files changed +186 -251
tests/entrypoints/openai/test_chat_error.py
+1 -2
@@ -80,10 +80,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
80 return dict(engine_prompt), {}
81
82 async def _fake_preprocess_chat(*args, **kwargs):
83 - # return conversation, request_prompts, engine_prompts
83 + # return conversation, engine_prompts
84 return (
85 [{"role": "user", "content": "Test"}],
86 - [[1, 2, 3]],
86 [{"prompt_token_ids": [1, 2, 3]}],
87 )
88
tests/entrypoints/openai/test_serving_chat.py
+13 -13
@@ -877,7 +877,7 @@ class TestServingChatWithHarmony:
877
878 # Test the Harmony messages for the first turn's input
879 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
880 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
880 + input_messages, _ = serving_chat._make_request_with_harmony(req)
881 verify_harmony_messages(
882 input_messages,
883 [
@@ -905,7 +905,7 @@ class TestServingChatWithHarmony:
905
906 # Test the Harmony messages for the second turn's input
907 req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
908 - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
908 + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
909 verify_harmony_messages(
910 input_messages_2,
911 [
@@ -927,7 +927,7 @@ class TestServingChatWithHarmony:
927
928 # Test the Harmony messages for the first turn's input
929 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
930 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
930 + input_messages, _ = serving_chat._make_request_with_harmony(req)
931 verify_harmony_messages(
932 input_messages,
933 [
@@ -971,7 +971,7 @@ class TestServingChatWithHarmony:
971
972 # Test the Harmony messages for the second turn's input
973 req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
974 - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
974 + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
975 verify_harmony_messages(
976 input_messages_2,
977 [
@@ -1008,7 +1008,7 @@ class TestServingChatWithHarmony:
1008
1009 # Test the Harmony messages for the first turn's input
1010 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1011 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1011 + input_messages, _ = serving_chat._make_request_with_harmony(req)
1012 verify_harmony_messages(
1013 input_messages,
1014 [
@@ -1052,7 +1052,7 @@ class TestServingChatWithHarmony:
1052
1053 # Test the Harmony messages for the second turn's input
1054 req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1055 - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
1055 + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1056 verify_harmony_messages(
1057 input_messages_2,
1058 [
@@ -1089,7 +1089,7 @@ class TestServingChatWithHarmony:
1089
1090 # Test the Harmony messages for the first turn's input
1091 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1092 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1092 + input_messages, _ = serving_chat._make_request_with_harmony(req)
1093 verify_harmony_messages(
1094 input_messages,
1095 [
@@ -1133,7 +1133,7 @@ class TestServingChatWithHarmony:
1133
1134 # Test the Harmony messages for the second turn's input
1135 req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1136 - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
1136 + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1137 verify_harmony_messages(
1138 input_messages_2,
1139 [
@@ -1183,7 +1183,7 @@ class TestServingChatWithHarmony:
1183
1184 # Test the Harmony messages for the third turn's input
1185 req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1186 - input_messages_3, _, _ = serving_chat._make_request_with_harmony(req_3)
1186 + input_messages_3, _ = serving_chat._make_request_with_harmony(req_3)
1187 verify_harmony_messages(
1188 input_messages_3,
1189 [
@@ -1246,7 +1246,7 @@ class TestServingChatWithHarmony:
1246
1247 # Test the Harmony messages for the fourth turn's input
1248 req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1249 - input_messages_4, _, _ = serving_chat._make_request_with_harmony(req_4)
1249 + input_messages_4, _ = serving_chat._make_request_with_harmony(req_4)
1250 verify_harmony_messages(
1251 input_messages_4,
1252 [
@@ -1295,7 +1295,7 @@ class TestServingChatWithHarmony:
1295 },
1296 ]
1297 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1298 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1298 + input_messages, _ = serving_chat._make_request_with_harmony(req)
1299
1300 verify_harmony_messages(
1301 input_messages,
@@ -1327,7 +1327,7 @@ class TestServingChatWithHarmony:
1327 },
1328 ]
1329 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1330 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1330 + input_messages, _ = serving_chat._make_request_with_harmony(req)
1331
1332 verify_harmony_messages(
1333 input_messages,
@@ -1357,7 +1357,7 @@ class TestServingChatWithHarmony:
1357 },
1358 ]
1359 req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1360 - input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1360 + input_messages, _ = serving_chat._make_request_with_harmony(req)
1361
1362 verify_harmony_messages(
1363 input_messages,
tests/entrypoints/openai/test_serving_responses.py
+3 -3
@@ -21,7 +21,7 @@ from vllm.entrypoints.openai.serving_responses import (
21 extract_tool_types,
22 )
23 from vllm.entrypoints.tool_server import ToolServer
24 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
24 +from vllm.inputs.data import TokensPrompt
25
26
27 class MockConversationContext(ConversationContext):
@@ -237,7 +237,7 @@ class TestValidateGeneratorInput:
237 """Test _validate_generator_input with valid prompt length"""
238 # Create an engine prompt with valid length (less than max_model_len)
239 valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len
240 - engine_prompt = EngineTokensPrompt(prompt_token_ids=valid_prompt_token_ids)
240 + engine_prompt = TokensPrompt(prompt_token_ids=valid_prompt_token_ids)
241
242 # Call the method
243 result = serving_responses_instance._validate_generator_input(engine_prompt)
@@ -247,7 +247,7 @@ class TestValidateGeneratorInput:
247
248 # create an invalid engine prompt
249 invalid_prompt_token_ids = list(range(200)) # 100 tokens >= 100 max_model_len
250 - engine_prompt = EngineTokensPrompt(prompt_token_ids=invalid_prompt_token_ids)
250 + engine_prompt = TokensPrompt(prompt_token_ids=invalid_prompt_token_ids)
251
252 # Call the method
253 result = serving_responses_instance._validate_generator_input(engine_prompt)
vllm/entrypoints/openai/serving_chat.py
+36 -19
@@ -61,7 +61,7 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser
61 from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
62 from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
63 from vllm.entrypoints.utils import get_max_tokens, should_include_usage
64 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
64 +from vllm.inputs.data import TokensPrompt
65 from vllm.logger import init_logger
66 from vllm.logprobs import Logprob
67 from vllm.outputs import CompletionOutput, RequestOutput
@@ -234,11 +234,7 @@ class OpenAIServingChat(OpenAIServing):
234 )
235 if error_check_ret is not None:
236 return error_check_ret
237 - (
238 - conversation,
239 - request_prompts,
240 - engine_prompts,
241 - ) = await self._preprocess_chat(
237 + conversation, engine_prompts = await self._preprocess_chat(
238 request,
239 tokenizer,
240 request.messages,
@@ -254,11 +250,7 @@ class OpenAIServingChat(OpenAIServing):
250 )
251 else:
252 # For GPT-OSS.
257 - (
258 - conversation,
259 - request_prompts,
260 - engine_prompts,
261 - ) = self._make_request_with_harmony(request)
253 + conversation, engine_prompts = self._make_request_with_harmony(request)
254 except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
255 logger.exception("Error in preprocessing prompt inputs")
256 return self.create_error_response(f"{e} {e.__cause__}")
@@ -278,7 +270,7 @@ class OpenAIServingChat(OpenAIServing):
270 generators: list[AsyncGenerator[RequestOutput, None]] = []
271 try:
272 for i, engine_prompt in enumerate(engine_prompts):
281 - prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
273 + prompt_text, _, _ = self._get_prompt_components(engine_prompt)
274 # If we are creating sub requests for multiple prompts, ensure that they
275 # have unique request ids.
276 sub_request_id = (
@@ -313,7 +305,7 @@ class OpenAIServingChat(OpenAIServing):
305
306 self._log_inputs(
307 sub_request_id,
316 - request_prompts[i],
308 + engine_prompt,
309 params=sampling_params,
310 lora_request=lora_request,
311 )
@@ -537,7 +529,7 @@ class OpenAIServingChat(OpenAIServing):
529 request_id: str,
530 model_name: str,
531 conversation: list[ConversationMessage],
540 - tokenizer: TokenizerLike,
532 + tokenizer: TokenizerLike | None,
533 request_metadata: RequestResponseMetadata,
534 ) -> AsyncGenerator[str, None]:
535 created_time = int(time.time())
@@ -591,6 +583,11 @@ class OpenAIServingChat(OpenAIServing):
583
584 try:
585 if self.reasoning_parser:
586 + if tokenizer is None:
587 + raise ValueError(
588 + "Tokenizer not available when `skip_tokenizer_init=True`"
589 + )
590 +
591 reasoning_parser = self.reasoning_parser(
592 tokenizer,
593 chat_template_kwargs=request.chat_template_kwargs, # type: ignore
@@ -604,6 +601,11 @@ class OpenAIServingChat(OpenAIServing):
601 # Prepare the tool parser if it's needed
602 try:
603 if tool_choice_auto and self.tool_parser:
604 + if tokenizer is None:
605 + raise ValueError(
606 + "Tokenizer not available when `skip_tokenizer_init=True`"
607 + )
608 +
609 tool_parsers: list[ToolParser | None] = [
610 self.tool_parser(tokenizer)
611 ] * num_choices
@@ -1317,7 +1319,7 @@ class OpenAIServingChat(OpenAIServing):
1319 request_id: str,
1320 model_name: str,
1321 conversation: list[ConversationMessage],
1320 - tokenizer: TokenizerLike,
1322 + tokenizer: TokenizerLike | None,
1323 request_metadata: RequestResponseMetadata,
1324 ) -> ErrorResponse | ChatCompletionResponse:
1325 created_time = int(time.time())
@@ -1367,6 +1369,11 @@ class OpenAIServingChat(OpenAIServing):
1369 reasoning = None
1370
1371 if self.tool_parser is not None:
1372 + if tokenizer is None:
1373 + raise ValueError(
1374 + "Tokenizer not available when `skip_tokenizer_init=True`"
1375 + )
1376 +
1377 tool_parser = self.tool_parser(tokenizer)
1378 # NOTE: We use token_ids for openai tool parser
1379 tool_call_info = tool_parser.extract_tool_calls(
@@ -1409,6 +1416,11 @@ class OpenAIServingChat(OpenAIServing):
1416
1417 if self.reasoning_parser:
1418 try:
1419 + if tokenizer is None:
1420 + raise ValueError(
1421 + "Tokenizer not available when `skip_tokenizer_init=True`"
1422 + )
1423 +
1424 reasoning_parser = self.reasoning_parser(
1425 tokenizer,
1426 chat_template_kwargs=request.chat_template_kwargs, # type: ignore
@@ -1648,7 +1660,7 @@ class OpenAIServingChat(OpenAIServing):
1660 self,
1661 logprobs: dict[int, Logprob],
1662 top_logprobs: int | None,
1651 - tokenizer: TokenizerLike,
1663 + tokenizer: TokenizerLike | None,
1664 should_return_as_token_id: bool,
1665 ) -> list[ChatCompletionLogProb]:
1666 return [
@@ -1672,7 +1684,7 @@ class OpenAIServingChat(OpenAIServing):
1684 self,
1685 token_ids: GenericSequence[int],
1686 top_logprobs: GenericSequence[dict[int, Logprob] | None],
1675 - tokenizer: TokenizerLike,
1687 + tokenizer: TokenizerLike | None,
1688 num_output_top_logprobs: int | None = None,
1689 return_as_token_id: bool | None = None,
1690 ) -> ChatCompletionLogProbs:
@@ -1690,6 +1702,11 @@ class OpenAIServingChat(OpenAIServing):
1702 if should_return_as_token_id:
1703 token = f"token_id:{token_id}"
1704 else:
1705 + if tokenizer is None:
1706 + raise ValueError(
1707 + "Tokenizer not available when `skip_tokenizer_init=True`"
1708 + )
1709 +
1710 token = tokenizer.decode(token_id)
1711
1712 logprobs_content.append(
@@ -1800,10 +1817,10 @@ class OpenAIServingChat(OpenAIServing):
1817
1818 # Render prompt token ids.
1819 prompt_token_ids = render_for_completion(messages)
1803 - engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1820 + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1821
1822 # Add cache_salt if provided in the request
1823 if request.cache_salt is not None:
1824 engine_prompt["cache_salt"] = request.cache_salt
1825
1809 - return messages, [prompt_token_ids], [engine_prompt]
1826 + return messages, [engine_prompt]
vllm/entrypoints/openai/serving_engine.py
+72 -125
@@ -5,60 +5,19 @@ import json
5 import sys
6 import time
7 import traceback
8 -from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
8 +from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
9 from concurrent.futures import ThreadPoolExecutor
10 from dataclasses import dataclass, field
11 from http import HTTPStatus
12 from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
13
14 import numpy as np
15 -import torch
15 from fastapi import Request
17 -from pydantic import ConfigDict, TypeAdapter
18 -from starlette.datastructures import Headers
19 -from typing_extensions import TypeIs
20 -
21 -from vllm.entrypoints.context import (
22 - HarmonyContext,
23 - ParsableContext,
24 - StreamingHarmonyContext,
25 -)
26 -from vllm.entrypoints.openai.protocol import (
27 - FunctionCall,
28 - ResponseInputOutputItem,
29 - ResponsesRequest,
30 -)
31 -from vllm.entrypoints.pooling.classify.protocol import (
32 - ClassificationChatRequest,
33 - ClassificationCompletionRequest,
34 - ClassificationRequest,
35 - ClassificationResponse,
36 -)
37 -from vllm.entrypoints.pooling.embed.protocol import (
38 - EmbeddingChatRequest,
39 - EmbeddingCompletionRequest,
40 - EmbeddingRequest,
41 - EmbeddingResponse,
42 -)
43 -from vllm.entrypoints.pooling.pooling.protocol import (
44 - IOProcessorRequest,
45 - PoolingResponse,
46 -)
47 -from vllm.entrypoints.pooling.score.protocol import (
48 - RerankRequest,
49 - ScoreRequest,
50 - ScoreResponse,
51 -)
52 -from vllm.transformers_utils.tokenizer import AnyTokenizer
53 -
54 -if sys.version_info >= (3, 12):
55 - from typing import TypedDict
56 -else:
57 - from typing_extensions import TypedDict
58 -
16 from openai.types.responses import (
17 ToolChoiceFunction,
18 )
19 +from pydantic import ConfigDict, TypeAdapter
20 +from starlette.datastructures import Headers
21
22 import vllm.envs as envs
23 from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
@@ -72,7 +31,12 @@ from vllm.entrypoints.chat_utils import (
31 parse_chat_messages_futures,
32 resolve_chat_template_content_format,
33 )
75 -from vllm.entrypoints.context import ConversationContext
34 +from vllm.entrypoints.context import (
35 + ConversationContext,
36 + HarmonyContext,
37 + ParsableContext,
38 + StreamingHarmonyContext,
39 +)
40 from vllm.entrypoints.logger import RequestLogger
41 from vllm.entrypoints.openai.protocol import (
42 ChatCompletionNamedToolChoiceParam,
@@ -83,7 +47,10 @@ from vllm.entrypoints.openai.protocol import (
47 DetokenizeRequest,
48 ErrorInfo,
49 ErrorResponse,
50 + FunctionCall,
51 FunctionDefinition,
52 + ResponseInputOutputItem,
53 + ResponsesRequest,
54 TokenizeChatRequest,
55 TokenizeCompletionRequest,
56 TokenizeResponse,
@@ -93,14 +60,34 @@ from vllm.entrypoints.openai.protocol import (
60 )
61 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
62 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
63 +from vllm.entrypoints.pooling.classify.protocol import (
64 + ClassificationChatRequest,
65 + ClassificationCompletionRequest,
66 + ClassificationRequest,
67 + ClassificationResponse,
68 +)
69 +from vllm.entrypoints.pooling.embed.protocol import (
70 + EmbeddingChatRequest,
71 + EmbeddingCompletionRequest,
72 + EmbeddingRequest,
73 + EmbeddingResponse,
74 +)
75 +from vllm.entrypoints.pooling.pooling.protocol import (
76 + IOProcessorRequest,
77 + PoolingResponse,
78 +)
79 +from vllm.entrypoints.pooling.score.protocol import (
80 + RerankRequest,
81 + ScoreRequest,
82 + ScoreResponse,
83 +)
84 from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
85 from vllm.entrypoints.responses_utils import (
86 construct_input_messages,
87 )
88 from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
89 from vllm.entrypoints.utils import _validate_truncation_size
102 -from vllm.inputs.data import PromptType
103 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
90 +from vllm.inputs.data import PromptType, TokensPrompt
91 from vllm.inputs.parse import (
92 PromptComponents,
93 get_prompt_components,
@@ -109,10 +96,7 @@ from vllm.inputs.parse import (
96 from vllm.logger import init_logger
97 from vllm.logprobs import Logprob, PromptLogprobs
98 from vllm.lora.request import LoRARequest
112 -from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin
113 - MultiModalDataDict,
114 - MultiModalUUIDDict,
115 -)
99 +from vllm.multimodal import MultiModalDataDict
100 from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
101 from vllm.pooling_params import PoolingParams
102 from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -185,34 +169,6 @@ AnyResponse: TypeAlias = (
169 )
170
171
188 -class TextTokensPrompt(TypedDict):
189 - prompt: str
190 - prompt_token_ids: list[int]
191 -
192 -
193 -class EmbedsPrompt(TypedDict):
194 - prompt_embeds: torch.Tensor
195 -
196 -
197 -RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt
198 -
199 -
200 -def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]:
201 - return (
202 - isinstance(prompt, dict)
203 - and "prompt_token_ids" in prompt
204 - and "prompt_embeds" not in prompt
205 - )
206 -
207 -
208 -def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:
209 - return (
210 - isinstance(prompt, dict)
211 - and "prompt_token_ids" not in prompt
212 - and "prompt_embeds" in prompt
213 - )
214 -
215 -
172 RequestT = TypeVar("RequestT", bound=AnyRequest)
173
174
@@ -223,8 +179,7 @@ class RequestProcessingMixin:
179 handling prompt preparation and engine input.
180 """
181
226 - request_prompts: Sequence[RequestPrompt] | None = field(default_factory=list)
227 - engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list)
182 + engine_prompts: list[TokensPrompt] | None = field(default_factory=list)
183
184
185 @dataclass(kw_only=True)
@@ -425,7 +380,7 @@ class OpenAIServing:
380 prompts_batch, lora_req_batch = zip(
381 *[
382 (
428 - EngineTokensPrompt(
383 + TokensPrompt(
384 prompt_token_ids=beam.tokens,
385 multi_modal_data=beam.multi_modal_data,
386 mm_processor_kwargs=beam.mm_processor_kwargs,
@@ -947,7 +902,7 @@ class OpenAIServing:
902 prompt: str,
903 tokenizer: TokenizerLike,
904 add_special_tokens: bool,
950 - ) -> TextTokensPrompt:
905 + ) -> TokensPrompt:
906 async_tokenizer = self._get_async_tokenizer(tokenizer)
907
908 if (
@@ -988,7 +943,7 @@ class OpenAIServing:
943 request: AnyRequest,
944 prompt_ids: list[int],
945 tokenizer: TokenizerLike | None,
991 - ) -> TextTokensPrompt:
946 + ) -> TokensPrompt:
947 truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
948
949 if truncate_prompt_tokens is None:
@@ -1011,7 +966,7 @@ class OpenAIServing:
966 request: AnyRequest,
967 input_ids: list[int],
968 input_text: str,
1014 - ) -> TextTokensPrompt:
969 + ) -> TokensPrompt:
970 token_num = len(input_ids)
971
972 # Note: EmbeddingRequest, ClassificationRequest,
@@ -1042,7 +997,7 @@ class OpenAIServing:
997 f"{token_num} tokens in the input for {operation}. "
998 f"Please reduce the length of the input."
999 )
1045 - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
1000 + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
1001
1002 # Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
1003 # and does not require model context length validation
@@ -1050,7 +1005,7 @@ class OpenAIServing:
1005 request,
1006 (TokenizeCompletionRequest, TokenizeChatRequest, DetokenizeRequest),
1007 ):
1053 - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
1008 + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
1009
1010 # chat completion endpoint supports max_completion_tokens
1011 if isinstance(request, ChatCompletionRequest):
@@ -1078,7 +1033,7 @@ class OpenAIServing:
1033 f" - {token_num})."
1034 )
1035
1081 - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
1036 + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
1037
1038 async def _tokenize_prompt_input_async(
1039 self,
@@ -1086,7 +1041,7 @@ class OpenAIServing:
1041 tokenizer: TokenizerLike,
1042 prompt_input: str | list[int],
1043 add_special_tokens: bool = True,
1089 - ) -> TextTokensPrompt:
1044 + ) -> TokensPrompt:
1045 """
1046 A simpler implementation that tokenizes a single prompt input.
1047 """
@@ -1105,7 +1060,7 @@ class OpenAIServing:
1060 tokenizer: TokenizerLike,
1061 prompt_inputs: Iterable[str | list[int]],
1062 add_special_tokens: bool = True,
1108 - ) -> AsyncGenerator[TextTokensPrompt, None]:
1063 + ) -> AsyncGenerator[TokensPrompt, None]:
1064 """
1065 A simpler implementation that tokenizes multiple prompt inputs.
1066 """
@@ -1158,11 +1113,7 @@ class OpenAIServing:
1113 chat_template_kwargs: dict[str, Any] | None = None,
1114 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
1115 add_special_tokens: bool = False,
1161 - ) -> tuple[
1162 - list[ConversationMessage],
1163 - Sequence[RequestPrompt],
1164 - list[EngineTokensPrompt],
1165 - ]:
1116 + ) -> tuple[list[ConversationMessage], list[TokensPrompt]]:
1117 model_config = self.model_config
1118
1119 resolved_content_format = resolve_chat_template_content_format(
@@ -1235,9 +1186,7 @@ class OpenAIServing:
1186 "Prompt has to be a string",
1187 "when the tokenizer is not initialised",
1188 )
1238 - prompt_inputs = TextTokensPrompt(
1239 - prompt=request_prompt, prompt_token_ids=[1]
1240 - )
1189 + prompt_inputs = TokensPrompt(prompt=request_prompt, prompt_token_ids=[1])
1190 elif isinstance(request_prompt, str):
1191 prompt_inputs = await self._tokenize_prompt_input_async(
1192 request,
@@ -1250,14 +1199,15 @@ class OpenAIServing:
1199 assert is_list_of(request_prompt, int), (
1200 "Prompt has to be either a string or a list of token ids"
1201 )
1253 - prompt_inputs = TextTokensPrompt(
1202 + prompt_inputs = TokensPrompt(
1203 prompt=tokenizer.decode(request_prompt),
1204 prompt_token_ids=request_prompt,
1205 )
1206
1258 - engine_prompt = EngineTokensPrompt(
1259 - prompt_token_ids=prompt_inputs["prompt_token_ids"]
1260 - )
1207 + engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["prompt_token_ids"])
1208 + if "prompt" in prompt_inputs:
1209 + engine_prompt["prompt"] = prompt_inputs["prompt"]
1210 +
1211 if mm_data is not None:
1212 engine_prompt["multi_modal_data"] = mm_data
1213
@@ -1270,7 +1220,7 @@ class OpenAIServing:
1220 if hasattr(request, "cache_salt") and request.cache_salt is not None:
1221 engine_prompt["cache_salt"] = request.cache_salt
1222
1273 - return conversation, [request_prompt], [engine_prompt]
1223 + return conversation, [engine_prompt]
1224
1225 async def _process_inputs(
1226 self,
@@ -1302,7 +1252,7 @@ class OpenAIServing:
1252 async def _render_next_turn(
1253 self,
1254 request: ResponsesRequest,
1305 - tokenizer: AnyTokenizer,
1255 + tokenizer: TokenizerLike | None,
1256 messages: list[ResponseInputOutputItem],
1257 tool_dicts: list[dict[str, Any]] | None,
1258 tool_parser,
@@ -1313,7 +1263,7 @@ class OpenAIServing:
1263 request_input=messages,
1264 )
1265
1316 - _, request_prompts, engine_prompts = await self._preprocess_chat(
1266 + _, engine_prompts = await self._preprocess_chat(
1267 request,
1268 tokenizer,
1269 new_messages,
@@ -1322,20 +1272,20 @@ class OpenAIServing:
1272 chat_template=chat_template,
1273 chat_template_content_format=chat_template_content_format,
1274 )
1325 - return request_prompts, engine_prompts
1275 + return engine_prompts
1276
1277 async def _generate_with_builtin_tools(
1278 self,
1279 request_id: str,
1330 - request_prompt: RequestPrompt,
1331 - engine_prompt: EngineTokensPrompt,
1280 + engine_prompt: TokensPrompt,
1281 sampling_params: SamplingParams,
1282 context: ConversationContext,
1283 lora_request: LoRARequest | None = None,
1284 priority: int = 0,
1285 **kwargs,
1286 ):
1338 - prompt_text, _, _ = self._get_prompt_components(request_prompt)
1287 + prompt_text, _, _ = self._get_prompt_components(engine_prompt)
1288 +
1289 orig_priority = priority
1290 sub_request = 0
1291 while True:
@@ -1343,7 +1293,7 @@ class OpenAIServing:
1293 sub_request_id = f"{request_id}_{sub_request}"
1294 self._log_inputs(
1295 sub_request_id,
1346 - request_prompt,
1296 + engine_prompt,
1297 params=sampling_params,
1298 lora_request=lora_request,
1299 )
@@ -1388,10 +1338,9 @@ class OpenAIServing:
1338 # Render the next prompt token ids.
1339 if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
1340 prompt_token_ids = context.render_for_completion()
1391 - engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1392 - request_prompt = prompt_token_ids
1341 + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1342 elif isinstance(context, ParsableContext):
1394 - request_prompts, engine_prompts = await self._render_next_turn(
1343 + engine_prompts = await self._render_next_turn(
1344 context.request,
1345 context.tokenizer,
1346 context.parser.response_messages,
@@ -1401,8 +1350,7 @@ class OpenAIServing:
1350 context.chat_template_content_format,
1351 )
1352 engine_prompt = engine_prompts[0]
1404 - request_prompt = request_prompts[0]
1405 - prompt_text, _, _ = self._get_prompt_components(request_prompt)
1353 + prompt_text, _, _ = self._get_prompt_components(engine_prompt)
1354
1355 # Update the sampling params.
1356 sampling_params.max_tokens = self.max_model_len - len(
@@ -1412,19 +1360,13 @@ class OpenAIServing:
1360 priority = orig_priority - 1
1361 sub_request += 1
1362
1415 - def _get_prompt_components(
1416 - self,
1417 - prompt: RequestPrompt | PromptType,
1418 - ) -> PromptComponents:
1419 - if isinstance(prompt, list):
1420 - return PromptComponents(token_ids=prompt)
1421 -
1422 - return get_prompt_components(prompt) # type: ignore[arg-type]
1363 + def _get_prompt_components(self, prompt: PromptType) -> PromptComponents:
1364 + return get_prompt_components(prompt)
1365
1366 def _log_inputs(
1367 self,
1368 request_id: str,
1427 - inputs: RequestPrompt | PromptType,
1369 + inputs: PromptType,
1370 params: SamplingParams | PoolingParams | BeamSearchParams | None,
1371 lora_request: LoRARequest | None,
1372 ) -> None:
@@ -1486,7 +1428,7 @@ class OpenAIServing:
1428 @staticmethod
1429 def _parse_tool_calls_from_content(
1430 request: ResponsesRequest | ChatCompletionRequest,
1489 - tokenizer: TokenizerLike,
1431 + tokenizer: TokenizerLike | None,
1432 enable_auto_tools: bool,
1433 tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
1434 content: str | None = None,
@@ -1526,6 +1468,11 @@ class OpenAIServing:
1468 and enable_auto_tools
1469 and (request.tool_choice == "auto" or request.tool_choice is None)
1470 ):
1471 + if tokenizer is None:
1472 + raise ValueError(
1473 + "Tokenizer not available when `skip_tokenizer_init=True`"
1474 + )
1475 +
1476 # Automatic Tool Call Parsing
1477 try:
1478 tool_parser = tool_parser_cls(tokenizer)
vllm/entrypoints/openai/serving_responses.py
+10 -11
@@ -107,7 +107,7 @@ from vllm.entrypoints.responses_utils import (
107 make_response_output_items_from_parsable_context,
108 )
109 from vllm.entrypoints.tool_server import ToolServer
110 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
110 +from vllm.inputs.data import TokensPrompt
111 from vllm.logger import init_logger
112 from vllm.logprobs import Logprob as SampleLogprob
113 from vllm.logprobs import SampleLogprobs
@@ -258,7 +258,7 @@ class OpenAIServingResponses(OpenAIServing):
258 self.tool_server = tool_server
259
260 def _validate_generator_input(
261 - self, engine_prompt: EngineTokensPrompt
261 + self, engine_prompt: TokensPrompt
262 ) -> ErrorResponse | None:
263 """Add validations to the input to the generator here."""
264 if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
@@ -353,11 +353,11 @@ class OpenAIServingResponses(OpenAIServing):
353 tokenizer = await self.engine_client.get_tokenizer()
354
355 if self.use_harmony:
356 - messages, request_prompts, engine_prompts = (
357 - self._make_request_with_harmony(request, prev_response)
356 + messages, engine_prompts = self._make_request_with_harmony(
357 + request, prev_response
358 )
359 else:
360 - messages, request_prompts, engine_prompts = await self._make_request(
360 + messages, engine_prompts = await self._make_request(
361 request, prev_response, tokenizer
362 )
363
@@ -393,7 +393,7 @@ class OpenAIServingResponses(OpenAIServing):
393 assert len(builtin_tool_list) == 0
394 available_tools = []
395 try:
396 - for i, engine_prompt in enumerate(engine_prompts):
396 + for engine_prompt in engine_prompts:
397 maybe_error = self._validate_generator_input(engine_prompt)
398 if maybe_error is not None:
399 return maybe_error
@@ -449,7 +449,6 @@ class OpenAIServingResponses(OpenAIServing):
449 )
450 generator = self._generate_with_builtin_tools(
451 request_id=request.request_id,
452 - request_prompt=request_prompts[i],
452 engine_prompt=engine_prompt,
453 sampling_params=sampling_params,
454 context=context,
@@ -564,7 +563,7 @@ class OpenAIServingResponses(OpenAIServing):
563 prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
564 prev_response_output=prev_response.output if prev_response else None,
565 )
567 - _, request_prompts, engine_prompts = await self._preprocess_chat(
566 + _, engine_prompts = await self._preprocess_chat(
567 request,
568 tokenizer,
569 messages,
@@ -573,7 +572,7 @@ class OpenAIServingResponses(OpenAIServing):
572 chat_template=self.chat_template,
573 chat_template_content_format=self.chat_template_content_format,
574 )
576 - return messages, request_prompts, engine_prompts
575 + return messages, engine_prompts
576
577 def _make_request_with_harmony(
578 self,
@@ -586,13 +585,13 @@ class OpenAIServingResponses(OpenAIServing):
585 )
586 messages = self._construct_input_messages_with_harmony(request, prev_response)
587 prompt_token_ids = render_for_completion(messages)
589 - engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
588 + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
589
590 # Add cache_salt if provided in the request
591 if request.cache_salt is not None:
592 engine_prompt["cache_salt"] = request.cache_salt
593
595 - return messages, [prompt_token_ids], [engine_prompt]
594 + return messages, [engine_prompt]
595
596 async def _initialize_tool_sessions(
597 self,
vllm/entrypoints/pooling/classify/serving.py
+1 -5
@@ -72,11 +72,7 @@ class ClassificationMixin(OpenAIServing):
72 if ret:
73 return ret
74
75 - (
76 - _,
77 - _,
78 - engine_prompts,
79 - ) = await self._preprocess_chat(
75 + _, engine_prompts = await self._preprocess_chat(
76 cast(ChatCompletionRequest, chat_request),
77 ctx.tokenizer,
78 messages,
vllm/entrypoints/pooling/embed/serving.py
+20 -39
@@ -20,7 +20,6 @@ from vllm.entrypoints.openai.serving_engine import (
20 EmbeddingServeContext,
21 OpenAIServing,
22 ServeContext,
23 - TextTokensPrompt,
23 )
24 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
25 from vllm.entrypoints.pooling.embed.protocol import (
@@ -32,7 +31,7 @@ from vllm.entrypoints.pooling.embed.protocol import (
31 EmbeddingResponseData,
32 )
33 from vllm.entrypoints.renderer import RenderConfig
35 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
34 +from vllm.inputs.data import TokensPrompt
35 from vllm.logger import init_logger
36 from vllm.outputs import (
37 EmbeddingRequestOutput,
@@ -83,11 +82,7 @@ class EmbeddingMixin(OpenAIServing):
82 renderer = self._get_renderer(tokenizer)
83
84 if isinstance(ctx.request, EmbeddingChatRequest):
86 - (
87 - _,
88 - _,
89 - ctx.engine_prompts,
90 - ) = await self._preprocess_chat(
85 + _, ctx.engine_prompts = await self._preprocess_chat(
86 ctx.request,
87 tokenizer,
88 ctx.request.messages,
@@ -209,14 +204,13 @@ class EmbeddingMixin(OpenAIServing):
204 async def _process_chunked_request(
205 self,
206 ctx: EmbeddingServeContext,
212 - original_prompt: TextTokensPrompt,
207 + token_ids: list[int],
208 pooling_params,
209 trace_headers,
210 prompt_idx: int,
211 ) -> list[AsyncGenerator[PoolingRequestOutput, None]]:
212 """Process a single prompt using chunked processing."""
213 generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
219 - token_ids = original_prompt["prompt_token_ids"]
214
215 # Split into chunks using max_position_embeddings
216 max_pos_embeddings = self._get_max_position_embeddings()
@@ -228,18 +222,12 @@ class EmbeddingMixin(OpenAIServing):
222 chunk_request_id = f"{ctx.request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
223
224 # Create engine prompt for this chunk
231 - chunk_engine_prompt = EngineTokensPrompt(prompt_token_ids=chunk_tokens)
232 -
233 - # Create chunk request prompt for logging
234 - chunk_text = ""
235 - chunk_request_prompt = TextTokensPrompt(
236 - prompt=chunk_text, prompt_token_ids=chunk_tokens
237 - )
225 + chunk_engine_prompt = TokensPrompt(prompt_token_ids=chunk_tokens)
226
227 # Log the chunk
228 self._log_inputs(
229 chunk_request_id,
242 - chunk_request_prompt,
230 + chunk_engine_prompt,
231 params=pooling_params,
232 lora_request=ctx.lora_request,
233 )
@@ -263,7 +251,7 @@ class EmbeddingMixin(OpenAIServing):
251 request,
252 input_ids: list[int],
253 input_text: str,
266 - ) -> TextTokensPrompt:
254 + ) -> TokensPrompt:
255 """Override to support chunked processing for embedding requests."""
256 token_num = len(input_ids)
257
@@ -328,23 +316,15 @@ class EmbeddingMixin(OpenAIServing):
316 )
317 )
318
331 - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
319 + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
320
321 # For other request types, use the parent's implementation
322 return super()._validate_input(request, input_ids, input_text)
323
336 - def _is_text_tokens_prompt(self, prompt) -> bool:
337 - """Check if a prompt is a TextTokensPrompt (has prompt_token_ids)."""
338 - return (
339 - isinstance(prompt, dict)
340 - and "prompt_token_ids" in prompt
341 - and "prompt_embeds" not in prompt
342 - )
343 -
324 async def _create_single_prompt_generator(
325 self,
326 ctx: EmbeddingServeContext,
347 - engine_prompt: EngineTokensPrompt,
327 + engine_prompt: TokensPrompt,
328 pooling_params: PoolingParams,
329 trace_headers: Mapping[str, str] | None,
330 prompt_index: int,
@@ -413,14 +393,16 @@ class EmbeddingMixin(OpenAIServing):
393
394 for i, engine_prompt in enumerate(ctx.engine_prompts):
395 # Check if this specific prompt needs chunked processing
416 - if self._is_text_tokens_prompt(engine_prompt):
417 - # Cast to TextTokensPrompt since we've verified
418 - # prompt_token_ids
419 - text_tokens_prompt = cast(TextTokensPrompt, engine_prompt)
420 - if len(text_tokens_prompt["prompt_token_ids"]) > max_pos_embeddings:
396 + if "prompt_token_ids" in engine_prompt:
397 + prompt_token_ids = engine_prompt["prompt_token_ids"]
398 + if len(prompt_token_ids) > max_pos_embeddings:
399 # Use chunked processing for this prompt
400 chunk_generators = await self._process_chunked_request(
423 - ctx, text_tokens_prompt, pooling_params, trace_headers, i
401 + ctx,
402 + prompt_token_ids,
403 + pooling_params,
404 + trace_headers,
405 + i,
406 )
407 generators.extend(chunk_generators)
408 continue
@@ -578,14 +560,13 @@ class EmbeddingMixin(OpenAIServing):
560
561 # Get original prompt token IDs for this prompt
562 original_prompt = ctx.engine_prompts[prompt_idx]
581 - if not self._is_text_tokens_prompt(original_prompt):
563 + if "prompt_token_ids" not in original_prompt:
564 return self.create_error_response(
583 - f"Chunked prompt {prompt_idx} is not a TextTokensPrompt"
565 + f"Chunked prompt {prompt_idx} does not contain "
566 + "token IDs"
567 )
568
586 - original_token_ids = cast(TextTokensPrompt, original_prompt)[
587 - "prompt_token_ids"
588 - ]
569 + original_token_ids = original_prompt["prompt_token_ids"]
570
571 pooling_request_output = PoolingRequestOutput(
572 request_id=aggregator["request_id"],
vllm/entrypoints/pooling/pooling/serving.py
+2 -5
@@ -137,11 +137,8 @@ class OpenAIServingPooling(OpenAIServing):
137 )
138 if error_check_ret is not None:
139 return error_check_ret
140 - (
141 - _,
142 - _,
143 - engine_prompts,
144 - ) = await self._preprocess_chat(
140 +
141 + _, engine_prompts = await self._preprocess_chat(
142 request,
143 tokenizer,
144 request.messages,
vllm/entrypoints/renderer.py
+18 -20
@@ -12,9 +12,7 @@ import torch
12 from pydantic import Field
13
14 from vllm.config import ModelConfig
15 -from vllm.inputs.data import EmbedsPrompt as EngineEmbedsPrompt
16 -from vllm.inputs.data import TextPrompt as EngineTextPrompt
17 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
15 +from vllm.inputs.data import EmbedsPrompt, TextPrompt, TokensPrompt
16 from vllm.inputs.parse import get_prompt_components, parse_raw_prompts
17 from vllm.tokenizers import TokenizerLike
18 from vllm.utils.async_utils import AsyncMicrobatchTokenizer
@@ -97,7 +95,7 @@ class BaseRenderer(ABC):
95 *,
96 prompt_or_prompts: str | list[str] | list[int] | list[list[int]],
97 config: RenderConfig,
100 - ) -> list[EngineTokensPrompt]:
98 + ) -> list[TokensPrompt]:
99 """
100 Convert text or token inputs into engine-ready TokensPrompt objects.
101
@@ -115,7 +113,7 @@ class BaseRenderer(ABC):
113 (e.g., tokenization and length handling).
114
115 Returns:
118 - list[EngineTokensPrompt]: Engine-ready token prompts.
116 + list[TokensPrompt]: Engine-ready token prompts.
117
118 Raises:
119 ValueError: If input formats are invalid or length limits exceeded.
@@ -129,7 +127,7 @@ class BaseRenderer(ABC):
127 prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,
128 prompt_embeds: bytes | list[bytes] | None = None,
129 config: RenderConfig,
132 - ) -> list[EngineTokensPrompt | EngineEmbedsPrompt]:
130 + ) -> list[TokensPrompt | EmbedsPrompt]:
131 """
132 Convert text/token and/or base64-encoded embeddings inputs into
133 engine-ready prompt objects using a unified RenderConfig.
@@ -146,7 +144,7 @@ class BaseRenderer(ABC):
144 (e.g., tokenization and length handling).
145
146 Returns:
149 - list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
147 + list[Union[TokensPrompt, EmbedsPrompt]]:
148 Engine-ready prompt objects.
149
150 Raises:
@@ -161,14 +159,14 @@ class BaseRenderer(ABC):
159 prompt_embeds: bytes | list[bytes],
160 truncate_prompt_tokens: Annotated[int, Field(ge=0)] | None = None,
161 cache_salt: str | None = None,
164 - ) -> list[EngineEmbedsPrompt]:
162 + ) -> list[EmbedsPrompt]:
163 """Load and validate base64-encoded embeddings into prompt objects."""
164 if not self.model_config.enable_prompt_embeds:
165 raise ValueError(
166 "You must set `--enable-prompt-embeds` to input `prompt_embeds`."
167 )
168
171 - def _load_and_validate_embed(embed: bytes) -> EngineEmbedsPrompt:
169 + def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:
170 tensor = torch.load(
171 io.BytesIO(pybase64.b64decode(embed, validate=True)),
172 weights_only=True,
@@ -185,7 +183,7 @@ class BaseRenderer(ABC):
183 assert tensor.dim() == 2
184 if truncate_prompt_tokens is not None:
185 tensor = tensor[-truncate_prompt_tokens:]
188 - embeds_prompt = EngineEmbedsPrompt(prompt_embeds=tensor)
186 + embeds_prompt = EmbedsPrompt(prompt_embeds=tensor)
187 if cache_salt is not None:
188 embeds_prompt["cache_salt"] = cache_salt
189 return embeds_prompt
@@ -213,7 +211,7 @@ class CompletionRenderer(BaseRenderer):
211 *,
212 prompt_or_prompts: str | list[str] | list[int] | list[list[int]],
213 config: RenderConfig,
216 - ) -> list[EngineTokensPrompt]:
214 + ) -> list[TokensPrompt]:
215 """Implementation of prompt rendering for completion-style requests.
216
217 Uses async tokenizer pooling for improved performance. See base class
@@ -240,7 +238,7 @@ class CompletionRenderer(BaseRenderer):
238 prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,
239 prompt_embeds: bytes | list[bytes] | None = None,
240 config: RenderConfig,
243 - ) -> list[EngineTokensPrompt | EngineEmbedsPrompt]:
241 + ) -> list[TokensPrompt | EmbedsPrompt]:
242 """
243 Render text/token prompts and/or precomputed embedding prompts. At
244 least one of `prompt_or_prompts` or `prompt_embeds` must be provided.
@@ -249,7 +247,7 @@ class CompletionRenderer(BaseRenderer):
247 if truncate_prompt_tokens == 0:
248 return []
249
252 - rendered: list[EngineTokensPrompt | EngineEmbedsPrompt] = []
250 + rendered: list[TokensPrompt | EmbedsPrompt] = []
251
252 if prompt_embeds is not None:
253 rendered.extend(
@@ -281,10 +279,10 @@ class CompletionRenderer(BaseRenderer):
279
280 async def _create_prompt(
281 self,
284 - prompt_input: EngineTextPrompt | EngineTokensPrompt,
282 + prompt_input: TextPrompt | TokensPrompt,
283 config: RenderConfig,
284 truncate_prompt_tokens: int | None,
287 - ) -> EngineTokensPrompt:
285 + ) -> TokensPrompt:
286 prompt, prompt_token_ids, _ = get_prompt_components(prompt_input)
287
288 if prompt_token_ids is not None:
@@ -317,7 +315,7 @@ class CompletionRenderer(BaseRenderer):
315 truncate_prompt_tokens: int | None,
316 add_special_tokens: bool,
317 cache_salt: str | None,
320 - ) -> EngineTokensPrompt:
318 + ) -> TokensPrompt:
319 """Tokenize text input asynchronously."""
320 async_tokenizer = self._get_async_tokenizer()
321
@@ -350,7 +348,7 @@ class CompletionRenderer(BaseRenderer):
348 truncate_prompt_tokens: int | None,
349 cache_salt: str | None,
350 needs_detokenization: bool | None = False,
353 - ) -> EngineTokensPrompt:
351 + ) -> TokensPrompt:
352 """Optionally detokenize token IDs and build a tokens prompt."""
353 token_ids = self._maybe_apply_truncation(token_ids, truncate_prompt_tokens)
354
@@ -392,8 +390,8 @@ class CompletionRenderer(BaseRenderer):
390 max_length: int | None = None,
391 cache_salt: str | None = None,
392 prompt: str | None = None,
395 - ) -> EngineTokensPrompt:
396 - """Create validated EngineTokensPrompt."""
393 + ) -> TokensPrompt:
394 + """Create validated TokensPrompt."""
395 if max_length is not None and len(token_ids) > max_length:
396 raise ValueError(
397 f"This model's maximum context length is {max_length} tokens. "
@@ -401,7 +399,7 @@ class CompletionRenderer(BaseRenderer):
399 "Please reduce the length of the input messages."
400 )
401
404 - tokens_prompt = EngineTokensPrompt(prompt_token_ids=token_ids)
402 + tokens_prompt = TokensPrompt(prompt_token_ids=token_ids)
403 if cache_salt is not None:
404 tokens_prompt["cache_salt"] = cache_salt
405 if prompt is not None:
vllm/entrypoints/serve/disagg/serving.py
+3 -3
@@ -27,7 +27,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
27 GenerateResponse,
28 GenerateResponseChoice,
29 )
30 -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
30 +from vllm.inputs.data import TokensPrompt
31 from vllm.logger import init_logger
32 from vllm.logprobs import Logprob
33 from vllm.outputs import RequestOutput
@@ -99,7 +99,7 @@ class ServingTokens(OpenAIServing):
99
100 # TODO(NickLucche): Change to EngineCoreRequest once Renderer work is
101 # completed
102 - engine_prompt = EngineTokensPrompt(prompt_token_ids=request.token_ids)
102 + engine_prompt = TokensPrompt(prompt_token_ids=request.token_ids)
103 if request.features is not None:
104 engine_prompt["multi_modal_data"] = None
105
@@ -115,7 +115,7 @@ class ServingTokens(OpenAIServing):
115
116 self._log_inputs(
117 request_id,
118 - request.token_ids,
118 + TokensPrompt(prompt_token_ids=request.token_ids),
119 params=sampling_params,
120 lora_request=lora_request,
121 )
vllm/entrypoints/serve/tokenize/serving.py
+7 -6
@@ -21,6 +21,7 @@ from vllm.entrypoints.openai.protocol import (
21 from vllm.entrypoints.openai.serving_engine import OpenAIServing
22 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
23 from vllm.entrypoints.renderer import RenderConfig
24 +from vllm.inputs import TokensPrompt
25 from vllm.logger import init_logger
26 from vllm.tokenizers import TokenizerLike
27
@@ -80,11 +81,8 @@ class OpenAIServingTokenization(OpenAIServing):
81 )
82 if error_check_ret is not None:
83 return error_check_ret
83 - (
84 - _,
85 - _,
86 - engine_prompts,
87 - ) = await self._preprocess_chat(
84 +
85 + _, engine_prompts = await self._preprocess_chat(
86 request,
87 tokenizer,
88 request.messages,
@@ -141,7 +139,10 @@ class OpenAIServingTokenization(OpenAIServing):
139 tokenizer = await self.engine_client.get_tokenizer()
140
141 self._log_inputs(
144 - request_id, request.tokens, params=None, lora_request=lora_request
142 + request_id,
143 + TokensPrompt(prompt_token_ids=request.tokens),
144 + params=None,
145 + lora_request=lora_request,
146 )
147
148 prompt_input = await self._tokenize_prompt_input_async(