A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 08:45 UTC
[Refactor] Call renderer for online IO processor request (#34490)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Cyrus Leung committed
Feb 13, 2026 at 14:48 UTC
ec090c2429d179309641cba9e7793eab34e19f8d
4 files changed
+39
-18
vllm/entrypoints/llm.py
+8
-8
@@ -500,7 +500,7 @@ class LLM:
500
engine_prompts: Sequence[DictPrompt | TokPrompt] = [
501
engine_prompt
502
for prompt, param in zip(seq_prompts, seq_params)
503
- for engine_prompt in self._preprocess_completion(
503
+ for engine_prompt in self._preprocess_cmpl(
504
[prompt],
505
tokenization_kwargs=merge_kwargs(
506
tokenization_kwargs,
@@ -509,7 +509,7 @@ class LLM:
509
)
510
]
511
else:
512
- engine_prompts = self._preprocess_completion(
512
+ engine_prompts = self._preprocess_cmpl(
513
seq_prompts,
514
tokenization_kwargs=tokenization_kwargs,
515
)
@@ -889,7 +889,7 @@ class LLM:
889
add_special_tokens=not model_config.is_encoder_decoder,
890
).with_kwargs(tokenization_kwargs)
891
892
- def _preprocess_completion(
892
+ def _preprocess_cmpl(
893
self,
894
prompts: Sequence[PromptType],
895
tokenization_kwargs: dict[str, Any] | None = None,
@@ -901,7 +901,7 @@ class LLM:
901
Refer to [LLM.generate][] for a complete description of the arguments.
902
903
Returns:
904
- A list of `TokensPrompts` objects containing the tokenized prompt
904
+ A list of `TokPrompt` objects containing the tokenized prompt
905
after chat template interpolation, and the raw multi-modal inputs.
906
"""
907
renderer = self.renderer
@@ -943,7 +943,7 @@ class LLM:
943
Refer to [LLM.chat][] for a complete description of the arguments.
944
945
Returns:
946
- A list of `TokensPrompts` objects containing the tokenized prompt
946
+ A list of `TokPrompt` objects containing the tokenized prompt
947
after chat template interpolation, and the raw multi-modal inputs.
948
"""
949
renderer = self.renderer
@@ -1823,11 +1823,11 @@ class LLM:
1823
if any(param.truncate_prompt_tokens is not None for param in seq_params):
1824
# TODO: Remove this after deprecating `param.truncate_prompt_tokens`
1825
# Then, move the code from the `else` block to the top and let
1826
- # `self._preprocess_completion` handle prompt normalization
1826
+ # `self._preprocess_cmpl` handle prompt normalization
1827
engine_prompts: Sequence[DictPrompt | TokPrompt] = [
1828
engine_prompt
1829
for prompt, param in zip(seq_prompts, seq_params)
1830
- for engine_prompt in self._preprocess_completion(
1830
+ for engine_prompt in self._preprocess_cmpl(
1831
[prompt],
1832
tokenization_kwargs=merge_kwargs(
1833
tokenization_kwargs,
@@ -1836,7 +1836,7 @@ class LLM:
1836
)
1837
]
1838
else:
1839
- engine_prompts = self._preprocess_completion(
1839
+ engine_prompts = self._preprocess_cmpl(
1840
seq_prompts,
1841
tokenization_kwargs=tokenization_kwargs,
1842
)
vllm/entrypoints/openai/engine/serving.py
+11
-4
@@ -5,7 +5,7 @@ import json
5
import sys
6
import time
7
import traceback
8
-from collections.abc import AsyncGenerator, Callable, Mapping
8
+from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
9
from dataclasses import dataclass, field
10
from http import HTTPStatus
11
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
@@ -959,15 +959,22 @@ class OpenAIServing:
959
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
960
prompt_embeds: bytes | list[bytes] | None,
961
) -> list[TokPrompt]:
962
- renderer = self.renderer
963
- model_config = self.model_config
964
-
962
prompts = list[SingletonPrompt | bytes]()
963
if prompt_embeds is not None: # embeds take higher priority
964
prompts.extend(prompt_to_seq(prompt_embeds))
965
if prompt_input is not None:
966
prompts.extend(prompt_to_seq(prompt_input))
967
968
+ return await self._preprocess_cmpl(request, prompts)
969
+
970
+ async def _preprocess_cmpl(
971
+ self,
972
+ request: RendererRequest,
973
+ prompts: Sequence[PromptType | bytes],
974
+ ) -> list[TokPrompt]:
975
+ renderer = self.renderer
976
+ model_config = self.model_config
977
+
978
parsed_prompts = [
979
(
980
prompt
vllm/entrypoints/pooling/pooling/protocol.py
+12
@@ -100,6 +100,18 @@ class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic
100
data: T
101
task: PoolingTask = "plugin"
102
103
+ def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
104
+ encoder_config = model_config.encoder_config or {}
105
+
106
+ return TokenizeParams(
107
+ max_total_tokens=model_config.max_model_len,
108
+ max_output_tokens=0,
109
+ truncate_prompt_tokens=self.truncate_prompt_tokens,
110
+ do_lower_case=encoder_config.get("do_lower_case", False),
111
+ add_special_tokens=not model_config.is_encoder_decoder,
112
+ max_total_tokens_param="max_model_len",
113
+ )
114
+
115
116
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
117
request_id: str | None = None
vllm/entrypoints/pooling/pooling/serving.py
+8
-6
@@ -6,7 +6,7 @@ import json
6
import time
7
from collections.abc import AsyncGenerator, Callable, Sequence
8
from functools import partial
9
-from typing import Any, Final, Literal, cast
9
+from typing import Final, Literal, cast
10
11
import jinja2
12
from fastapi import Request
@@ -108,7 +108,10 @@ class OpenAIServingPooling(OpenAIServing):
108
raw_prompts = await self.io_processor.pre_process_async(
109
prompt=validated_prompt, request_id=request_id
110
)
111
- engine_prompts = prompt_to_seq(raw_prompts)
111
+ engine_prompts = await self._preprocess_cmpl(
112
+ request,
113
+ prompt_to_seq(raw_prompts),
114
+ )
115
elif isinstance(request, PoolingChatRequest):
116
error_check_ret = self._validate_chat_template(
117
request_chat_template=request.chat_template,
@@ -146,12 +149,11 @@ class OpenAIServingPooling(OpenAIServing):
149
pooling_params = self.io_processor.merge_pooling_params()
150
if pooling_params.task is None:
151
pooling_params.task = "plugin"
149
-
150
- tokenization_kwargs: dict[str, Any] = {}
152
else:
153
pooling_params = request.to_pooling_params() # type: ignore
153
- tok_params = request.build_tok_params(self.model_config) # type: ignore
154
- tokenization_kwargs = tok_params.get_encode_kwargs()
154
+
155
+ tok_params = request.build_tok_params(self.model_config)
156
+ tokenization_kwargs = tok_params.get_encode_kwargs()
157
158
for i, engine_prompt in enumerate(engine_prompts):
159
request_id_item = f"{request_id}-{i}"