A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Renderer] Move Processor out of LLMEngine (#26165)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 3, 2025 at 23:08 UTC
d78fda7cda87426df6e7e96c7c766191b212492e
4 files changed
+107
-52
vllm/entrypoints/llm.py
+57
-13
@@ -37,6 +37,7 @@ from vllm.entrypoints.utils import (_validate_truncation_size,
37
log_non_default_args)
38
from vllm.inputs import (DataPrompt, PromptType, SingletonPrompt, TextPrompt,
39
TokensPrompt)
40
+from vllm.inputs.parse import get_prompt_components
41
from vllm.logger import init_logger
42
from vllm.lora.request import LoRARequest
43
from vllm.model_executor.layers.quantization import QuantizationMethods
@@ -49,10 +50,13 @@ from vllm.sampling_params import (BeamSearchParams, RequestOutputKind,
50
SamplingParams)
51
from vllm.tasks import PoolingTask
52
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
52
- get_cached_tokenizer)
53
+ get_cached_tokenizer,
54
+ init_tokenizer_from_configs)
55
from vllm.usage.usage_lib import UsageContext
56
from vllm.utils import Counter, Device, as_iter, is_list_of
57
+from vllm.v1.engine import EngineCoreRequest
58
from vllm.v1.engine.llm_engine import LLMEngine
59
+from vllm.v1.engine.processor import Processor
60
from vllm.v1.sample.logits_processor import LogitsProcessor
61
62
if TYPE_CHECKING:
@@ -312,6 +316,10 @@ class LLM:
316
self.io_processor = get_io_processor(self.llm_engine.vllm_config,
317
io_processor_plugin)
318
319
+ @property
320
+ def model_config(self):
321
+ return self.llm_engine.model_config
322
+
323
def get_tokenizer(self) -> AnyTokenizer:
324
return self.llm_engine.get_tokenizer()
325
@@ -324,6 +332,16 @@ class LLM:
332
else:
333
self.llm_engine.tokenizer = get_cached_tokenizer(tokenizer)
334
335
+ def _get_processor(self) -> Processor:
336
+ if not hasattr(self, "_processor"):
337
+ vllm_config = self.llm_engine.vllm_config
338
+ if self.model_config.skip_tokenizer_init:
339
+ tokenizer = None
340
+ else:
341
+ tokenizer = init_tokenizer_from_configs(self.model_config)
342
+ self._processor = Processor(vllm_config, tokenizer)
343
+ return self._processor
344
+
345
def get_default_sampling_params(self) -> SamplingParams:
346
if self.default_sampling_params is None:
347
self.default_sampling_params = (
@@ -1497,8 +1515,6 @@ class LLM:
1515
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
1516
it = tqdm_func(it, desc="Adding requests")
1517
1500
- model_config = self.llm_engine.model_config
1501
-
1518
for i, prompt in enumerate(it):
1519
1520
if isinstance(prompt, dict):
@@ -1506,17 +1522,9 @@ class LLM:
1522
prompt.get("multi_modal_data"),
1523
prompt.get("multi_modal_uuids"))
1524
1509
- param = params[i] if isinstance(params, Sequence) else params
1510
-
1511
- tokenization_kwargs: dict[str, Any] = {}
1512
- _validate_truncation_size(model_config.max_model_len,
1513
- param.truncate_prompt_tokens,
1514
- tokenization_kwargs)
1515
-
1525
self._add_request(
1526
prompt,
1527
params[i] if isinstance(params, Sequence) else params,
1519
- tokenization_kwargs=tokenization_kwargs,
1528
lora_request=lora_request[i] if isinstance(
1529
lora_request, Sequence) else lora_request,
1530
priority=priority[i] if priority else 0,
@@ -1557,22 +1565,58 @@ class LLM:
1565
raise ValueError(f"Multi-modal data for {modality} is None"
1566
f" but UUID is not provided")
1567
1568
+ def _process_inputs(
1569
+ self,
1570
+ request_id: str,
1571
+ engine_prompt: PromptType,
1572
+ params: Union[SamplingParams, PoolingParams],
1573
+ *,
1574
+ lora_request: Optional[LoRARequest],
1575
+ priority: int,
1576
+ ) -> tuple[EngineCoreRequest, dict[str, Any]]:
1577
+ """Use the Processor to process inputs for LLMEngine."""
1578
+ tokenization_kwargs: dict[str, Any] = {}
1579
+ _validate_truncation_size(self.model_config.max_model_len,
1580
+ params.truncate_prompt_tokens,
1581
+ tokenization_kwargs)
1582
+
1583
+ processor = self._get_processor()
1584
+ engine_request = processor.process_inputs(
1585
+ request_id,
1586
+ engine_prompt,
1587
+ params,
1588
+ lora_request=lora_request,
1589
+ tokenization_kwargs=tokenization_kwargs,
1590
+ priority=priority,
1591
+ )
1592
+ return engine_request, tokenization_kwargs
1593
+
1594
def _add_request(
1595
self,
1596
prompt: PromptType,
1597
params: Union[SamplingParams, PoolingParams],
1564
- tokenization_kwargs: Optional[dict[str, Any]] = None,
1598
lora_request: Optional[LoRARequest] = None,
1599
priority: int = 0,
1600
) -> None:
1601
+ prompt_text, _, _ = get_prompt_components(prompt)
1602
request_id = str(next(self.request_counter))
1569
- self.llm_engine.add_request(
1603
+
1604
+ engine_request, tokenization_kwargs = self._process_inputs(
1605
request_id,
1606
prompt,
1607
params,
1608
lora_request=lora_request,
1609
+ priority=priority,
1610
+ )
1611
+
1612
+ self.llm_engine.add_request(
1613
+ request_id,
1614
+ engine_request,
1615
+ params,
1616
+ lora_request=lora_request,
1617
tokenization_kwargs=tokenization_kwargs,
1618
priority=priority,
1619
+ prompt_text=prompt_text,
1620
)
1621
1622
def _run_engine(
vllm/entrypoints/openai/serving_engine.py
+10
-31
@@ -7,8 +7,7 @@ import traceback
7
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
8
from concurrent.futures import ThreadPoolExecutor
9
from http import HTTPStatus
10
-from typing import (Any, Callable, ClassVar, Generic, NamedTuple, Optional,
11
- TypeVar, Union)
10
+from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union
11
12
import torch
13
from fastapi import Request
@@ -69,6 +68,7 @@ from vllm.entrypoints.renderer import (BaseRenderer, CompletionRenderer,
68
# yapf: enable
69
from vllm.inputs.data import PromptType
70
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
71
+from vllm.inputs.parse import PromptComponents, get_prompt_components
72
from vllm.logger import init_logger
73
from vllm.logprobs import Logprob, PromptLogprobs
74
from vllm.lora.request import LoRARequest
@@ -140,12 +140,6 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:
140
and "prompt_embeds" in prompt)
141
142
143
-class PromptComponents(NamedTuple):
144
- text: Optional[str] = None
145
- token_ids: Optional[list[int]] = None
146
- embeds: Optional[torch.Tensor] = None
147
-
148
-
143
RequestT = TypeVar("RequestT", bound=AnyRequest)
144
145
@@ -876,25 +870,23 @@ class OpenAIServing:
870
self,
871
request_id: str,
872
engine_prompt: PromptType,
879
- sampling_params: SamplingParams,
873
+ params: Union[SamplingParams, PoolingParams],
874
*,
875
lora_request: Optional[LoRARequest],
876
trace_headers: Optional[Mapping[str, str]],
877
priority: int,
878
) -> tuple[EngineCoreRequest, dict[str, Any]]:
885
- """
886
- using the Processor to process inputs for AsyncLLM
887
- """
879
+ """Use the Processor to process inputs for AsyncLLM."""
880
tokenization_kwargs: dict[str, Any] = {}
881
_validate_truncation_size(self.max_model_len,
890
- sampling_params.truncate_prompt_tokens,
882
+ params.truncate_prompt_tokens,
883
tokenization_kwargs)
884
885
processor = await self._get_processor()
886
engine_request = processor.process_inputs(
887
request_id,
888
engine_prompt,
897
- sampling_params,
889
+ params,
890
lora_request=lora_request,
891
tokenization_kwargs=tokenization_kwargs,
892
trace_headers=trace_headers,
@@ -973,25 +965,12 @@ class OpenAIServing:
965
966
def _get_prompt_components(
967
self,
976
- inputs: Union[RequestPrompt, PromptType],
968
+ prompt: Union[RequestPrompt, PromptType],
969
) -> PromptComponents:
978
- if isinstance(inputs, str):
979
- return PromptComponents(text=inputs)
980
- if isinstance(inputs, list):
981
- return PromptComponents(token_ids=inputs)
982
- if isinstance(inputs, dict):
983
- return PromptComponents(
984
- text=inputs.get("prompt"), # type: ignore[arg-type]
985
- token_ids=inputs.get(
986
- "prompt_token_ids"), # type: ignore[arg-type]
987
- embeds=inputs.get("prompt_embeds"),
988
- )
970
+ if isinstance(prompt, list):
971
+ return PromptComponents(token_ids=prompt)
972
990
- return PromptComponents(
991
- text=getattr(inputs, "prompt", None),
992
- token_ids=getattr(inputs, "prompt_token_ids", None),
993
- embeds=getattr(inputs, "prompt_embeds", None),
994
- )
973
+ return get_prompt_components(prompt) # type: ignore[arg-type]
974
975
def _log_inputs(
976
self,
vllm/inputs/parse.py
+25
-1
@@ -1,7 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Sequence
4
-from typing import Literal, Optional, TypedDict, Union, cast, overload
4
+from typing import (TYPE_CHECKING, Literal, NamedTuple, Optional, TypedDict,
5
+ Union, cast, overload)
6
7
from typing_extensions import TypeIs
8
@@ -11,6 +12,9 @@ from .data import (EmbedsPrompt, ExplicitEncoderDecoderPrompt, ProcessorInputs,
12
PromptType, SingletonInputs, SingletonPrompt, TextPrompt,
13
TokensPrompt)
14
15
+if TYPE_CHECKING:
16
+ import torch
17
+
18
19
class ParsedText(TypedDict):
20
content: str
@@ -149,3 +153,23 @@ def split_enc_dec_inputs(
153
)
154
155
return None, inputs
156
+
157
+
158
+class PromptComponents(NamedTuple):
159
+ text: Optional[str] = None
160
+ token_ids: Optional[list[int]] = None
161
+ embeds: Optional["torch.Tensor"] = None
162
+
163
+
164
+def get_prompt_components(prompt: PromptType) -> PromptComponents:
165
+ if isinstance(prompt, str):
166
+ return PromptComponents(text=prompt)
167
+
168
+ if (encoder_prompt := prompt.get("encoder_prompt")):
169
+ return get_prompt_components(encoder_prompt) # type: ignore[arg-type]
170
+
171
+ return PromptComponents(
172
+ text=prompt.get("prompt"), # type: ignore[arg-type]
173
+ token_ids=prompt.get("prompt_token_ids"), # type: ignore[arg-type]
174
+ embeds=prompt.get("prompt_embeds"),
175
+ )
vllm/v1/engine/llm_engine.py
+15
-7
@@ -27,6 +27,7 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer,
27
init_tokenizer_from_configs)
28
from vllm.usage.usage_lib import UsageContext
29
from vllm.utils import Device
30
+from vllm.v1.engine import EngineCoreRequest
31
from vllm.v1.engine.core_client import EngineCoreClient
32
from vllm.v1.engine.output_processor import OutputProcessor
33
from vllm.v1.engine.parallel_sampling import ParentRequest
@@ -213,13 +214,14 @@ class LLMEngine:
214
def add_request(
215
self,
216
request_id: str,
216
- prompt: PromptType,
217
+ prompt: Union[EngineCoreRequest, PromptType],
218
params: Union[SamplingParams, PoolingParams],
219
arrival_time: Optional[float] = None,
220
lora_request: Optional[LoRARequest] = None,
221
tokenization_kwargs: Optional[dict[str, Any]] = None,
222
trace_headers: Optional[Mapping[str, str]] = None,
223
priority: int = 0,
224
+ prompt_text: Optional[str] = None,
225
) -> None:
226
# Validate the request_id type.
227
if not isinstance(request_id, str):
@@ -227,12 +229,18 @@ class LLMEngine:
229
f"request_id must be a string, got {type(request_id)}")
230
231
# Process raw inputs into the request.
230
- request = self.processor.process_inputs(request_id, prompt, params,
231
- arrival_time, lora_request,
232
- tokenization_kwargs,
233
- trace_headers, priority)
234
- prompt_text = prompt if isinstance(prompt,
235
- str) else prompt.get("prompt")
232
+ if isinstance(prompt, EngineCoreRequest):
233
+ request = prompt
234
+ else:
235
+ assert prompt_text is None
236
+ logger.warning_once("Processor has been moved under LLM and will "
237
+ "be removed from LLMEngine in v0.13.")
238
+ request = self.processor.process_inputs(request_id, prompt, params,
239
+ arrival_time, lora_request,
240
+ tokenization_kwargs,
241
+ trace_headers, priority)
242
+ prompt_text = (prompt if isinstance(prompt, str) else
243
+ prompt.get("prompt"))
244
245
n = params.n if isinstance(params, SamplingParams) else 1
246