A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Hardware] Add processor inputs to platform validation (#16680)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Apr 16, 2025 at 18:28 UTC
e1b004839a2c6d1f1771b7ab9c97acd0ed0c7aa2
3 files changed
+10
-8
vllm/platforms/interface.py
+2
-1
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
8
import numpy as np
9
import torch
10
11
-from vllm.inputs import PromptType
11
+from vllm.inputs import ProcessorInputs, PromptType
12
from vllm.logger import init_logger
13
14
if TYPE_CHECKING:
@@ -400,6 +400,7 @@ class Platform:
400
cls,
401
prompt: PromptType,
402
params: Union[SamplingParams, PoolingParams],
403
+ processed_inputs: ProcessorInputs,
404
) -> None:
405
"""Raises if this request is unsupported on this platform"""
406
vllm/platforms/tpu.py
+2
-1
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional, Union
5
import torch
6
7
import vllm.envs as envs
8
-from vllm.inputs import PromptType
8
+from vllm.inputs import ProcessorInputs, PromptType
9
from vllm.logger import init_logger
10
from vllm.sampling_params import SamplingParams, SamplingType
11
@@ -150,6 +150,7 @@ class TpuPlatform(Platform):
150
cls,
151
prompt: PromptType,
152
params: Union[SamplingParams, PoolingParams],
153
+ processed_inputs: ProcessorInputs,
154
) -> None:
155
"""Raises if this request is unsupported on this platform"""
156
if isinstance(params, SamplingParams):
vllm/v1/engine/processor.py
+6
-6
@@ -202,12 +202,6 @@ class Processor:
202
203
# TODO(woosuk): Support pooling models.
204
# TODO(woosuk): Support encoder-decoder models.
205
-
206
- from vllm.platforms import current_platform
207
- current_platform.validate_request(
208
- prompt=prompt,
209
- params=params,
210
- )
205
self._validate_lora(lora_request)
206
self._validate_params(params)
207
if priority != 0:
@@ -231,6 +225,12 @@ class Processor:
225
prompt_adapter_request=prompt_adapter_request,
226
return_mm_hashes=self.use_hash,
227
)
228
+ from vllm.platforms import current_platform
229
+ current_platform.validate_request(
230
+ prompt=prompt,
231
+ params=params,
232
+ processed_inputs=processed_inputs,
233
+ )
234
eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
235
236
self._validate_model_inputs(processed_inputs, lora_request)