A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Hardware] add platform-specific request validation api (#16291)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Apr 9, 2025 at 21:50 UTC
cb391d85dc91645108fc22807b42f218df16d13c
9 files changed
+38
-41
vllm/platforms/cpu.py
-4
@@ -180,7 +180,3 @@ class CpuPlatform(Platform):
180
Get device specific communicator class for distributed communication.
181
"""
182
return "vllm.distributed.device_communicators.cpu_communicator.CpuCommunicator" # noqa
183
-
184
- @classmethod
185
- def supports_structured_output(cls) -> bool:
186
- return True
vllm/platforms/cuda.py
-4
@@ -308,10 +308,6 @@ class CudaPlatformBase(Platform):
308
def supports_v1(cls, model_config: ModelConfig) -> bool:
309
return True
310
311
- @classmethod
312
- def supports_structured_output(cls) -> bool:
313
- return True
314
-
311
@classmethod
312
def use_custom_allreduce(cls) -> bool:
313
return True
vllm/platforms/hpu.py
-4
@@ -92,7 +92,3 @@ class HpuPlatform(Platform):
92
@classmethod
93
def get_device_communicator_cls(cls) -> str:
94
return "vllm.distributed.device_communicators.hpu_communicator.HpuCommunicator" # noqa
95
-
96
- @classmethod
97
- def supports_structured_output(cls) -> bool:
98
- return True
vllm/platforms/interface.py
+15
-8
@@ -1,5 +1,4 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-
2
import enum
3
import platform
4
import random
@@ -9,14 +8,21 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
8
import numpy as np
9
import torch
10
11
+from vllm.inputs import PromptType
12
from vllm.logger import init_logger
13
14
if TYPE_CHECKING:
15
from vllm.config import ModelConfig, VllmConfig
16
+ from vllm.lora.request import LoRARequest
17
+ from vllm.pooling_params import PoolingParams
18
+ from vllm.sampling_params import SamplingParams
19
from vllm.utils import FlexibleArgumentParser
20
else:
21
ModelConfig = None
22
VllmConfig = None
23
+ LoRARequest = None
24
+ PoolingParams = None
25
+ SamplingParams = None
26
FlexibleArgumentParser = None
27
28
logger = init_logger(__name__)
@@ -379,13 +385,6 @@ class Platform:
385
"""
386
return False
387
382
- @classmethod
383
- def supports_structured_output(cls) -> bool:
384
- """
385
- Returns whether the current platform can support structured output.
386
- """
387
- return False
388
-
388
@classmethod
389
def use_custom_allreduce(cls) -> bool:
390
"""
@@ -393,6 +392,14 @@ class Platform:
392
"""
393
return False
394
395
+ @classmethod
396
+ def validate_request(
397
+ cls,
398
+ prompt: PromptType,
399
+ params: Union[SamplingParams, PoolingParams],
400
+ ) -> None:
401
+ """Raises if this request is unsupported on this platform"""
402
+
403
404
class UnspecifiedPlatform(Platform):
405
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/neuron.py
-4
@@ -67,7 +67,3 @@ class NeuronPlatform(Platform):
67
@classmethod
68
def use_all_gather(cls) -> bool:
69
return True
70
-
71
- @classmethod
72
- def supports_structured_output(cls) -> bool:
73
- return True
vllm/platforms/rocm.py
-4
@@ -303,10 +303,6 @@ class RocmPlatform(Platform):
303
# V1 support on AMD gpus is experimental
304
return True
305
306
- @classmethod
307
- def supports_structured_output(cls) -> bool:
308
- return True
309
-
306
@classmethod
307
def use_custom_allreduce(cls) -> bool:
308
# We only enable custom allreduce for MI300 series
vllm/platforms/tpu.py
+18
-4
@@ -1,19 +1,26 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import TYPE_CHECKING, Optional
3
+from typing import TYPE_CHECKING, Optional, Union
4
5
import torch
6
7
import vllm.envs as envs
8
+from vllm.inputs import PromptType
9
from vllm.logger import init_logger
10
11
from .interface import Platform, PlatformEnum, _Backend
12
13
if TYPE_CHECKING:
14
from vllm.config import ModelConfig, VllmConfig
15
+ from vllm.lora.request import LoRARequest
16
+ from vllm.pooling_params import PoolingParams
17
+ from vllm.sampling_params import SamplingParams
18
else:
19
ModelConfig = None
20
VllmConfig = None
21
+ LoRARequest = None
22
+ PoolingParams = None
23
+ SamplingParams = None
24
25
logger = init_logger(__name__)
26
@@ -135,6 +142,13 @@ class TpuPlatform(Platform):
142
return True
143
144
@classmethod
138
- def supports_structured_output(cls) -> bool:
139
- # Structured output is not supported on TPU.
140
- return False
145
+ def validate_request(
146
+ cls,
147
+ prompt: PromptType,
148
+ params: Union[SamplingParams, PoolingParams],
149
+ ) -> None:
150
+ """Raises if this request is unsupported on this platform"""
151
+ if isinstance(params,
152
+ SamplingParams) and params.guided_decoding is not None:
153
+ raise ValueError("Structured output is not supported on "
154
+ f"{cls.device_name}.")
vllm/platforms/xpu.py
-4
@@ -140,7 +140,3 @@ class XPUPlatform(Platform):
140
@classmethod
141
def get_device_communicator_cls(cls) -> str:
142
return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa
143
-
144
- @classmethod
145
- def supports_structured_output(cls) -> bool:
146
- return True
vllm/v1/engine/processor.py
+5
-5
@@ -141,11 +141,6 @@ class Processor:
141
else:
142
params.guided_decoding.backend = engine_level_backend
143
144
- from vllm.platforms import current_platform
145
- if not current_platform.supports_structured_output():
146
- raise ValueError("Structured output is not supported on "
147
- f"{current_platform.device_name}.")
148
-
144
# Request content validation
145
if engine_level_backend.startswith("xgrammar"):
146
# xgrammar with no fallback
@@ -187,6 +182,11 @@ class Processor:
182
# TODO(woosuk): Support pooling models.
183
# TODO(woosuk): Support encoder-decoder models.
184
185
+ from vllm.platforms import current_platform
186
+ current_platform.validate_request(
187
+ prompt=prompt,
188
+ params=params,
189
+ )
190
self._validate_lora(lora_request)
191
self._validate_params(params)
192
if priority != 0: