A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[bugfix] add supports_v1 platform interface (#15417)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Mar 25, 2025 at 15:00 UTC
5f063a80bda621da09e73fc63d4c59320259131d
5 files changed
+31
-7
vllm/engine/arg_utils.py
+2
-3
@@ -1666,9 +1666,8 @@ class EngineArgs:
1666
_raise_or_fallback(feature_name=name, recommend_to_remove=True)
1667
return False
1668
1669
- # No support for device type other than CUDA, AMD (experiemntal) or
1670
- # TPU (experimental) so far.
1671
- if not (current_platform.is_cuda_alike() or current_platform.is_tpu()):
1669
+ # Platforms must decide if they can support v1 for this model
1670
+ if not current_platform.supports_v1(model_config=model_config):
1671
_raise_or_fallback(
1672
feature_name=f"device type={current_platform.device_type}",
1673
recommend_to_remove=False)
vllm/platforms/cuda.py
+6
-1
@@ -20,8 +20,9 @@ from vllm.utils import import_pynvml
20
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
21
22
if TYPE_CHECKING:
23
- from vllm.config import VllmConfig
23
+ from vllm.config import ModelConfig, VllmConfig
24
else:
25
+ ModelConfig = None
26
VllmConfig = None
27
28
logger = init_logger(__name__)
@@ -303,6 +304,10 @@ class CudaPlatformBase(Platform):
304
def supports_fp8(cls) -> bool:
305
return cls.has_device_capability(89)
306
307
+ @classmethod
308
+ def supports_v1(cls, model_config: ModelConfig) -> bool:
309
+ return True
310
+
311
312
# NVML utils
313
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/interface.py
+9
-1
@@ -12,9 +12,10 @@ import torch
12
from vllm.logger import init_logger
13
14
if TYPE_CHECKING:
15
- from vllm.config import VllmConfig
15
+ from vllm.config import ModelConfig, VllmConfig
16
from vllm.utils import FlexibleArgumentParser
17
else:
18
+ ModelConfig = None
19
VllmConfig = None
20
FlexibleArgumentParser = None
21
@@ -371,6 +372,13 @@ class Platform:
372
or parallel_config.distributed_executor_backend
373
== "external_launcher")
374
375
+ @classmethod
376
+ def supports_v1(cls, model_config: ModelConfig) -> bool:
377
+ """Returns whether the current platform can support v1 for the supplied
378
+ model configuration.
379
+ """
380
+ return False
381
+
382
383
class UnspecifiedPlatform(Platform):
384
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/rocm.py
+7
-1
@@ -12,8 +12,9 @@ from vllm.logger import init_logger
12
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
13
14
if TYPE_CHECKING:
15
- from vllm.config import VllmConfig
15
+ from vllm.config import ModelConfig, VllmConfig
16
else:
17
+ ModelConfig = None
18
VllmConfig = None
19
20
logger = init_logger(__name__)
@@ -249,3 +250,8 @@ class RocmPlatform(Platform):
250
return torch.float8_e4m3fnuz
251
else:
252
return torch.float8_e4m3fn
253
+
254
+ @classmethod
255
+ def supports_v1(cls, model_config: ModelConfig) -> bool:
256
+ # V1 support on AMD gpus is experimental
257
+ return True
vllm/platforms/tpu.py
+7
-1
@@ -10,8 +10,9 @@ from vllm.logger import init_logger
10
from .interface import Platform, PlatformEnum, _Backend
11
12
if TYPE_CHECKING:
13
- from vllm.config import VllmConfig
13
+ from vllm.config import ModelConfig, VllmConfig
14
else:
15
+ ModelConfig = None
16
VllmConfig = None
17
18
logger = init_logger(__name__)
@@ -127,3 +128,8 @@ class TpuPlatform(Platform):
128
@classmethod
129
def use_all_gather(cls) -> bool:
130
return True
131
+
132
+ @classmethod
133
+ def supports_v1(cls, model_config: ModelConfig) -> bool:
134
+ # V1 support on TPU is experimental
135
+ return True