A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[platform] Add verify_quantization in platform. (#10757)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Nov 29, 2024 at 23:22 UTC
661175bc826f4caba04182a1faeeca9e7a3259ac
10 files changed
+38
-27
vllm/config.py
+1
-27
@@ -393,17 +393,11 @@ class ModelConfig:
393
394
def _verify_quantization(self) -> None:
395
supported_quantization = QUANTIZATION_METHODS
396
- rocm_supported_quantization = [
397
- "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
398
- "fbgemm_fp8", "gguf"
399
- ]
396
optimized_quantization_methods = [
397
"fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
398
"awq_marlin", "fbgemm_fp8", "compressed_tensors",
399
"compressed-tensors", "experts_int8"
400
]
405
- tpu_supported_quantization = ["tpu_int8"]
406
- neuron_supported_quantization = ["neuron_quant"]
401
if self.quantization is not None:
402
self.quantization = self.quantization.lower()
403
@@ -438,32 +432,12 @@ class ModelConfig:
432
raise ValueError(
433
f"Unknown quantization method: {self.quantization}. Must "
434
f"be one of {supported_quantization}.")
441
- if current_platform.is_rocm(
442
- ) and self.quantization not in rocm_supported_quantization:
443
- raise ValueError(
444
- f"{self.quantization} quantization is currently not "
445
- f"supported in ROCm.")
446
- if current_platform.is_tpu(
447
- ) and self.quantization not in tpu_supported_quantization:
448
- raise ValueError(
449
- f"{self.quantization} quantization is currently not "
450
- f"supported in TPU Backend.")
435
+ current_platform.verify_quantization(self.quantization)
436
if self.quantization not in optimized_quantization_methods:
437
logger.warning(
438
"%s quantization is not fully "
439
"optimized yet. The speed can be slower than "
440
"non-quantized models.", self.quantization)
456
- if (self.quantization == "awq" and current_platform.is_rocm()
457
- and not envs.VLLM_USE_TRITON_AWQ):
458
- logger.warning(
459
- "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
460
- " is not set, enabling VLLM_USE_TRITON_AWQ.")
461
- envs.VLLM_USE_TRITON_AWQ = True
462
- if current_platform.is_neuron(
463
- ) and self.quantization not in neuron_supported_quantization:
464
- raise ValueError(
465
- f"{self.quantization} quantization is currently not "
466
- f"supported in Neuron Backend.")
441
442
def _verify_cuda_graph(self) -> None:
443
if self.max_seq_len_to_capture is None:
vllm/platforms/cpu.py
+1
@@ -19,6 +19,7 @@ logger = init_logger(__name__)
19
20
class CpuPlatform(Platform):
21
_enum = PlatformEnum.CPU
22
+ device_name: str = "cpu"
23
device_type: str = "cpu"
24
dispatch_key: str = "CPU"
25
vllm/platforms/cuda.py
+1
@@ -72,6 +72,7 @@ def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
72
73
class CudaPlatformBase(Platform):
74
_enum = PlatformEnum.CUDA
75
+ device_name: str = "cuda"
76
device_type: str = "cuda"
77
dispatch_key: str = "CUDA"
78
vllm/platforms/hpu.py
+1
@@ -12,6 +12,7 @@ else:
12
13
class HpuPlatform(Platform):
14
_enum = PlatformEnum.HPU
15
+ device_name: str = "hpu"
16
device_type: str = "hpu"
17
dispatch_key: str = "HPU"
18
vllm/platforms/interface.py
+13
@@ -56,11 +56,13 @@ class DeviceCapability(NamedTuple):
56
57
class Platform:
58
_enum: PlatformEnum
59
+ device_name: str
60
device_type: str
61
# available dispatch keys:
62
# check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
63
# use "CPU" as a fallback for platforms not registered in PyTorch
64
dispatch_key: str = "CPU"
65
+ supported_quantization: list[str] = []
66
67
def is_cuda(self) -> bool:
68
return self._enum == PlatformEnum.CUDA
@@ -171,6 +173,17 @@ class Platform:
173
"""
174
pass
175
176
+ @classmethod
177
+ def verify_quantization(cls, quant: str) -> None:
178
+ """
179
+ Verify whether the quantization is supported by the current platform.
180
+ """
181
+ if cls.supported_quantization and \
182
+ quant not in cls.supported_quantization:
183
+ raise ValueError(
184
+ f"{quant} quantization is currently not supported in "
185
+ f"{cls.device_name}.")
186
+
187
188
class UnspecifiedPlatform(Platform):
189
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/neuron.py
+2
@@ -10,7 +10,9 @@ else:
10
11
class NeuronPlatform(Platform):
12
_enum = PlatformEnum.NEURON
13
+ device_name: str = "neuron"
14
device_type: str = "neuron"
15
+ supported_quantization: list[str] = ["neuron_quant"]
16
17
@classmethod
18
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/openvino.py
+1
@@ -23,6 +23,7 @@ except ImportError as e:
23
24
class OpenVinoPlatform(Platform):
25
_enum = PlatformEnum.OPENVINO
26
+ device_name: str = "openvino"
27
device_type: str = "openvino"
28
dispatch_key: str = "CPU"
29
vllm/platforms/rocm.py
+15
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
4
5
import torch
6
7
+import vllm.envs as envs
8
from vllm.logger import init_logger
9
10
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
@@ -35,8 +36,13 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
36
37
class RocmPlatform(Platform):
38
_enum = PlatformEnum.ROCM
39
+ device_name: str = "rocm"
40
device_type: str = "cuda"
41
dispatch_key: str = "CUDA"
42
+ supported_quantization: list[str] = [
43
+ "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
44
+ "fbgemm_fp8", "gguf"
45
+ ]
46
47
@classmethod
48
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
@@ -79,3 +85,12 @@ class RocmPlatform(Platform):
85
"vllm.spec_decode.spec_decode_worker.create_spec_worker"
86
else:
87
parallel_config.worker_cls = "vllm.worker.worker.Worker"
88
+
89
+ @classmethod
90
+ def verify_quantization(cls, quant: str) -> None:
91
+ super().verify_quantization(quant)
92
+ if quant == "awq" and not envs.VLLM_USE_TRITON_AWQ:
93
+ logger.warning(
94
+ "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
95
+ " is not set, enabling VLLM_USE_TRITON_AWQ.")
96
+ envs.VLLM_USE_TRITON_AWQ = True
vllm/platforms/tpu.py
+2
@@ -16,8 +16,10 @@ logger = init_logger(__name__)
16
17
class TpuPlatform(Platform):
18
_enum = PlatformEnum.TPU
19
+ device_name: str = "tpu"
20
device_type: str = "tpu"
21
dispatch_key: str = "XLA"
22
+ supported_quantization: list[str] = ["tpu_int8"]
23
24
@classmethod
25
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/xpu.py
+1
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16
17
class XPUPlatform(Platform):
18
_enum = PlatformEnum.XPU
19
+ device_name: str = "xpu"
20
device_type: str = "xpu"
21
dispatch_key: str = "XPU"
22