@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 661175bc82
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[platform] Add verify_quantization in platform. (#10757)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Nov 29, 2024 at 23:22 UTC 661175bc826f4caba04182a1faeeca9e7a3259ac
10 files changed +38 -27
vllm/config.py
+1 -27
@@ -393,17 +393,11 @@ class ModelConfig:
393
394 def _verify_quantization(self) -> None:
395 supported_quantization = QUANTIZATION_METHODS
396 - rocm_supported_quantization = [
397 - "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
398 - "fbgemm_fp8", "gguf"
399 - ]
396 optimized_quantization_methods = [
397 "fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
398 "awq_marlin", "fbgemm_fp8", "compressed_tensors",
399 "compressed-tensors", "experts_int8"
400 ]
405 - tpu_supported_quantization = ["tpu_int8"]
406 - neuron_supported_quantization = ["neuron_quant"]
401 if self.quantization is not None:
402 self.quantization = self.quantization.lower()
403
@@ -438,32 +432,12 @@ class ModelConfig:
432 raise ValueError(
433 f"Unknown quantization method: {self.quantization}. Must "
434 f"be one of {supported_quantization}.")
441 - if current_platform.is_rocm(
442 - ) and self.quantization not in rocm_supported_quantization:
443 - raise ValueError(
444 - f"{self.quantization} quantization is currently not "
445 - f"supported in ROCm.")
446 - if current_platform.is_tpu(
447 - ) and self.quantization not in tpu_supported_quantization:
448 - raise ValueError(
449 - f"{self.quantization} quantization is currently not "
450 - f"supported in TPU Backend.")
435 + current_platform.verify_quantization(self.quantization)
436 if self.quantization not in optimized_quantization_methods:
437 logger.warning(
438 "%s quantization is not fully "
439 "optimized yet. The speed can be slower than "
440 "non-quantized models.", self.quantization)
456 - if (self.quantization == "awq" and current_platform.is_rocm()
457 - and not envs.VLLM_USE_TRITON_AWQ):
458 - logger.warning(
459 - "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
460 - " is not set, enabling VLLM_USE_TRITON_AWQ.")
461 - envs.VLLM_USE_TRITON_AWQ = True
462 - if current_platform.is_neuron(
463 - ) and self.quantization not in neuron_supported_quantization:
464 - raise ValueError(
465 - f"{self.quantization} quantization is currently not "
466 - f"supported in Neuron Backend.")
441
442 def _verify_cuda_graph(self) -> None:
443 if self.max_seq_len_to_capture is None:
vllm/platforms/cpu.py
+1
@@ -19,6 +19,7 @@ logger = init_logger(__name__)
19
20 class CpuPlatform(Platform):
21 _enum = PlatformEnum.CPU
22 + device_name: str = "cpu"
23 device_type: str = "cpu"
24 dispatch_key: str = "CPU"
25
vllm/platforms/cuda.py
+1
@@ -72,6 +72,7 @@ def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
72
73 class CudaPlatformBase(Platform):
74 _enum = PlatformEnum.CUDA
75 + device_name: str = "cuda"
76 device_type: str = "cuda"
77 dispatch_key: str = "CUDA"
78
vllm/platforms/hpu.py
+1
@@ -12,6 +12,7 @@ else:
12
13 class HpuPlatform(Platform):
14 _enum = PlatformEnum.HPU
15 + device_name: str = "hpu"
16 device_type: str = "hpu"
17 dispatch_key: str = "HPU"
18
vllm/platforms/interface.py
+13
@@ -56,11 +56,13 @@ class DeviceCapability(NamedTuple):
56
57 class Platform:
58 _enum: PlatformEnum
59 + device_name: str
60 device_type: str
61 # available dispatch keys:
62 # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
63 # use "CPU" as a fallback for platforms not registered in PyTorch
64 dispatch_key: str = "CPU"
65 + supported_quantization: list[str] = []
66
67 def is_cuda(self) -> bool:
68 return self._enum == PlatformEnum.CUDA
@@ -171,6 +173,17 @@ class Platform:
173 """
174 pass
175
176 + @classmethod
177 + def verify_quantization(cls, quant: str) -> None:
178 + """
179 + Verify whether the quantization is supported by the current platform.
180 + """
181 + if cls.supported_quantization and \
182 + quant not in cls.supported_quantization:
183 + raise ValueError(
184 + f"{quant} quantization is currently not supported in "
185 + f"{cls.device_name}.")
186 +
187
188 class UnspecifiedPlatform(Platform):
189 _enum = PlatformEnum.UNSPECIFIED
vllm/platforms/neuron.py
+2
@@ -10,7 +10,9 @@ else:
10
11 class NeuronPlatform(Platform):
12 _enum = PlatformEnum.NEURON
13 + device_name: str = "neuron"
14 device_type: str = "neuron"
15 + supported_quantization: list[str] = ["neuron_quant"]
16
17 @classmethod
18 def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/openvino.py
+1
@@ -23,6 +23,7 @@ except ImportError as e:
23
24 class OpenVinoPlatform(Platform):
25 _enum = PlatformEnum.OPENVINO
26 + device_name: str = "openvino"
27 device_type: str = "openvino"
28 dispatch_key: str = "CPU"
29
vllm/platforms/rocm.py
+15
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
4
5 import torch
6
7 +import vllm.envs as envs
8 from vllm.logger import init_logger
9
10 from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
@@ -35,8 +36,13 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
36
37 class RocmPlatform(Platform):
38 _enum = PlatformEnum.ROCM
39 + device_name: str = "rocm"
40 device_type: str = "cuda"
41 dispatch_key: str = "CUDA"
42 + supported_quantization: list[str] = [
43 + "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
44 + "fbgemm_fp8", "gguf"
45 + ]
46
47 @classmethod
48 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
@@ -79,3 +85,12 @@ class RocmPlatform(Platform):
85 "vllm.spec_decode.spec_decode_worker.create_spec_worker"
86 else:
87 parallel_config.worker_cls = "vllm.worker.worker.Worker"
88 +
89 + @classmethod
90 + def verify_quantization(cls, quant: str) -> None:
91 + super().verify_quantization(quant)
92 + if quant == "awq" and not envs.VLLM_USE_TRITON_AWQ:
93 + logger.warning(
94 + "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
95 + " is not set, enabling VLLM_USE_TRITON_AWQ.")
96 + envs.VLLM_USE_TRITON_AWQ = True
vllm/platforms/tpu.py
+2
@@ -16,8 +16,10 @@ logger = init_logger(__name__)
16
17 class TpuPlatform(Platform):
18 _enum = PlatformEnum.TPU
19 + device_name: str = "tpu"
20 device_type: str = "tpu"
21 dispatch_key: str = "XLA"
22 + supported_quantization: list[str] = ["tpu_int8"]
23
24 @classmethod
25 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/xpu.py
+1
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16
17 class XPUPlatform(Platform):
18 _enum = PlatformEnum.XPU
19 + device_name: str = "xpu"
20 device_type: str = "xpu"
21 dispatch_key: str = "XPU"
22