A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Explicitly explain quant method override ordering and ensure all overrides are ordered (#17256)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 28, 2025 at 17:55 UTC
c7941cca18722ab28f8806eb6b81d33f0d15524b
2 files changed
+39
-9
vllm/config.py
+34
-2
@@ -28,6 +28,7 @@ import vllm.envs as envs
28
from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
29
from vllm.logger import init_logger
30
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
31
+ QuantizationMethods,
32
get_quantization_config)
33
from vllm.model_executor.models import ModelRegistry
34
from vllm.platforms import CpuArchEnum, current_platform
@@ -767,12 +768,43 @@ class ModelConfig:
768
"compressed-tensors")
769
quant_cfg["quant_method"] = quant_method
770
771
+ # Quantization methods which are overrides (i.e. they have a
772
+ # `override_quantization_method` method) must be checked in order
773
+ # of preference (this is particularly important for GPTQ).
774
+ overrides = [
775
+ "marlin",
776
+ "bitblas",
777
+ "gptq_marlin_24",
778
+ "gptq_marlin",
779
+ "gptq_bitblas",
780
+ "awq_marlin",
781
+ "ipex",
782
+ "moe_wna16",
783
+ ]
784
+ quantization_methods = [
785
+ q for q in supported_quantization if q not in overrides
786
+ ]
787
+ # Any custom overrides will be in quantization_methods so we place
788
+ # them at the start of the list so custom overrides have preference
789
+ # over the built in ones.
790
+ quantization_methods = quantization_methods + overrides
791
+
792
# Detect which checkpoint is it
771
- for name in QUANTIZATION_METHODS:
793
+ for name in quantization_methods:
794
method = get_quantization_config(name)
795
quantization_override = method.override_quantization_method(
796
quant_cfg, self.quantization)
775
- if quantization_override:
797
+ if quantization_override is not None:
798
+ # Raise error if the override is not custom (custom would
799
+ # be in QUANTIZATION_METHODS but not QuantizationMethods)
800
+ # and hasn't been added to the overrides list.
801
+ if (name in get_args(QuantizationMethods)
802
+ and name not in overrides):
803
+ raise ValueError(
804
+ f"Quantization method {name} is an override but "
805
+ "is has not been added to the `overrides` list "
806
+ "above. This is necessary to ensure that the "
807
+ "overrides are checked in order of preference.")
808
quant_method = quantization_override
809
self.quantization = quantization_override
810
break
vllm/model_executor/layers/quantization/__init__.py
+5
-7
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import Dict, List, Type
3
+from typing import Literal, Type, get_args
4
5
from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
7
8
-QUANTIZATION_METHODS: List[str] = [
8
+QuantizationMethods = Literal[
9
"aqlm",
10
"awq",
11
"deepspeedfp",
@@ -15,8 +15,6 @@ QUANTIZATION_METHODS: List[str] = [
15
"fbgemm_fp8",
16
"modelopt",
17
"nvfp4",
18
- # The order of gptq methods is important for config.py iteration over
19
- # override_quantization_method(..)
18
"marlin",
19
"bitblas",
20
"gguf",
@@ -36,6 +34,7 @@ QUANTIZATION_METHODS: List[str] = [
34
"moe_wna16",
35
"torchao",
36
]
37
+QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods))
38
39
# The customized quantization methods which will be added to this dict.
40
_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {}
@@ -111,7 +110,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
110
from .torchao import TorchAOConfig
111
from .tpu_int8 import Int8TpuConfig
112
114
- method_to_config: Dict[str, Type[QuantizationConfig]] = {
113
+ method_to_config: dict[str, Type[QuantizationConfig]] = {
114
"aqlm": AQLMConfig,
115
"awq": AWQConfig,
116
"deepspeedfp": DeepSpeedFPConfig,
@@ -120,8 +119,6 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
119
"fbgemm_fp8": FBGEMMFp8Config,
120
"modelopt": ModelOptFp8Config,
121
"nvfp4": ModelOptNvFp4Config,
123
- # The order of gptq methods is important for config.py iteration over
124
- # override_quantization_method(..)
122
"marlin": MarlinConfig,
123
"bitblas": BitBLASConfig,
124
"gguf": GGUFConfig,
@@ -150,6 +147,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
147
148
__all__ = [
149
"QuantizationConfig",
150
+ "QuantizationMethods",
151
"get_quantization_config",
152
"QUANTIZATION_METHODS",
153
]
\ No newline at end of file