A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 11:30 UTC
[Quantization] Deprecate Long Tail of Schemes (#31688)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Robert Shaw committed
Jan 8, 2026 at 19:07 UTC
5825bbc1f792d3fe041f845148a9e9ad837eef30
8 files changed
+61
-5
tests/compile/fullgraph/test_full_graph.py
+4
-1
@@ -62,7 +62,10 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
62
TEST_MODELS.append(
63
(
64
"alexm-nm/tinyllama-24-marlin24-4bit-g128",
65
- {"quantization": "gptq_marlin_24"},
65
+ {
66
+ "quantization": "gptq_marlin_24",
67
+ "allow_deprecated_quantization": True,
68
+ },
69
)
70
)
71
tests/models/quantization/test_gptq_marlin_24.py
+4
-1
@@ -63,7 +63,10 @@ def test_models(
63
num_logprobs: int,
64
) -> None:
65
with vllm_runner(
66
- model_pair.model_marlin, dtype=dtype, quantization="gptq_marlin_24"
66
+ model_pair.model_marlin,
67
+ dtype=dtype,
68
+ quantization="gptq_marlin_24",
69
+ allow_deprecated_quantization=True,
70
) as marlin_24_model:
71
marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
72
example_prompts, max_tokens, num_logprobs
tests/quantization/test_auto_round.py
+3
-1
@@ -26,7 +26,9 @@ MODELS = [
26
)
27
@pytest.mark.parametrize("model", MODELS)
28
def test_auto_round(vllm_runner, model):
29
- with vllm_runner(model, enforce_eager=True) as llm:
29
+ with vllm_runner(
30
+ model, enforce_eager=True, allow_deprecated_quantization=True
31
+ ) as llm:
32
output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
33
assert output
34
print(f"{output[0][1]}")
tests/quantization/test_experts_int8.py
+5
-1
@@ -34,6 +34,10 @@ def test_model_experts_int8_startup(
34
model_info.check_transformers_version(on_fail="skip")
35
36
with vllm_runner(
37
- model, dtype=dtype, enforce_eager=True, quantization="experts_int8"
37
+ model,
38
+ dtype=dtype,
39
+ enforce_eager=True,
40
+ quantization="experts_int8",
41
+ allow_deprecated_quantization=True,
42
) as vllm_model:
43
vllm_model.generate_greedy(example_prompts, max_tokens)
tests/quantization/test_rtn.py
+5
-1
@@ -30,6 +30,10 @@ def test_model_rtn_startup(
30
max_tokens: int,
31
) -> None:
32
with vllm_runner(
33
- model, enforce_eager=True, dtype=dtype, quantization="rtn"
33
+ model,
34
+ enforce_eager=True,
35
+ dtype=dtype,
36
+ quantization="rtn",
37
+ allow_deprecated_quantization=True,
38
) as vllm_model:
39
vllm_model.generate_greedy(example_prompts, max_tokens)
vllm/config/model.py
+17
@@ -191,6 +191,8 @@ class ModelConfig:
191
`quantization_config` attribute in the model config file. If that is
192
`None`, we assume the model weights are not quantized and use `dtype` to
193
determine the data type of the weights."""
194
+ allow_deprecated_quantization: bool = False
195
+ """Whether to allow deprecated quantization methods."""
196
enforce_eager: bool = False
197
"""Whether to always use eager-mode PyTorch. If True, we will disable CUDA
198
graph and always execute the model in eager mode. If False, we will use
@@ -940,6 +942,21 @@ class ModelConfig:
942
943
current_platform.verify_quantization(self.quantization)
944
945
+ if self.quantization in me_quant.DEPRECATED_QUANTIZATION_METHODS:
946
+ if self.allow_deprecated_quantization:
947
+ logger.warning(
948
+ "The quantization method %s is deprecated "
949
+ "and will be removed in future versions of vLLM.",
950
+ self.quantization,
951
+ )
952
+ else:
953
+ raise ValueError(
954
+ "The quantization method %s is deprecated "
955
+ "and will be removed in future versions of vLLM. To bypass, "
956
+ "set `--allow-deprecated-quantization`.",
957
+ self.quantization,
958
+ )
959
+
960
def _verify_cuda_graph(self) -> None:
961
# CUDAGraph capture not supported for encoder-decoder models on ROCm
962
unsupported_rocm = self.is_encoder_decoder
vllm/engine/arg_utils.py
+6
@@ -451,6 +451,7 @@ class EngineArgs:
451
hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
452
tokenizer_revision: str | None = ModelConfig.tokenizer_revision
453
quantization: QuantizationMethods | None = ModelConfig.quantization
454
+ allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
455
enforce_eager: bool = ModelConfig.enforce_eager
456
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
457
limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
@@ -648,6 +649,10 @@ class EngineArgs:
649
)
650
model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
651
model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
652
+ model_group.add_argument(
653
+ "--allow-deprecated-quantization",
654
+ **model_kwargs["allow_deprecated_quantization"],
655
+ )
656
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
657
model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
658
model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
@@ -1225,6 +1230,7 @@ class EngineArgs:
1230
tokenizer_revision=self.tokenizer_revision,
1231
max_model_len=self.max_model_len,
1232
quantization=self.quantization,
1233
+ allow_deprecated_quantization=self.allow_deprecated_quantization,
1234
enforce_eager=self.enforce_eager,
1235
max_logprobs=self.max_logprobs,
1236
logprobs_mode=self.logprobs_mode,
vllm/model_executor/layers/quantization/__init__.py
+17
@@ -41,6 +41,23 @@ QuantizationMethods = Literal[
41
]
42
QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods))
43
44
+DEPRECATED_QUANTIZATION_METHODS = [
45
+ "deepspeedfp",
46
+ "tpu_int8",
47
+ "ptpc_fp8",
48
+ "fbgemm_fp8",
49
+ "fp_quant",
50
+ "bitblas",
51
+ "gptq_marlin_24",
52
+ "gptq_bitblas",
53
+ "hqq",
54
+ "experts_int8",
55
+ "ipex",
56
+ "auto-round",
57
+ "rtn",
58
+ "petit_nvfp4",
59
+]
60
+
61
# The customized quantization methods which will be added to this dict.
62
_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {}
63