A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Hardware][AMD][CI][Bugfix] Fix AMD Quantization test group (#31713)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Matt committed
Jan 11, 2026 at 01:19 UTC
bde57ab2edb67533158a762823fba31e32f71a85
12 files changed
+114
-52
.buildkite/test-amd.yaml
+1
-1
@@ -731,7 +731,7 @@ steps:
731
732
- label: Quantization Test # 70min
733
timeout_in_minutes: 90
734
- mirror_hardwares: [amdexperimental]
734
+ mirror_hardwares: [amdexperimental, amdproduction]
735
agent_pool: mi325_1
736
# grade: Blocking
737
source_file_dependencies:
tests/quantization/test_compressed_tensors.py
+7
-1
@@ -644,6 +644,9 @@ def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):
644
assert output
645
646
647
+@pytest.mark.skipif(
648
+ not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
649
+)
650
@pytest.mark.parametrize(
651
"args",
652
[
@@ -762,7 +765,10 @@ def test_compressed_tensors_fp8_block_enabled(vllm_runner):
765
766
input_quant_op = qkv_proj.scheme.w8a8_block_fp8_linear.input_quant_op
767
assert isinstance(input_quant_op, QuantFP8)
765
- assert input_quant_op._forward_method == input_quant_op.forward_cuda
768
+ assert input_quant_op._forward_method in (
769
+ input_quant_op.forward_cuda,
770
+ input_quant_op.forward_hip,
771
+ )
772
773
llm.apply_model(check_model)
774
tests/quantization/test_configs.py
+31
-6
@@ -10,6 +10,7 @@ from dataclasses import dataclass
10
import pytest
11
12
from vllm.config import ModelConfig
13
+from vllm.platforms import current_platform
14
15
16
@dataclass
@@ -23,20 +24,44 @@ MODEL_ARG_EXPTYPES = [
24
# AUTOGPTQ
25
# compat: autogptq <=0.7.1 is_marlin_format: bool
26
# Model Serialized in Exllama Format.
26
- ("TheBloke/Llama-2-7B-Chat-GPTQ", None, "gptq_marlin"),
27
- ("TheBloke/Llama-2-7B-Chat-GPTQ", "marlin", "gptq_marlin"),
27
+ (
28
+ "TheBloke/Llama-2-7B-Chat-GPTQ",
29
+ None,
30
+ "gptq_marlin" if current_platform.is_cuda() else "gptq",
31
+ ),
32
+ (
33
+ "TheBloke/Llama-2-7B-Chat-GPTQ",
34
+ "marlin",
35
+ "gptq_marlin" if current_platform.is_cuda() else "ERROR",
36
+ ),
37
("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "gptq"),
38
("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
39
# compat: autogptq >=0.8.0 use checkpoint_format: str
40
# Model Serialized in Exllama Format.
32
- ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "gptq_marlin"),
33
- ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "marlin", "gptq_marlin"),
41
+ (
42
+ "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
43
+ None,
44
+ "gptq_marlin" if current_platform.is_cuda() else "gptq",
45
+ ),
46
+ (
47
+ "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
48
+ "marlin",
49
+ "gptq_marlin" if current_platform.is_cuda() else "ERROR",
50
+ ),
51
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "gptq"),
52
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
53
# AUTOAWQ
37
- ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, "awq_marlin"),
54
+ (
55
+ "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
56
+ None,
57
+ "awq_marlin" if current_platform.is_cuda() else "awq",
58
+ ),
59
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
39
- ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", "awq_marlin"),
60
+ (
61
+ "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
62
+ "marlin",
63
+ "awq_marlin" if current_platform.is_cuda() else "ERROR",
64
+ ),
65
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
66
]
67
tests/quantization/test_cpu_offload.py
+1
-1
@@ -66,7 +66,7 @@ def test_cpu_offload_compressed_tensors(monkeypatch):
66
monkeypatch.setenv("VLLM_TEST_FORCE_LOAD_FORMAT", "auto")
67
# Test wNa16
68
compare_two_settings(
69
- "nm-testing/tinyllama-oneshot-w4a16-channel-v2",
69
+ "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16",
70
["--enforce_eager"],
71
["--enforce_eager", "--cpu-offload-gb", "1"],
72
max_wait_seconds=480,
tests/quantization/test_fp8.py
+12
-4
@@ -36,7 +36,9 @@ MODELS = [
36
reason="FP8 is not supported on this GPU type.",
37
)
38
@pytest.mark.parametrize("model_id", MODELS)
39
-@pytest.mark.parametrize("force_marlin", [False, True])
39
+@pytest.mark.parametrize(
40
+ "force_marlin", [False] if current_platform.is_rocm() else [False, True]
41
+)
42
@pytest.mark.parametrize(
43
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
44
)
@@ -125,7 +127,9 @@ def test_kv_cache_model_load_and_run(
127
reason="FP8 is not supported on this GPU type.",
128
)
129
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
128
-@pytest.mark.parametrize("force_marlin", [False, True])
130
+@pytest.mark.parametrize(
131
+ "force_marlin", [False] if current_platform.is_rocm() else [False, True]
132
+)
133
@pytest.mark.parametrize(
134
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
135
)
@@ -197,10 +201,10 @@ def test_scaled_fp8_quant(dtype) -> None:
201
def quantize_ref(tensor, inv_scale):
202
# The reference implementation that fully aligns to
203
# the kernel being tested.
200
- finfo = torch.finfo(torch.float8_e4m3fn)
204
+ finfo = torch.finfo(current_platform.fp8_dtype())
205
scale = inv_scale.reciprocal()
206
qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max)
203
- qweight = qweight.to(torch.float8_e4m3fn)
207
+ qweight = qweight.to(current_platform.fp8_dtype())
208
return qweight
209
210
def per_tensor_dequantize(tensor, inv_scale, dtype):
@@ -267,6 +271,10 @@ def test_scaled_fp8_quant(dtype) -> None:
271
)
272
273
274
+@pytest.mark.skipif(
275
+ current_platform.is_fp8_fnuz(),
276
+ reason="FP8 e4m3fn weight reloading is not supported on e4m3fnuz platforms",
277
+)
278
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
279
# FP8 weight reloading does not support online quantization
280
@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False
tests/quantization/test_gptq_dynamic.py
+5
-1
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinLinear
14
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
15
get_dynamic_override,
16
)
17
+from vllm.platforms import current_platform
18
19
PROMPT = "On the surface of Mars, we found"
20
@@ -21,7 +22,10 @@ PROMPT = "On the surface of Mars, we found"
22
# The second layer is quantized using bits=8, group_size=32
23
# All other layers (layer index >= 2) are not quantized
24
MODEL_QUANT = [
24
- ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue", True),
25
+ (
26
+ "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
27
+ current_platform.is_cuda(),
28
+ ),
29
(
30
"ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
31
False,
tests/quantization/test_ptpc_fp8.py
+12
-25
@@ -6,18 +6,12 @@ Run `pytest tests/quantization/test_ptpc_fp8.py --forked`.
6
"""
7
8
import pytest
9
-import torch
9
10
from tests.quantization.utils import is_quant_method_supported
11
from vllm.model_executor.layers.quantization.fp8 import Fp8KVCacheMethod
12
from vllm.model_executor.layers.quantization.ptpc_fp8 import PTPCFp8LinearMethod
13
from vllm.platforms import current_platform
14
16
-UNSUPPORTED_STR = (
17
- "Currently torch._scaled_mm (hipBLASLt) rowwise gemm only "
18
- "support output dtype of bfloat16. torch.float16 is specified."
19
-)
20
-
15
16
@pytest.fixture(scope="function", autouse=True)
17
def enable_pickle(monkeypatch):
@@ -30,24 +24,17 @@ def enable_pickle(monkeypatch):
24
reason="PTPC FP8 is not supported on this GPU type.",
25
)
26
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test is for ROCm GPU.")
33
-@pytest.mark.parametrize("dtype", ["auto", "bfloat16", "float16"])
34
-@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
27
+@pytest.mark.parametrize("dtype", ["bfloat16"])
28
+@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
29
def test_ptpc_fp8_rocm(vllm_runner, dtype: str, kv_cache_dtype: str) -> None:
36
- try:
37
- llm = vllm_runner(
38
- "facebook/opt-125m",
39
- dtype=dtype,
40
- quantization="ptpc_fp8",
41
- enforce_eager=True,
42
- kv_cache_dtype=kv_cache_dtype,
43
- )
44
- except AssertionError as e:
45
- if str(e) == UNSUPPORTED_STR:
46
- # If the error message matches, the test passes
47
- return
48
- else:
49
- # If the error message does not match, re-raise the exception
50
- raise
30
+ llm = vllm_runner(
31
+ "facebook/opt-125m",
32
+ dtype=dtype,
33
+ quantization="ptpc_fp8",
34
+ enforce_eager=True,
35
+ kv_cache_dtype=kv_cache_dtype,
36
+ allow_deprecated_quantization=True,
37
+ )
38
39
with llm:
40
@@ -60,9 +47,9 @@ def test_ptpc_fp8_rocm(vllm_runner, dtype: str, kv_cache_dtype: str) -> None:
47
assert attn._k_scale == 1.0
48
assert attn._v_scale == 1.0
49
50
+ # For GPUs with hardware support, we keep weights in fp8
51
if current_platform.has_device_capability(94):
64
- # For GPUs with hardware support, we keep weights in fp8
65
- assert fc1.weight.dtype == torch.float8_e4m3fnuz
52
+ assert fc1.weight.dtype == current_platform.fp8_dtype()
53
54
llm.apply_model(check_model)
55
tests/quantization/utils.py
+5
@@ -10,6 +10,11 @@ def is_quant_method_supported(quant_method: str) -> bool:
10
if not (current_platform.is_cuda() or current_platform.is_rocm()):
11
return False
12
13
+ try:
14
+ current_platform.verify_quantization(quant_method)
15
+ except ValueError:
16
+ return False
17
+
18
capability = current_platform.get_device_capability()
19
assert capability is not None
20
vllm/model_executor/layers/quantization/__init__.py
+4
@@ -5,6 +5,7 @@ from typing import Literal, get_args
5
6
from vllm.logger import init_logger
7
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
8
+from vllm.platforms import current_platform
9
10
logger = init_logger(__name__)
11
@@ -98,6 +99,9 @@ def register_quantization_config(quantization: str):
99
)
100
else:
101
QUANTIZATION_METHODS.append(quantization)
102
+ # Automatically assume the custom quantization config is supported
103
+ if sq := current_platform.supported_quantization:
104
+ sq.append(quantization)
105
106
if not issubclass(quant_config_cls, QuantizationConfig):
107
raise ValueError(
vllm/model_executor/layers/quantization/kernels/scaled_mm/triton.py
+17
@@ -9,6 +9,9 @@ from vllm.model_executor.layers.quantization.compressed_tensors.triton_scaled_mm
9
triton_scaled_mm,
10
)
11
from vllm.model_executor.layers.quantization.utils import replace_parameter
12
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
13
+ convert_to_channelwise,
14
+)
15
from vllm.platforms import current_platform
16
17
from .ScaledMMLinearKernel import ScaledMMLinearKernel, ScaledMMLinearLayerConfig
@@ -37,6 +40,20 @@ class TritonScaledMMLinearKernel(ScaledMMLinearKernel):
40
torch.nn.Parameter(weight.t().data, requires_grad=False),
41
)
42
43
+ # WEIGHT SCALE
44
+ # Triton kernel supports only per-tensor and per-channel.
45
+ # If we have a fused module (QKV, MLP) with per tensor scales (thus N
46
+ # scales being passed to the kernel), convert to the per-channel case.
47
+ is_fused_module = len(layer.logical_widths) > 1
48
+ weight_scale = getattr(layer, self.w_s_name)
49
+ if is_fused_module and not self.config.is_channelwise:
50
+ weight_scale = convert_to_channelwise(weight_scale, layer.logical_widths)
51
+ replace_parameter(
52
+ layer,
53
+ self.w_s_name,
54
+ torch.nn.Parameter(weight_scale.data, requires_grad=False),
55
+ )
56
+
57
# INPUT SCALE
58
if self.config.is_static_input_scheme:
59
input_scale = getattr(layer, self.i_s_name)
vllm/model_executor/layers/quantization/ptpc_fp8.py
+17
-13
@@ -103,21 +103,25 @@ class PTPCFp8LinearMethod(Fp8LinearMethod):
103
)
104
105
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
106
- layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
107
-
108
- assert layer.weight.data.dtype == torch.bfloat16, (
109
- f"Currently torch._scaled_mm (hipBLASLt) rowwise gemm only support output dtype of bfloat16. {str(layer.weight.data.dtype)} is specified." # noqa: E501
110
- )
111
- # Quantize the weights.
112
- qweight, weight_scale = ops.scaled_fp8_quant(
113
- layer.weight, scale=None, use_per_token_if_dynamic=True
106
+ assert layer.weight.data.dtype not in (torch.float16, torch.float32), (
107
+ "Currently torch._scaled_mm (hipBLASLt) rowwise gemm only support "
108
+ f"output dtype of bfloat16. {layer.weight.data.dtype} is specified."
109
)
110
116
- # Update the layer with the new values.
117
- layer.weight = Parameter(
118
- qweight.t(), requires_grad=False
119
- ) # Pretranspose the weight
120
- layer.weight_scale = Parameter(weight_scale, requires_grad=False)
111
+ if layer.weight.data.dtype == torch.bfloat16:
112
+ # Quantize the weights.
113
+ qweight, weight_scale = ops.scaled_fp8_quant(
114
+ layer.weight, scale=None, use_per_token_if_dynamic=True
115
+ )
116
+
117
+ # Update the layer with the new values.
118
+ layer.weight = Parameter(
119
+ qweight.t(), requires_grad=False
120
+ ) # Pretranspose the weight
121
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
122
+ else:
123
+ assert layer.weight.data.dtype == current_platform.fp8_dtype()
124
+ assert getattr(layer, "weight_scale", None) is not None
125
layer.input_scale = None
126
127
def apply(
vllm/platforms/rocm.py
+2
@@ -170,7 +170,9 @@ class RocmPlatform(Platform):
170
171
supported_quantization: list[str] = [
172
"awq",
173
+ "awq_marlin", # will be overwritten with awq
174
"gptq",
175
+ "gptq_marlin", # will be overwritten with gptq
176
"fp8",
177
"compressed-tensors",
178
"fbgemm_fp8",