A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 00:30 UTC
[Bugfix] Clear error message for FP8 torchao quantization on unsupported GPUs (#36854)
Signed-off-by: haosdent <haosdent@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
haosdent committed
May 23, 2026 at 04:09 UTC
4e597b749144d4b3f0716994b95486ab734b185f
1 file changed
+33
vllm/model_executor/layers/quantization/torchao.py
+33
@@ -99,6 +99,38 @@ else:
99
convert_to_packed_tensor_based_on_current_hardware = lambda t: t
100
101
102
+def _check_torchao_fp8_activation_capability(torchao_config) -> None:
103
+ """Check if the current GPU supports FP8 activation quantization.
104
+
105
+ FP8 activation configs (e.g., Float8DynamicActivationFloat8WeightConfig)
106
+ require GPU compute capability >= 8.9 (Ada Lovelace / Hopper) on NVIDIA,
107
+ or MI300+ on AMD. This check provides a clear error message before
108
+ torchao's internal assertion fires with a confusing message.
109
+ """
110
+ config_name = type(torchao_config).__name__
111
+ if "Float8" not in config_name or "Activation" not in config_name:
112
+ return
113
+
114
+ from vllm.platforms import current_platform
115
+
116
+ if current_platform.supports_fp8():
117
+ return
118
+
119
+ capability = current_platform.get_device_capability()
120
+ capability_str = (
121
+ f" (current GPU compute capability: {capability.major}.{capability.minor})"
122
+ if capability is not None
123
+ else ""
124
+ )
125
+ raise ValueError(
126
+ f"torchao FP8 activation quantization config '{config_name}' "
127
+ f"requires GPU compute capability >= 8.9 (e.g., NVIDIA Ada Lovelace "
128
+ f"/ Hopper or AMD MI300+){capability_str}. "
129
+ f"For older GPUs, consider using a non-FP8 config such as "
130
+ f"Int8WeightOnlyConfig or Int4WeightOnlyConfig."
131
+ )
132
+
133
+
134
class TorchAOConfig(QuantizationConfig):
135
"""Config class for torchao."""
136
@@ -269,6 +301,7 @@ def torchao_quantize_param_data(
301
from torchao.quantization import quantize_
302
303
assert isinstance(torchao_config, AOBaseConfig), f"{torchao_config}"
304
+ _check_torchao_fp8_activation_capability(torchao_config)
305
"""
306
Avoid real weight allocation for faster load, since we will
307
end up setting it to param.