[ROCm][Bugfix] Convert ModelOpt FP8 per-channel weights to e4m3fnuz on MI300/MI325 (#47201)
Signed-off-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Micah Williamson committed
Jul 7, 2026 at 01:24 UTC
dd5c299fbe23b68ef8b7b7bd126053e838a7a689
2 files changed
+11
-6
tests/quantization/test_modelopt.py
+5
-4
@@ -249,10 +249,11 @@ def test_modelopt_fp8_pc_pt_checkpoint_setup(default_vllm_config, vllm_runner):
249
assert isinstance(gate_up_proj.quant_method, ModelOptFp8PcPtLinearMethod)
250
assert isinstance(down_proj.quant_method, ModelOptFp8PcPtLinearMethod)
251
252
- assert qkv_proj.weight.dtype == torch.float8_e4m3fn
253
- assert o_proj.weight.dtype == torch.float8_e4m3fn
254
- assert gate_up_proj.weight.dtype == torch.float8_e4m3fn
255
- assert down_proj.weight.dtype == torch.float8_e4m3fn
252
+ fp8_dtype = current_platform.fp8_dtype()
253
+ assert qkv_proj.weight.dtype == fp8_dtype
254
+ assert o_proj.weight.dtype == fp8_dtype
255
+ assert gate_up_proj.weight.dtype == fp8_dtype
256
+ assert down_proj.weight.dtype == fp8_dtype
257
258
# Per-channel scales; activations are dynamically scaled per token.
259
assert hasattr(qkv_proj, "weight_scale")
vllm/model_executor/layers/quantization/modelopt.py
+6
-2
@@ -63,6 +63,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
63
)
64
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
65
process_fp8_input_tensor_strategy_moe,
66
+ process_fp8_weight_channel_strategy,
67
process_fp8_weight_tensor_strategy_moe,
68
)
69
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -604,8 +605,11 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase):
605
)
606
607
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
607
- layer.weight = Parameter(layer.weight.t(), requires_grad=False)
608
- layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
608
+ weight, weight_scale, _ = process_fp8_weight_channel_strategy(
609
+ layer.weight, layer.weight_scale.data
610
+ )
611
+ layer.weight = Parameter(weight.t(), requires_grad=False)
612
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
613
self.fp8_linear.process_weights_after_loading(layer)
614
615
def apply(