A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[CI][Bugfix][AMD][ Ensure weights created when using emulating OCP MXFP4 (#36993)
Signed-off-by: Randall Smith <Randall.Smith@amd.com>
rasmith committed
Apr 7, 2026 at 11:37 UTC
83d09d36b5951a8de5205438d0742768ad191c4d
1 file changed
+13
-7
vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py
+13
-7
@@ -267,20 +267,26 @@ class QuarkOCP_MX(QuarkScheme):
267
def get_min_capability(cls) -> int:
268
return 70
269
270
+ def process_dynamic_mxfp4_weights_after_loading(
271
+ self, layer: torch.nn.Module
272
+ ) -> None:
273
+ w_q, w_s = dynamic_mxfp4_quant(layer.weight)
274
+ layer.weight_scale = torch.nn.Parameter(w_s.T.contiguous(), requires_grad=False)
275
+ layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
276
+
277
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
278
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
279
280
if self.emulate:
274
- layer.weight_scale = torch.nn.Parameter(
275
- layer.weight_scale.data, requires_grad=False
276
- )
277
- else:
281
if self.dynamic_mxfp4_quant:
279
- w_q, w_s = dynamic_mxfp4_quant(layer.weight)
282
+ self.process_dynamic_mxfp4_weights_after_loading(layer)
283
+ else:
284
layer.weight_scale = torch.nn.Parameter(
281
- w_s.T.contiguous(), requires_grad=False
285
+ layer.weight_scale.data, requires_grad=False
286
)
283
- layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
287
+ else:
288
+ if self.dynamic_mxfp4_quant:
289
+ self.process_dynamic_mxfp4_weights_after_loading(layer)
290
elif self.rocm_use_aiter_fp4_asm_gemm:
291
# shuffle weight scale
292
weight_scale_shuffle = layer.weight_scale.data