fix: NVFP4 quantization out_dtype should match model dtype, not torch default (#48861)
Signed-off-by: Chris Fontes <2224082+fattchris@users.noreply.github.com> Co-authored-by: Chris Fontes <chris@fontes.io> Co-authored-by: Chris Fontes <chris@fontes.dev> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Chris Fontes committed
Aug 3, 2026 at 16:47 UTC
0b37d8389f4b8378adab0d3dfa1beffbb152e303
1 file changed
+3
-3
vllm/model_executor/layers/quantization/modelopt.py
+3
-3
@@ -450,7 +450,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
450
451
def __init__(self, quant_config: ModelOptFp8Config) -> None:
452
self.quant_config = quant_config
453
- self.out_dtype = torch.get_default_dtype()
453
+ self.out_dtype = get_current_vllm_config().model_config.dtype
454
self.input_dtype = get_current_vllm_config().model_config.dtype
455
456
def create_weights(
@@ -543,7 +543,7 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase):
543
544
def __init__(self, quant_config: ModelOptFp8Config) -> None:
545
self.quant_config = quant_config
546
- self.out_dtype = torch.get_default_dtype()
546
+ self.out_dtype = get_current_vllm_config().model_config.dtype
547
self.input_dtype = get_current_vllm_config().model_config.dtype
548
549
def create_weights(
@@ -640,7 +640,7 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
640
static=True, group_shape=GroupShape(block_n, block_k)
641
)
642
643
- self.out_dtype = torch.get_default_dtype()
643
+ self.out_dtype = get_current_vllm_config().model_config.dtype
644
self.input_dtype = get_current_vllm_config().model_config.dtype
645
646
def create_weights(