A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 06:15 UTC
[torch.compile] Disable ar-rms fusion for ds3-fp4 & DP, fix CI test (#34392)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Luka Govedič committed
Feb 15, 2026 at 09:33 UTC
23d825aba11afcc6713e9b11acb54c473a734501
3 files changed
+46
-3
vllm/config/model.py
+14
@@ -1687,6 +1687,20 @@ class ModelConfig:
1687
def is_quantized(self) -> bool:
1688
return getattr(self.hf_config, "quantization_config", None) is not None
1689
1690
+ def is_nvfp4_quantized(self) -> bool:
1691
+ # ModelOpt NVFP4 checkpoints resolve to modelopt_fp4 quantization method
1692
+ if self.quantization in ("modelopt_fp4",):
1693
+ return True
1694
+
1695
+ # For Compressed Tensors we look for `"format": "nvfp4-pack-quantized"`
1696
+ # in the quantization config
1697
+ quant_config = self.model_arch_config.quantization_config
1698
+ return (
1699
+ self.quantization == "compressed-tensors"
1700
+ and quant_config is not None
1701
+ and "nvfp4" in quant_config.get("format", "").lower()
1702
+ )
1703
+
1704
1705
def get_served_model_name(model: str, served_model_name: str | list[str] | None):
1706
"""
vllm/config/vllm.py
+8
-2
@@ -103,15 +103,21 @@ def enable_act_fusion(cfg: "VllmConfig") -> bool:
103
104
105
def enable_allreduce_rms_fusion(cfg: "VllmConfig") -> bool:
106
- """Enable if TP > 1 and Hopper+ and flashinfer installed."""
106
+ """Enable if TP > 1 and Hopper/Blackwell and flashinfer installed."""
107
from vllm.platforms import current_platform
108
from vllm.utils.flashinfer import has_flashinfer
109
110
return (
111
cfg.parallel_config.tensor_parallel_size > 1
112
and current_platform.is_cuda()
113
- and current_platform.has_device_capability(90)
113
and has_flashinfer()
114
+ and (
115
+ current_platform.is_device_capability(100)
116
+ or current_platform.is_device_capability(90)
117
+ )
118
+ # tp-dp combination broken:
119
+ # https://github.com/vllm-project/vllm/issues/34458
120
+ and cfg.parallel_config.data_parallel_size == 1
121
)
122
123
vllm/model_executor/models/config.py
+24
-1
@@ -536,12 +536,34 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
536
)
537
538
539
-class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
539
+class DeepseekV3ForCausalLM(VerifyAndUpdateConfig):
540
+ @classmethod
541
+ def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
542
+ """Disable AR-RMS-Quant fusion for DeepSeekV3 in NVFP4"""
543
+ # TODO: https://github.com/vllm-project/vllm/issues/34395
544
+
545
+ # disable AR-rms-fp4 fusion for DSv3+
546
+ ar_rms_enabled = vllm_config.compilation_config.pass_config.fuse_allreduce_rms
547
+ nvfp4 = vllm_config.model_config.is_nvfp4_quantized()
548
+
549
+ # Disable by default, warn if manually enabled:
550
+ if ar_rms_enabled is None and nvfp4:
551
+ vllm_config.compilation_config.pass_config.fuse_allreduce_rms = False
552
+ if ar_rms_enabled and nvfp4:
553
+ logger.warning(
554
+ "Allreduce-rms fusion broken for DeepSeekV3 with NVFP4 quant,"
555
+ "see https://github.com/vllm-project/vllm/issues/34395."
556
+ )
557
+
558
+
559
+class DeepseekV32ForCausalLM(DeepseekV3ForCausalLM):
560
@classmethod
561
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
562
"""
563
Updated fp8 cache to custom "fp8_ds_mla" format for DeepSeekV32
564
"""
565
+ super().verify_and_update_config(vllm_config)
566
+
567
hf_config = vllm_config.model_config.hf_config
568
569
# Mirror the check in vllm/model_executor/models/deepseek_v2.py
@@ -632,6 +654,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
654
"MambaForCausalLM": MambaModelConfig,
655
"Mamba2ForCausalLM": MambaModelConfig,
656
"FalconMambaForCausalLM": MambaModelConfig,
657
+ "DeepseekV3ForCausalLM": DeepseekV3ForCausalLM,
658
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
659
"NemotronHForCausalLM": NemotronHForCausalLMConfig,
660
"NemotronHPuzzleForCausalLM": NemotronHForCausalLMConfig,