A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 00:00 UTC
Bump Flashinfer Version and Re-enable DeepSeek NVFP4 AR+Norm Fusion (#34899)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Wei Zhao committed
Feb 20, 2026 at 16:37 UTC
ea5f903f80fec5afd4960a3846b8a84b0e53ca6e
5 files changed
+6
-29
docker/Dockerfile
+1
-1
@@ -582,7 +582,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
582
# This is ~1.1GB and only changes when FlashInfer version bumps
583
# https://docs.flashinfer.ai/installation.html
584
# From versions.json: .flashinfer.version
585
-ARG FLASHINFER_VERSION=0.6.3
585
+ARG FLASHINFER_VERSION=0.6.4
586
RUN --mount=type=cache,target=/root/.cache/uv \
587
uv pip install --system flashinfer-cubin==${FLASHINFER_VERSION} \
588
&& uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
docker/Dockerfile.nightly_torch
+2
-2
@@ -217,13 +217,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
217
218
219
# build flashinfer for torch nightly from source around 10 mins
220
-# release version: v0.6.3
220
+# release version: v0.6.4
221
# todo(elainewy): cache flashinfer build result for faster build
222
ENV CCACHE_DIR=/root/.cache/ccache
223
RUN --mount=type=cache,target=/root/.cache/ccache \
224
--mount=type=cache,target=/root/.cache/uv \
225
echo "git clone flashinfer..." \
226
- && git clone --depth 1 --branch v0.6.3 --recursive https://github.com/flashinfer-ai/flashinfer.git \
226
+ && git clone --depth 1 --branch v0.6.4 --recursive https://github.com/flashinfer-ai/flashinfer.git \
227
&& cd flashinfer \
228
&& git submodule update --init --recursive \
229
&& echo "finish git clone flashinfer..." \
docker/versions.json
+1
-1
@@ -68,7 +68,7 @@
68
"default": "true"
69
},
70
"FLASHINFER_VERSION": {
71
- "default": "0.6.3"
71
+ "default": "0.6.4"
72
},
73
"GDRCOPY_CUDA_VERSION": {
74
"default": "12.8"
requirements/cuda.txt
+1
-1
@@ -10,4 +10,4 @@ torchaudio==2.10.0
10
# These must be updated alongside torch
11
torchvision==0.25.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
12
# FlashInfer should be updated together with the Dockerfile
13
-flashinfer-python==0.6.3
13
+flashinfer-python==0.6.4
vllm/model_executor/models/config.py
+1
-24
@@ -536,34 +536,12 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
536
)
537
538
539
-class DeepseekV3ForCausalLM(VerifyAndUpdateConfig):
540
- @classmethod
541
- def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
542
- """Disable AR-RMS-Quant fusion for DeepSeekV3 in NVFP4"""
543
- # TODO: https://github.com/vllm-project/vllm/issues/34395
544
-
545
- # disable AR-rms-fp4 fusion for DSv3+
546
- ar_rms_enabled = vllm_config.compilation_config.pass_config.fuse_allreduce_rms
547
- nvfp4 = vllm_config.model_config.is_nvfp4_quantized()
548
-
549
- # Disable by default, warn if manually enabled:
550
- if ar_rms_enabled is None and nvfp4:
551
- vllm_config.compilation_config.pass_config.fuse_allreduce_rms = False
552
- if ar_rms_enabled and nvfp4:
553
- logger.warning(
554
- "Allreduce-rms fusion broken for DeepSeekV3 with NVFP4 quant,"
555
- "see https://github.com/vllm-project/vllm/issues/34395."
556
- )
557
-
558
-
559
-class DeepseekV32ForCausalLM(DeepseekV3ForCausalLM):
539
+class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
540
@classmethod
541
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
542
"""
543
Updated fp8 cache to custom "fp8_ds_mla" format for DeepSeekV32
544
"""
565
- super().verify_and_update_config(vllm_config)
566
-
545
hf_config = vllm_config.model_config.hf_config
546
547
# Mirror the check in vllm/model_executor/models/deepseek_v2.py
@@ -654,7 +632,6 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
632
"MambaForCausalLM": MambaModelConfig,
633
"Mamba2ForCausalLM": MambaModelConfig,
634
"FalconMambaForCausalLM": MambaModelConfig,
657
- "DeepseekV3ForCausalLM": DeepseekV3ForCausalLM,
635
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
636
"NemotronHForCausalLM": NemotronHForCausalLMConfig,
637
"NemotronHPuzzleForCausalLM": NemotronHForCausalLMConfig,