A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 13:00 UTC
[CI/Build] Bump flashinfer to v0.6.11.post2 (#41711)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Artem Perevedentsev committed
May 17, 2026 at 00:55 UTC
0867497368f390212a3f9684e2e05f698f8d1149
6 files changed
+30
-23
docker/Dockerfile
+1
-1
@@ -639,7 +639,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
639
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
640
# https://docs.flashinfer.ai/installation.html
641
# From versions.json: .flashinfer.version
642
-ARG FLASHINFER_VERSION=0.6.8.post1
642
+ARG FLASHINFER_VERSION=0.6.11.post2
643
RUN --mount=type=cache,target=/root/.cache/uv \
644
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
645
--extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
docker/Dockerfile.nightly_torch
+2
-2
@@ -217,13 +217,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
217
218
219
# build flashinfer for torch nightly from source around 10 mins
220
-# release version: v0.6.8.post1
220
+# release version: v0.6.11.post2
221
# todo(elainewy): cache flashinfer build result for faster build
222
ENV CCACHE_DIR=/root/.cache/ccache
223
RUN --mount=type=cache,target=/root/.cache/ccache \
224
--mount=type=cache,target=/root/.cache/uv \
225
echo "git clone flashinfer..." \
226
- && git clone --depth 1 --branch v0.6.8.post1 --recursive https://github.com/flashinfer-ai/flashinfer.git \
226
+ && git clone --depth 1 --branch v0.6.11.post2 --recursive https://github.com/flashinfer-ai/flashinfer.git \
227
&& cd flashinfer \
228
&& git submodule update --init --recursive \
229
&& echo "finish git clone flashinfer..." \
docker/versions.json
+1
-1
@@ -68,7 +68,7 @@
68
"default": "true"
69
},
70
"FLASHINFER_VERSION": {
71
- "default": "0.6.8.post1"
71
+ "default": "0.6.11.post2"
72
},
73
"GDRCOPY_CUDA_VERSION": {
74
"default": "12.8"
requirements/cuda.txt
+2
-2
@@ -9,8 +9,8 @@ torchaudio==2.11.0
9
# These must be updated alongside torch
10
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
11
# FlashInfer should be updated together with the Dockerfile
12
-flashinfer-python==0.6.8.post1
13
-flashinfer-cubin==0.6.8.post1
12
+flashinfer-python==0.6.11.post2
13
+flashinfer-cubin==0.6.11.post2
14
apache-tvm-ffi==0.1.9
15
tilelang==0.1.9
16
# Cap nvidia-cudnn-frontend (transitive dep of flashinfer) due to
tests/kernels/moe/test_cutedsl_moe.py
+6
-2
@@ -453,11 +453,15 @@ def test_flashinfer_cutedsl_moe_masked(
453
)
454
455
# reference
456
- a_fp4, a_scale_interleaved = fp4_quantize(hidden_states, input_global_scale)
456
+ # input_global_scale is per-expert ([num_experts]); fp4_quantize and
457
+ # dequantize_nvfp4_to_dtype are non-grouped APIs that expect [1] or
458
+ # [num_tokens]. Use a single element since all values are uniform here.
459
+ a_global = input_global_scale[:1].contiguous()
460
+ a_fp4, a_scale_interleaved = fp4_quantize(hidden_states, a_global)
461
a_in_dtype = dequantize_nvfp4_to_dtype(
462
a_fp4,
463
a_scale_interleaved,
460
- input_global_scale,
464
+ a_global,
465
dtype=hidden_states.dtype,
466
device=hidden_states.device,
467
block_size=16,
vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
+18
-15
@@ -907,24 +907,27 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
907
else:
908
assert mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16
909
910
- def _interleave_mxfp4_cutlass_sm90(w):
911
- w_shape = w.shape
912
- w_interleaved = w.reshape(w_shape[0], w_shape[1], (w_shape[2] // 4), 4)
913
- w_interleaved = w_interleaved.permute(0, 2, 1, 3)
914
- w_interleaved = w_interleaved.reshape(
915
- w_shape[0], w_shape[2] // 4, w_shape[1] * 4
916
- )
917
- return w_interleaved
918
-
919
- w31_scales = w13_scale_swapped.to(torch.uint8)
920
- w31_scales_interleaved = _interleave_mxfp4_cutlass_sm90(w31_scales)
910
+ from flashinfer.fused_moe import (
911
+ interleave_moe_scales_for_sm90_mixed_gemm,
912
+ interleave_moe_weights_for_sm90_mixed_gemm,
913
+ )
914
922
- w2_scale = w2_weight_scale.data.to(torch.uint8)
923
- w2_scale_interleaved = _interleave_mxfp4_cutlass_sm90(w2_scale)
915
+ w13_weight_interleaved = interleave_moe_weights_for_sm90_mixed_gemm(
916
+ w13_weight_swapped.contiguous(), "fp4"
917
+ )
918
+ w2_weight_interleaved = interleave_moe_weights_for_sm90_mixed_gemm(
919
+ w2_weight.contiguous(), "fp4"
920
+ )
921
+ w31_scales_interleaved = interleave_moe_scales_for_sm90_mixed_gemm(
922
+ w13_scale_swapped.to(torch.uint8)
923
+ )
924
+ w2_scale_interleaved = interleave_moe_scales_for_sm90_mixed_gemm(
925
+ w2_weight_scale.data.to(torch.uint8)
926
+ )
927
928
return (
926
- w13_weight_swapped,
927
- w2_weight,
929
+ w13_weight_interleaved,
930
+ w2_weight_interleaved,
931
w31_scales_interleaved,
932
w2_scale_interleaved,
933
w13_bias_swapped,