A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 11:00 UTC
[BugFix] Fix `plan` API Mismatch when using latest FlashInfer (#29426)
Signed-off-by: Andrii Skliar <askliar@askliar-mlt.client.nvidia.com> Co-authored-by: Andrii Skliar <askliar@askliar-mlt.client.nvidia.com>
Andrii Skliar committed
Nov 27, 2025 at 20:34 UTC
a5345bf49df74cd394a07797649f51cd67c6c697
3 files changed
+5
-4
docker/Dockerfile
+2
-2
@@ -398,8 +398,8 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
398
# Install FlashInfer pre-compiled kernel cache and binaries
399
# https://docs.flashinfer.ai/installation.html
400
RUN --mount=type=cache,target=/root/.cache/uv \
401
- uv pip install --system flashinfer-cubin==0.5.2 \
402
- && uv pip install --system flashinfer-jit-cache==0.5.2 \
401
+ uv pip install --system flashinfer-cubin==0.5.3 \
402
+ && uv pip install --system flashinfer-jit-cache==0.5.3 \
403
--extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
404
&& flashinfer show-config
405
requirements/cuda.txt
+1
-1
@@ -10,4 +10,4 @@ torchaudio==2.9.0
10
# These must be updated alongside torch
11
torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
12
# FlashInfer should be updated together with the Dockerfile
13
-flashinfer-python==0.5.2
13
+flashinfer-python==0.5.3
vllm/v1/attention/backends/flashinfer.py
+2
-1
@@ -1508,7 +1508,7 @@ def fast_plan_decode(
1508
qo_indptr_host = _get_range_buf(batch_size + 1, "cpu")
1509
1510
try:
1511
- # Make sure we pass exactly 18 arguments for tensor core version
1511
+ # Make sure we pass exactly 19 arguments for tensor core version
1512
self._plan_info = self._cached_module.plan(
1513
self._float_workspace_buffer,
1514
self._int_workspace_buffer,
@@ -1528,6 +1528,7 @@ def fast_plan_decode(
1528
window_left,
1529
fixed_split_size,
1530
disable_split_kv,
1531
+ 0,
1532
)
1533
except Exception as e:
1534
raise RuntimeError(f"Error in tensor core plan: {e}") from e