A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 14:30 UTC
Update Flashinfer from `v0.4.1` to `v0.5.2` (#27952)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Nov 7, 2025 at 16:24 UTC
811df41ee90125a028f54261d4b3687dc6787b8d
4 files changed
+11
-13
docker/Dockerfile
+4
-8
@@ -132,9 +132,7 @@ WORKDIR /workspace
132
COPY requirements/common.txt requirements/common.txt
133
COPY requirements/cuda.txt requirements/cuda.txt
134
RUN --mount=type=cache,target=/root/.cache/uv \
135
- # TODO: remove apache-tvm-ffi once FlashInfer is fixed https://github.com/flashinfer-ai/flashinfer/issues/1962
136
- uv pip install --python /opt/venv/bin/python3 --pre apache-tvm-ffi==0.1.0b15 \
137
- && uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
135
+ uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
136
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
137
138
# cuda arch list used by torch
@@ -356,16 +354,14 @@ RUN --mount=type=cache,target=/root/.cache/uv \
354
# Install vllm wheel first, so that torch etc will be installed.
355
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
356
--mount=type=cache,target=/root/.cache/uv \
359
- # TODO: remove apache-tvm-ffi once FlashInfer is fixed https://github.com/flashinfer-ai/flashinfer/issues/1962
360
- uv pip install --system --pre apache-tvm-ffi==0.1.0b15 \
361
- && uv pip install --system dist/*.whl --verbose \
357
+ uv pip install --system dist/*.whl --verbose \
358
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
359
360
# Install FlashInfer pre-compiled kernel cache and binaries
361
# https://docs.flashinfer.ai/installation.html
362
RUN --mount=type=cache,target=/root/.cache/uv \
367
- uv pip install --system flashinfer-cubin==0.4.1 \
368
- && uv pip install --system flashinfer-jit-cache==0.4.1 \
363
+ uv pip install --system flashinfer-cubin==0.5.2 \
364
+ && uv pip install --system flashinfer-jit-cache==0.5.2 \
365
--extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
366
&& flashinfer show-config
367
docker/Dockerfile.nightly_torch
+2
-2
@@ -246,7 +246,7 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
246
247
248
# build flashinfer for torch nightly from source around 10 mins
249
-# release version: v0.4.1
249
+# release version: v0.5.2
250
# todo(elainewy): cache flashinfer build result for faster build
251
ENV CCACHE_DIR=/root/.cache/ccache
252
RUN --mount=type=cache,target=/root/.cache/ccache \
@@ -254,7 +254,7 @@ RUN --mount=type=cache,target=/root/.cache/ccache \
254
echo "git clone flashinfer..." \
255
&& git clone --recursive https://github.com/flashinfer-ai/flashinfer.git \
256
&& cd flashinfer \
257
- && git checkout v0.4.1\
257
+ && git checkout v0.5.2 \
258
&& git submodule update --init --recursive \
259
&& echo "finish git clone flashinfer..." \
260
&& rm -rf build \
requirements/cuda.txt
+1
-1
@@ -12,4 +12,4 @@ torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytor
12
# Build from https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1
13
xformers==0.0.33+5d4b92a5.d20251029; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.9
14
# FlashInfer should be updated together with the Dockerfile
15
-flashinfer-python==0.4.1
15
+flashinfer-python==0.5.2
tests/kernels/attention/test_flashinfer_trtllm_attention.py
+4
-2
@@ -238,9 +238,11 @@ def test_flashinfer_trtllm_decode_with_baseline(
238
if q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP4_DTYPE:
239
rtol, atol = 7e-2, 9e-2
240
elif q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP8_DTYPE:
241
- rtol, atol = 2e-2, 4e-2
241
+ rtol, atol = 3e-2, 4e-2
242
elif q_quant_dtype == FP8_DTYPE and o_quant_dtype == dtype:
243
- rtol, atol = 1e-2, 2e-2
243
+ rtol, atol = 2e-2, 2e-2
244
+ elif kv_quant_dtype == FP8_DTYPE:
245
+ rtol, atol = 4e-2, 6e-2
246
else:
247
rtol, atol = 1e-2, 1e-2
248