A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
Add `flashinfer_python` to CUDA wheel requirements (#21389)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jul 29, 2025 at 15:51 UTC
a33ea28b1be64f0b57e9eb90389dd36715c60ecb
2 files changed
+5
-1
docker/Dockerfile
+3
-1
@@ -386,6 +386,8 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
386
387
# Install FlashInfer from source
388
ARG FLASHINFER_GIT_REPO="https://github.com/flashinfer-ai/flashinfer.git"
389
+# Keep this in sync with https://github.com/vllm-project/vllm/blob/main/requirements/cuda.txt
390
+# We use `--force-reinstall --no-deps` to avoid issues with the existing FlashInfer wheel.
391
ARG FLASHINFER_GIT_REF="v0.2.9rc2"
392
RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'
393
. /etc/environment
@@ -408,7 +410,7 @@ RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'
410
TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}" \
411
python3 -m flashinfer.aot
412
TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}" \
411
- uv pip install --system --no-build-isolation .
413
+ uv pip install --system --no-build-isolation --force-reinstall --no-deps .
414
popd
415
rm -rf flashinfer
416
BASH
requirements/cuda.txt
+2
@@ -12,3 +12,5 @@ torchaudio==2.7.1
12
torchvision==0.22.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
13
# https://github.com/facebookresearch/xformers/releases/tag/v0.0.31
14
xformers==0.0.31; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.7
15
+# FlashInfer should be updated together with the Dockerfile
16
+flashinfer_python==0.2.9rc2
\ No newline at end of file