A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
[UX] Add flashinfer-cubin as CUDA default dep (#37233)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Mar 24, 2026 at 22:13 UTC
0f0e03890ed7edd41eb0b5e2de21447b3f70586c
2 files changed
+3
-4
docker/Dockerfile
+2
-4
@@ -587,14 +587,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \
587
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') && \
588
rm /tmp/requirements-cuda.txt /tmp/common.txt
589
590
-# Install FlashInfer pre-compiled kernel cache and binaries
591
-# This is ~1.1GB and only changes when FlashInfer version bumps
590
+# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
591
# https://docs.flashinfer.ai/installation.html
592
# From versions.json: .flashinfer.version
593
ARG FLASHINFER_VERSION=0.6.6
594
RUN --mount=type=cache,target=/root/.cache/uv \
596
- uv pip install --system flashinfer-cubin==${FLASHINFER_VERSION} \
597
- && uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
595
+ uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
596
--extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
597
&& flashinfer show-config
598
requirements/cuda.txt
+1
@@ -10,6 +10,7 @@ torchaudio==2.10.0
10
torchvision==0.25.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
11
# FlashInfer should be updated together with the Dockerfile
12
flashinfer-python==0.6.6
13
+flashinfer-cubin==0.6.6
14
# Cap nvidia-cudnn-frontend (transitive dep of flashinfer) due to
15
# breaking changes in 1.19.0
16
nvidia-cudnn-frontend>=1.13.0,<1.19.0