A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 21:00 UTC
[CI Bugfix] Pre-download missing FlashInfer headers in Docker build (#38391)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Mar 28, 2026 at 14:09 UTC
2bf5b70ae86261431b4b92276828b40b9c0903b6
1 file changed
+19
docker/Dockerfile
+19
@@ -596,6 +596,25 @@ RUN --mount=type=cache,target=/root/.cache/uv \
596
--extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
597
&& flashinfer show-config
598
599
+# Pre-download FlashInfer TRTLLM BMM headers for air-gapped environments.
600
+# At runtime, MoE JIT compilation downloads these from edge.urm.nvidia.com
601
+# which fails without internet. This step caches them at build time.
602
+RUN python3 <<'PYEOF'
603
+from flashinfer.jit import env as jit_env
604
+from flashinfer.jit.cubin_loader import download_trtllm_headers, get_cubin
605
+from flashinfer.artifacts import ArtifactPath, CheckSumHash
606
+
607
+download_trtllm_headers(
608
+ 'bmm',
609
+ jit_env.FLASHINFER_CUBIN_DIR / 'flashinfer' / 'trtllm' / 'batched_gemm' / 'trtllmGen_bmm_export',
610
+ f'{ArtifactPath.TRTLLM_GEN_BMM}/include/trtllmGen_bmm_export',
611
+ ArtifactPath.TRTLLM_GEN_BMM,
612
+ get_cubin(f'{ArtifactPath.TRTLLM_GEN_BMM}/checksums.txt', CheckSumHash.TRTLLM_GEN_BMM),
613
+)
614
+
615
+print('FlashInfer TRTLLM BMM headers downloaded successfully')
616
+PYEOF
617
+
618
# ============================================================
619
# OPENAI API SERVER DEPENDENCIES
620
# Pre-install these to avoid reinstalling on every vLLM wheel rebuild