A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 09:45 UTC
[Docker] Add fastsafetensors to NVIDIA Dockerfile (#38950)
Zhewen Li committed
Apr 8, 2026 at 22:21 UTC
9e78555743c19207db796f432cc213d118873a08
5 files changed
+14
-4
docker/Dockerfile
+3
-1
@@ -538,7 +538,9 @@ RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \
538
cuda-nvrtc-${CUDA_VERSION_DASH} \
539
cuda-cuobjdump-${CUDA_VERSION_DASH} \
540
libcurand-dev-${CUDA_VERSION_DASH} \
541
- libcublas-${CUDA_VERSION_DASH} && \
541
+ libcublas-${CUDA_VERSION_DASH} \
542
+ # Required by fastsafetensors (fixes #20384)
543
+ libnuma-dev && \
544
# Fixes nccl_allocator requiring nccl.h at runtime
545
# https://github.com/vllm-project/vllm/blob/1336a1ea244fa8bfd7e72751cabbdb5b68a0c11a/vllm/distributed/device_communicators/pynccl_allocator.py#L22
546
# NCCL packages don't use the cuda-MAJOR-MINOR naming convention,
docker/Dockerfile.rocm
+2
-1
@@ -19,7 +19,8 @@ ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
19
# Install some basic utilities
20
RUN apt-get update -q -y && apt-get install -q -y \
21
sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev \
22
- apt-transport-https ca-certificates wget curl
22
+ apt-transport-https ca-certificates wget curl \
23
+ libnuma-dev
24
RUN python3 -m pip install --upgrade pip
25
# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_base)
26
ARG USE_SCCACHE
requirements/cuda.txt
+3
@@ -15,6 +15,9 @@ flashinfer-cubin==0.6.7
15
# breaking changes in 1.19.0
16
nvidia-cudnn-frontend>=1.13.0,<1.19.0
17
18
+# Required for faster safetensors model loading
19
+fastsafetensors >= 0.2.2
20
+
21
# QuACK and Cutlass DSL for FA4 (cute-DSL implementation)
22
nvidia-cutlass-dsl>=4.4.2
23
quack-kernels>=0.3.3
requirements/rocm-test.txt
+3
-1
@@ -276,7 +276,9 @@ fastar==0.9.0
276
fastparquet==2026.3.0
277
# via genai-perf
278
fastsafetensors==0.2.2
279
- # via -r requirements/rocm-test.in
279
+ # via
280
+ # -c requirements/rocm.txt
281
+ # -r requirements/rocm-test.in
282
filelock==3.25.2
283
# via
284
# -c requirements/common.txt
requirements/rocm.txt
+3
-1
@@ -20,4 +20,6 @@ conch-triton-kernels==1.2.1
20
timm>=1.0.17
21
# amd-quark: required for Quark quantization on ROCm
22
# To be consistent with test_quark.py
23
-amd-quark>=0.8.99
\ No newline at end of file
23
+amd-quark>=0.8.99
24
+# Required for faster safetensors model loading
25
+fastsafetensors >= 0.2.2
\ No newline at end of file