A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[ROCm] Effort to reduce the number of environment variables in command line (#17229)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Hongxia Yang committed
May 1, 2025 at 02:27 UTC
90d0a54c4dae03f3e8c111babd359907e719355f
1 file changed
+9
docker/Dockerfile.rocm
+9
@@ -114,6 +114,15 @@ COPY --from=export_vllm /examples ${COMMON_WORKDIR}/vllm/examples
114
ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
115
ENV TOKENIZERS_PARALLELISM=false
116
117
+# ENV that can improve safe tensor loading, and end-to-end time
118
+ENV SAFETENSORS_FAST_GPU=1
119
+
120
+# User-friendly environment setting for multi-processing to avoid below RuntimeError.
121
+# RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing,
122
+# you must use the 'spawn' start method
123
+# See https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
124
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
125
+
126
# Performance environment variable.
127
ENV HIP_FORCE_DEV_KERNARG=1
128