A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 14:30 UTC
Enhance run_cluster.sh for multi-NIC support (#28328)
Signed-off-by: Ev Lacey <elacey@nvidia.com>
Ev Lacey committed
Nov 8, 2025 at 14:04 UTC
77d702a22b607b92165d1ddac290e058a0375b83
1 file changed
+24
examples/online_serving/run_cluster.sh
+24
@@ -83,6 +83,29 @@ else
83
RAY_START_CMD+=" --address=${HEAD_NODE_ADDRESS}:6379"
84
fi
85
86
+# Parse VLLM_HOST_IP from additional args if present.
87
+# This is needed for multi-NIC configurations where Ray needs explicit IP bindings.
88
+VLLM_HOST_IP=""
89
+for arg in "${ADDITIONAL_ARGS[@]}"; do
90
+ if [[ $arg == "-e" ]]; then
91
+ continue
92
+ fi
93
+ if [[ $arg == VLLM_HOST_IP=* ]]; then
94
+ VLLM_HOST_IP="${arg#VLLM_HOST_IP=}"
95
+ break
96
+ fi
97
+done
98
+
99
+# Build Ray IP environment variables if VLLM_HOST_IP is set.
100
+# These variables ensure Ray binds to the correct network interface on multi-NIC systems.
101
+RAY_IP_VARS=()
102
+if [ -n "${VLLM_HOST_IP}" ]; then
103
+ RAY_IP_VARS=(
104
+ -e "RAY_NODE_IP_ADDRESS=${VLLM_HOST_IP}"
105
+ -e "RAY_OVERRIDE_NODE_IP_ADDRESS=${VLLM_HOST_IP}"
106
+ )
107
+fi
108
+
109
# Launch the container with the assembled parameters.
110
# --network host: Allows Ray nodes to communicate directly via host networking
111
# --shm-size 10.24g: Increases shared memory
@@ -95,5 +118,6 @@ docker run \
118
--shm-size 10.24g \
119
--gpus all \
120
-v "${PATH_TO_HF_HOME}:/root/.cache/huggingface" \
121
+ "${RAY_IP_VARS[@]}" \
122
"${ADDITIONAL_ARGS[@]}" \
123
"${DOCKER_IMAGE}" -c "${RAY_START_CMD}"