A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 15:00 UTC
[FIX]Patch run-cluster.sh (fix for #28328) (#30002)
Signed-off-by: elacey <elacey@nvidia.com> Signed-off-by: Ev Lacey <github@everettlacey.com>
Ev Lacey committed
Dec 11, 2025 at 15:36 UTC
d527cf0b3d4210c4277f258c9d26286cec726a6f
1 file changed
+34
-26
examples/online_serving/run_cluster.sh
+34
-26
@@ -21,7 +21,7 @@
21
# --worker \
22
# /abs/path/to/huggingface/cache \
23
# -e VLLM_HOST_IP=<worker_node_ip>
24
-#
24
+#
25
# Each worker requires a unique VLLM_HOST_IP value.
26
# Keep each terminal session open. Closing a session stops the associated Ray
27
# node and thereby shuts down the entire cluster.
@@ -59,6 +59,34 @@ if [ "${NODE_TYPE}" != "--head" ] && [ "${NODE_TYPE}" != "--worker" ]; then
59
exit 1
60
fi
61
62
+# Extract VLLM_HOST_IP from ADDITIONAL_ARGS (e.g. "-e VLLM_HOST_IP=...").
63
+VLLM_HOST_IP=""
64
+for ((i = 0; i < ${#ADDITIONAL_ARGS[@]}; i++)); do
65
+ arg="${ADDITIONAL_ARGS[$i]}"
66
+ case "${arg}" in
67
+ -e)
68
+ next="${ADDITIONAL_ARGS[$((i + 1))]:-}"
69
+ if [[ "${next}" == VLLM_HOST_IP=* ]]; then
70
+ VLLM_HOST_IP="${next#VLLM_HOST_IP=}"
71
+ break
72
+ fi
73
+ ;;
74
+ -eVLLM_HOST_IP=* | VLLM_HOST_IP=*)
75
+ VLLM_HOST_IP="${arg#*=}"
76
+ break
77
+ ;;
78
+ esac
79
+done
80
+
81
+# For the head node, HEAD_NODE_ADDRESS and VLLM_HOST_IP should be consistent.
82
+if [[ "${NODE_TYPE}" == "--head" && -n "${VLLM_HOST_IP}" ]]; then
83
+ if [[ "${VLLM_HOST_IP}" != "${HEAD_NODE_ADDRESS}" ]]; then
84
+ echo "Warning: VLLM_HOST_IP (${VLLM_HOST_IP}) differs from head_node_ip (${HEAD_NODE_ADDRESS})."
85
+ echo "Using VLLM_HOST_IP as the head node address."
86
+ HEAD_NODE_ADDRESS="${VLLM_HOST_IP}"
87
+ fi
88
+fi
89
+
90
# Generate a unique container name with random suffix.
91
# Docker container names must be unique on each host.
92
# The random suffix allows multiple Ray containers to run simultaneously on the same machine,
@@ -74,36 +102,17 @@ cleanup() {
102
trap cleanup EXIT
103
104
# Build the Ray start command based on the node role.
77
-# The head node manages the cluster and accepts connections on port 6379,
105
+# The head node manages the cluster and accepts connections on port 6379,
106
# while workers connect to the head's address.
107
RAY_START_CMD="ray start --block"
108
if [ "${NODE_TYPE}" == "--head" ]; then
81
- RAY_START_CMD+=" --head --port=6379"
109
+ RAY_START_CMD+=" --head --node-ip-address=${HEAD_NODE_ADDRESS} --port=6379"
110
else
83
- RAY_START_CMD+=" --address=${HEAD_NODE_ADDRESS}:6379"
84
-fi
111
86
-# Parse VLLM_HOST_IP from additional args if present.
87
-# This is needed for multi-NIC configurations where Ray needs explicit IP bindings.
88
-VLLM_HOST_IP=""
89
-for arg in "${ADDITIONAL_ARGS[@]}"; do
90
- if [[ $arg == "-e" ]]; then
91
- continue
92
- fi
93
- if [[ $arg == VLLM_HOST_IP=* ]]; then
94
- VLLM_HOST_IP="${arg#VLLM_HOST_IP=}"
95
- break
112
+ RAY_START_CMD+=" --address=${HEAD_NODE_ADDRESS}:6379"
113
+ if [ -n "${VLLM_HOST_IP}" ]; then
114
+ RAY_START_CMD+=" --node-ip-address=${VLLM_HOST_IP}"
115
fi
97
-done
98
-
99
-# Build Ray IP environment variables if VLLM_HOST_IP is set.
100
-# These variables ensure Ray binds to the correct network interface on multi-NIC systems.
101
-RAY_IP_VARS=()
102
-if [ -n "${VLLM_HOST_IP}" ]; then
103
- RAY_IP_VARS=(
104
- -e "RAY_NODE_IP_ADDRESS=${VLLM_HOST_IP}"
105
- -e "RAY_OVERRIDE_NODE_IP_ADDRESS=${VLLM_HOST_IP}"
106
- )
116
fi
117
118
# Launch the container with the assembled parameters.
@@ -118,6 +127,5 @@ docker run \
127
--shm-size 10.24g \
128
--gpus all \
129
-v "${PATH_TO_HF_HOME}:/root/.cache/huggingface" \
121
- "${RAY_IP_VARS[@]}" \
130
"${ADDITIONAL_ARGS[@]}" \
131
"${DOCKER_IMAGE}" -c "${RAY_START_CMD}"