[CI] Recover empty multi-node Docker networks and finish partial cleanup (#55454)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Codex <noreply@openai.com>
Kevin H. Luu committed
Sep 7, 2026 at 12:53 UTC
537af2c3a4ba7462ddc9bc94ec7a4ea496da6d2e
2 files changed
+39
-29
.buildkite/scripts/hardware_ci/run-amd-test.sh
-14
@@ -121,18 +121,6 @@ clear_ci_orchestration_env() {
121
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
122
}
123
124
-cleanup_network() {
125
- local max_nodes=${NUM_NODES:-2}
126
- for node in $(seq 0 $((max_nodes - 1))); do
127
- if docker ps -a -q -f name="node${node}" | grep -q .; then
128
- docker stop "node${node}" || true
129
- fi
130
- done
131
- if docker network ls | grep -q docker-net; then
132
- docker network rm docker-net || true
133
- fi
134
-}
135
-
124
amd_ci_teardown_log() {
125
local event=$1
126
shift
@@ -1704,13 +1692,11 @@ if is_multi_node "$commands"; then
1692
1693
/bin/bash -c "${composite_command}"
1694
exit_code=$?
1707
- cleanup_network
1695
handle_pytest_exit "$exit_code"
1696
else
1697
echo "Multi-node job detected but failed to parse bracket command syntax."
1698
echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]"
1699
echo "Got: $commands"
1713
- cleanup_network
1700
exit 111
1701
fi
1702
else
.buildkite/scripts/run-multi-node-test.sh
+39
-15
@@ -38,8 +38,21 @@ for command in "${COMMANDS[@]}"; do
38
done
39
40
41
+STARTED_CONTAINERS=()
42
+NETWORK_CREATED=0
43
+RUN_DIR=$(mktemp -d)
44
+NETWORK_NAME="docker-net-${RUN_DIR##*/}"
45
+
46
start_network() {
42
- docker network create --subnet=192.168.10.0/24 docker-net
47
+ if docker network inspect docker-net > /dev/null 2>&1; then
48
+ if [ "$(docker network inspect --format '{{len .Containers}}' docker-net)" != 0 ]; then
49
+ echo "docker-net still has attached containers; refusing to reuse it." >&2
50
+ return 1
51
+ fi
52
+ docker network rm docker-net
53
+ fi
54
+ docker network create --subnet=192.168.10.0/24 "$NETWORK_NAME"
55
+ NETWORK_CREATED=1
56
}
57
58
start_nodes() {
@@ -67,20 +80,21 @@ start_nodes() {
80
# 3. map the huggingface cache directory to the container
81
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
82
# starting from 192.168.10.11)
70
- docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
71
- -v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
72
- --network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
73
- /bin/bash -c "tail -f /dev/null"
83
+ CONTAINER_ID=$(docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
84
+ -v ~/.cache/huggingface:/root/.cache/huggingface --name "$NETWORK_NAME-node$node" \
85
+ --network "$NETWORK_NAME" --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
86
+ /bin/bash -c "tail -f /dev/null")
87
+ STARTED_CONTAINERS+=("$CONTAINER_ID")
88
89
# organize containers into a ray cluster
90
if [ "$node" -eq 0 ]; then
91
# start the ray head node
78
- docker exec -d "node$node" /bin/bash -c "ray start --head --port=6379 --block"
92
+ docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --head --port=6379 --block"
93
# wait for the head node to be ready
94
sleep 10
95
else
96
# start the ray worker nodes, and connect them to the head node
83
- docker exec -d "node$node" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
97
+ docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
98
fi
99
done
100
@@ -88,7 +102,7 @@ start_nodes() {
102
sleep 10
103
104
# print the cluster status
91
- docker exec node0 /bin/bash -c "ray status"
105
+ docker exec "${STARTED_CONTAINERS[0]}" /bin/bash -c "ray status"
106
}
107
108
run_nodes() {
@@ -106,22 +120,32 @@ run_nodes() {
120
done
121
echo "Running node$node with GPU devices: $DEVICE_LIST"
122
if [ "$node" -ne 0 ]; then
109
- docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
123
+ docker exec -d "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
124
else
125
# Allocate a TTY (-t -i) for the foreground head node so its output
126
# keeps ANSI color in the Buildkite log (see run-amd-test.sh).
113
- docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
127
+ docker exec -t -i "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
128
fi
129
done
130
}
131
cleanup() {
118
- for node in $(seq 0 $(($NUM_NODES-1))); do
119
- docker stop "node$node"
120
- done
121
- docker network rm docker-net
132
+ local status=$?
133
+ local cleanup_status=0
134
+ if [ "${#STARTED_CONTAINERS[@]}" -gt 0 ]; then
135
+ for container_id in "${STARTED_CONTAINERS[@]}"; do
136
+ docker stop "$container_id" || cleanup_status=$?
137
+ done
138
+ fi
139
+ if [ "$NETWORK_CREATED" -eq 1 ]; then
140
+ docker network rm "$NETWORK_NAME" || cleanup_status=$?
141
+ fi
142
+ rmdir "$RUN_DIR" || cleanup_status=$?
143
+ if [ "$status" -ne 0 ]; then
144
+ return "$status"
145
+ fi
146
+ return "$cleanup_status"
147
}
148
trap cleanup EXIT
149
start_network
150
start_nodes
151
run_nodes
127
-