[CI][AMD] Honor single-node Docker workload timeout (#52547)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Andreas Karatzas committed
Aug 22, 2026 at 22:37 UTC
d569e71bd031693ac945c590731a36f0e70789cc
1 file changed
+66
-5
.buildkite/scripts/hardware_ci/run-amd-test.sh
+66
-5
@@ -83,6 +83,8 @@ export PYTHONFAULTHANDLER
83
# depend on their current working directory.
84
export PYTHONPATH="${PYTHONPATH:-..}"
85
86
+ci_started_at=$SECONDS
87
+
88
###############################################################################
89
# Helper Functions
90
###############################################################################
@@ -130,6 +132,66 @@ cleanup_network() {
132
fi
133
}
134
135
+amd_ci_teardown_log() {
136
+ local event=$1
137
+ shift
138
+
139
+ printf '[amd-ci-teardown] event=%s' "${event}" >&2
140
+ if (($#)); then
141
+ printf ' %s' "$@" >&2
142
+ fi
143
+ printf '\n' >&2
144
+}
145
+
146
+run_docker_with_ci_timeout() {
147
+ local raw_timeout=${CONTAINER_TIMEOUT_S:-0}
148
+ local configured_timeout=0
149
+ local elapsed=0
150
+ local remaining_timeout=0
151
+ local status=0
152
+
153
+ if [[ ! "${raw_timeout}" =~ ^(0|[1-9][0-9]{0,5})$ ]]; then
154
+ amd_ci_teardown_log configuration_error \
155
+ "reason=invalid_timeout" "expected=integer_0_to_604800"
156
+ return 2
157
+ fi
158
+ configured_timeout=$((10#${raw_timeout}))
159
+ if ((configured_timeout > 604800)); then
160
+ amd_ci_teardown_log configuration_error \
161
+ "reason=invalid_timeout" "expected=integer_0_to_604800"
162
+ return 2
163
+ fi
164
+
165
+ remaining_timeout=${configured_timeout}
166
+ if ((configured_timeout > 0)); then
167
+ if ! command -v timeout >/dev/null 2>&1; then
168
+ amd_ci_teardown_log configuration_error \
169
+ "reason=timeout_command_not_found"
170
+ return 127
171
+ fi
172
+ elapsed=$((SECONDS - ci_started_at))
173
+ if ((elapsed >= configured_timeout)); then
174
+ amd_ci_teardown_log deadline_exhausted \
175
+ "mode=docker" "configured_s=${configured_timeout}" "elapsed_s=${elapsed}"
176
+ return 124
177
+ fi
178
+ remaining_timeout=$((configured_timeout - elapsed))
179
+ fi
180
+
181
+ amd_ci_teardown_log workload_started \
182
+ "mode=docker" "timeout_s=${remaining_timeout}"
183
+ if ((remaining_timeout == 0)); then
184
+ "$@" || status=$?
185
+ else
186
+ # Docker runs interactively, so keep it in the foreground process group.
187
+ # --verbose records both TERM and any KILL escalation caused by timeout.
188
+ timeout --verbose --foreground --signal=TERM --kill-after=10s \
189
+ "${remaining_timeout}s" "$@" || status=$?
190
+ fi
191
+ amd_ci_teardown_log workload_finished "mode=docker" "status=${status}"
192
+ return "${status}"
193
+}
194
+
195
prepare_artifact_image() {
196
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
197
return 1
@@ -1599,8 +1661,9 @@ else
1661
echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
1662
fi
1663
1664
+ exit_code=0
1665
# shellcheck disable=SC2086 # word splitting is intentional: both hold multiple docker flags
1603
- docker run \
1666
+ run_docker_with_ci_timeout docker run \
1667
"${docker_run_terminal_args[@]}" \
1668
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
1669
$RDMA_FLAGS \
@@ -1636,8 +1699,6 @@ else
1699
"${standalone_merge_base_env[@]}" \
1700
--name "${container_name}" \
1701
"${image_name}" \
1639
- /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
1640
-
1641
- exit_code=$?
1642
- handle_pytest_exit "$exit_code"
1702
+ /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}" || exit_code=$?
1703
+ handle_pytest_exit "${exit_code}"
1704
fi