[CI][AMD] Preserve diagnostics for unwritable checkouts (#53437)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Sep 2, 2026 at 03:18 UTC
01eeb798b6c770284f7bb43e14fa6054fb98bb9d
1 file changed
+78
-1
.buildkite/scripts/hardware_ci/run-amd-test.sh
+78
-1
@@ -58,6 +58,7 @@ amd_diagnostics_collected=0
58
amd_diagnostics_memory_events_path=""
59
amd_diagnostics_probe_budget_seconds=25
60
amd_diagnostics_command_timeout_seconds=5
61
+amd_diagnostics_upload_timeout_seconds=20
62
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
63
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
64
fi
@@ -705,6 +706,46 @@ run_failure_diagnostic() {
706
return 0
707
}
708
709
+upload_failure_diagnostics_artifact() {
710
+ local upload_root=$1 upload_path=$2 agent_bin=""
711
+ local agent_upload_help=""
712
+ local -a upload_options=()
713
+
714
+ if [[ -n "${BUILDKITE_BIN_PATH:-}" \
715
+ && -x "${BUILDKITE_BIN_PATH}/buildkite-agent" ]]; then
716
+ agent_bin="${BUILDKITE_BIN_PATH}/buildkite-agent"
717
+ else
718
+ agent_bin=$(command -v buildkite-agent 2>/dev/null || true)
719
+ fi
720
+ if [[ -z "${agent_bin}" && -x /workspace/buildkite-agent ]]; then
721
+ agent_bin=/workspace/buildkite-agent
722
+ fi
723
+ [[ -n "${agent_bin}" && -n "${BUILDKITE_JOB_ID:-}" ]] || return 1
724
+ command -v timeout >/dev/null 2>&1 || return 1
725
+
726
+ # Older AMD runners predate the literal upload options. A restricted path
727
+ # has no glob or delimiter characters, so it is safe with either agent.
728
+ if [[ ! "${upload_path}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*(/[A-Za-z0-9_][A-Za-z0-9_.-]*)*$ ]]; then
729
+ agent_upload_help=$("${agent_bin}" artifact upload --help 2>&1 || true)
730
+ if [[ "${agent_upload_help}" != *"--literal"* \
731
+ || "${agent_upload_help}" != *"--delimiter"* ]]; then
732
+ return 1
733
+ fi
734
+ upload_options=(--literal --delimiter "")
735
+ fi
736
+
737
+ (
738
+ cd "${upload_root}" || exit 1
739
+ BUILDKITE_AGENT_DEBUG=false \
740
+ BUILDKITE_AGENT_DEBUG_HTTP=false \
741
+ BUILDKITE_AGENT_TRACE_HTTP=false \
742
+ BUILDKITE_AGENT_LOG_LEVEL=error \
743
+ timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \
744
+ "${agent_bin}" artifact upload \
745
+ "${upload_options[@]}" "./${upload_path}"
746
+ ) >/dev/null 2>&1
747
+}
748
+
749
append_failure_diagnostic_file() {
750
local log_file=$1
751
local label=$2
@@ -970,6 +1011,9 @@ collect_rocm_failure_diagnostics() {
1011
local diagnostics_relative_path=""
1012
local diagnostics_path=""
1013
local diagnostics_parent=""
1014
+ local diagnostics_fallback_root=""
1015
+ local diagnostics_storage="checkout"
1016
+ local diagnostics_artifact_summary=""
1017
local checkout_real=""
1018
local diagnostics_parent_real=""
1019
local exit_signal=""
@@ -1037,6 +1081,23 @@ collect_rocm_failure_diagnostics() {
1081
|| ! mkdir -p "${diagnostics_parent}" \
1082
|| [[ -L "${diagnostics_path}" ]] \
1083
|| ! (set -o noclobber; : > "${diagnostics_path}") 2>/dev/null; then
1084
+ echo "WARNING: unable to create AMD CI diagnostics in the checkout; using temporary storage."
1085
+ diagnostics_path=""
1086
+ if diagnostics_fallback_root=$(mktemp -d -t \
1087
+ vllm-amd-diagnostics.XXXXXX 2>/dev/null); then
1088
+ diagnostics_path="${diagnostics_fallback_root}/${diagnostics_relative_path}"
1089
+ diagnostics_parent=$(dirname "${diagnostics_path}")
1090
+ if (umask 077; mkdir -p "${diagnostics_parent}" \
1091
+ && (set -o noclobber; : > "${diagnostics_path}")) 2>/dev/null; then
1092
+ diagnostics_storage="temporary-fallback"
1093
+ else
1094
+ rm -rf -- "${diagnostics_fallback_root}" || true
1095
+ diagnostics_fallback_root=""
1096
+ diagnostics_path=""
1097
+ fi
1098
+ fi
1099
+ fi
1100
+ if [[ -z "${diagnostics_path}" ]]; then
1101
echo "WARNING: unable to create AMD CI diagnostics at ${diagnostics_relative_path}."
1102
printf '\nAMD CI failure summary\n'
1103
printf '%-22s | %s\n' \
@@ -1068,6 +1129,7 @@ collect_rocm_failure_diagnostics() {
1129
echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}"
1130
echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}"
1131
echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}"
1132
+ echo "diagnostics_storage=${diagnostics_storage}"
1133
echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}"
1134
echo "container_hostname=${HOSTNAME:-unknown}"
1135
echo "k8s_pod=${k8s_pod}"
@@ -1182,7 +1244,22 @@ collect_rocm_failure_diagnostics() {
1244
if [[ -n "${oom_kill_count}" ]]; then
1245
summary_rows+=("Cgroup OOM kills" "${oom_kill_count}")
1246
fi
1185
- summary_rows+=("Diagnostics artifact" "${diagnostics_relative_path}")
1247
+ diagnostics_artifact_summary="${diagnostics_relative_path}"
1248
+ if [[ -n "${diagnostics_fallback_root}" ]]; then
1249
+ if upload_failure_diagnostics_artifact \
1250
+ "${diagnostics_fallback_root}" "${diagnostics_relative_path}"; then
1251
+ echo "Uploaded AMD CI diagnostics artifact from temporary storage: ${diagnostics_relative_path}"
1252
+ diagnostics_artifact_summary="${diagnostics_relative_path} (direct upload)"
1253
+ else
1254
+ echo "WARNING: failed to upload temporary AMD CI diagnostics artifact: ${diagnostics_relative_path}"
1255
+ echo "--- AMD CI diagnostics (artifact upload failed)"
1256
+ cat "${diagnostics_path}" || true
1257
+ diagnostics_artifact_summary="job log only: direct upload failed"
1258
+ fi
1259
+ rm -rf -- "${diagnostics_fallback_root}" || \
1260
+ echo "WARNING: unable to remove temporary AMD CI diagnostics storage."
1261
+ fi
1262
+ summary_rows+=("Diagnostics artifact" "${diagnostics_artifact_summary}")
1263
1264
printf '\nAMD CI failure summary\n'
1265
printf '%-22s | %s\n' "${summary_rows[@]}"