60c3a31b12 · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -44,11 +44,14 @@ amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}"

44 amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}"

45 amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}"

46 amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}"

47 -amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}"

47 +amd_diagnostics_workspace="${VLLM_CI_WORKSPACE:-/vllm-workspace}"

48 +amd_diagnostics_pod_name="${VLLM_CI_K8S_POD_NAME:-${POD_NAME:-${HOSTNAME:-}}}"

49 +amd_diagnostics_k8s_namespace="${VLLM_CI_K8S_NAMESPACE:-${POD_NAMESPACE:-unknown}}"

50 +amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-${NODE_NAME:-}}"

51 amd_diagnostics_collected=0

52 +amd_diagnostics_memory_events_path=""

53 amd_diagnostics_probe_budget_seconds=25

54 amd_diagnostics_command_timeout_seconds=5

51 -amd_diagnostics_upload_timeout_seconds=20

55 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then

56 PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"

57 fi

@@ -101,6 +104,8 @@ clear_ci_orchestration_env() {

104 VLLM_CI_ARTIFACT_GLOB \

105 VLLM_CI_ARTIFACT_CHECKSUM_GLOB \

106 VLLM_CI_EXPECTED_GPU_COUNT \

107 + VLLM_CI_K8S_POD_NAME \

108 + VLLM_CI_K8S_NAMESPACE \

109 VLLM_CI_K8S_NODE_NAME \

110 VLLM_CI_USE_ARTIFACTS \

111 VLLM_CI_RESULTS_ROOT \

@@ -577,7 +582,24 @@ is_multi_node() {

582 return 1

583 }

584

580 -run_amd_diagnostic() {

585 +append_failure_diagnostic_section() {

586 + local log_file=$1

587 + local title=$2

588 +

589 + printf '\n===============================================================================\n' \

590 + >> "${log_file}"

591 + printf '%s\n' "${title}" >> "${log_file}"

592 + printf '===============================================================================\n' \

593 + >> "${log_file}"

594 +}

595 +

596 +append_failure_diagnostic_note() {

597 + local log_file=$1

598 + shift

599 + printf '%s\n' "$@" >> "${log_file}"

600 +}

601 +

602 +run_failure_diagnostic() {

603 local log_file=$1

604 local probe_deadline=$2

605 shift 2

@@ -589,23 +611,285 @@ run_amd_diagnostic() {

611 printf '\n$'

612 printf ' %q' "$@"

613 printf '\n'

592 - } | tee -a "${log_file}"

614 + } >> "${log_file}"

615

616 + if ! command -v timeout >/dev/null 2>&1; then

617 + echo "[timeout unavailable; bounded diagnostic command skipped]" \

618 + >> "${log_file}"

619 + return 0

620 + fi

621 if [[ "${remaining_seconds}" -le 0 ]]; then

622 echo "[diagnostic probe budget exhausted; command skipped]" \

596 - | tee -a "${log_file}"

623 + >> "${log_file}"

624 return 0

625 fi

626 if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then

627 command_timeout_seconds=${remaining_seconds}

628 fi

629

603 - timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \

604 - | tee -a "${log_file}"

605 - command_status=${PIPESTATUS[0]}

630 + timeout --kill-after=1s "${command_timeout_seconds}s" "$@" \

631 + >> "${log_file}" 2>&1

632 + command_status=$?

633 if [[ "${command_status}" -ne 0 ]]; then

634 echo "[diagnostic command exited with status ${command_status}]" \

608 - | tee -a "${log_file}"

635 + >> "${log_file}"

636 + fi

637 + return 0

638 +}

639 +

640 +append_failure_diagnostic_file() {

641 + local log_file=$1

642 + local label=$2

643 + local path=$3

644 + local command_status=0

645 +

646 + printf '\n%s:\n' "${label}" >> "${log_file}"

647 + printf '$ cat %q\n' "${path}" >> "${log_file}"

648 + if [[ ! -r "${path}" ]]; then

649 + echo "[file unavailable]" >> "${log_file}"

650 + return 0

651 + fi

652 +

653 + cat "${path}" >> "${log_file}" 2>&1

654 + command_status=$?

655 + if [[ "${command_status}" -ne 0 ]]; then

656 + echo "[diagnostic file read exited with status ${command_status}]" \

657 + >> "${log_file}"

658 + fi

659 + return 0

660 +}

661 +

662 +decode_mountinfo_path() {

663 + local value=$1

664 +

665 + value=${value//\\040/ }

666 + value=${value//\\011/$'\t'}

667 + value=${value//\\012/$'\n'}

668 + value=${value//\\134/\\}

669 + printf '%s\n' "${value}"

670 +}

671 +

672 +resolve_current_cgroup_v2_dir() {

673 + local cgroup_path=""

674 + local mount_root=""

675 + local mount_point=""

676 + local relative_path=""

677 + local candidate=""

678 +

679 + cgroup_path=$(awk -F: '$2 == "" { print $3; exit }' \

680 + /proc/self/cgroup 2>/dev/null)

681 + [[ -n "${cgroup_path}" ]] || return 1

682 +

683 + while IFS=$'\t' read -r mount_root mount_point; do

684 + mount_root=$(decode_mountinfo_path "${mount_root}")

685 + mount_point=$(decode_mountinfo_path "${mount_point}")

686 + if [[ "${cgroup_path}" == "/" ]]; then

687 + relative_path=""

688 + elif [[ "${mount_root}" == "/" ]]; then

689 + relative_path="${cgroup_path}"

690 + elif [[ "${cgroup_path}" == "${mount_root}" ]]; then

691 + relative_path=""

692 + elif [[ "${cgroup_path}" == "${mount_root}/"* ]]; then

693 + relative_path="/${cgroup_path#"${mount_root}/"}"

694 + else

695 + continue

696 + fi

697 +

698 + candidate="${mount_point%/}${relative_path}"

699 + if [[ -d "${candidate}" ]]; then

700 + printf '%s\n' "${candidate}"

701 + return 0

702 + fi

703 + done < <(

704 + awk '

705 + {

706 + for (i = 6; i <= NF; i++) {

707 + if ($i == "-" && $(i + 1) == "cgroup2") {

708 + print $4 "\t" $5

709 + break

710 + }

711 + }

712 + }

713 + ' /proc/self/mountinfo 2>/dev/null

714 + )

715 + return 1

716 +}

717 +

718 +collect_cgroup_files() {

719 + local log_file=$1

720 + local title=$2

721 + local cgroup_dir=$3

722 + shift 3

723 + local cgroup_file=""

724 + local files_collected=0

725 +

726 + append_failure_diagnostic_section "${log_file}" "${title}"

727 + if [[ -z "${cgroup_dir}" || ! -d "${cgroup_dir}" ]]; then

728 + append_failure_diagnostic_note "${log_file}" \

729 + "resolved_cgroup_path=unavailable"

730 + return 0

731 + fi

732 +

733 + append_failure_diagnostic_note "${log_file}" \

734 + "resolved_cgroup_path=${cgroup_dir}"

735 + for cgroup_file in "$@"; do

736 + if [[ -r "${cgroup_dir}/${cgroup_file}" ]]; then

737 + append_failure_diagnostic_file "${log_file}" "${cgroup_file}" \

738 + "${cgroup_dir}/${cgroup_file}"

739 + files_collected=$((files_collected + 1))

740 + fi

741 + done

742 + if [[ "${files_collected}" -eq 0 ]]; then

743 + append_failure_diagnostic_note "${log_file}" \

744 + "[no requested cgroup files were readable]"

745 + fi

746 +}

747 +

748 +collect_cgroup_diagnostics() {

749 + local log_file=$1

750 + local cgroup_dir=""

751 +

752 + append_failure_diagnostic_section "${log_file}" "Cgroup membership"

753 + append_failure_diagnostic_file "${log_file}" "current_process" \

754 + /proc/self/cgroup

755 + append_failure_diagnostic_file "${log_file}" "pid_namespace_init_process" \

756 + /proc/1/cgroup

757 +

758 + cgroup_dir=$(resolve_current_cgroup_v2_dir 2>/dev/null || true)

759 + collect_cgroup_files "${log_file}" \

760 + "Cgroup v2 current-process resource state" "${cgroup_dir}" \

761 + cgroup.type cgroup.events \

762 + memory.current memory.peak memory.max memory.high memory.low memory.min \

763 + memory.oom.group memory.swap.current memory.swap.max \

764 + memory.events memory.events.local memory.swap.events memory.stat \

765 + memory.pressure \

766 + cpu.max cpu.max.burst cpu.weight cpu.stat cpu.pressure \

767 + cpuset.cpus cpuset.cpus.effective cpuset.mems cpuset.mems.effective \

768 + pids.current pids.max pids.events pids.events.local \

769 + io.stat io.max io.weight io.pressure

770 + if [[ -n "${cgroup_dir}" && -r "${cgroup_dir}/memory.events" ]]; then

771 + amd_diagnostics_memory_events_path="${cgroup_dir}/memory.events"

772 + fi

773 +}

774 +

775 +collect_process_diagnostics() {

776 + local log_file=$1

777 + local probe_deadline=$2

778 + local shell_proc_dir="/proc/$$"

779 +

780 + append_failure_diagnostic_section "${log_file}" \

781 + "Runner process constraints and PID-namespace-visible processes"

782 + append_failure_diagnostic_file "${log_file}" "diagnostic_shell_limits" \

783 + "${shell_proc_dir}/limits"

784 + if command -v awk >/dev/null 2>&1; then

785 + # shellcheck disable=SC2016 # The expression is evaluated by awk.

786 + run_failure_diagnostic "${log_file}" "${probe_deadline}" awk '

787 + $1 ~ /^(Pid:|PPid:|Threads:|VmPeak:|VmSize:|VmHWM:|VmRSS:|RssAnon:|RssFile:|RssShmem:|Cpus_allowed_list:|Mems_allowed_list:|voluntary_ctxt_switches:|nonvoluntary_ctxt_switches:)$/ {

788 + print

789 + }

790 + ' "${shell_proc_dir}/status"

791 + fi

792 + if command -v nproc >/dev/null 2>&1; then

793 + run_failure_diagnostic "${log_file}" "${probe_deadline}" nproc

794 + fi

795 + run_failure_diagnostic "${log_file}" "${probe_deadline}" \

796 + /bin/bash -c 'ulimit -a'

797 + if command -v ps >/dev/null 2>&1 && command -v head >/dev/null 2>&1; then

798 + append_failure_diagnostic_note "${log_file}" \

799 + "process_snapshot=top_100_by_rss"

800 + run_failure_diagnostic "${log_file}" "${probe_deadline}" \

801 + /bin/bash -c \

802 + 'ps -eo pid,ppid,stat,etimes,nlwp,pcpu,rss,vsz,comm --sort=-rss | head -n 101'

803 + fi

804 + return 0

805 +}

806 +

807 +collect_mount_diagnostic() {

808 + local log_file=$1

809 + local probe_deadline=$2

810 + local label=$3

811 + local path=$4

812 +

813 + printf '\n%s (%s):\n' "${label}" "${path}" >> "${log_file}"

814 + if [[ ! -e "${path}" ]]; then

815 + echo "[path unavailable]" >> "${log_file}"

816 + return 0

817 + fi

818 + if command -v findmnt >/dev/null 2>&1; then

819 + run_failure_diagnostic "${log_file}" "${probe_deadline}" findmnt \

820 + -n -T "${path}" -o TARGET,FSTYPE,VFS-OPTIONS

821 + fi

822 + if command -v df >/dev/null 2>&1; then

823 + run_failure_diagnostic "${log_file}" "${probe_deadline}" df -h \

824 + --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent,target \

825 + -- "${path}"

826 + fi

827 + return 0

828 +}

829 +

830 +collect_execution_resource_diagnostics() {

831 + local log_file=$1

832 + local probe_deadline=$2

833 +

834 + # free and /proc/meminfo generally describe the node from a pod. Cgroups are

835 + # the live source for this container's resource limits, usage, and events.

836 + collect_cgroup_diagnostics "${log_file}"

837 + collect_process_diagnostics "${log_file}" "${probe_deadline}"

838 +

839 + append_failure_diagnostic_section "${log_file}" \

840 + "Mount-visible filesystem capacity"

841 + append_failure_diagnostic_note "${log_file}" \

842 + "These values are filesystem views, not Kubernetes ephemeral-storage quotas."

843 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

844 + "checkout" "${amd_diagnostics_checkout_root}"

845 + if is_native_runtime; then

846 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

847 + "native workspace emptyDir" "${amd_diagnostics_workspace}"

848 + else

849 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

850 + "test workspace (if mounted)" "${amd_diagnostics_workspace}"

851 + fi

852 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

853 + "temporary directory" "${TMPDIR:-/tmp}"

854 + if [[ "${TMPDIR:-/tmp}" != "/tmp" ]]; then

855 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

856 + "system temporary directory" /tmp

857 + fi

858 + if is_native_runtime; then

859 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

860 + "shared-memory emptyDir" /dev/shm

861 + else

862 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

863 + "shared-memory mount" /dev/shm

864 + fi

865 + if [[ -n "${HF_HOME:-}" ]]; then

866 + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \

867 + "Hugging Face cache" "${HF_HOME}"

868 + fi

869 + return 0

870 +}

871 +

872 +collect_amd_device_nodes() {

873 + local log_file=$1

874 + local probe_deadline=$2

875 + local device=""

876 + local device_count=0

877 +

878 + for device in /dev/kfd /dev/dri/renderD*; do

879 + if [[ ! -e "${device}" ]]; then

880 + continue

881 + fi

882 + device_count=$((device_count + 1))

883 + if command -v stat >/dev/null 2>&1; then

884 + run_failure_diagnostic "${log_file}" "${probe_deadline}" stat -Lc \

885 + '%n type=%F mode=%a owner=%u:%g device=%t:%T' -- "${device}"

886 + else

887 + append_failure_diagnostic_note "${log_file}" "device=${device}"

888 + fi

889 + done

890 + if [[ "${device_count}" -eq 0 ]]; then

891 + append_failure_diagnostic_note "${log_file}" \

892 + "No /dev/kfd or /dev/dri/renderD* device nodes are visible."

893 fi

894 return 0

895 }

@@ -617,30 +901,55 @@ collect_rocm_failure_diagnostics() {

901 local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}"

902 local diagnostics_relative_path=""

903 local diagnostics_path=""

620 - local upload_root="${amd_diagnostics_checkout_root}"

621 - local upload_path=""

904 + local diagnostics_parent=""

905 + local checkout_real=""

906 + local diagnostics_parent_real=""

907 local exit_signal=""

908 local probe_deadline=0

909 + local runtime="single-node-docker"

910 + local diagnostics_scope="outer-runner-after-test-container-exit"

911 + local identity_label="runner"

912 + local identity_value="${HOSTNAME:-unknown}"

913 + local k8s_pod="unknown"

914 + local k8s_namespace="${amd_diagnostics_k8s_namespace}"

915 + local oom_kill_count=""

916 + local summary_identity_label="Runner"

917 + local -a summary_rows=()

918

919 if [[ "${amd_diagnostics_collected}" == "1" ]]; then

920 return 0

921 fi

922 amd_diagnostics_collected=1

923

630 - if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then

631 - echo "Skipping AMD GPU diagnostics for a CPU-only job."

632 - return 0

924 + if is_native_runtime; then

925 + runtime="native-kubernetes"

926 + diagnostics_scope="current-container-cgroup-and-namespaces"

927 + if [[ -n "${amd_diagnostics_pod_name}" ]]; then

928 + identity_label="pod"

929 + identity_value="${amd_diagnostics_pod_name}"

930 + k8s_pod="${amd_diagnostics_pod_name}"

931 + else

932 + identity_label="container_hostname"

933 + fi

934 + elif [[ "${amd_diagnostics_execution_mode}" == "multi-node" ]]; then

935 + runtime="multi-node-docker"

936 fi

634 -

635 - if ! command -v timeout >/dev/null 2>&1; then

636 - echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job."

637 - return 0

937 + if [[ "${k8s_namespace}" == "unknown" \

938 + && -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then

939 + k8s_namespace=$(

940 + tr -d '\r\n' \

941 + < /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null

942 + )

943 fi

944

945 job_id="${job_id//[^A-Za-z0-9_.-]/_}"

946 retry_count="${retry_count//[^A-Za-z0-9_.-]/_}"

947 parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}"

948

949 + if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then

950 + exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)

951 + fi

952 +

953 # Buildkite keeps the supplied artifact path. Restrict the configurable

954 # directory to a checkout-relative path so the UI shows a clean artifact key.

955 if [[ -z "${amd_diagnostics_dir}" \

@@ -649,26 +958,32 @@ collect_rocm_failure_diagnostics() {

958 echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR"

959 amd_diagnostics_dir="artifacts/amd-gpu-diagnostics"

960 fi

652 - diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log"

961 + diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/diagnostics.log"

962 diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}"

654 - upload_path="${diagnostics_relative_path}"

655 -

656 - if ! mkdir -p "$(dirname "${diagnostics_path}")" \

657 - || ! : > "${diagnostics_path}"; then

658 - diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || {

659 - echo "WARNING: unable to create an AMD GPU diagnostics log."

660 - return 0

661 - }

662 - upload_root=$(dirname "${diagnostics_path}")

663 - upload_path=$(basename "${diagnostics_path}")

664 - fi

665 -

666 - if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then

667 - exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)

963 + diagnostics_parent=$(dirname "${diagnostics_path}")

964 + checkout_real=$(readlink -m "${amd_diagnostics_checkout_root}" 2>/dev/null || true)

965 + diagnostics_parent_real=$(readlink -m "${diagnostics_parent}" 2>/dev/null || true)

966 + if [[ -z "${checkout_real}" || -z "${diagnostics_parent_real}" \

967 + || ("${diagnostics_parent_real}" != "${checkout_real}" \

968 + && "${diagnostics_parent_real}" != "${checkout_real}/"*) ]] \

969 + || ! mkdir -p "${diagnostics_parent}" \

970 + || [[ -L "${diagnostics_path}" ]] \

971 + || ! (set -o noclobber; : > "${diagnostics_path}") 2>/dev/null; then

972 + echo "WARNING: unable to create AMD CI diagnostics at ${diagnostics_relative_path}."

973 + printf '\nAMD CI failure summary\n'

974 + printf '%-22s | %s\n' \

975 + "Field" "Value" \

976 + "----------------------" "-----" \

977 + "Exit code" "${exit_code}" \

978 + "Signal" "${exit_signal:-none}" \

979 + "Runtime" "${runtime}" \

980 + "Pod/container" "${identity_value}" \

981 + "Diagnostics artifact" "unavailable: ${diagnostics_relative_path}"

982 + return 0

983 fi

984

985 {

671 - echo "AMD GPU failure diagnostics"

986 + echo "AMD CI failure diagnostics"

987 echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"

988 echo "exit_code=${exit_code}"

989 echo "exit_signal=${exit_signal:-none}"

@@ -680,105 +995,130 @@ collect_rocm_failure_diagnostics() {

995 echo "retry_count=${retry_count}"

996 echo "parallel_job=${parallel_job}"

997 echo "execution_mode=${amd_diagnostics_execution_mode}"

998 + echo "runtime=${runtime}"

999 + echo "diagnostics_scope=${diagnostics_scope}"

1000 echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}"

1001 echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}"

1002 echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}"

686 - echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}"

1003 echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}"

1004 + echo "container_hostname=${HOSTNAME:-unknown}"

1005 + echo "k8s_pod=${k8s_pod}"

1006 + echo "k8s_namespace=${k8s_namespace:-unknown}"

1007 echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}"

1008 echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)"

1009 } > "${diagnostics_path}"

1010

692 - echo "--- :rotating_light: AMD GPU failure diagnostics"

693 - cat "${diagnostics_path}"

694 -

1011 probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds))

1012

697 - # Collect the most useful evidence first so auxiliary probes cannot consume

698 - # the shared deadline before GPU health is captured.

699 - echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}"

700 - if command -v amd-smi >/dev/null 2>&1; then

701 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version

702 - # Bus data identifies a card within the public node without publishing its

703 - # persistent UUID, serial number, or process list.

704 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all

705 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all

706 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all

707 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all

708 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all

709 - elif command -v rocm-smi >/dev/null 2>&1; then

710 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \

711 - --showbus --showreplaycount --showrasinfo --showpagesinfo

712 - else

713 - echo "Neither amd-smi nor rocm-smi is available." \

714 - | tee -a "${diagnostics_path}"

715 - fi

1013 + # Cgroup, process, and mount state is fast and most useful for explaining OOM,

1014 + # throttling, PID exhaustion, and emptyDir pressure. Capture it before external

1015 + # probes can consume the shared deadline.

1016 + collect_execution_resource_diagnostics \

1017 + "${diagnostics_path}" "${probe_deadline}"

1018

717 - echo "Node resource diagnostics (runner-visible scope)" \

718 - | tee -a "${diagnostics_path}"

719 - if command -v df >/dev/null 2>&1; then

720 - echo "checkout_filesystem:" | tee -a "${diagnostics_path}"

721 - if [[ -d "${amd_diagnostics_checkout_root}" ]]; then

722 - (

723 - cd "${amd_diagnostics_checkout_root}" || exit 0

724 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \

725 - --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \

726 - -- .

727 - )

1019 + append_failure_diagnostic_section "${diagnostics_path}" \

1020 + "AMD GPU diagnostics (tool-visible devices)"

1021 + if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then

1022 + append_failure_diagnostic_note "${diagnostics_path}" \

1023 + "CPU-only job; AMD GPU probes skipped."

1024 + else

1025 + if [[ "${runtime}" == "native-kubernetes" ]]; then

1026 + append_failure_diagnostic_note "${diagnostics_path}" \

1027 + "SMI visibility may be broader than the Kubernetes device allocation."

1028 else

729 - echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}"

1029 + append_failure_diagnostic_note "${diagnostics_path}" \

1030 + "Outer-runner SMI visibility may be broader than the test container."

1031 fi

731 - echo "tmp_filesystem:" | tee -a "${diagnostics_path}"

732 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \

733 - --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \

734 - -- /tmp

1032 + collect_amd_device_nodes "${diagnostics_path}" "${probe_deadline}"

1033 fi

736 - if command -v free >/dev/null 2>&1; then

737 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h

738 - fi

739 - if [[ -r /sys/fs/cgroup/memory.events ]]; then

740 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \

741 - /sys/fs/cgroup/memory.events

1034 + if [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \

1035 + && command -v amd-smi >/dev/null 2>&1; then

1036 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \

1037 + amd-smi version

1038 + # Bus data identifies a card within the public node without publishing its

1039 + # persistent UUID, serial number, or process list.

1040 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \

1041 + amd-smi static -b -g all

1042 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \

1043 + amd-smi metric -e -k -P -x -g all

1044 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \

1045 + amd-smi bad-pages -p -r -u -g all

1046 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \

1047 + amd-smi metric -p -t -u -m -v -g all

1048 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \

1049 + amd-smi xgmi -l -g all

1050 + elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \

1051 + && command -v rocm-smi >/dev/null 2>&1; then

1052 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \

1053 + --showbus --showreplaycount --showrasinfo --showpagesinfo

1054 + elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]]; then

1055 + append_failure_diagnostic_note "${diagnostics_path}" \

1056 + "Neither amd-smi nor rocm-smi is available."

1057 fi

1058

744 - echo "Unauthenticated outer-runner network reachability diagnostics" \

745 - | tee -a "${diagnostics_path}"

1059 + if [[ "${runtime}" == "native-kubernetes" ]]; then

1060 + append_failure_diagnostic_section "${diagnostics_path}" \

1061 + "Unauthenticated pod network reachability"

1062 + else

1063 + append_failure_diagnostic_section "${diagnostics_path}" \

1064 + "Unauthenticated outer-runner network reachability"

1065 + fi

1066 if command -v curl >/dev/null 2>&1; then

747 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \

1067 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \

1068 --silent --location --proto '=https' --proto-redir '=https' \

1069 --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \

1070 --write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \

1071 https://huggingface.co/api/models/gpt2

752 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \

1072 + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \

1073 --silent --location --proto '=https' --proto-redir '=https' \

1074 --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \

1075 --write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \

1076 'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack'

1077 else

758 - echo "curl unavailable; reachability probes skipped." \

759 - | tee -a "${diagnostics_path}"

760 - fi

761 -

762 - if command -v buildkite-agent >/dev/null 2>&1 \

763 - && [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then

764 - if (

765 - cd "${upload_root}" \

766 - && BUILDKITE_AGENT_DEBUG=false \

767 - BUILDKITE_AGENT_DEBUG_HTTP=false \

768 - BUILDKITE_AGENT_TRACE_HTTP=false \

769 - BUILDKITE_AGENT_LOG_LEVEL=error \

770 - timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \

771 - buildkite-agent artifact upload "${upload_path}" \

772 - >/dev/null 2>&1

773 - ); then

774 - echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}"

775 - else

776 - echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}"

1078 + append_failure_diagnostic_note "${diagnostics_path}" \

1079 + "curl unavailable; reachability probes skipped."

1080 + fi

1081 +

1082 + if [[ "${runtime}" == "native-kubernetes" \

1083 + && -n "${amd_diagnostics_memory_events_path}" \

1084 + && -r "${amd_diagnostics_memory_events_path}" ]]; then

1085 + oom_kill_count=$(

1086 + awk '$1 == "oom_kill" { print $2; exit }' \

1087 + "${amd_diagnostics_memory_events_path}" 2>/dev/null

1088 + )

1089 + if [[ ! "${oom_kill_count}" =~ ^[0-9]+$ ]]; then

1090 + oom_kill_count=""

1091 fi

778 - else

779 - echo "Buildkite agent unavailable; diagnostics artifact was not uploaded."

1092 fi

1093

1094 + case "${identity_label}" in

1095 + pod)

1096 + summary_identity_label="Kubernetes pod"

1097 + ;;

1098 + container_hostname)

1099 + summary_identity_label="Container hostname"

1100 + ;;

1101 + esac

1102 +

1103 + summary_rows=(

1104 + "Field" "Value"

1105 + "----------------------" "-----"

1106 + "Exit code" "${exit_code}"

1107 + "Signal" "${exit_signal:-none}"

1108 + "Runtime" "${runtime}"

1109 + "${summary_identity_label}" "${identity_value}"

1110 + )

1111 + if [[ -n "${amd_diagnostics_k8s_node_name}" ]]; then

1112 + summary_rows+=("Kubernetes node" "${amd_diagnostics_k8s_node_name}")

1113 + fi

1114 + if [[ -n "${oom_kill_count}" ]]; then

1115 + summary_rows+=("Cgroup OOM kills" "${oom_kill_count}")

1116 + fi

1117 + summary_rows+=("Diagnostics artifact" "${diagnostics_relative_path}")

1118 +

1119 + printf '\nAMD CI failure summary\n'

1120 + printf '%-22s | %s\n' "${summary_rows[@]}"

1121 +

1122 return 0

1123 }

1124

@@ -788,9 +1128,6 @@ handle_pytest_exit() {

1128 echo "Pytest exit code 5 (no tests collected) - treating as success."

1129 exit 0

1130 fi

791 - if [[ "${exit_code}" -ne 0 ]]; then

792 - collect_rocm_failure_diagnostics "${exit_code}"

793 - fi

1131 exit "$exit_code"

1132 }

1133

@@ -959,7 +1296,7 @@ handle_amd_runner_exit() {

1296 }

1297

1298 # Catch both test failures and wrapper/setup failures. Runtime-specific cleanup

962 -# traps below replace this trap but call the same handler before cleaning up.

1299 +# traps below replace this trap and call the same handler after cleanup.

1300 trap handle_amd_runner_exit EXIT

1301

1302 ###############################################################################

@@ -973,10 +1310,10 @@ if is_native_runtime; then

1310 # shellcheck disable=SC2317 # Called indirectly by the EXIT trap.

1311 cleanup_native_workspace() {

1312 local exit_code=$?

976 - handle_amd_runner_exit "${exit_code}"

1313 if [[ -n "${artifact_work_dir}" ]]; then

1314 rm -rf "${artifact_work_dir}"

1315 fi

1316 + handle_amd_runner_exit "${exit_code}"

1317 }

1318 trap cleanup_native_workspace EXIT

1319

@@ -1020,6 +1357,7 @@ if is_native_runtime; then

1357

1358 echo "Native test commands: $commands"

1359 run_native_preflight || exit 1

1360 + echo "--- Test log"

1361 # Keep AMD CI orchestration variables out of vLLM's runtime environment.

1362 clear_ci_orchestration_env

1363 /bin/bash -o pipefail -c "${commands}"

@@ -1042,7 +1380,6 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea

1380 # shellcheck disable=SC2317 # Called indirectly by the EXIT trap.

1381 remove_docker_container() {

1382 local exit_code=$?

1045 - handle_amd_runner_exit "${exit_code}"

1383 if docker container inspect "${container_name}" >/dev/null 2>&1; then

1384 docker rm -f "${container_name}" || true

1385 fi

@@ -1055,6 +1392,7 @@ remove_docker_container() {

1392 if [[ -n "${artifact_work_dir}" ]]; then

1393 rm -rf "${artifact_work_dir}"

1394 fi

1395 + handle_amd_runner_exit "${exit_code}"

1396 }

1397 trap remove_docker_container EXIT

1398

[CI][AMD] Improve Kubernetes failure diagnostics (#52264)