| 1 | #!/bin/sh |
| 2 | |
| 3 | [ "${_CI_INFRA_OTEL_INITIALIZED:-0}" = "1" ] && return 0 |
| 4 | |
| 5 | # No-op fallbacks keep every generated wrapper safe when setup is unavailable. |
| 6 | ci_otel_start() { :; } |
| 7 | ci_otel_finish() { :; } |
| 8 | ci_otel_run() { shift 2; env "$@"; return $?; } |
| 9 | |
| 10 | if [ -z "${CI_INFRA_OTEL_DIR:-}" ]; then |
| 11 | echo "vLLM CI OTel: helper directory is unset; tracing disabled" >&2 || : |
| 12 | return 0 |
| 13 | fi |
| 14 | export CI_INFRA_OTEL_DIR |
| 15 | |
| 16 | _CI_INFRA_OTEL_OWNS_RUNTIME=0 |
| 17 | if [ -z "${CI_INFRA_OTEL_RUNTIME_DIR:-}" ]; then |
| 18 | CI_INFRA_OTEL_RUNTIME_DIR="$(mktemp -d 2>/dev/null || :)" |
| 19 | [ -n "${CI_INFRA_OTEL_RUNTIME_DIR}" ] && _CI_INFRA_OTEL_OWNS_RUNTIME=1 |
| 20 | fi |
| 21 | CI_INFRA_OTEL_SPOOL_DIR="${CI_INFRA_OTEL_SPOOL_DIR:-${CI_INFRA_OTEL_RUNTIME_DIR}/spans}" |
| 22 | export CI_INFRA_OTEL_RUNTIME_DIR CI_INFRA_OTEL_SPOOL_DIR |
| 23 | |
| 24 | _CI_INFRA_OTEL_PYTHON="$(command -v python3 2>/dev/null || :)" |
| 25 | if [ -z "${CI_INFRA_OTEL_RUNTIME_DIR}" ] || |
| 26 | [ -z "${_CI_INFRA_OTEL_PYTHON}" ] || |
| 27 | [ ! -f "${CI_INFRA_OTEL_DIR}/ci_otel.py" ] || |
| 28 | ! mkdir -p "${CI_INFRA_OTEL_SPOOL_DIR}" || |
| 29 | ! PYTHONDONTWRITEBYTECODE=1 \ |
| 30 | PYTHONPATH="${CI_INFRA_OTEL_DIR}${PYTHONPATH:+:${PYTHONPATH}}" \ |
| 31 | "${_CI_INFRA_OTEL_PYTHON}" -c "import ci_otel" >/dev/null 2>&1; then |
| 32 | echo "vLLM CI OTel: tracing unavailable; test command will run normally" >&2 || : |
| 33 | if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then |
| 34 | rm -rf -- "${CI_INFRA_OTEL_RUNTIME_DIR}" || : |
| 35 | fi |
| 36 | return 0 |
| 37 | fi |
| 38 | |
| 39 | _CI_INFRA_OTEL_SHIM_DIR="${CI_INFRA_OTEL_RUNTIME_DIR}/bin" |
| 40 | if mkdir -p "${_CI_INFRA_OTEL_SHIM_DIR}"; then |
| 41 | _CI_INFRA_OTEL_SHIM="${_CI_INFRA_OTEL_SHIM_DIR}/pytest" |
| 42 | if [ ! -e "${_CI_INFRA_OTEL_SHIM}" ] && [ ! -L "${_CI_INFRA_OTEL_SHIM}" ]; then |
| 43 | ln -s "${CI_INFRA_OTEL_DIR}/ci_pytest.sh" "${_CI_INFRA_OTEL_SHIM}" || : |
| 44 | fi |
| 45 | if [ -L "${_CI_INFRA_OTEL_SHIM}" ] && |
| 46 | [ "$(readlink "${_CI_INFRA_OTEL_SHIM}" 2>/dev/null || :)" = \ |
| 47 | "${CI_INFRA_OTEL_DIR}/ci_pytest.sh" ]; then |
| 48 | case ":${CI_INFRA_OTEL_SHIM_PATHS:-}:" in |
| 49 | *":${_CI_INFRA_OTEL_SHIM_DIR}:"*) ;; |
| 50 | *) |
| 51 | CI_INFRA_OTEL_SHIM_PATHS="${CI_INFRA_OTEL_SHIM_PATHS:+${CI_INFRA_OTEL_SHIM_PATHS}:}${_CI_INFRA_OTEL_SHIM_DIR}" |
| 52 | PATH="${_CI_INFRA_OTEL_SHIM_DIR}:${PATH}" |
| 53 | ;; |
| 54 | esac |
| 55 | export CI_INFRA_OTEL_SHIM_PATHS PATH |
| 56 | hash -r 2>/dev/null || : |
| 57 | fi |
| 58 | fi |
| 59 | |
| 60 | _ci_otel_python() { |
| 61 | if command -v timeout >/dev/null 2>&1; then |
| 62 | PYTHONDONTWRITEBYTECODE=1 timeout 2s "${_CI_INFRA_OTEL_PYTHON}" "$@" |
| 63 | else |
| 64 | PYTHONDONTWRITEBYTECODE=1 "${_CI_INFRA_OTEL_PYTHON}" "$@" |
| 65 | fi |
| 66 | } |
| 67 | |
| 68 | # The Docker test commands run as root, while /workdir is owned by the |
| 69 | # Buildkite checkout user. Run the GPU sampler as that unprivileged identity so |
| 70 | # any future project-code import cannot leave root-owned files in the checkout. |
| 71 | _ci_otel_prepare_gpu_user() { |
| 72 | _CI_INFRA_GPU_UID="" |
| 73 | _CI_INFRA_GPU_GID="" |
| 74 | [ "$(id -u 2>/dev/null || :)" = "0" ] || return 0 |
| 75 | command -v setpriv >/dev/null 2>&1 || return 1 |
| 76 | |
| 77 | _CI_INFRA_OTEL_WORKSPACE_DIR="${CI_INFRA_OTEL_WORKSPACE_DIR:-/workdir}" |
| 78 | [ -d "${_CI_INFRA_OTEL_WORKSPACE_DIR}" ] || return 1 |
| 79 | # The owner fields are numeric and whitespace-free on GNU stat. |
| 80 | # shellcheck disable=SC2046 |
| 81 | set -- $(stat -c '%u %g' -- "${_CI_INFRA_OTEL_WORKSPACE_DIR}" 2>/dev/null || :) |
| 82 | [ "$#" -eq 2 ] || return 1 |
| 83 | case "$1:$2" in |
| 84 | *[!0-9:]* | 0:* | *:0) return 1 ;; |
| 85 | esac |
| 86 | |
| 87 | _CI_INFRA_GPU_UID="$1" |
| 88 | _CI_INFRA_GPU_GID="$2" |
| 89 | chown "${_CI_INFRA_GPU_UID}:${_CI_INFRA_GPU_GID}" \ |
| 90 | "${CI_INFRA_OTEL_SPOOL_DIR}" || return 1 |
| 91 | chmod 0700 "${CI_INFRA_OTEL_SPOOL_DIR}" || return 1 |
| 92 | if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then |
| 93 | # Permit traversal to the private child spool, but not directory listing. |
| 94 | chmod 0711 "${CI_INFRA_OTEL_RUNTIME_DIR}" || return 1 |
| 95 | fi |
| 96 | setpriv --reuid="${_CI_INFRA_GPU_UID}" \ |
| 97 | --regid="${_CI_INFRA_GPU_GID}" --clear-groups --no-new-privs \ |
| 98 | -- test -w "${CI_INFRA_OTEL_SPOOL_DIR}" |
| 99 | } |
| 100 | |
| 101 | ci_otel_start() { |
| 102 | _CI_INFRA_OTEL_COMMAND_INDEX="$1" |
| 103 | _CI_INFRA_OTEL_COMMAND_LABEL="${2:-command ${_CI_INFRA_OTEL_COMMAND_INDEX}}" |
| 104 | _CI_INFRA_OTEL_CONTEXT="$(_ci_otel_python "${CI_INFRA_OTEL_DIR}/ci_otel.py" new-context)" || |
| 105 | return 0 |
| 106 | # The helper prints four whitespace-free fields. |
| 107 | # shellcheck disable=SC2086 |
| 108 | set -- ${_CI_INFRA_OTEL_CONTEXT} |
| 109 | [ "$#" -eq 4 ] || return 0 |
| 110 | |
| 111 | _CI_INFRA_OTEL_TRACE_ID="$1" |
| 112 | _CI_INFRA_OTEL_COMMAND_SPAN_ID="$2" |
| 113 | _CI_INFRA_OTEL_PARENT_SPAN_ID="$3" |
| 114 | _CI_INFRA_OTEL_START_NS="$4" |
| 115 | CI_INFRA_TRACE_ID="${_CI_INFRA_OTEL_TRACE_ID}" |
| 116 | CI_INFRA_COMMAND_SPAN_ID="${_CI_INFRA_OTEL_COMMAND_SPAN_ID}" |
| 117 | export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID |
| 118 | _CI_INFRA_OTEL_ACTIVE=1 |
| 119 | _CI_INFRA_GPU_PID="" |
| 120 | if [ "${CI_INFRA_GPU_SAMPLING:-1}" != "0" ] && |
| 121 | [ -f "${CI_INFRA_OTEL_DIR}/ci_gpu.py" ] && |
| 122 | command -v nvidia-smi >/dev/null 2>&1; then |
| 123 | if _ci_otel_prepare_gpu_user; then |
| 124 | if [ -n "${_CI_INFRA_GPU_UID}" ]; then |
| 125 | PYTHONDONTWRITEBYTECODE=1 setpriv \ |
| 126 | --reuid="${_CI_INFRA_GPU_UID}" --regid="${_CI_INFRA_GPU_GID}" \ |
| 127 | --clear-groups --no-new-privs -- \ |
| 128 | "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" \ |
| 129 | </dev/null & |
| 130 | else |
| 131 | PYTHONDONTWRITEBYTECODE=1 \ |
| 132 | "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" \ |
| 133 | </dev/null & |
| 134 | fi |
| 135 | _CI_INFRA_GPU_PID=$! |
| 136 | else |
| 137 | echo "vLLM CI OTel: non-root GPU sampler unavailable; sampling disabled" \ |
| 138 | >&2 || : |
| 139 | fi |
| 140 | fi |
| 141 | } |
| 142 | |
| 143 | ci_otel_finish() { |
| 144 | _CI_INFRA_OTEL_COMMAND_STATUS="${1:-0}" |
| 145 | [ "${_CI_INFRA_OTEL_ACTIVE:-0}" = "1" ] || return 0 |
| 146 | _CI_INFRA_OTEL_ACTIVE=0 |
| 147 | _ci_otel_python "${CI_INFRA_OTEL_DIR}/ci_otel.py" record-command \ |
| 148 | "${_CI_INFRA_OTEL_TRACE_ID}" "${_CI_INFRA_OTEL_COMMAND_SPAN_ID}" \ |
| 149 | "${_CI_INFRA_OTEL_PARENT_SPAN_ID}" "${_CI_INFRA_OTEL_START_NS}" \ |
| 150 | "${_CI_INFRA_OTEL_COMMAND_INDEX}" "${_CI_INFRA_OTEL_COMMAND_STATUS}" \ |
| 151 | "${_CI_INFRA_OTEL_COMMAND_LABEL}" || : |
| 152 | if [ -n "${_CI_INFRA_GPU_PID:-}" ]; then |
| 153 | kill -TERM "${_CI_INFRA_GPU_PID}" 2>/dev/null || : |
| 154 | _CI_INFRA_GPU_WAIT=0 |
| 155 | while kill -0 "${_CI_INFRA_GPU_PID}" 2>/dev/null && |
| 156 | [ "${_CI_INFRA_GPU_WAIT}" -lt 30 ]; do |
| 157 | sleep 0.1 |
| 158 | _CI_INFRA_GPU_WAIT=$((_CI_INFRA_GPU_WAIT + 1)) |
| 159 | done |
| 160 | kill -KILL "${_CI_INFRA_GPU_PID}" 2>/dev/null || : |
| 161 | wait "${_CI_INFRA_GPU_PID}" 2>/dev/null || : |
| 162 | _CI_INFRA_GPU_PID="" |
| 163 | fi |
| 164 | CI_INFRA_TRACE_ID="" |
| 165 | CI_INFRA_COMMAND_SPAN_ID="" |
| 166 | export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID |
| 167 | } |
| 168 | |
| 169 | # Run a simple command with tracing. Only for commands that do not modify |
| 170 | # shell state (export, cd, etc.) — those need the explicit start/finish pair. |
| 171 | # Assignment-prefixed arguments (VAR=value) are routed through env so the |
| 172 | # shell does not try to execute the assignment as a program. |
| 173 | ci_otel_run() { |
| 174 | _CI_INFRA_OTEL_RUN_INDEX="$1" |
| 175 | _CI_INFRA_OTEL_RUN_LABEL="$2" |
| 176 | shift 2 |
| 177 | ci_otel_start "${_CI_INFRA_OTEL_RUN_INDEX}" "${_CI_INFRA_OTEL_RUN_LABEL}" || : |
| 178 | env "$@" |
| 179 | _CI_INFRA_OTEL_RUN_STATUS=$? |
| 180 | ci_otel_finish "${_CI_INFRA_OTEL_RUN_STATUS}" || : |
| 181 | return "${_CI_INFRA_OTEL_RUN_STATUS}" |
| 182 | } |
| 183 | |
| 184 | _ci_otel_on_exit() { |
| 185 | _CI_INFRA_OTEL_EXIT_STATUS=$? |
| 186 | trap - 0 |
| 187 | ci_otel_finish "${_CI_INFRA_OTEL_EXIT_STATUS}" || : |
| 188 | if command -v timeout >/dev/null 2>&1; then |
| 189 | PYTHONDONTWRITEBYTECODE=1 timeout 4s \ |
| 190 | "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_otel.py" flush || : |
| 191 | else |
| 192 | PYTHONDONTWRITEBYTECODE=1 \ |
| 193 | "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_otel.py" flush || : |
| 194 | fi |
| 195 | if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then |
| 196 | rm -rf -- "${CI_INFRA_OTEL_RUNTIME_DIR}" || : |
| 197 | fi |
| 198 | exit "${_CI_INFRA_OTEL_EXIT_STATUS}" |
| 199 | } |
| 200 | |
| 201 | trap _ci_otel_on_exit 0 |
| 202 | _CI_INFRA_OTEL_INITIALIZED=1 |
| 203 | : |