| 1 | #!/bin/bash |
| 2 | # Fetch vLLM Buildkite CI logs (public; no login required). |
| 3 | # |
| 4 | # Usage: |
| 5 | # ci-fetch-log.sh [--soft|--all] --pr [<PR>] failed jobs in the PR's latest |
| 6 | # build (current branch if omitted) |
| 7 | # ci-fetch-log.sh [--soft|--all] <build_url> failed jobs in that build |
| 8 | # ci-fetch-log.sh <job_url> [output] one job; both #<job_uuid> and |
| 9 | # ?sid=<id> URL forms work |
| 10 | # ci-fetch-log.sh <build> <job_uuid> [output] |
| 11 | # |
| 12 | # --soft also fetches soft-failed jobs; --all fetches every finished job. |
| 13 | # Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and |
| 14 | # prints "<file>\t<job name>" per job. [output] is single-job only; "-" |
| 15 | # streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches. |
| 16 | |
| 17 | set -euo pipefail |
| 18 | |
| 19 | ORG="vllm" |
| 20 | PIPELINE="ci" |
| 21 | UA="vllm-ci-fetch-log" |
| 22 | UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' |
| 23 | |
| 24 | usage() { |
| 25 | sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//' |
| 26 | exit 1 |
| 27 | } |
| 28 | |
| 29 | die() { |
| 30 | echo "$1" >&2 |
| 31 | exit 1 |
| 32 | } |
| 33 | |
| 34 | BUILD="" JOB="" SID="" OUT="" |
| 35 | SCOPE="failed" |
| 36 | |
| 37 | while :; do |
| 38 | case "${1:-}" in |
| 39 | --soft) SCOPE="soft" ;; |
| 40 | --all) SCOPE="all" ;; |
| 41 | *) break ;; |
| 42 | esac |
| 43 | shift |
| 44 | done |
| 45 | |
| 46 | case "${1:-}" in |
| 47 | --pr) |
| 48 | PR="${2:-}" |
| 49 | # gh pr checks exits non-zero when checks are failing; that is the |
| 50 | # expected case here. |
| 51 | URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null | |
| 52 | grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" | |
| 53 | sort -t/ -k7 -n | tail -1 || true) |
| 54 | [ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}" |
| 55 | BUILD="${URL##*/}" |
| 56 | ;; |
| 57 | https://*) |
| 58 | BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p') |
| 59 | JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true) |
| 60 | SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true) |
| 61 | OUT="${2:-}" |
| 62 | [ -n "$BUILD" ] || die "Could not parse build number from: $1" |
| 63 | ;; |
| 64 | [0-9]*) |
| 65 | [ $# -ge 2 ] || usage |
| 66 | BUILD="$1" |
| 67 | JOB="$2" |
| 68 | OUT="${3:-}" |
| 69 | ;; |
| 70 | *) |
| 71 | usage |
| 72 | ;; |
| 73 | esac |
| 74 | |
| 75 | COOKIES=$(mktemp) |
| 76 | JOBS_TSV=$(mktemp) |
| 77 | trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT |
| 78 | |
| 79 | # Buildkite issues a session cookie on first hit; later requests need it. |
| 80 | curl -fsSL -c "$COOKIES" -A "$UA" \ |
| 81 | "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null |
| 82 | |
| 83 | # The build's job list (id, step uuid, state, name) is served as JSON from |
| 84 | # the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering: |
| 85 | # job_id step_uuid failed soft_failed finished slug name |
| 86 | curl -fsSL -b "$COOKIES" -A "$UA" \ |
| 87 | "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" | |
| 88 | python3 -c ' |
| 89 | import json, re, sys |
| 90 | |
| 91 | data = json.load(sys.stdin) |
| 92 | if data.get("has_next_page"): |
| 93 | print("warning: job list is paginated; some jobs not shown", file=sys.stderr) |
| 94 | for r in data["records"]: |
| 95 | if r.get("type") != "script": |
| 96 | continue |
| 97 | name = (r.get("name") or "").replace("\t", " ").replace("\n", " ") |
| 98 | slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60] |
| 99 | print("\t".join([ |
| 100 | r["id"], |
| 101 | r.get("step_uuid") or "", |
| 102 | str(r.get("passed") is False), |
| 103 | str(bool(r.get("soft_failed"))), |
| 104 | str(bool(r.get("finished_at"))), |
| 105 | slug, |
| 106 | name, |
| 107 | ])) |
| 108 | ' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}" |
| 109 | |
| 110 | if [ -n "$SID" ] && [ -z "$JOB" ]; then |
| 111 | # The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid. |
| 112 | JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV") |
| 113 | [ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}" |
| 114 | fi |
| 115 | |
| 116 | fetch_job() { # <job_uuid> <output_file> |
| 117 | curl -fsSL -b "$COOKIES" -A "$UA" \ |
| 118 | "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \ |
| 119 | -o "$2" |
| 120 | bash "$(dirname "$0")/ci-clean-log.sh" "$2" |
| 121 | } |
| 122 | |
| 123 | if [ -n "$JOB" ]; then |
| 124 | # Single-job mode. |
| 125 | NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV") |
| 126 | SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV") |
| 127 | [ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log" |
| 128 | if [ "$OUT" = "-" ]; then |
| 129 | TMP=$(mktemp) |
| 130 | fetch_job "$JOB" "$TMP" |
| 131 | cat "$TMP" |
| 132 | rm -f "$TMP" |
| 133 | exit 0 |
| 134 | fi |
| 135 | if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then |
| 136 | die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)." |
| 137 | fi |
| 138 | fetch_job "$JOB" "$OUT" |
| 139 | printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}" |
| 140 | exit 0 |
| 141 | fi |
| 142 | |
| 143 | # Build-wide mode: fetch finished jobs matching $SCOPE. |
| 144 | [ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job." |
| 145 | |
| 146 | # shellcheck disable=SC2016 # single-quoted: these are awk field refs ($3/$4/$5), not shell vars |
| 147 | case "$SCOPE" in |
| 148 | failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;; |
| 149 | soft) FILTER='$3 == "True" && $5 == "True"' ;; |
| 150 | all) FILTER='$5 == "True"' ;; |
| 151 | esac |
| 152 | |
| 153 | if [ "$SCOPE" = "failed" ]; then |
| 154 | SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l) |
| 155 | [ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2 |
| 156 | fi |
| 157 | |
| 158 | FOUND=0 |
| 159 | EMITTED=" " |
| 160 | while IFS=$'\t' read -r job_id _ _ _ _ slug name; do |
| 161 | FOUND=$((FOUND + 1)) |
| 162 | out="ci-${BUILD}-${slug:-${job_id:0:13}}.log" |
| 163 | # Retries share a name with the original job; disambiguate by uuid. |
| 164 | case "$EMITTED" in |
| 165 | *" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;; |
| 166 | esac |
| 167 | EMITTED="${EMITTED}${out} " |
| 168 | if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then |
| 169 | echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2 |
| 170 | elif ! fetch_job "$job_id" "$out"; then |
| 171 | echo "Failed to download log for job ${job_id} (${name})." >&2 |
| 172 | continue |
| 173 | fi |
| 174 | printf '%s\t%s\n' "$out" "$name" |
| 175 | done < <(awk -F'\t' "$FILTER" "$JOBS_TSV") |
| 176 | |
| 177 | if [ "$FOUND" -eq 0 ]; then |
| 178 | echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2 |
| 179 | fi |