A high-throughput and memory-efficient inference and serving engine for LLMs
[CI] ci-fetch-log.sh: fetch all failed jobs from a build URL or PR number (#45274)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Michael Goin committed
Jun 12, 2026 at 08:42 UTC
bd59c913bc0338b90bdabdb0e83e5061ce31f9c1
4 files changed
+174
-50
.buildkite/scripts/ci-clean-log.sh
+3
@@ -13,5 +13,8 @@ INPUT_FILE="$1"
13
# Strip timestamps
14
sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
15
16
+# Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
17
+sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
18
+
19
# Strip colorization
20
sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
.buildkite/scripts/ci-fetch-log.sh
+149
-45
@@ -1,74 +1,178 @@
1
#!/bin/bash
2
-# Usage: ./ci-fetch-log.sh <buildkite_job_url> [output_file]
3
-# ./ci-fetch-log.sh <build_number> <job_uuid> [output_file]
2
+# Fetch vLLM Buildkite CI logs (public; no login required).
3
#
5
-# Downloads the raw log for a Buildkite job from the public, unauthenticated
6
-# /organizations/<org>/pipelines/<pipeline>/builds/<n>/jobs/<uuid>/download
7
-# endpoint, then strips ANSI/timestamps via ci-clean-log.sh.
4
+# Usage:
5
+# ci-fetch-log.sh [--soft|--all] --pr [<PR>] failed jobs in the PR's latest
6
+# build (current branch if omitted)
7
+# ci-fetch-log.sh [--soft|--all] <build_url> failed jobs in that build
8
+# ci-fetch-log.sh <job_url> [output] one job; both #<job_uuid> and
9
+# ?sid=<id> URL forms work
10
+# ci-fetch-log.sh <build> <job_uuid> [output]
11
#
9
-# Find <build_number> and <job_uuid> via:
10
-# gh pr checks <PR> --repo vllm-project/vllm
11
-# Each failing row's URL is .../builds/<build_number>#<job_uuid>.
12
-#
13
-# Default output path: ci-<build>-<uuid_first_13_chars>.log (e.g.
14
-# ci-68478-019e6b07-daae.log). Jobs in the same build share the UUID's
15
-# first 8 chars, so the second segment is needed for uniqueness when
16
-# fetching multiple jobs in parallel. The script refuses to overwrite an
17
-# existing output file; pass an explicit path or set CI_FETCH_LOG_FORCE=1
18
-# to override.
12
+# --soft also fetches soft-failed jobs; --all fetches every finished job.
13
+# Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and
14
+# prints "<file>\t<job name>" per job. [output] is single-job only; "-"
15
+# streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches.
16
17
set -euo pipefail
18
19
ORG="vllm"
20
PIPELINE="ci"
21
+UA="vllm-ci-fetch-log"
22
+UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
23
24
usage() {
26
- echo "Usage: $0 <buildkite_job_url> [output_file]"
27
- echo " $0 <build_number> <job_uuid> [output_file]"
25
+ sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'
26
+ exit 1
27
+}
28
+
29
+die() {
30
+ echo "$1" >&2
31
exit 1
32
}
33
31
-if [ $# -lt 1 ]; then usage; fi
34
+BUILD="" JOB="" SID="" OUT=""
35
+SCOPE="failed"
36
+
37
+while :; do
38
+ case "${1:-}" in
39
+ --soft) SCOPE="soft" ;;
40
+ --all) SCOPE="all" ;;
41
+ *) break ;;
42
+ esac
43
+ shift
44
+done
45
33
-if [[ "$1" == https://* ]]; then
46
+case "${1:-}" in
47
+--pr)
48
+ PR="${2:-}"
49
+ # gh pr checks exits non-zero when checks are failing; that is the
50
+ # expected case here.
51
+ URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null |
52
+ grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" |
53
+ sort -t/ -k7 -n | tail -1 || true)
54
+ [ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}"
55
+ BUILD="${URL##*/}"
56
+ ;;
57
+https://*)
58
BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p')
35
- JOB=$(echo "$1" | grep -oE '[0-9a-f]{8}-[0-9a-f-]+' | head -n 1)
59
+ JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true)
60
+ SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true)
61
OUT="${2:-}"
37
-else
38
- if [ $# -lt 2 ]; then usage; fi
62
+ [ -n "$BUILD" ] || die "Could not parse build number from: $1"
63
+ ;;
64
+[0-9]*)
65
+ [ $# -ge 2 ] || usage
66
BUILD="$1"
67
JOB="$2"
68
OUT="${3:-}"
42
-fi
43
-
44
-if [ -z "$BUILD" ] || [ -z "$JOB" ]; then
45
- echo "Could not parse build number or job UUID from: $1" >&2
69
+ ;;
70
+*)
71
usage
47
-fi
72
+ ;;
73
+esac
74
49
-# Jobs in the same build share the UUID's first segment, so include the
50
-# second segment (chars 9-13, e.g. "019e6b07-daae") to keep default filenames
51
-# unique when fetching multiple jobs from one build in parallel.
52
-if [ -z "$OUT" ]; then
53
- OUT="ci-${BUILD}-${JOB:0:13}.log"
75
+COOKIES=$(mktemp)
76
+JOBS_TSV=$(mktemp)
77
+trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT
78
+
79
+# Buildkite issues a session cookie on first hit; later requests need it.
80
+curl -fsSL -c "$COOKIES" -A "$UA" \
81
+ "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
82
+
83
+# The build's job list (id, step uuid, state, name) is served as JSON from
84
+# the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering:
85
+# job_id step_uuid failed soft_failed finished slug name
86
+curl -fsSL -b "$COOKIES" -A "$UA" \
87
+ "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" |
88
+ python3 -c '
89
+import json, re, sys
90
+
91
+data = json.load(sys.stdin)
92
+if data.get("has_next_page"):
93
+ print("warning: job list is paginated; some jobs not shown", file=sys.stderr)
94
+for r in data["records"]:
95
+ if r.get("type") != "script":
96
+ continue
97
+ name = (r.get("name") or "").replace("\t", " ").replace("\n", " ")
98
+ slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60]
99
+ print("\t".join([
100
+ r["id"],
101
+ r.get("step_uuid") or "",
102
+ str(r.get("passed") is False),
103
+ str(bool(r.get("soft_failed"))),
104
+ str(bool(r.get("finished_at"))),
105
+ slug,
106
+ name,
107
+ ]))
108
+' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}"
109
+
110
+if [ -n "$SID" ] && [ -z "$JOB" ]; then
111
+ # The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid.
112
+ JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV")
113
+ [ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}"
114
fi
115
56
-if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
57
- echo "Refusing to overwrite existing $OUT (set CI_FETCH_LOG_FORCE=1 or pass an explicit output path)." >&2
58
- exit 1
116
+fetch_job() { # <job_uuid> <output_file>
117
+ curl -fsSL -b "$COOKIES" -A "$UA" \
118
+ "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \
119
+ -o "$2"
120
+ bash "$(dirname "$0")/ci-clean-log.sh" "$2"
121
+}
122
+
123
+if [ -n "$JOB" ]; then
124
+ # Single-job mode.
125
+ NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV")
126
+ SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV")
127
+ [ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log"
128
+ if [ "$OUT" = "-" ]; then
129
+ TMP=$(mktemp)
130
+ fetch_job "$JOB" "$TMP"
131
+ cat "$TMP"
132
+ rm -f "$TMP"
133
+ exit 0
134
+ fi
135
+ if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
136
+ die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)."
137
+ fi
138
+ fetch_job "$JOB" "$OUT"
139
+ printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}"
140
+ exit 0
141
fi
142
61
-COOKIES=$(mktemp)
62
-trap 'rm -f "$COOKIES"' EXIT
143
+# Build-wide mode: fetch finished jobs matching $SCOPE.
144
+[ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
145
64
-# Buildkite issues a session cookie on first hit; subsequent /download needs it.
65
-curl -fsSL -c "$COOKIES" -A "vllm-ci-fetch-log" \
66
- "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
146
+case "$SCOPE" in
147
+failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
148
+soft) FILTER='$3 == "True" && $5 == "True"' ;;
149
+all) FILTER='$5 == "True"' ;;
150
+esac
151
68
-curl -fsSL -b "$COOKIES" -A "vllm-ci-fetch-log" \
69
- "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/${JOB}/download" \
70
- -o "$OUT"
152
+if [ "$SCOPE" = "failed" ]; then
153
+ SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l)
154
+ [ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2
155
+fi
156
72
-bash "$(dirname "$0")/ci-clean-log.sh" "$OUT"
157
+FOUND=0
158
+EMITTED=" "
159
+while IFS=$'\t' read -r job_id _ _ _ _ slug name; do
160
+ FOUND=$((FOUND + 1))
161
+ out="ci-${BUILD}-${slug:-${job_id:0:13}}.log"
162
+ # Retries share a name with the original job; disambiguate by uuid.
163
+ case "$EMITTED" in
164
+ *" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;;
165
+ esac
166
+ EMITTED="${EMITTED}${out} "
167
+ if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
168
+ echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2
169
+ elif ! fetch_job "$job_id" "$out"; then
170
+ echo "Failed to download log for job ${job_id} (${name})." >&2
171
+ continue
172
+ fi
173
+ printf '%s\t%s\n' "$out" "$name"
174
+done < <(awk -F'\t' "$FILTER" "$JOBS_TSV")
175
74
-echo "$OUT"
176
+if [ "$FOUND" -eq 0 ]; then
177
+ echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2
178
+fi
AGENTS.md
+11
@@ -105,6 +105,17 @@ The line length limit for Python code is 88 characters. If you are not sure, use
105
106
Use [Google-style docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) (`Args:`/`Returns:`/`Raises:` sections), not reStructuredText/Sphinx fields (`:param:`, `:return:`, `:rtype:`).
107
108
+### Diagnosing CI failures
109
+
110
+Buildkite logs are public; no login needed. Details: [docs/contributing/ci/failures.md](docs/contributing/ci/failures.md).
111
+
112
+```bash
113
+# All failed-job logs for a PR's latest build (current branch's PR if omitted):
114
+.buildkite/scripts/ci-fetch-log.sh --pr <PR>
115
+# Any Buildkite build or job URL also works:
116
+.buildkite/scripts/ci-fetch-log.sh "<buildkite_url>"
117
+```
118
+
119
### Commit messages
120
121
Add attribution using commit trailers such as `Co-authored-by:` (other projects use `Assisted-by:` or `Generated-by:`). For example:
docs/contributing/ci/failures.md
+11
-5
@@ -60,15 +60,21 @@ the failure?
60
61
## Logs Wrangling
62
63
-Download a job's log (no Buildkite login required):
64
-
63
+Logs are public; no Buildkite login needed.
64
[.buildkite/scripts/ci-fetch-log.sh](../../../.buildkite/scripts/ci-fetch-log.sh)
65
+saves each log as `ci-<build>-<job-name>.log`, stripped of timestamps and
66
+ANSI codes:
67
68
```bash
68
-# Find the failing job. Each row's URL is .../builds/<N>#<job_uuid>:
69
-gh pr checks <PR> --repo vllm-project/vllm
69
+# All failed jobs in a PR's latest build (current branch's PR if omitted):
70
+.buildkite/scripts/ci-fetch-log.sh --pr <PR>
71
+
72
+# All failed jobs in a build (--soft also includes soft-failed jobs;
73
+# --all fetches every finished job):
74
+.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>"
75
71
-# Download + strip timestamps/ANSI in one step:
76
+# One job — `gh pr checks` URLs (#<job_uuid>) and web UI URLs (?sid=) both
77
+# work; pass "-" as a second argument to stream to stdout:
78
.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>#<job_uuid>"
79
```
80