[CI] Report torch-nightly results to PyTorch CRCR (#51830)
Signed-off-by: Andrey Talman <atalman@fb.com>
Andrey Talman committed
Aug 24, 2026 at 21:00 UTC
e3dde1ee9bfab341592fbfae1f170795568b83ac
3 files changed
+239
.buildkite/scripts/crcr-report.sh
new
+72
@@ -0,0 +1,72 @@
1
+#!/bin/bash
2
+# Report this build's per-job results to the PyTorch Cross-Repo CI Relay (CRCR).
3
+#
4
+# Only runs in the torch-nightly lane on main. CRCR's nightly path is a
5
+# self-report: unlike PR-triggered callbacks there is no upstream dispatch to
6
+# correlate with, so the relay accepts a single "completed" callback per job and
7
+# forwards it to HUD (hud.pytorch.org/crcr).
8
+#
9
+# Reporting is best-effort. A relay outage, an expired mapping or a missing
10
+# token must never fail the nightly build, so every failure path here exits 0
11
+# after logging. The step is also marked soft_fail in the pipeline.
12
+
13
+set -uo pipefail
14
+
15
+# --- Gating -----------------------------------------------------------------
16
+# Matches the image_build.sh convention: the lane is selected inside the script
17
+# rather than in pipeline YAML.
18
+if [[ "${TORCH_NIGHTLY:-0}" != "1" ]]; then
19
+ echo "TORCH_NIGHTLY != 1 -- not the nightly lane, nothing to report"
20
+ exit 0
21
+fi
22
+
23
+# Defence in depth. The relay decides what a Buildkite pipeline may claim via
24
+# ci_providers.yml, and a pipeline that builds fork PRs should constrain
25
+# build_branch there. Refusing to even mint a token off main means a fork PR
26
+# cannot report as vllm-project/vllm even if that mapping is ever relaxed.
27
+if [[ "${BUILDKITE_BRANCH:-}" != "main" ]]; then
28
+ echo "branch '${BUILDKITE_BRANCH:-}' is not main -- refusing to report"
29
+ exit 0
30
+fi
31
+
32
+CALLBACK_URL="${CRCR_CALLBACK_URL:-}"
33
+if [[ -z "${CALLBACK_URL}" ]]; then
34
+ echo "CRCR_CALLBACK_URL unset -- skipping report"
35
+ exit 0
36
+fi
37
+
38
+# read_builds only. Needed because a job cannot see its siblings' outcomes:
39
+# the agent exposes only its own step, so the job list comes from the REST API.
40
+BK_TOKEN="${BUILDKITE_API_TOKEN:-}"
41
+if [[ -z "${BK_TOKEN}" ]]; then
42
+ echo "BUILDKITE_API_TOKEN unset -- skipping report"
43
+ exit 0
44
+fi
45
+
46
+AUDIENCE="pytorch-cross-repo-ci-relay"
47
+OIDC_TOKEN="$(buildkite-agent oidc request-token --audience "${AUDIENCE}" 2>/dev/null)"
48
+if [[ -z "${OIDC_TOKEN}" ]]; then
49
+ echo "could not mint a Buildkite OIDC token -- skipping report"
50
+ exit 0
51
+fi
52
+
53
+BUILD_JSON="$(mktemp)"
54
+trap 'rm -f "${BUILD_JSON}"' EXIT
55
+http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
56
+ -H "Authorization: Bearer ${BK_TOKEN}" \
57
+ "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")"
58
+if [[ "${http_code}" != "200" ]]; then
59
+ echo "buildkite API returned ${http_code} -- skipping report"
60
+ exit 0
61
+fi
62
+
63
+# One callback per job, matching HUD's per-job crcr_workflow_job schema.
64
+# delivery_id is synthetic: the nightly path has no upstream dispatch to borrow
65
+# one from, so build+job is used as the idempotency key.
66
+python3 .buildkite/scripts/crcr_report.py \
67
+ --build-json "${BUILD_JSON}" \
68
+ --callback-url "${CALLBACK_URL}" \
69
+ --oidc-token "${OIDC_TOKEN}" \
70
+ || echo "crcr report failed -- continuing (reporting is best-effort)"
71
+
72
+exit 0
.buildkite/scripts/crcr_report.py
new
+130
@@ -0,0 +1,130 @@
1
+#!/usr/bin/env python3
2
+# SPDX-License-Identifier: Apache-2.0
3
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4
+
5
+"""Post one CRCR nightly callback per Buildkite job.
6
+
7
+Separated from crcr-report.sh because the payload needs real JSON encoding: job
8
+labels routinely contain quotes, colons and emoji, and hand-building JSON in
9
+shell corrupts them silently.
10
+
11
+CRCR's nightly contract (aws/lambda/cross_repo_ci_relay/callback/callback_handler.py):
12
+
13
+ {"delivery_id": str,
14
+ "event_type": "nightly",
15
+ "workflow": {"status": "completed", # nightly rejects any other status
16
+ "conclusion": str,
17
+ "run_id": int,
18
+ "run_attempt": int,
19
+ "name": str,
20
+ "job_name": str}}
21
+"""
22
+
23
+from __future__ import annotations
24
+
25
+import argparse
26
+import json
27
+import sys
28
+import urllib.error
29
+import urllib.request
30
+
31
+# Buildkite job states -> the conclusion vocabulary HUD already stores for
32
+# GitHub Actions callbacks, so nightly rows aggregate with the rest.
33
+_CONCLUSION = {
34
+ "passed": "success",
35
+ "failed": "failure",
36
+ "broken": "failure",
37
+ "canceled": "cancelled",
38
+ "canceling": "cancelled",
39
+ "timed_out": "timed_out",
40
+ "timing_out": "timed_out",
41
+ "skipped": "skipped",
42
+ "not_run": "skipped",
43
+}
44
+
45
+
46
+def post(url: str, token: str, body: dict, timeout: int = 30) -> tuple[int, str]:
47
+ req = urllib.request.Request(
48
+ url,
49
+ data=json.dumps(body).encode(),
50
+ method="POST",
51
+ headers={
52
+ "Authorization": f"Bearer {token}",
53
+ "Content-Type": "application/json",
54
+ },
55
+ )
56
+ try:
57
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
58
+ return resp.status, resp.read().decode("utf-8", "replace")[:200]
59
+ except urllib.error.HTTPError as exc:
60
+ return exc.code, exc.read().decode("utf-8", "replace")[:200]
61
+ except Exception as exc: # network, DNS, TLS
62
+ return 0, f"{type(exc).__name__}: {exc}"
63
+
64
+
65
+def main() -> int:
66
+ p = argparse.ArgumentParser(description=__doc__)
67
+ p.add_argument("--build-json", required=True)
68
+ p.add_argument("--callback-url", required=True)
69
+ p.add_argument("--oidc-token", required=True)
70
+ p.add_argument("--dry-run", action="store_true")
71
+ args = p.parse_args()
72
+
73
+ with open(args.build_json) as f:
74
+ build = json.load(f)
75
+ build_number = build.get("number")
76
+ commit = build.get("commit", "")
77
+
78
+ # Only script jobs are real work. Buildkite also returns waiter/trigger/
79
+ # manual pseudo-jobs, which would otherwise show up in HUD as phantom
80
+ # zero-duration entries.
81
+ jobs = [
82
+ j
83
+ for j in build.get("jobs") or []
84
+ if j.get("type") == "script" and j.get("state") in _CONCLUSION
85
+ ]
86
+ if not jobs:
87
+ print("no terminal script jobs in this build; nothing to report")
88
+ return 0
89
+
90
+ ok = failed = 0
91
+ for job in jobs:
92
+ name = job.get("name") or job.get("command") or "<unnamed>"
93
+ body = {
94
+ # Stable per (build, job) so a re-run of this reporting step is
95
+ # idempotent from HUD's point of view rather than double-counting.
96
+ "delivery_id": f"buildkite-{build_number}-{job['id']}",
97
+ "event_type": "nightly",
98
+ "pytorch_head_sha": commit,
99
+ "workflow": {
100
+ "status": "completed",
101
+ "conclusion": _CONCLUSION[job["state"]],
102
+ "run_id": int(build_number),
103
+ "run_attempt": 1,
104
+ "name": build.get("pipeline", {}).get("slug", "ci"),
105
+ "job_name": name,
106
+ "url": job.get("web_url", ""),
107
+ "started_at": job.get("started_at"),
108
+ "completed_at": job.get("finished_at"),
109
+ },
110
+ }
111
+ if args.dry_run:
112
+ print(f"[dry-run] {_CONCLUSION[job['state']]:<10} {name[:70]}")
113
+ ok += 1
114
+ continue
115
+ status, detail = post(args.callback_url, args.oidc_token, body)
116
+ if 200 <= status < 300:
117
+ ok += 1
118
+ else:
119
+ failed += 1
120
+ # Printed once per job on purpose: a 403 means the pipeline mapping
121
+ # is missing, and a 200 {"status":"ignored"} means the repo is not
122
+ # allowlisted -- two failures that otherwise look identical.
123
+ print(f" {status} for {name[:60]}: {detail}", file=sys.stderr)
124
+
125
+ print(f"reported {ok} job(s), {failed} failed")
126
+ return 0
127
+
128
+
129
+if __name__ == "__main__":
130
+ sys.exit(main())
.buildkite/test_areas/crcr_report.yaml
new
+37
@@ -0,0 +1,37 @@
1
+group: CRCR Report
2
+# Runs only in the torch-nightly lane; the script exits 0 immediately otherwise,
3
+# following the same in-script gating as image_build.sh.
4
+#
5
+# ORDERING (needs confirmation against the pipeline generator): this must be the
6
+# last step to run, because it reads final job states from the Buildkite REST
7
+# API and silently skips any job still in a non-terminal state. On build 83159
8
+# four jobs were still `scheduled` when the build was already 320 jobs deep, so
9
+# running early under-reports rather than erroring.
10
+#
11
+# depends_on across every group is brittle to maintain. If the generator emits a
12
+# `wait` between groups, placing this group last is enough. If not, the more
13
+# robust option is an agent pre-exit hook or a follow-up scheduled step, since
14
+# `CONTINUE_ON_FAILURE=1` means failures must not stop the report either way.
15
+steps:
16
+ - label: ":satellite: Report nightly results to CRCR"
17
+ key: crcr-report
18
+ # Belt and braces with the script's own TORCH_NIGHTLY check: this keeps the
19
+ # step out of non-nightly builds entirely, while the script gate still holds
20
+ # if the generator ever drops this key.
21
+ if: build.env("TORCH_NIGHTLY") == "1"
22
+ # Reporting is observability, never a gate: a relay outage, an expired
23
+ # pipeline mapping or a missing secret must not turn the nightly red.
24
+ soft_fail: true
25
+ allow_dependency_failure: true
26
+ timeout_in_minutes: 15
27
+ # Reporting is a couple of REST calls and a curl: no GPU needed, so keep it
28
+ # off the accelerator agents the test groups queue for.
29
+ device: cpu-small
30
+ # No source_file_dependencies: this step is lane-gated, not change-gated,
31
+ # so it must not be skipped by the file-based step selection.
32
+ commands:
33
+ - bash .buildkite/scripts/crcr-report.sh
34
+ # CRCR_CALLBACK_URL and BUILDKITE_API_TOKEN are deliberately not declared
35
+ # here. Buildkite interpolates a step-level env block at pipeline-upload
36
+ # time, so declaring them would substitute empty strings and shadow the
37
+ # values the agent already has from the pipeline/schedule environment.