| 1 | #!/usr/bin/env python3 |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | |
| 5 | """Post one CRCR nightly callback per Buildkite job. |
| 6 | |
| 7 | Separated from crcr-report.sh because the payload needs real JSON encoding: job |
| 8 | labels routinely contain quotes, colons and emoji, and hand-building JSON in |
| 9 | shell corrupts them silently. |
| 10 | |
| 11 | CRCR's nightly contract (aws/lambda/cross_repo_ci_relay/callback/callback_handler.py): |
| 12 | |
| 13 | {"delivery_id": str, |
| 14 | "event_type": "nightly", |
| 15 | "workflow": {"status": "completed", # nightly rejects any other status |
| 16 | "conclusion": str, |
| 17 | "run_id": int, |
| 18 | "run_attempt": int, |
| 19 | "name": str, |
| 20 | "job_name": str}} |
| 21 | """ |
| 22 | |
| 23 | from __future__ import annotations |
| 24 | |
| 25 | import argparse |
| 26 | import json |
| 27 | import sys |
| 28 | import urllib.error |
| 29 | import urllib.request |
| 30 | |
| 31 | # Buildkite job states -> the conclusion vocabulary HUD already stores for |
| 32 | # GitHub Actions callbacks, so nightly rows aggregate with the rest. |
| 33 | _CONCLUSION = { |
| 34 | "passed": "success", |
| 35 | "failed": "failure", |
| 36 | "broken": "failure", |
| 37 | "canceled": "cancelled", |
| 38 | "canceling": "cancelled", |
| 39 | "timed_out": "timed_out", |
| 40 | "timing_out": "timed_out", |
| 41 | "skipped": "skipped", |
| 42 | "not_run": "skipped", |
| 43 | } |
| 44 | |
| 45 | |
| 46 | def post(url: str, token: str, body: dict, timeout: int = 30) -> tuple[int, str]: |
| 47 | req = urllib.request.Request( |
| 48 | url, |
| 49 | data=json.dumps(body).encode(), |
| 50 | method="POST", |
| 51 | headers={ |
| 52 | "Authorization": f"Bearer {token}", |
| 53 | "Content-Type": "application/json", |
| 54 | }, |
| 55 | ) |
| 56 | try: |
| 57 | with urllib.request.urlopen(req, timeout=timeout) as resp: |
| 58 | return resp.status, resp.read().decode("utf-8", "replace")[:200] |
| 59 | except urllib.error.HTTPError as exc: |
| 60 | return exc.code, exc.read().decode("utf-8", "replace")[:200] |
| 61 | except Exception as exc: # network, DNS, TLS |
| 62 | return 0, f"{type(exc).__name__}: {exc}" |
| 63 | |
| 64 | |
| 65 | def main() -> int: |
| 66 | p = argparse.ArgumentParser(description=__doc__) |
| 67 | p.add_argument("--build-json", required=True) |
| 68 | p.add_argument("--callback-url", required=True) |
| 69 | p.add_argument("--oidc-token", required=True) |
| 70 | p.add_argument("--dry-run", action="store_true") |
| 71 | args = p.parse_args() |
| 72 | |
| 73 | with open(args.build_json) as f: |
| 74 | build = json.load(f) |
| 75 | build_number = build.get("number") |
| 76 | commit = build.get("commit", "") |
| 77 | |
| 78 | # Only script jobs are real work. Buildkite also returns waiter/trigger/ |
| 79 | # manual pseudo-jobs, which would otherwise show up in HUD as phantom |
| 80 | # zero-duration entries. |
| 81 | jobs = [ |
| 82 | j |
| 83 | for j in build.get("jobs") or [] |
| 84 | if j.get("type") == "script" and j.get("state") in _CONCLUSION |
| 85 | ] |
| 86 | if not jobs: |
| 87 | print("no terminal script jobs in this build; nothing to report") |
| 88 | return 0 |
| 89 | |
| 90 | ok = failed = 0 |
| 91 | for job in jobs: |
| 92 | name = job.get("name") or job.get("command") or "<unnamed>" |
| 93 | body = { |
| 94 | # Stable per (build, job) so a re-run of this reporting step is |
| 95 | # idempotent from HUD's point of view rather than double-counting. |
| 96 | "delivery_id": f"buildkite-{build_number}-{job['id']}", |
| 97 | "event_type": "nightly", |
| 98 | "pytorch_head_sha": commit, |
| 99 | "workflow": { |
| 100 | "status": "completed", |
| 101 | "conclusion": _CONCLUSION[job["state"]], |
| 102 | "run_id": int(build_number), |
| 103 | "run_attempt": 1, |
| 104 | "name": build.get("pipeline", {}).get("slug", "ci"), |
| 105 | "job_name": name, |
| 106 | "url": job.get("web_url", ""), |
| 107 | "started_at": job.get("started_at"), |
| 108 | "completed_at": job.get("finished_at"), |
| 109 | }, |
| 110 | } |
| 111 | if args.dry_run: |
| 112 | print(f"[dry-run] {_CONCLUSION[job['state']]:<10} {name[:70]}") |
| 113 | ok += 1 |
| 114 | continue |
| 115 | status, detail = post(args.callback_url, args.oidc_token, body) |
| 116 | if 200 <= status < 300: |
| 117 | ok += 1 |
| 118 | else: |
| 119 | failed += 1 |
| 120 | # Printed once per job on purpose: a 403 means the pipeline mapping |
| 121 | # is missing, and a 200 {"status":"ignored"} means the repo is not |
| 122 | # allowlisted -- two failures that otherwise look identical. |
| 123 | print(f" {status} for {name[:60]}: {detail}", file=sys.stderr) |
| 124 | |
| 125 | print(f"reported {ok} job(s), {failed} failed") |
| 126 | return 0 |
| 127 | |
| 128 | |
| 129 | if __name__ == "__main__": |
| 130 | sys.exit(main()) |