@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 504bb8b0c3

[CI] Add repository-local OTel tracing helpers (#52851)

Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Codex <noreply@openai.com>

Kevin H. Luu committed Sep 1, 2026 at 02:36 UTC 504bb8b0c39dbde713a7344772bdb7005adbb214

5 files changed +1661 -1

.buildkite/image_build/image_build.sh

+48 -1

@@ -160,6 +160,42 @@ print_bake_config() {

160 (cd "$(dirname "${BAKE_CONFIG_FILE}")" && buildkite-agent artifact upload "$(basename "${BAKE_CONFIG_FILE}")")

161 }

162

163 +record_buildkit_trace() (

164 + local metadata_file="$1"

165 + local helper=".buildkite/scripts/ci-otel/ci_otel.py"

166 + local trace_file="${BUILD_TMP_DIR}/buildkit-trace.json"

167 + local build_ref record_id

168 +

169 + if [[ "${BUILDKITE:-}" != "true" || ! -f "${helper}" ]] ||

170 + ! command -v timeout >/dev/null 2>&1 ||

171 + ! command -v python3 >/dev/null 2>&1; then

172 + return 0

173 + fi

174 + build_ref="$(timeout 3s python3 "${helper}" build-ref "${metadata_file}" "${TARGET}" 2>/dev/null)" || {

175 + echo "vLLM CI OTel: BuildKit metadata unavailable; trace skipped" >&2

176 + return 0

177 + }

178 + record_id="${build_ref##*/}"

179 + if [[ -z "${record_id}" ]]; then

180 + return 0

181 + fi

182 +

183 + echo "--- :mag: Recording BuildKit trace"

184 + if ! timeout 20s docker buildx history trace "${record_id}" >"${trace_file}"; then

185 + echo "vLLM CI OTel: BuildKit history trace unavailable; upload skipped" >&2

186 + return 0

187 + fi

188 +

189 + export CI_INFRA_BUILDX_REF="${build_ref}"

190 + export CI_INFRA_OTEL_SPOOL_DIR="${BUILD_TMP_DIR}/otel-spans"

191 + timeout 5s python3 "${helper}" record-buildkit "${trace_file}" || {

192 + echo "vLLM CI OTel: BuildKit trace conversion skipped" >&2

193 + return 0

194 + }

195 + timeout 5s python3 "${helper}" flush ||

196 + echo "vLLM CI OTel: BuildKit trace upload skipped" >&2

197 +)

198 +

199 #################################

200 # Main Script #

201 #################################

@@ -269,7 +305,18 @@ export PARENT_COMMIT

305 print_bake_config

306

307 echo "--- :docker: Building ${TARGET}"

272 -docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}"

308 +BUILD_TMP_DIR="$(mktemp -d)"

309 +trap 'rm -rf -- "${BUILD_TMP_DIR}"' EXIT

310 +BUILD_METADATA_FILE="${BUILD_TMP_DIR}/build-metadata.json"

311 +BUILD_STATUS=0

312 +docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" \

313 + --progress plain --metadata-file "${BUILD_METADATA_FILE}" "${TARGET}" || BUILD_STATUS=$?

314 +

315 +record_buildkit_trace "${BUILD_METADATA_FILE}" || true

316 +

317 +if [[ "${BUILD_STATUS}" -ne 0 ]]; then

318 + exit "${BUILD_STATUS}"

319 +fi

320

321 echo "--- :white_check_mark: Build complete"

322

.buildkite/scripts/ci-otel/ci_otel.py new

+650

@@ -0,0 +1,650 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +"""Dependency-free OTLP exporter and pytest timing plugin for CI."""

5 +

6 +from __future__ import annotations

7 +

8 +import hashlib

9 +import json

10 +import os

11 +import secrets

12 +import struct

13 +import sys

14 +import time

15 +import urllib.request

16 +from collections.abc import Iterable

17 +from dataclasses import asdict, dataclass

18 +from pathlib import Path

19 +

20 +ENDPOINT = os.getenv("CI_INFRA_OTEL_ENDPOINT", "https://ci.vllm.ai/api/otel/v1/traces")

21 +AUDIENCE = os.getenv("CI_INFRA_OTEL_AUDIENCE", "https://ci.vllm.ai/api/otel")

22 +MAX_BATCH_SIZE = 2_000

23 +TEST_SPOOL_BATCH_SIZE = 100

24 +

25 +

26 +def _parse_dockerfile_step(name: str) -> tuple[str, int, int, str] | None:

27 + """Parse "[stage index/total] label" without regex."""

28 + if not name.startswith("["):

29 + return None

30 + close = name.find("]")

31 + if close == -1:

32 + return None

33 + header = name[1:close]

34 + label = name[close + 1 :].lstrip()

35 + if not label:

36 + return None

37 + parts = header.rsplit(None, 1)

38 + if len(parts) != 2:

39 + return None

40 + stage, step = parts

41 + if "/" not in step:

42 + return None

43 + index_str, total_str = step.split("/", 1)

44 + if not index_str.isdigit() or not total_str.isdigit():

45 + return None

46 + return stage, int(index_str), int(total_str), label

47 +

48 +

49 +@dataclass(frozen=True)

50 +class Span:

51 + trace_id: str

52 + span_id: str

53 + parent_span_id: str | None

54 + name: str

55 + start_ns: int

56 + end_ns: int

57 + attributes: dict[str, str | int | bool]

58 + status_code: int = 1

59 +

60 +

61 +def _varint(value: int) -> bytes:

62 + if value < 0:

63 + value += 1 << 64

64 + encoded = bytearray()

65 + while value > 0x7F:

66 + encoded.append((value & 0x7F) | 0x80)

67 + value >>= 7

68 + encoded.append(value)

69 + return bytes(encoded)

70 +

71 +

72 +def _field(number: int, wire_type: int) -> bytes:

73 + return _varint((number << 3) | wire_type)

74 +

75 +

76 +def _bytes_field(number: int, value: bytes) -> bytes:

77 + return _field(number, 2) + _varint(len(value)) + value

78 +

79 +

80 +def _string_field(number: int, value: str) -> bytes:

81 + return _bytes_field(number, value.encode())

82 +

83 +

84 +def _varint_field(number: int, value: int) -> bytes:

85 + return _field(number, 0) + _varint(value)

86 +

87 +

88 +def _fixed64_field(number: int, value: int) -> bytes:

89 + return _field(number, 1) + struct.pack("<Q", value)

90 +

91 +

92 +def _attribute(name: str, value: str | int | bool) -> bytes:

93 + if isinstance(value, bool):

94 + encoded_value = _varint_field(2, int(value))

95 + elif isinstance(value, int):

96 + encoded_value = _varint_field(3, value)

97 + else:

98 + encoded_value = _string_field(1, value)

99 + return _string_field(1, name) + _bytes_field(2, encoded_value)

100 +

101 +

102 +def _resource_attributes() -> dict[str, str | int | bool]:

103 + build_number = os.getenv("BUILDKITE_BUILD_NUMBER", "0")

104 + attributes: dict[str, str | int | bool] = {

105 + "service.name": "vllm-ci",

106 + "buildkite.organization.slug": os.getenv("BUILDKITE_ORGANIZATION_SLUG", ""),

107 + "buildkite.pipeline.slug": os.getenv("BUILDKITE_PIPELINE_SLUG", ""),

108 + "buildkite.build.id": os.getenv("BUILDKITE_BUILD_ID", ""),

109 + "buildkite.build.number": int(build_number) if build_number.isdigit() else 0,

110 + "buildkite.build.branch": os.getenv("BUILDKITE_BRANCH", ""),

111 + "buildkite.build.commit": os.getenv("BUILDKITE_COMMIT", ""),

112 + "buildkite.job.id": os.getenv("BUILDKITE_JOB_ID", ""),

113 + "buildkite.job.label": os.getenv("BUILDKITE_LABEL", ""),

114 + "buildkite.agent.queue": os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", ""),

115 + }

116 + return {name: value for name, value in attributes.items() if value != ""}

117 +

118 +

119 +def _encode_span(span: Span) -> bytes:

120 + encoded = b"".join(

121 + (

122 + _bytes_field(1, bytes.fromhex(span.trace_id)),

123 + _bytes_field(2, bytes.fromhex(span.span_id)),

124 + _bytes_field(4, bytes.fromhex(span.parent_span_id))

125 + if span.parent_span_id

126 + else b"",

127 + _string_field(5, span.name),

128 + _varint_field(6, 1),

129 + _fixed64_field(7, span.start_ns),

130 + _fixed64_field(8, span.end_ns),

131 + )

132 + )

133 + attributes = dict(span.attributes)

134 + if os.getenv("BUILDKITE_BUILD_URL") and os.getenv("BUILDKITE_JOB_ID"):

135 + attributes["buildkite.job.web_url"] = (

136 + f"{os.environ['BUILDKITE_BUILD_URL']}#{os.environ['BUILDKITE_JOB_ID']}"

137 + )

138 + for name, value in attributes.items():

139 + encoded += _bytes_field(9, _attribute(name, value))

140 + return encoded + _bytes_field(15, _varint_field(3, span.status_code))

141 +

142 +

143 +def encode_request(spans: Iterable[Span]) -> bytes:

144 + resource = b"".join(

145 + _bytes_field(1, _attribute(name, value))

146 + for name, value in _resource_attributes().items()

147 + )

148 + scope_spans = _bytes_field(1, _string_field(1, "vllm.ci"))

149 + for span in spans:

150 + scope_spans += _bytes_field(2, _encode_span(span))

151 + return _bytes_field(1, _bytes_field(1, resource) + _bytes_field(2, scope_spans))

152 +

153 +

154 +def new_context() -> tuple[str, str, str | None]:

155 + parts = os.getenv("TRACEPARENT", "").lower().split("-")

156 + valid = (

157 + len(parts) == 4

158 + and [len(part) for part in parts] == [2, 32, 16, 2]

159 + and all(all(char in "0123456789abcdef" for char in part) for part in parts)

160 + and parts[1] != "0" * 32

161 + and parts[2] != "0" * 16

162 + )

163 + if valid:

164 + return parts[1], secrets.token_hex(8), parts[2]

165 + build_id = os.getenv("BUILDKITE_BUILD_ID", "local")

166 + trace_id = hashlib.sha256(build_id.encode()).hexdigest()[:32]

167 + return trace_id, secrets.token_hex(8), None

168 +

169 +

170 +def _spool_dir() -> Path | None:

171 + value = os.getenv("CI_INFRA_OTEL_SPOOL_DIR")

172 + return Path(value) if value else None

173 +

174 +

175 +def record_spans(spans: Iterable[Span]) -> bool:

176 + try:

177 + spool_dir = _spool_dir()

178 + records = [json.dumps(asdict(span), separators=(",", ":")) for span in spans]

179 + if spool_dir is None or not records:

180 + return False

181 + spool_dir.mkdir(mode=0o700, parents=True, exist_ok=True)

182 + with (spool_dir / f"spans-{os.getpid()}.jsonl").open(

183 + "a", encoding="utf-8"

184 + ) as output:

185 + output.write("\n".join(records) + "\n")

186 + return True

187 + except Exception as error:

188 + print(f"CI timing spool skipped: {error}", file=sys.stderr)

189 + return False

190 +

191 +

192 +def _read_spool() -> tuple[list[Span], list[Path]]:

193 + spool_dir = _spool_dir()

194 + if spool_dir is None or not spool_dir.is_dir():

195 + return [], []

196 + spans: list[Span] = []

197 + loaded_files: list[Path] = []

198 + for spool_file in sorted(spool_dir.glob("spans-*.jsonl")):

199 + try:

200 + spans.extend(

201 + Span(**json.loads(record))

202 + for record in spool_file.read_text(encoding="utf-8").splitlines()

203 + if record

204 + )

205 + loaded_files.append(spool_file)

206 + except Exception as error:

207 + print(

208 + f"CI timing spool ignored {spool_file.name}: {error}", file=sys.stderr

209 + )

210 + return spans, loaded_files

211 +

212 +

213 +def load_spans() -> list[Span]:

214 + return _read_spool()[0]

215 +

216 +

217 +def buildx_ref(metadata_path: str, target: str) -> str:

218 + """Read the exact Buildx history reference emitted by bake."""

219 + metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))

220 + target_metadata = metadata.get(target, metadata)

221 + reference = target_metadata.get("buildx.build.ref")

222 + if not isinstance(reference, str) or not reference:

223 + raise ValueError(f"Buildx metadata has no history reference for {target}")

224 + return reference

225 +

226 +

227 +def _buildkit_tags(span: dict) -> dict[str, str | int | bool]:

228 + tags: dict[str, str | int | bool] = {}

229 + for tag in span.get("tags", []):

230 + key = tag.get("key")

231 + value = tag.get("value")

232 + if isinstance(key, str) and isinstance(value, (str, int, bool)):

233 + tags[key] = value

234 + return tags

235 +

236 +

237 +def _buildkit_parent_ids(span: dict) -> list[str]:

238 + return [

239 + reference["spanID"]

240 + for reference in span.get("references", [])

241 + if reference.get("refType") == "CHILD_OF"

242 + and isinstance(reference.get("spanID"), str)

243 + ]

244 +

245 +

246 +def _buildkit_span_id(trace_id: str, value: str) -> str:

247 + return hashlib.sha256(f"{trace_id}:{value}".encode()).hexdigest()[:16]

248 +

249 +

250 +def buildkit_spans(payload: dict) -> list[Span]:

251 + """Normalize Buildx's Jaeger JSON into dashboard-oriented OTLP spans."""

252 + trace_id, _, _ = new_context()

253 + traces = payload.get("data", [])

254 + native_trace_id = (

255 + traces[0].get("traceID", "buildkit")

256 + if traces and isinstance(traces[0], dict)

257 + else "buildkit"

258 + )

259 + span_scope = (

260 + os.getenv("CI_INFRA_BUILDX_REF")

261 + or os.getenv("BUILDKITE_JOB_ID")

262 + or str(native_trace_id)

263 + )

264 + steps: list[tuple[str, int, int, str, dict, int, int, int]] = []

265 +

266 + for trace in traces:

267 + raw_spans = [

268 + span

269 + for span in trace.get("spans", [])

270 + if isinstance(span, dict) and isinstance(span.get("spanID"), str)

271 + ]

272 + spans_by_id = {span["spanID"]: span for span in raw_spans}

273 + children: dict[str, list[dict]] = {}

274 + for span in raw_spans:

275 + for parent_id in _buildkit_parent_ids(span):

276 + if parent_id in spans_by_id:

277 + children.setdefault(parent_id, []).append(span)

278 +

279 + def envelope(

280 + span: dict, child_spans: dict[str, list[dict]] = children

281 + ) -> tuple[int, int]:

282 + start_us = int(span["startTime"])

283 + end_us = start_us + int(span["duration"])

284 + pending = list(child_spans.get(span["spanID"], []))

285 + seen: set[str] = set()

286 + while pending:

287 + child = pending.pop()

288 + child_id = child["spanID"]

289 + if child_id in seen:

290 + continue

291 + seen.add(child_id)

292 + child_start = int(child["startTime"])

293 + start_us = min(start_us, child_start)

294 + end_us = max(end_us, child_start + int(child["duration"]))

295 + pending.extend(child_spans.get(child_id, []))

296 + return start_us * 1_000, end_us * 1_000

297 +

298 + selected: dict[str, tuple[dict, int, int]] = {}

299 + for span in raw_spans:

300 + name = span.get("operationName")

301 + if not isinstance(name, str) or name.startswith("cache request: "):

302 + continue

303 + step = _parse_dockerfile_step(name)

304 + is_internal = name.startswith("[internal] ")

305 + is_export = name.startswith(("exporting to ", "export to "))

306 + if not (step or is_internal or is_export):

307 + continue

308 + try:

309 + start_ns, end_ns = envelope(span)

310 + except (KeyError, TypeError, ValueError):

311 + continue

312 + vertex = _buildkit_tags(span).get("vertex")

313 + key = str(vertex) if vertex else f"{name}:{span['spanID']}"

314 + previous = selected.get(key)

315 + if previous is None or end_ns - start_ns > previous[2] - previous[1]:

316 + selected[key] = (span, start_ns, end_ns)

317 +

318 + for span, start_ns, end_ns in selected.values():

319 + name = span["operationName"]

320 + step = _parse_dockerfile_step(name)

321 + tags = _buildkit_tags(span)

322 + failed = (

323 + tags.get("error") is True or tags.get("otel.status_code") == "ERROR"

324 + )

325 + if step:

326 + stage, step_index, step_total, label = step

327 + else:

328 + stage = "BuildKit internals"

329 + label = name.removeprefix("[internal] ")

330 + step_index = step_total = 0

331 + steps.append(

332 + (

333 + stage,

334 + step_index,

335 + step_total,

336 + label,

337 + tags,

338 + start_ns,

339 + end_ns,

340 + 2 if failed else 1,

341 + )

342 + )

343 +

344 + output: list[Span] = []

345 + build_ref = os.getenv("CI_INFRA_BUILDX_REF", "")

346 + for stage in dict.fromkeys(

347 + step[0] for step in steps if step[0] != "BuildKit internals"

348 + ):

349 + stage_steps = sorted(

350 + (step for step in steps if step[0] == stage), key=lambda step: step[5]

351 + )

352 + stage_id = _buildkit_span_id(trace_id, f"{span_scope}:stage:{stage}")

353 + stage_start = min(step[5] for step in stage_steps)

354 + stage_end = max(step[6] for step in stage_steps)

355 + stage_failed = any(step[7] == 2 for step in stage_steps)

356 + stage_attributes: dict[str, str | int | bool] = {

357 + "ci.span.kind": "docker-stage",

358 + "docker.stage": stage,

359 + "docker.step.count": len(stage_steps),

360 + }

361 + if build_ref:

362 + stage_attributes["docker.build.ref"] = build_ref

363 + output.append(

364 + Span(

365 + trace_id=trace_id,

366 + span_id=stage_id,

367 + parent_span_id=None,

368 + name="docker.stage",

369 + start_ns=stage_start,

370 + end_ns=stage_end,

371 + attributes=stage_attributes,

372 + status_code=2 if stage_failed else 1,

373 + )

374 + )

375 + for (

376 + stage_name,

377 + index,

378 + total,

379 + label,

380 + tags,

381 + start_ns,

382 + end_ns,

383 + status,

384 + ) in stage_steps:

385 + vertex = str(tags.get("vertex", ""))

386 + attributes: dict[str, str | int | bool] = {

387 + "ci.span.kind": "docker-instruction",

388 + "docker.stage": stage_name,

389 + "docker.step.label": label,

390 + }

391 + if index:

392 + attributes["docker.step.index"] = index

393 + attributes["docker.step.total"] = total

394 + attributes["docker.instruction"] = label.split(None, 1)[0]

395 + if vertex:

396 + attributes["docker.vertex"] = vertex

397 + output.append(

398 + Span(

399 + trace_id=trace_id,

400 + span_id=_buildkit_span_id(

401 + trace_id,

402 + f"{span_scope}:step:{stage_name}:{index}:{vertex}:{label}",

403 + ),

404 + parent_span_id=stage_id,

405 + name="docker.instruction",

406 + start_ns=start_ns,

407 + end_ns=end_ns,

408 + attributes=attributes,

409 + status_code=status,

410 + )

411 + )

412 + for stage_name, index, total, label, tags, start_ns, end_ns, status in (

413 + step for step in steps if step[0] == "BuildKit internals"

414 + ):

415 + vertex = str(tags.get("vertex", ""))

416 + attributes: dict[str, str | int | bool] = {

417 + "ci.span.kind": "docker-internal",

418 + "docker.stage": stage_name,

419 + "docker.step.label": label,

420 + }

421 + if vertex:

422 + attributes["docker.vertex"] = vertex

423 + output.append(

424 + Span(

425 + trace_id=trace_id,

426 + span_id=_buildkit_span_id(

427 + trace_id, f"{span_scope}:internal:{vertex}:{label}"

428 + ),

429 + parent_span_id=None,

430 + name="docker.internal",

431 + start_ns=start_ns,

432 + end_ns=end_ns,

433 + attributes=attributes,

434 + status_code=status,

435 + )

436 + )

437 + return output

438 +

439 +

440 +def record_buildkit_trace(trace_path: str) -> bool:

441 + payload = json.loads(Path(trace_path).read_text(encoding="utf-8"))

442 + return record_spans(buildkit_spans(payload))

443 +

444 +

445 +def _remaining_seconds(deadline: float) -> float:

446 + remaining = deadline - time.monotonic()

447 + if remaining <= 0:

448 + raise TimeoutError("CI timing upload deadline exceeded")

449 + return remaining

450 +

451 +

452 +def _oidc_token(deadline: float) -> str:

453 + access_token = os.getenv("BUILDKITE_AGENT_ACCESS_TOKEN", "")

454 + job_id = os.getenv("BUILDKITE_JOB_ID", "")

455 + if not access_token or not job_id:

456 + raise RuntimeError("Buildkite job credentials are unavailable")

457 + endpoint = os.getenv(

458 + "BUILDKITE_AGENT_ENDPOINT", "https://agent.buildkite.com/v3"

459 + ).rstrip("/")

460 + request = urllib.request.Request(

461 + f"{endpoint}/jobs/{job_id}/oidc/tokens",

462 + data=json.dumps({"audience": AUDIENCE, "lifetime": 300}).encode(),

463 + method="POST",

464 + headers={

465 + "Authorization": f"Token {access_token}",

466 + "Content-Type": "application/json",

467 + "User-Agent": "vllm-ci-otel/1",

468 + },

469 + )

470 + with urllib.request.urlopen(

471 + request, timeout=_remaining_seconds(deadline)

472 + ) as response:

473 + token = json.loads(response.read()).get("token")

474 + if not isinstance(token, str) or not token:

475 + raise RuntimeError("Buildkite OIDC response did not contain a token")

476 + return token

477 +

478 +

479 +def export_spans(spans: list[Span], timeout_seconds: float = 3.0) -> bool:

480 + if not spans or os.getenv("BUILDKITE") != "true":

481 + return False

482 + try:

483 + deadline = time.monotonic() + max(timeout_seconds, 0.1)

484 + token = _oidc_token(deadline)

485 + for offset in range(0, len(spans), MAX_BATCH_SIZE):

486 + request = urllib.request.Request(

487 + ENDPOINT,

488 + data=encode_request(spans[offset : offset + MAX_BATCH_SIZE]),

489 + method="POST",

490 + headers={

491 + "Authorization": f"Bearer {token}",

492 + "Content-Type": "application/x-protobuf",

493 + "User-Agent": "vllm-ci-otel/1",

494 + },

495 + )

496 + with urllib.request.urlopen(

497 + request, timeout=_remaining_seconds(deadline)

498 + ) as response:

499 + if response.status != 200:

500 + raise RuntimeError(f"OTLP endpoint returned {response.status}")

501 + return True

502 + except Exception as error:

503 + print(f"CI timing upload skipped: {error}", file=sys.stderr)

504 + return False

505 +

506 +

507 +def flush_spans(timeout_seconds: float = 3.0) -> bool:

508 + spans, spool_files = _read_spool()

509 + if not spool_files:

510 + return True

511 + if not spans or not export_spans(spans, timeout_seconds):

512 + return False

513 + for spool_file in spool_files:

514 + try:

515 + spool_file.unlink()

516 + except FileNotFoundError:

517 + pass

518 + except Exception as error:

519 + print(f"CI timing spool cleanup skipped: {error}", file=sys.stderr)

520 + return True

521 +

522 +

523 +_test_runs: dict[str, tuple[int, str]] = {}

524 +_test_spans: list[Span] = []

525 +

526 +

527 +def _spool_test_spans() -> None:

528 + if not _test_spans:

529 + return

530 + record_spans(_test_spans)

531 + _test_spans.clear()

532 +

533 +

534 +def _test_span(nodeid: str, start_ns: int, outcome: str) -> Span:

535 + return Span(

536 + trace_id=os.environ["CI_INFRA_TRACE_ID"],

537 + span_id=secrets.token_hex(8),

538 + parent_span_id=os.environ["CI_INFRA_COMMAND_SPAN_ID"],

539 + name="pytest.test",

540 + start_ns=start_ns,

541 + end_ns=time.time_ns(),

542 + attributes={

543 + "ci.span.kind": "test",

544 + "test.nodeid": nodeid,

545 + "test.file": nodeid.split("::", 1)[0],

546 + "test.outcome": outcome,

547 + },

548 + status_code=2 if outcome == "failed" else 1,

549 + )

550 +

551 +

552 +def pytest_runtest_logstart(nodeid: str, location: tuple[str, int | None, str]):

553 + try:

554 + if os.getenv("CI_INFRA_TRACE_ID") and os.getenv("CI_INFRA_COMMAND_SPAN_ID"):

555 + _test_runs[nodeid] = (time.time_ns(), "unknown")

556 + except Exception:

557 + pass

558 +

559 +

560 +def pytest_runtest_logreport(report):

561 + try:

562 + if report.nodeid not in _test_runs:

563 + return

564 + start_ns, outcome = _test_runs[report.nodeid]

565 + if report.failed:

566 + outcome = "failed"

567 + elif report.skipped and outcome != "failed":

568 + outcome = "skipped"

569 + elif report.when == "call" and outcome == "unknown":

570 + outcome = "passed"

571 + _test_runs[report.nodeid] = (start_ns, outcome)

572 + except Exception:

573 + pass

574 +

575 +

576 +def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]):

577 + try:

578 + run = _test_runs.pop(nodeid, None)

579 + if run:

580 + _test_spans.append(_test_span(nodeid, *run))

581 + if len(_test_spans) >= TEST_SPOOL_BATCH_SIZE:

582 + _spool_test_spans()

583 + except Exception:

584 + pass

585 +

586 +

587 +def pytest_sessionfinish(session, exitstatus: int):

588 + try:

589 + for nodeid, run in list(_test_runs.items()):

590 + _test_spans.append(_test_span(nodeid, *run))

591 + _test_runs.clear()

592 + _spool_test_spans()

593 + except Exception:

594 + pass

595 +

596 +

597 +def _record_command(arguments: list[str]) -> None:

598 + trace_id, span_id, parent_span_id, start_ns, index, exit_code, label = arguments

599 + exit_status = int(exit_code)

600 + record_spans(

601 + [

602 + Span(

603 + trace_id=trace_id,

604 + span_id=span_id,

605 + parent_span_id=None if parent_span_id == "-" else parent_span_id,

606 + name="ci.command",

607 + start_ns=int(start_ns),

608 + end_ns=time.time_ns(),

609 + attributes={

610 + "ci.span.kind": "command",

611 + "ci.command.index": int(index),

612 + "ci.command.label": label,

613 + "process.exit.code": exit_status,

614 + },

615 + status_code=1 if exit_status == 0 else 2,

616 + )

617 + ]

618 + )

619 +

620 +

621 +def main(arguments: list[str] | None = None) -> int:

622 + arguments = sys.argv[1:] if arguments is None else arguments

623 + try:

624 + if arguments == ["new-context"]:

625 + trace_id, span_id, parent_span_id = new_context()

626 + print(trace_id, span_id, parent_span_id or "-", time.time_ns())

627 + return 0

628 + if arguments == ["flush"]:

629 + return 0 if flush_spans() else 1

630 + if len(arguments) == 3 and arguments[0] == "build-ref":

631 + print(buildx_ref(arguments[1], arguments[2]))

632 + return 0

633 + if len(arguments) == 2 and arguments[0] == "record-buildkit":

634 + return 0 if record_buildkit_trace(arguments[1]) else 1

635 + if len(arguments) == 8 and arguments[0] == "record-command":

636 + _record_command(arguments[1:])

637 + return 0

638 + print(

639 + "usage: ci_otel.py {new-context|flush|build-ref ...|"

640 + "record-buildkit ...|record-command ...}",

641 + file=sys.stderr,

642 + )

643 + return 2

644 + except Exception as error:

645 + print(f"CI timing helper failed: {error}", file=sys.stderr)

646 + return 1

647 +

648 +

649 +if __name__ == "__main__":

650 + raise SystemExit(main())

.buildkite/scripts/ci-otel/ci_otel.sh new

+133

@@ -0,0 +1,133 @@

1 +#!/bin/sh

2 +

3 +[ "${_CI_INFRA_OTEL_INITIALIZED:-0}" = "1" ] && return 0

4 +

5 +# No-op fallbacks keep every generated wrapper safe when setup is unavailable.

6 +ci_otel_start() { :; }

7 +ci_otel_finish() { :; }

8 +ci_otel_run() { shift 2; env "$@"; return $?; }

9 +

10 +if [ -z "${CI_INFRA_OTEL_DIR:-}" ]; then

11 + echo "vLLM CI OTel: helper directory is unset; tracing disabled" >&2 || :

12 + return 0

13 +fi

14 +export CI_INFRA_OTEL_DIR

15 +

16 +_CI_INFRA_OTEL_OWNS_RUNTIME=0

17 +if [ -z "${CI_INFRA_OTEL_RUNTIME_DIR:-}" ]; then

18 + CI_INFRA_OTEL_RUNTIME_DIR="$(mktemp -d 2>/dev/null || :)"

19 + [ -n "${CI_INFRA_OTEL_RUNTIME_DIR}" ] && _CI_INFRA_OTEL_OWNS_RUNTIME=1

20 +fi

21 +CI_INFRA_OTEL_SPOOL_DIR="${CI_INFRA_OTEL_SPOOL_DIR:-${CI_INFRA_OTEL_RUNTIME_DIR}/spans}"

22 +export CI_INFRA_OTEL_RUNTIME_DIR CI_INFRA_OTEL_SPOOL_DIR

23 +

24 +_CI_INFRA_OTEL_PYTHON="$(command -v python3 2>/dev/null || :)"

25 +if [ -z "${CI_INFRA_OTEL_RUNTIME_DIR}" ] ||

26 + [ -z "${_CI_INFRA_OTEL_PYTHON}" ] ||

27 + [ ! -f "${CI_INFRA_OTEL_DIR}/ci_otel.py" ] ||

28 + ! mkdir -p "${CI_INFRA_OTEL_SPOOL_DIR}" ||

29 + ! PYTHONPATH="${CI_INFRA_OTEL_DIR}${PYTHONPATH:+:${PYTHONPATH}}" \

30 + "${_CI_INFRA_OTEL_PYTHON}" -c "import ci_otel" >/dev/null 2>&1; then

31 + echo "vLLM CI OTel: tracing unavailable; test command will run normally" >&2 || :

32 + if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then

33 + rm -rf -- "${CI_INFRA_OTEL_RUNTIME_DIR}" || :

34 + fi

35 + return 0

36 +fi

37 +

38 +_CI_INFRA_OTEL_SHIM_DIR="${CI_INFRA_OTEL_RUNTIME_DIR}/bin"

39 +if mkdir -p "${_CI_INFRA_OTEL_SHIM_DIR}"; then

40 + _CI_INFRA_OTEL_SHIM="${_CI_INFRA_OTEL_SHIM_DIR}/pytest"

41 + if [ ! -e "${_CI_INFRA_OTEL_SHIM}" ] && [ ! -L "${_CI_INFRA_OTEL_SHIM}" ]; then

42 + ln -s "${CI_INFRA_OTEL_DIR}/ci_pytest.sh" "${_CI_INFRA_OTEL_SHIM}" || :

43 + fi

44 + if [ -L "${_CI_INFRA_OTEL_SHIM}" ] &&

45 + [ "$(readlink "${_CI_INFRA_OTEL_SHIM}" 2>/dev/null || :)" = \

46 + "${CI_INFRA_OTEL_DIR}/ci_pytest.sh" ]; then

47 + case ":${CI_INFRA_OTEL_SHIM_PATHS:-}:" in

48 + *":${_CI_INFRA_OTEL_SHIM_DIR}:"*) ;;

49 + *)

50 + CI_INFRA_OTEL_SHIM_PATHS="${CI_INFRA_OTEL_SHIM_PATHS:+${CI_INFRA_OTEL_SHIM_PATHS}:}${_CI_INFRA_OTEL_SHIM_DIR}"

51 + PATH="${_CI_INFRA_OTEL_SHIM_DIR}:${PATH}"

52 + ;;

53 + esac

54 + export CI_INFRA_OTEL_SHIM_PATHS PATH

55 + hash -r 2>/dev/null || :

56 + fi

57 +fi

58 +

59 +_ci_otel_python() {

60 + if command -v timeout >/dev/null 2>&1; then

61 + timeout 2s "${_CI_INFRA_OTEL_PYTHON}" "$@"

62 + else

63 + "${_CI_INFRA_OTEL_PYTHON}" "$@"

64 + fi

65 +}

66 +

67 +ci_otel_start() {

68 + _CI_INFRA_OTEL_COMMAND_INDEX="$1"

69 + _CI_INFRA_OTEL_COMMAND_LABEL="${2:-command ${_CI_INFRA_OTEL_COMMAND_INDEX}}"

70 + _CI_INFRA_OTEL_CONTEXT="$(_ci_otel_python "${CI_INFRA_OTEL_DIR}/ci_otel.py" new-context)" ||

71 + return 0

72 + # The helper prints four whitespace-free fields.

73 + # shellcheck disable=SC2086

74 + set -- ${_CI_INFRA_OTEL_CONTEXT}

75 + [ "$#" -eq 4 ] || return 0

76 +

77 + _CI_INFRA_OTEL_TRACE_ID="$1"

78 + _CI_INFRA_OTEL_COMMAND_SPAN_ID="$2"

79 + _CI_INFRA_OTEL_PARENT_SPAN_ID="$3"

80 + _CI_INFRA_OTEL_START_NS="$4"

81 + CI_INFRA_TRACE_ID="${_CI_INFRA_OTEL_TRACE_ID}"

82 + CI_INFRA_COMMAND_SPAN_ID="${_CI_INFRA_OTEL_COMMAND_SPAN_ID}"

83 + export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID

84 + _CI_INFRA_OTEL_ACTIVE=1

85 +}

86 +

87 +ci_otel_finish() {

88 + _CI_INFRA_OTEL_COMMAND_STATUS="${1:-0}"

89 + [ "${_CI_INFRA_OTEL_ACTIVE:-0}" = "1" ] || return 0

90 + _CI_INFRA_OTEL_ACTIVE=0

91 + _ci_otel_python "${CI_INFRA_OTEL_DIR}/ci_otel.py" record-command \

92 + "${_CI_INFRA_OTEL_TRACE_ID}" "${_CI_INFRA_OTEL_COMMAND_SPAN_ID}" \

93 + "${_CI_INFRA_OTEL_PARENT_SPAN_ID}" "${_CI_INFRA_OTEL_START_NS}" \

94 + "${_CI_INFRA_OTEL_COMMAND_INDEX}" "${_CI_INFRA_OTEL_COMMAND_STATUS}" \

95 + "${_CI_INFRA_OTEL_COMMAND_LABEL}" || :

96 + CI_INFRA_TRACE_ID=""

97 + CI_INFRA_COMMAND_SPAN_ID=""

98 + export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID

99 +}

100 +

101 +# Run a simple command with tracing. Only for commands that do not modify

102 +# shell state (export, cd, etc.) — those need the explicit start/finish pair.

103 +# Assignment-prefixed arguments (VAR=value) are routed through env so the

104 +# shell does not try to execute the assignment as a program.

105 +ci_otel_run() {

106 + _CI_INFRA_OTEL_RUN_INDEX="$1"

107 + _CI_INFRA_OTEL_RUN_LABEL="$2"

108 + shift 2

109 + ci_otel_start "${_CI_INFRA_OTEL_RUN_INDEX}" "${_CI_INFRA_OTEL_RUN_LABEL}" || :

110 + env "$@"

111 + _CI_INFRA_OTEL_RUN_STATUS=$?

112 + ci_otel_finish "${_CI_INFRA_OTEL_RUN_STATUS}" || :

113 + return "${_CI_INFRA_OTEL_RUN_STATUS}"

114 +}

115 +

116 +_ci_otel_on_exit() {

117 + _CI_INFRA_OTEL_EXIT_STATUS=$?

118 + trap - 0

119 + ci_otel_finish "${_CI_INFRA_OTEL_EXIT_STATUS}" || :

120 + if command -v timeout >/dev/null 2>&1; then

121 + timeout 4s "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_otel.py" flush || :

122 + else

123 + "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_otel.py" flush || :

124 + fi

125 + if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then

126 + rm -rf -- "${CI_INFRA_OTEL_RUNTIME_DIR}" || :

127 + fi

128 + exit "${_CI_INFRA_OTEL_EXIT_STATUS}"

129 +}

130 +

131 +trap _ci_otel_on_exit 0

132 +_CI_INFRA_OTEL_INITIALIZED=1

133 +:

.buildkite/scripts/ci-otel/ci_pytest.sh new

+42

@@ -0,0 +1,42 @@

1 +#!/bin/sh

2 +

3 +real_pytest=""

4 +old_ifs="${IFS}"

5 +IFS=:

6 +for path_dir in ${PATH}; do

7 + path_dir="${path_dir:-.}"

8 + case ":${CI_INFRA_OTEL_SHIM_PATHS:-}:" in

9 + *":${path_dir}:"*) continue ;;

10 + esac

11 + if [ -x "${path_dir}/pytest" ]; then

12 + real_pytest="${path_dir}/pytest"

13 + break

14 + fi

15 +done

16 +IFS="${old_ifs}"

17 +

18 +if [ -z "${real_pytest}" ]; then

19 + echo "vLLM CI OTel: pytest executable not found" >&2 || :

20 + exit 127

21 +fi

22 +

23 +helper_dir="${CI_INFRA_OTEL_DIR:-}"

24 +otel_pythonpath="${helper_dir}${PYTHONPATH:+:${PYTHONPATH}}"

25 +otel_available() {

26 + if command -v timeout >/dev/null 2>&1; then

27 + timeout 2s env PYTHONPATH="${otel_pythonpath}" python3 -c "import ci_otel" \

28 + >/dev/null 2>&1

29 + else

30 + PYTHONPATH="${otel_pythonpath}" python3 -c "import ci_otel" >/dev/null 2>&1

31 + fi

32 +}

33 +

34 +if [ -n "${helper_dir}" ] &&

35 + [ -n "${CI_INFRA_TRACE_ID:-}" ] &&

36 + [ -n "${CI_INFRA_COMMAND_SPAN_ID:-}" ] &&

37 + otel_available; then

38 + PYTHONPATH="${otel_pythonpath}" exec "${real_pytest}" -p ci_otel "$@"

39 +fi

40 +

41 +echo "vLLM CI OTel: pytest tracing skipped; running pytest normally" >&2 || :

42 +exec "${real_pytest}" "$@"

.buildkite/scripts/ci-otel/tests/test_ci_otel.py new

+788

@@ -0,0 +1,788 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from __future__ import annotations

5 +

6 +import json

7 +import os

8 +import shlex

9 +import subprocess

10 +import sys

11 +import time

12 +from pathlib import Path

13 +

14 +import pytest

15 +

16 +SCRIPTS_DIR = Path(__file__).resolve().parents[1]

17 +sys.path.insert(0, str(SCRIPTS_DIR))

18 +

19 +import ci_otel # noqa: E402

20 +from ci_otel import Span # noqa: E402

21 +

22 +

23 +def _span(**attributes) -> Span:

24 + return Span(

25 + trace_id="01" * 16,

26 + span_id="02" * 8,

27 + parent_span_id=None,

28 + name="ci.command",

29 + start_ns=100,

30 + end_ns=200,

31 + attributes=attributes,

32 + )

33 +

34 +

35 +def _quoted(value: str) -> str:

36 + return shlex.quote(value)

37 +

38 +

39 +def test_context_continues_traceparent(monkeypatch):

40 + monkeypatch.setenv(

41 + "TRACEPARENT",

42 + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",

43 + )

44 +

45 + trace_id, span_id, parent_span_id = ci_otel.new_context()

46 +

47 + assert trace_id == "4bf92f3577b34da6a3ce929d0e0e4736"

48 + assert len(span_id) == 16

49 + assert parent_span_id == "00f067aa0ba902b7"

50 +

51 +

52 +def test_otlp_payload_contains_dashboard_identity(monkeypatch):

53 + monkeypatch.setenv("BUILDKITE_ORGANIZATION_SLUG", "vllm")

54 + monkeypatch.setenv("BUILDKITE_PIPELINE_SLUG", "ci")

55 + monkeypatch.setenv("BUILDKITE_BUILD_ID", "build-id")

56 + monkeypatch.setenv("BUILDKITE_BUILD_NUMBER", "42")

57 + monkeypatch.setenv("BUILDKITE_JOB_ID", "job-id")

58 +

59 + payload = ci_otel.encode_request([_span(**{"ci.span.kind": "command"})])

60 +

61 + for value in (b"ci.command", b"ci.span.kind", b"buildkite.job.id", b"job-id"):

62 + assert value in payload

63 +

64 +

65 +def test_otlp_payload_accepts_negative_integer_attributes():

66 + payload = ci_otel.encode_request([_span(**{"process.exit.code": -1})])

67 +

68 + assert b"process.exit.code" in payload

69 +

70 +

71 +def test_spool_round_trip_is_local(monkeypatch, tmp_path):

72 + monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path))

73 + monkeypatch.setattr(

74 + ci_otel,

75 + "_oidc_token",

76 + lambda deadline: (_ for _ in ()).throw(AssertionError("unexpected upload")),

77 + )

78 + span = _span(**{"ci.command.index": 1})

79 +

80 + assert ci_otel.record_spans([span]) is True

81 + assert ci_otel.load_spans() == [span]

82 +

83 +

84 +def test_buildx_ref_supports_bake_metadata(tmp_path):

85 + metadata = tmp_path / "metadata.json"

86 + metadata.write_text(

87 + json.dumps({"test-ci": {"buildx.build.ref": "builder/node/record-id"}})

88 + )

89 +

90 + assert ci_otel.buildx_ref(str(metadata), "test-ci") == "builder/node/record-id"

91 +

92 +

93 +def test_buildkit_trace_groups_stages_and_includes_descendant_time(monkeypatch):

94 + monkeypatch.setenv("BUILDKITE_BUILD_ID", "build-id")

95 + payload = {

96 + "data": [

97 + {

98 + "spans": [

99 + {

100 + "spanID": "01" * 8,

101 + "operationName": "[base 1/2] FROM alpine:3.21",

102 + "startTime": 100,

103 + "duration": 10,

104 + "tags": [{"key": "vertex", "value": "sha256:from"}],

105 + },

106 + {

107 + "spanID": "02" * 8,

108 + "operationName": "registry pull",

109 + "references": [{"refType": "CHILD_OF", "spanID": "01" * 8}],

110 + "startTime": 105,

111 + "duration": 95,

112 + },

113 + {

114 + "spanID": "03" * 8,

115 + "operationName": "[base 2/2] RUN uv pip install vllm",

116 + "startTime": 210,

117 + "duration": 40,

118 + "tags": [{"key": "vertex", "value": "sha256:run"}],

119 + },

120 + {

121 + "spanID": "04" * 8,

122 + "operationName": "[test 1/1] COPY --from=base /opt /opt",

123 + "startTime": 260,

124 + "duration": 20,

125 + "tags": [{"key": "vertex", "value": "sha256:copy"}],

126 + },

127 + {

128 + "spanID": "05" * 8,

129 + "operationName": "[internal] load build context",

130 + "startTime": 90,

131 + "duration": 30,

132 + "tags": [{"key": "vertex", "value": "sha256:context"}],

133 + },

134 + {

135 + "spanID": "06" * 8,

136 + "operationName": (

137 + "cache request: [base 2/2] RUN uv pip install vllm"

138 + ),

139 + "startTime": 205,

140 + "duration": 2,

141 + "tags": [{"key": "vertex", "value": "sha256:run"}],

142 + },

143 + ]

144 + }

145 + ]

146 + }

147 +

148 + spans = ci_otel.buildkit_spans(payload)

149 + stages = {

150 + span.attributes.get("docker.stage"): span

151 + for span in spans

152 + if span.name == "docker.stage"

153 + }

154 + instructions = [span for span in spans if span.name == "docker.instruction"]

155 +

156 + assert set(stages) == {"base", "test"}

157 + assert stages["base"].start_ns == 100_000

158 + assert stages["base"].end_ns == 250_000

159 + assert len(instructions) == 3

160 + assert len([span for span in spans if span.name == "docker.internal"]) == 1

161 + from_span = next(

162 + span for span in instructions if span.attributes["docker.instruction"] == "FROM"

163 + )

164 + assert from_span.end_ns == 200_000

165 + assert from_span.parent_span_id == stages["base"].span_id

166 + assert all(span.trace_id == stages["base"].trace_id for span in spans)

167 +

168 +

169 +def test_record_buildkit_trace_spools_normalized_spans(monkeypatch, tmp_path):

170 + trace_file = tmp_path / "trace.json"

171 + trace_file.write_text(

172 + json.dumps(

173 + {

174 + "data": [

175 + {

176 + "spans": [

177 + {

178 + "spanID": "01" * 8,

179 + "operationName": "[base 1/1] RUN true",

180 + "startTime": 100,

181 + "duration": 20,

182 + }

183 + ]

184 + }

185 + ]

186 + }

187 + )

188 + )

189 + monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path / "spool"))

190 +

191 + assert ci_otel.record_buildkit_trace(str(trace_file)) is True

192 + assert {span.attributes["ci.span.kind"] for span in ci_otel.load_spans()} == {

193 + "docker-stage",

194 + "docker-instruction",

195 + }

196 +

197 +

198 +def test_export_uses_buildkite_oidc_and_otlp(monkeypatch):

199 + for name, value in {

200 + "BUILDKITE": "true",

201 + "BUILDKITE_AGENT_ACCESS_TOKEN": "agent-token",

202 + "BUILDKITE_AGENT_ENDPOINT": "https://agent.example/v3",

203 + "BUILDKITE_JOB_ID": "job-id",

204 + }.items():

205 + monkeypatch.setenv(name, value)

206 + requests = []

207 +

208 + class Response:

209 + status = 200

210 +

211 + def __init__(self, body=b""):

212 + self.body = body

213 +

214 + def read(self):

215 + return self.body

216 +

217 + def __enter__(self):

218 + return self

219 +

220 + def __exit__(self, *args):

221 + return False

222 +

223 + def open_request(request, timeout):

224 + requests.append((request, timeout))

225 + if request.full_url.endswith("/oidc/tokens"):

226 + return Response(b'{"token":"oidc-token"}')

227 + return Response()

228 +

229 + monkeypatch.setattr(ci_otel.urllib.request, "urlopen", open_request)

230 +

231 + assert ci_otel.export_spans([_span()], timeout_seconds=0.5) is True

232 + oidc_request, upload_request = (request for request, _ in requests)

233 + assert oidc_request.full_url.endswith("/jobs/job-id/oidc/tokens")

234 + assert oidc_request.get_header("Authorization") == "Token agent-token"

235 + assert json.loads(oidc_request.data)["audience"] == ci_otel.AUDIENCE

236 + assert upload_request.get_header("Authorization") == "Bearer oidc-token"

237 + assert upload_request.get_header("Content-type") == "application/x-protobuf"

238 + assert all(0 < timeout <= 0.5 for _, timeout in requests)

239 +

240 +

241 +def test_export_failure_is_soft_and_bounded(monkeypatch):

242 + monkeypatch.setenv("BUILDKITE", "true")

243 + monkeypatch.setattr(

244 + ci_otel,

245 + "_oidc_token",

246 + lambda deadline: (_ for _ in ()).throw(TimeoutError("unavailable")),

247 + )

248 +

249 + started = time.monotonic()

250 + assert ci_otel.export_spans([_span()], timeout_seconds=0.1) is False

251 + assert time.monotonic() - started < 0.5

252 +

253 +

254 +def test_export_batches_with_one_oidc_token(monkeypatch):

255 + monkeypatch.setenv("BUILDKITE", "true")

256 + monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2)

257 + token_calls = []

258 + requests = []

259 +

260 + class Response:

261 + status = 200

262 +

263 + def __enter__(self):

264 + return self

265 +

266 + def __exit__(self, *args):

267 + return False

268 +

269 + def token(deadline):

270 + token_calls.append(deadline)

271 + return "token"

272 +

273 + def open_request(request, timeout):

274 + requests.append(request)

275 + return Response()

276 +

277 + monkeypatch.setattr(ci_otel, "_oidc_token", token)

278 + monkeypatch.setattr(ci_otel.urllib.request, "urlopen", open_request)

279 +

280 + assert ci_otel.export_spans([_span() for _ in range(5)]) is True

281 + assert len(token_calls) == 1

282 + assert len(requests) == 3

283 +

284 +

285 +def test_successful_flush_removes_spool_and_does_not_duplicate(monkeypatch, tmp_path):

286 + monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path))

287 + uploaded = []

288 + assert ci_otel.record_spans([_span()]) is True

289 + monkeypatch.setattr(

290 + ci_otel, "export_spans", lambda spans, timeout: uploaded.extend(spans) or True

291 + )

292 +

293 + assert ci_otel.flush_spans() is True

294 + assert len(uploaded) == 1

295 + assert list(tmp_path.glob("spans-*.jsonl")) == []

296 + assert ci_otel.flush_spans() is True

297 + assert len(uploaded) == 1

298 +

299 +

300 +def test_failed_flush_preserves_spool(monkeypatch, tmp_path):

301 + monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path))

302 + assert ci_otel.record_spans([_span()]) is True

303 + monkeypatch.setattr(ci_otel, "export_spans", lambda spans, timeout: False)

304 +

305 + assert ci_otel.flush_spans() is False

306 + assert len(list(tmp_path.glob("spans-*.jsonl"))) == 1

307 +

308 +

309 +def test_cli_rejects_bad_arguments_without_traceback(capsys):

310 + assert ci_otel.main(["bogus"]) == 2

311 + assert ci_otel.main(["record-command", "too", "short"]) == 2

312 + assert (

313 + ci_otel.main(["record-command", "t", "s", "-", "not-an-int", "1", "0", "label"])

314 + == 1

315 + )

316 +

317 + stderr = capsys.readouterr().err

318 + assert "usage:" in stderr

319 + assert "CI timing helper failed:" in stderr

320 + assert "Traceback" not in stderr

321 +

322 +

323 +def test_pytest_spans_are_spooled_incrementally(monkeypatch):

324 + recorded = []

325 + monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16)

326 + monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8)

327 + monkeypatch.setattr(ci_otel, "TEST_SPOOL_BATCH_SIZE", 1)

328 + monkeypatch.setattr(

329 + ci_otel, "record_spans", lambda spans: recorded.extend(spans) or True

330 + )

331 + ci_otel._test_runs.clear()

332 + ci_otel._test_spans.clear()

333 + ci_otel._test_runs["test_sample.py::test_one"] = (time.time_ns(), "passed")

334 +

335 + ci_otel.pytest_runtest_logfinish("test_sample.py::test_one", ("", 0, ""))

336 +

337 + assert len(recorded) == 1

338 + assert ci_otel._test_spans == []

339 +

340 +

341 +def test_shell_wrapper_records_commands_without_changing_shell_state(tmp_path):

342 + first = f"ci_otel_start 1 {_quoted('export VALUE=ready')}"

343 + second = f"ci_otel_start 2 {_quoted('check VALUE')}"

344 + shell = (

345 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; {first}; '

346 + f"export VALUE=ready; ci_otel_finish 0; {second}; "

347 + 'test "$VALUE" = ready; ci_otel_finish 0'

348 + )

349 +

350 + result = subprocess.run(

351 + ["/bin/sh", "-c", shell],

352 + check=False,

353 + capture_output=True,

354 + text=True,

355 + env={

356 + **os.environ,

357 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

358 + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path),

359 + },

360 + )

361 +

362 + assert result.returncode == 0, result.stderr

363 + records = "".join(path.read_text() for path in tmp_path.glob("spans-*.jsonl"))

364 + assert "export VALUE=ready" in records

365 + assert "check VALUE" in records

366 +

367 +

368 +def test_shell_wrapper_preserves_failure_status(tmp_path):

369 + shell = (

370 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

371 + f"ci_otel_start 1 {_quoted('false')}; false; status=$?; "

372 + 'ci_otel_finish "$status"; exit "$status"'

373 + )

374 +

375 + result = subprocess.run(

376 + ["/bin/sh", "-c", shell],

377 + check=False,

378 + capture_output=True,

379 + text=True,

380 + env={

381 + **os.environ,

382 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

383 + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path),

384 + },

385 + )

386 +

387 + assert result.returncode == 1

388 +

389 +

390 +def test_ci_otel_run_records_command_and_preserves_status(tmp_path):

391 + shell = (

392 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

393 + f"ci_otel_run 1 {_quoted('true')} true; "

394 + f"ci_otel_run 2 {_quoted('false')} false; "

395 + 'echo "status=$?"'

396 + )

397 +

398 + result = subprocess.run(

399 + ["/bin/sh", "-c", shell],

400 + check=False,

401 + capture_output=True,

402 + text=True,

403 + env={

404 + **os.environ,

405 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

406 + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path),

407 + },

408 + )

409 +

410 + assert result.returncode == 0, result.stderr

411 + assert "status=1" in result.stdout

412 + records = "".join(path.read_text() for path in tmp_path.glob("spans-*.jsonl"))

413 + assert '"ci.command.index":1' in records

414 + assert '"ci.command.index":2' in records

415 + assert '"process.exit.code":0' in records

416 + assert '"process.exit.code":1' in records

417 +

418 +

419 +def test_ci_otel_run_fail_open_when_tracing_unavailable(tmp_path):

420 + shell = (

421 + f'CI_INFRA_OTEL_DIR="{tmp_path / "missing"}"; export CI_INFRA_OTEL_DIR; '

422 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

423 + f"ci_otel_run 1 {_quoted('echo ran')} echo ran"

424 + )

425 +

426 + result = subprocess.run(

427 + ["/bin/sh", "-e", "-c", shell],

428 + check=False,

429 + capture_output=True,

430 + text=True,

431 + )

432 +

433 + assert result.returncode == 0, result.stderr

434 + assert result.stdout.strip() == "ran"

435 +

436 +

437 +def test_ci_otel_run_handles_assignment_prefixed_commands(tmp_path):

438 + shell = (

439 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

440 + f"ci_otel_run 1 {_quoted('MY_VAR=hello env')} "

441 + "MY_VAR=hello env"

442 + )

443 +

444 + result = subprocess.run(

445 + ["/bin/sh", "-c", shell],

446 + check=False,

447 + capture_output=True,

448 + text=True,

449 + env={

450 + **os.environ,

451 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

452 + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path),

453 + },

454 + )

455 +

456 + assert result.returncode == 0, result.stderr

457 + assert "MY_VAR=hello" in result.stdout

458 +

459 +

460 +def test_missing_helpers_do_not_block_the_test_command(tmp_path):

461 + output = tmp_path / "ran"

462 + shell = (

463 + f'CI_INFRA_OTEL_DIR="{tmp_path / "missing"}"; export CI_INFRA_OTEL_DIR; '

464 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; printf ran > "{output}"'

465 + )

466 +

467 + result = subprocess.run(

468 + ["/bin/sh", "-e", "-c", shell],

469 + check=False,

470 + capture_output=True,

471 + text=True,

472 + )

473 +

474 + assert result.returncode == 0, result.stderr

475 + assert output.read_text() == "ran"

476 +

477 +

478 +def test_unset_helper_dir_does_not_block_the_test_command():

479 + shell = f'unset CI_INFRA_OTEL_DIR; . "{SCRIPTS_DIR / "ci_otel.sh"}"; echo ran'

480 +

481 + result = subprocess.run(

482 + ["/bin/bash", "-e", "-c", shell],

483 + check=False,

484 + capture_output=True,

485 + text=True,

486 + )

487 +

488 + assert result.returncode == 0, result.stderr

489 + assert result.stdout.strip() == "ran"

490 +

491 +

492 +def test_helper_dir_assignment_is_exported():

493 + shell = (

494 + "unset CI_INFRA_OTEL_DIR; "

495 + f'CI_INFRA_OTEL_DIR="{SCRIPTS_DIR}" . "{SCRIPTS_DIR / "ci_otel.sh"}"; '

496 + f'/bin/sh -c \'test "$CI_INFRA_OTEL_DIR" = "{SCRIPTS_DIR}"\''

497 + )

498 +

499 + result = subprocess.run(

500 + ["/bin/sh", "-c", shell], check=False, capture_output=True, text=True

501 + )

502 +

503 + assert result.returncode == 0, result.stderr

504 +

505 +

506 +@pytest.mark.parametrize("failure", ["new-context", "record-command"])

507 +def test_helper_failure_cannot_change_job_status(tmp_path, failure):

508 + fake_bin = tmp_path / "bin"

509 + fake_bin.mkdir()

510 + fake_python = fake_bin / "python3"

511 + fake_python.write_text(

512 + "#!/bin/sh\n"

513 + '[ "$1" = "-c" ] && exit 0\n'

514 + '[ "$2" = "new-context" ] && {\n'

515 + ' [ "$CI_OTEL_TEST_FAILURE" = "new-context" ] && exit 99\n'

516 + ' echo "01010101010101010101010101010101 0202020202020202 - 1"\n'

517 + " exit 0\n"

518 + "}\n"

519 + '[ "$2" = "record-command" ] && exit 99\n'

520 + "exit 99\n"

521 + )

522 + fake_python.chmod(0o755)

523 + shell = (

524 + f'PATH="{fake_bin}:$PATH"; . "{SCRIPTS_DIR / "ci_otel.sh"}"; '

525 + f"ci_otel_start 1 {_quoted('true')}; ci_otel_finish 0; echo ran"

526 + )

527 +

528 + result = subprocess.run(

529 + ["/bin/sh", "-e", "-c", shell],

530 + check=False,

531 + capture_output=True,

532 + text=True,

533 + env={

534 + **os.environ,

535 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

536 + "CI_INFRA_OTEL_RUNTIME_DIR": str(tmp_path / "runtime"),

537 + "CI_OTEL_TEST_FAILURE": failure,

538 + },

539 + )

540 +

541 + assert result.returncode == 0, result.stderr

542 + assert result.stdout.strip() == "ran"

543 +

544 +

545 +def test_flush_failure_preserves_job_status(tmp_path):

546 + fake_bin = tmp_path / "bin"

547 + fake_bin.mkdir()

548 + fake_python = fake_bin / "python3"

549 + fake_python.write_text("#!/bin/sh\nexit 99\n")

550 + fake_python.chmod(0o755)

551 +

552 + def run(status: int):

553 + shell = (

554 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

555 + f'PATH="{fake_bin}"; export PATH; exit {status}'

556 + )

557 + return subprocess.run(

558 + ["/bin/sh", "-c", shell],

559 + check=False,

560 + capture_output=True,

561 + text=True,

562 + env={**os.environ, "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR)},

563 + )

564 +

565 + assert run(0).returncode == 0

566 + assert run(7).returncode == 7

567 +

568 +

569 +def test_shell_wrapper_uses_private_trace_ids(tmp_path):

570 + shell = (

571 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

572 + f"ci_otel_start 1 {_quoted('true')}; "

573 + 'expected="$CI_INFRA_TRACE_ID"; '

574 + "CI_INFRA_TRACE_ID=corrupted; CI_INFRA_COMMAND_SPAN_ID=corrupted; "

575 + 'ci_otel_finish 0; printf "%s" "$expected"'

576 + )

577 +

578 + result = subprocess.run(

579 + ["/bin/sh", "-c", shell],

580 + check=False,

581 + capture_output=True,

582 + text=True,

583 + env={

584 + **os.environ,

585 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

586 + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path),

587 + },

588 + )

589 +

590 + assert result.returncode == 0, result.stderr

591 + record = json.loads(next(tmp_path.glob("spans-*.jsonl")).read_text())

592 + assert record["trace_id"] == result.stdout

593 + assert record["span_id"] != "corrupted"

594 +

595 +

596 +def test_shell_setup_is_idempotent(tmp_path):

597 + runtime = tmp_path / "runtime"

598 + shell = (

599 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

600 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

601 + 'printf "%s\n%s" "$PATH" "$CI_INFRA_OTEL_SHIM_PATHS"'

602 + )

603 +

604 + result = subprocess.run(

605 + ["/bin/sh", "-c", shell],

606 + check=False,

607 + capture_output=True,

608 + text=True,

609 + env={

610 + **os.environ,

611 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

612 + "CI_INFRA_OTEL_RUNTIME_DIR": str(runtime),

613 + },

614 + )

615 +

616 + assert result.returncode == 0, result.stderr

617 + path, shim_paths = result.stdout.splitlines()

618 + assert path.split(":").count(str(runtime / "bin")) == 1

619 + assert shim_paths.split(":").count(str(runtime / "bin")) == 1

620 +

621 +

622 +def test_helper_owned_runtime_is_removed_on_exit(tmp_path):

623 + runtime_file = tmp_path / "runtime"

624 + shell = (

625 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '

626 + f'printf "%s" "$CI_INFRA_OTEL_RUNTIME_DIR" > "{runtime_file}"'

627 + )

628 +

629 + result = subprocess.run(

630 + ["/bin/sh", "-c", shell],

631 + check=False,

632 + capture_output=True,

633 + text=True,

634 + env={**os.environ, "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR)},

635 + )

636 +

637 + assert result.returncode == 0, result.stderr

638 + assert not Path(runtime_file.read_text()).exists()

639 +

640 +

641 +def test_caller_owned_runtime_is_preserved(tmp_path):

642 + runtime = tmp_path / "runtime"

643 + shell = f'. "{SCRIPTS_DIR / "ci_otel.sh"}"'

644 +

645 + result = subprocess.run(

646 + ["/bin/sh", "-c", shell],

647 + check=False,

648 + capture_output=True,

649 + text=True,

650 + env={

651 + **os.environ,

652 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

653 + "CI_INFRA_OTEL_RUNTIME_DIR": str(runtime),

654 + },

655 + )

656 +

657 + assert result.returncode == 0, result.stderr

658 + assert runtime.exists()

659 +

660 +

661 +def test_pytest_shim_recovers_from_stale_recorded_path(tmp_path):

662 + bin_dir = tmp_path / "bin"

663 + bin_dir.mkdir()

664 + real_pytest = bin_dir / "pytest"

665 + real_pytest.write_text("#!/bin/sh\nprintf 'CURRENT-PYTEST'")

666 + real_pytest.chmod(0o755)

667 +

668 + result = subprocess.run(

669 + [str(SCRIPTS_DIR / "ci_pytest.sh"), "--version"],

670 + check=False,

671 + capture_output=True,

672 + text=True,

673 + env={

674 + **os.environ,

675 + "PATH": str(bin_dir),

676 + "CI_INFRA_OTEL_REAL_PYTEST": "/missing/pytest",

677 + },

678 + )

679 +

680 + assert result.returncode == 0, result.stderr

681 + assert result.stdout == "CURRENT-PYTEST"

682 +

683 +

684 +def test_path_change_after_source_uses_new_pytest(tmp_path):

685 + old_bin = tmp_path / "old"

686 + new_bin = tmp_path / "new"

687 + runtime = tmp_path / "runtime"

688 + old_bin.mkdir()

689 + new_bin.mkdir()

690 + for directory, output in ((old_bin, "OLD"), (new_bin, "NEW")):

691 + executable = directory / "pytest"

692 + executable.write_text(f"#!/bin/sh\nprintf '{output}'")

693 + executable.chmod(0o755)

694 + shell = (

695 + f'PATH="{old_bin}:$PATH"; . "{SCRIPTS_DIR / "ci_otel.sh"}"; '

696 + f'PATH="{new_bin}:$PATH"; hash -r; pytest'

697 + )

698 +

699 + result = subprocess.run(

700 + ["/bin/sh", "-c", shell],

701 + check=False,

702 + capture_output=True,

703 + text=True,

704 + env={

705 + **os.environ,

706 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

707 + "CI_INFRA_OTEL_RUNTIME_DIR": str(runtime),

708 + },

709 + )

710 +

711 + assert result.returncode == 0, result.stderr

712 + assert result.stdout == "NEW"

713 +

714 +

715 +def test_pytest_shim_records_distinct_test_intervals_with_pythonpath_override(

716 + tmp_path,

717 +):

718 + test_file = tmp_path / "test_sample.py"

719 + test_file.write_text(

720 + "import time\n"

721 + "def test_first(): time.sleep(0.02)\n"

722 + "def test_second(): time.sleep(0.02)\n"

723 + )

724 + workspace = tmp_path / "workspace"

725 + workspace.mkdir()

726 + bin_dir = tmp_path / "bin"

727 + bin_dir.mkdir()

728 + pytest_executable = bin_dir / "pytest"

729 + pytest_executable.write_text(

730 + f'#!/bin/sh\nexec {shlex.quote(sys.executable)} -m pytest "$@"\n'

731 + )

732 + pytest_executable.chmod(0o755)

733 + shell = (

734 + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"\n'

735 + f"ci_otel_start 1 {_quoted('pytest tests')}\n"

736 + f"PYTHONPATH={shlex.quote(str(workspace))} "

737 + f"pytest -q {shlex.quote(str(test_file))}\n"

738 + "status=$?\n"

739 + 'ci_otel_finish "$status"\n'

740 + 'exit "$status"'

741 + )

742 +

743 + result = subprocess.run(

744 + ["/bin/sh", "-c", shell],

745 + check=False,

746 + capture_output=True,

747 + text=True,

748 + env={

749 + **os.environ,

750 + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),

751 + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path / "spans"),

752 + "PATH": f"{bin_dir}:{os.environ['PATH']}",

753 + "PYTEST_ADDOPTS": "",

754 + },

755 + )

756 +

757 + assert result.returncode == 0, result.stderr

758 + records = [

759 + json.loads(line)

760 + for path in (tmp_path / "spans").glob("spans-*.jsonl")

761 + for line in path.read_text().splitlines()

762 + ]

763 + tests = [record for record in records if record["name"] == "pytest.test"]

764 + assert len(tests) == 2

765 + assert tests[0]["start_ns"] < tests[1]["start_ns"]

766 + assert tests[0]["end_ns"] <= tests[1]["start_ns"]

767 +

768 +

769 +def test_pytest_plugin_failure_cannot_fail_pytest(tmp_path):

770 + test_file = tmp_path / "test_sample.py"

771 + test_file.write_text("def test_sample(): assert True\n")

772 +

773 + result = subprocess.run(

774 + [sys.executable, "-m", "pytest", "-q", "-p", "ci_otel", str(test_file)],

775 + check=False,

776 + capture_output=True,

777 + text=True,

778 + env={

779 + **os.environ,

780 + "PYTHONPATH": str(SCRIPTS_DIR),

781 + "CI_INFRA_TRACE_ID": "01" * 16,

782 + "CI_INFRA_COMMAND_SPAN_ID": "02" * 8,

783 + "CI_INFRA_OTEL_SPOOL_DIR": "/dev/null/spans",

784 + },

785 + )

786 +

787 + assert result.returncode == 0, result.stderr

788 + assert "1 passed" in result.stdout