@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 80650454dc

[CI] Give every Buildkite step an explicit key, and a hook to enforce it (#56329)

Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>

Tahsin Tunan committed Sep 12, 2026 at 07:08 UTC 80650454dcf1a0b758b31ce59c004f49897963db

12 files changed +139 -1

.buildkite/intel_jobs/basic_correctness_intel.yaml

@@ -3,6 +3,7 @@ depends_on:

3 - image-build-xpu

4 steps:

5 - label: XPU Sleep Mode

6 + key: xpu-sleep-mode

7 timeout_in_minutes: 45

8 device: intel_gpu

9 agent_tags:

.buildkite/intel_jobs/engine_intel.yaml

@@ -41,6 +41,7 @@ steps:

41 pytest -v -s engine/test_arg_utils.py test_sequence.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py'

43 - label: Engine (1 GPU)

44 + key: engine-1-gpu

45 timeout_in_minutes: 30

46 device: intel_gpu

47 agent_tags:

@@ -65,6 +66,7 @@ steps:

66 VLLM_XPU_ENABLE_XPU_GRAPH=1 pytest -v -s test_config.py'

68 - label: V1 e2e (2 GPUs)

69 + key: v1-e2e-2-gpus

70 timeout_in_minutes: 30

71 device: intel_gpu

72 agent_tags:

.buildkite/intel_jobs/kernels_intel.yaml

@@ -3,6 +3,7 @@ depends_on:

3 - image-build-xpu

4 steps:

5 - label: vLLM IR Tests

6 + key: vllm-ir-tests

7 timeout_in_minutes: 30

8 device: intel_gpu

9 agent_tags:

.buildkite/intel_jobs/lora_intel.yaml

@@ -3,6 +3,7 @@ depends_on:

3 - image-build-xpu

4 steps:

5 - label: LoRA Runtime + Utils

6 + key: lora-runtime-utils

7 timeout_in_minutes: 45

8 device: intel_gpu

9 agent_tags:

@@ -36,6 +37,7 @@ steps:

37 pytest -v -s lora/test_worker.py'

39 - label: LoRA Fused/MoE Kernels

40 + key: lora-fused-moe-kernels

41 timeout_in_minutes: 45

42 device: intel_gpu

43 agent_tags:

@@ -60,6 +62,7 @@ steps:

62 pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"'

64 - label: LoRA Punica Kernels

65 + key: lora-punica-kernels

66 timeout_in_minutes: 45

67 device: intel_gpu

68 agent_tags:

@@ -86,6 +89,7 @@ steps:

89 pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'

91 - label: LoRA Punica FP8/XPU Ops

92 + key: lora-punica-fp8-xpu-ops

93 timeout_in_minutes: 60

94 device: intel_gpu

95 agent_tags:

@@ -110,6 +114,7 @@ steps:

114 pytest -v -s lora/test_punica_xpu_ops.py'

115

116 - label: LoRA Models

117 + key: lora-models

118 timeout_in_minutes: 45

119 device: intel_gpu

120 agent_tags:

@@ -137,6 +142,7 @@ steps:

142 pytest -s -v lora/test_minicpmv_tp.py'

143

144 - label: LoRA Multimodal

145 + key: lora-multimodal

146 timeout_in_minutes: 45

147 device: intel_gpu

148 agent_tags:

.buildkite/intel_jobs/misc_intel.yaml

+7 -1

@@ -3,6 +3,7 @@ depends_on:

3 - image-build-xpu

4 steps:

5 - label: V1 Core + KV + Metrics

6 + key: v1-core-kv-metrics-intel

7 timeout_in_minutes: 45

8 device: intel_gpu

9 agent_tags:

@@ -32,9 +33,10 @@ steps:

33 export VLLM_WORKER_MULTIPROC_METHOD=spawn &&

34 cd tests &&

35 pytest -v -s v1/executor &&

35 - VLLM_BATCH_INVARIANT=1 pytest -v -s -m 'not cpu_test' v1/ec_connector/unit'

36 + VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit'

38 - label: V1 Sample + Logits

39 + key: v1-sample-logits-intel

40 timeout_in_minutes: 90

41 device: intel_gpu

42 agent_tags:

@@ -80,6 +82,7 @@ steps:

82 parallelism: 4

84 - label: Basic Models Tests (Initialization)

85 + key: basic-models-tests-initialization

86 timeout_in_minutes: 60

87 device: intel_gpu

88 agent_tags:

@@ -104,6 +107,7 @@ steps:

107 pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'

108

109 - label: XPU CPU Offload

110 + key: xpu-cpu-offload

111 timeout_in_minutes: 60

112 device: intel_gpu

113 agent_tags:

@@ -131,6 +135,7 @@ steps:

135 pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'

136

137 - label: NixlConnector PD accuracy (4 GPUs)

138 + key: nixlconnector-pd-accuracy-4-gpus

139 timeout_in_minutes: 60

140 num_devices: 4

141 device: intel_gpu

@@ -269,6 +274,7 @@ steps:

274 pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'

275

276 - label: Fusion Unit Tests

277 + key: fusion-unit-tests

278 timeout_in_minutes: 30

279 device: intel_gpu

280 agent_tags:

.buildkite/intel_jobs/model_runner_v2_intel.yaml

@@ -3,6 +3,7 @@ depends_on:

3 - image-build-xpu

4 steps:

5 - label: Model Runner V2 Core Tests (Intel)

6 + key: model-runner-v2-core-tests-intel

7 timeout_in_minutes: 45

8 device: intel_gpu

9 agent_tags:

@@ -34,6 +35,7 @@ steps:

35 pytest -v -s v1/e2e/general/test_min_tokens.py'

37 - label: Model Runner V2 Examples (Intel)

38 + key: model-runner-v2-examples-intel

39 timeout_in_minutes: 45

40 device: intel_gpu

41 agent_tags:

@@ -64,6 +66,7 @@ steps:

66 python3 features/automatic_prefix_caching/prefix_caching_offline.py'

68 - label: Model Runner V2 Distributed (2 GPUs)

69 + key: model-runner-v2-distributed-2-gpus

70 timeout_in_minutes: 50

71 device: intel_gpu

72 agent_tags:

@@ -90,6 +93,7 @@ steps:

93 TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m "distributed$num_gpus=2$" -k "not ray and not True"'

95 - label: Model Runner V2 Spec Decode

96 + key: model-runner-v2-spec-decode

97 timeout_in_minutes: 50

98 device: intel_gpu

99 agent_tags:

.buildkite/intel_jobs/test-intel.yaml

@@ -15,6 +15,7 @@ steps:

15 - exit_status: -10 # Agent was lost

16 limit: 2

17 - label: "XPU example Test"

18 + key: xpu-example-test

19 depends_on:

20 - image-build-xpu

21 timeout_in_minutes: 50

@@ -50,6 +51,7 @@ steps:

51 python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192

52 '

53 - label: "XPU W8A8 FP8 Linear Examples"

54 + key: xpu-w8a8-fp8-linear-examples

55 depends_on:

56 - image-build-xpu

57 timeout_in_minutes: 60

@@ -77,6 +79,7 @@ steps:

79 python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096

80 '

81 - label: "XPU V1 test"

82 + key: xpu-v1-test

83 depends_on:

84 - image-build-xpu

85 timeout_in_minutes: 70

@@ -107,6 +110,7 @@ steps:

110 pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'

111 parallelism: 2

112 - label: "XPU server test"

113 + key: xpu-server-test

114 depends_on:

115 - image-build-xpu

116 timeout_in_minutes: 45

@@ -130,6 +134,7 @@ steps:

134 pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&

135 pytest -v -s benchmarks/test_serve_cli.py'

136 - label: "XPU quantization test"

137 + key: xpu-quantization-test

138 depends_on:

139 - image-build-xpu

140 timeout_in_minutes: 30

@@ -158,6 +163,7 @@ steps:

163 pytest -v -s quantization/test_auto_round.py &&

164 pytest -v -s quantization/test_online.py'

165 - label: "XPU compressed tensors FP8 test"

166 + key: xpu-compressed-tensors-fp8-test

167 depends_on:

168 - image-build-xpu

169 timeout_in_minutes: 60

@@ -182,6 +188,7 @@ steps:

188 pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8 &&

189 pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8_block_enabled'

190 - label: "XPU Graph"

191 + key: xpu-graph

192 depends_on:

193 - image-build-xpu

194 timeout_in_minutes: 60

.buildkite/test_areas/docker.yaml

@@ -3,6 +3,7 @@ depends_on:

3 - image-build-cpu

4 steps:

5 - label: ":computer: (CPU) Docker Build Metadata"

6 + key: docker-build-metadata

7 timeout_in_minutes: 20

8 device: cpu-small

9 source_file_dependencies:

.buildkite/test_areas/lm_eval.yaml

@@ -470,6 +470,7 @@ steps:

470 - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"

471

472 - label: ":nvidia: (H200 MIG 35GB) MRCR Eval Small Models"

473 + key: mrcr-eval-small-models

474 device: h200_35gb

475 timeout_in_minutes: 25

476 source_file_dependencies:

.buildkite/test_areas/rust_frontend.yaml

@@ -3,6 +3,7 @@ depends_on:

3 - image-build

4 steps:

5 - label: ":nvidia: (H200 MIG 18GB) Rust Frontend OpenAI Coverage"

6 + key: rust-frontend-openai-coverage

7 timeout_in_minutes: 30

8 device: h200_18gb

9 working_dir: "/vllm-workspace/tests"

@@ -67,6 +68,7 @@ steps:

68 - vllm/platforms/rocm.py

70 - label: ":nvidia: (H200 MIG 18GB) Rust Frontend Serve/Admin Coverage"

71 + key: rust-frontend-serve-admin-coverage

72 timeout_in_minutes: 25

73 device: h200_18gb

74 working_dir: "/vllm-workspace/tests"

@@ -116,6 +118,7 @@ steps:

118 - vllm/platforms/rocm.py

119

120 - label: ":nvidia: (H200 MIG 18GB) Rust Frontend Core Correctness"

121 + key: rust-frontend-core-correctness

122 timeout_in_minutes: 20

123 device: h200_18gb

124 working_dir: "/vllm-workspace/tests"

@@ -130,6 +133,7 @@ steps:

133 - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine

134

135 - label: ":nvidia: (H200 MIG 35GB) Rust Frontend Tool Use"

136 + key: rust-frontend-tool-use

137 device: h200_35gb

138 timeout_in_minutes: 25

139 working_dir: "/vllm-workspace/tests"

@@ -145,6 +149,7 @@ steps:

149 - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"

150

151 - label: ":nvidia: (L4) Rust Frontend Distributed"

152 + key: rust-frontend-distributed

153 timeout_in_minutes: 25

154 device: l4

155 num_devices: 4

.pre-commit-config.yaml

@@ -177,6 +177,13 @@ repos:

177 files: ^\.github/mergify\.yml$

178 pass_filenames: false

179 additional_dependencies: [PyYAML, regex]

180 + - id: check-buildkite-step-keys

181 + name: Check every Buildkite step declares an explicit key

182 + language: python

183 + entry: python tools/pre_commit/check_buildkite_step_keys.py

184 + files: ^\.buildkite/.*\.ya?ml$

185 + pass_filenames: false

186 + additional_dependencies: [PyYAML]

187 - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward

188 name: Run mypy for Python 3.10

189 entry: python tools/pre_commit/mypy.py "3.10"

tools/pre_commit/check_buildkite_step_keys.py new

+97

@@ -0,0 +1,97 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Check that every Buildkite step the CI generator reads declares a key.

4 +

5 +A step with no `key:` is published under one the generator derives from its

6 +label, which makes the job's address a function of its title: retitling the

7 +step silently renames it and breaks retries, `depends_on:` and build history.

8 +The generator only reads the directories a pipeline config names in

9 +`job_dirs:`, so this walks the same files and fails on any step without a key.

10 +

11 +Usage:

12 + python tools/pre_commit/check_buildkite_step_keys.py

13 +"""

14 +

15 +import sys

16 +from pathlib import Path

17 +

18 +import yaml

19 +

20 +BUILDKITE = Path(".buildkite")

21 +CONFIG_GLOB = "ci_config*.yaml"

22 +JOB_GLOB = "*.yaml"

23 +

24 +

25 +def job_dirs() -> list[Path]:

26 + """Every directory the pipeline generator reads, per the pipeline configs."""

27 + configs = sorted(BUILDKITE.glob(CONFIG_GLOB))

28 + if not configs:

29 + raise SystemExit(f"no pipeline config matched {BUILDKITE / CONFIG_GLOB}")

30 + dirs: set[Path] = set()

31 + for config in configs:

32 + with open(config, encoding="utf-8") as f:

33 + doc = yaml.safe_load(f)

34 + declared = doc.get("job_dirs") if isinstance(doc, dict) else None

35 + if not isinstance(declared, list) or not declared:

36 + raise SystemExit(f"{config}: `job_dirs` is missing or not a list")

37 + for name in declared:

38 + job_dir = Path(name)

39 + if not job_dir.is_dir():

40 + raise SystemExit(f"{config}: job_dir `{name}` does not exist")

41 + dirs.add(job_dir)

42 + return sorted(dirs)

43 +

44 +

45 +def scan(steps: object, source: Path) -> list[str]:

46 + """Label of every step in a `steps:` list that declares no key."""

47 + if not isinstance(steps, list):

48 + raise SystemExit(f"{source}: `steps:` is not a list")

49 + keyless: list[str] = []

50 + for i, step in enumerate(steps):

51 + # A scalar shorthand such as `- wait` has nowhere to put a key. Every

52 + # other step does, so there are no other exemptions. A `mirror:` block

53 + # is not a step here: the generator publishes it as `<hw>-<parent key>`.

54 + if not isinstance(step, dict):

55 + continue

56 + label = step.get("label") or step.get("group") or f"step {i}"

57 + if "steps" in step:

58 + raise SystemExit(

59 + f"{source}: `{label}` nests its own `steps:`. The generator has "

60 + "no group step and drops the children without an error, so give "

61 + "each one its own entry."

62 + )

63 + key = step.get("key")

64 + if not (isinstance(key, str) and key.strip()):

65 + keyless.append(label)

66 + return keyless

67 +

68 +

69 +def keyless_steps(path: Path) -> list[str]:

70 + with open(path, encoding="utf-8") as f:

71 + doc = yaml.safe_load(f)

72 + if not isinstance(doc, dict) or "steps" not in doc:

73 + return []

74 + return scan(doc["steps"], path)

75 +

76 +

77 +def main() -> int:

78 + files = sorted({p for d in job_dirs() for p in d.rglob(JOB_GLOB)})

79 + if not files:

80 + raise SystemExit("no pipeline files under the declared job_dirs")

81 + keyless = [(path, label) for path in files for label in keyless_steps(path)]

82 + if not keyless:

83 + return 0

84 + print(f"{len(keyless)} Buildkite step(s) have no explicit key:\n", file=sys.stderr)

85 + for path, label in keyless:

86 + print(f" {path}: {label}", file=sys.stderr)

87 + print(

88 + "\nAdd a `key:` naming the test each step runs. Without one the pipeline"

89 + "\ngenerator derives the key from the label, so retitling the step renames"

90 + "\nits job and breaks retries and build history.",

91 + file=sys.stderr,

92 + )

93 + return 1

94 +

95 +

96 +if __name__ == "__main__":

97 + raise SystemExit(main())