[CI] Split long misc test groups by command (#52346)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Kimi Code <noreply@moonshot.cn> Co-authored-by: Codex <noreply@openai.com>
Kevin H. Luu committed
Sep 8, 2026 at 13:44 UTC
808353f477c21953a3a476b562775385e89da8e9
1 file changed
+138
-28
.buildkite/test_areas/misc.yaml
+138
-28
@@ -2,11 +2,11 @@ group: Miscellaneous
2
depends_on:
3
- image-build
4
steps:
5
-- label: ":nvidia: (H200 MIG 18GB) V1 Sample + Logits"
6
- key: v1-sample-logits
7
- timeout_in_minutes: 83
5
+- label: ":nvidia: (H200 MIG 18GB) V1 Sample"
6
+ key: v1-sample
7
+ timeout_in_minutes: 35
8
device: h200_18gb
9
- source_file_dependencies:
9
+ source_file_dependencies: &v1-sample-logits-deps
10
- vllm/config/
11
- vllm/distributed/
12
- "!vllm/distributed/kv_transfer/"
@@ -27,30 +27,42 @@ steps:
27
commands:
28
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
29
- pytest -v -s v1/sample
30
+ mirror:
31
+ amd:
32
+ label: ":amd: (MI300) V1 Sample"
33
+ dind: false
34
+ device: mi300_1
35
+ timeout_in_minutes: 70
36
+ depends_on:
37
+ - image-build-amd
38
+
39
+- label: ":nvidia: (H200 MIG 18GB) V1 Logits + Oracle"
40
+ key: v1-logits-oracle
41
+ timeout_in_minutes: 35
42
+ device: h200_18gb
43
+ source_file_dependencies: *v1-sample-logits-deps
44
+ commands:
45
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
46
+ # Keep these in separate pytest processes because the CUDA-initializing
47
+ # correctness tests interfere with the fork-based logits processor tests.
48
- pytest -v -s v1/logits_processors
49
- pytest -v -s v1/test_oracle.py
50
- pytest -v -s v1/test_request.py
51
- pytest -v -s v1/test_outputs.py
52
mirror:
53
amd:
36
- label: ":amd: (MI300) V1 Sample + Logits"
54
+ label: ":amd: (MI300) V1 Logits + Oracle"
55
dind: false
56
device: mi300_1
57
timeout_in_minutes: 70
58
depends_on:
59
- image-build-amd
60
43
-- label: ":nvidia: (H200 MIG 35GB) V1 Core + KV + Metrics"
61
+- label: ":nvidia: (H200 MIG 35GB) V1 Core"
62
device: h200_35gb
45
- key: v1-core-kv-metrics
46
- timeout_in_minutes: 100
47
- env:
48
- # GitHub intermittently rejects HTTP/2 upload-pack requests from the H200
49
- # CI image. Force Git's smart protocol to use HTTP/1.1 for source installs.
50
- GIT_CONFIG_COUNT: "1"
51
- GIT_CONFIG_KEY_0: http.version
52
- GIT_CONFIG_VALUE_0: HTTP/1.1
53
- source_file_dependencies:
63
+ key: v1-core
64
+ timeout_in_minutes: 45
65
+ source_file_dependencies: &v1-core-kv-metrics-deps
66
- vllm/config/
67
- vllm/distributed/
68
- "!vllm/distributed/kv_transfer/"
@@ -80,24 +92,95 @@ steps:
92
- tests/v1/metrics
93
- tests/entrypoints/openai/correctness/test_lmeval.py
94
commands:
83
- - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
95
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
85
- # split the test to avoid interference
96
- pytest -v -s -m 'not cpu_test' v1/core
97
+ mirror:
98
+ amd:
99
+ label: ":amd: (MI300) V1 Core"
100
+ dind: false
101
+ device: mi300_1
102
+ timeout_in_minutes: 45
103
+ depends_on:
104
+ - image-build-amd
105
+
106
+- label: ":nvidia: (H200 MIG 35GB) V1 Executor + Worker"
107
+ device: h200_35gb
108
+ key: v1-executor-worker
109
+ timeout_in_minutes: 45
110
+ source_file_dependencies: *v1-core-kv-metrics-deps
111
+ commands:
112
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
113
+ # split the test to avoid interference
114
- pytest -v -s v1/executor
88
- - pytest -v -s v1/kv_offload
89
- - pytest -v -s v1/simple_kv_offload
115
- pytest -v -s v1/worker
116
- pytest -v -s v1/streaming_input
117
+ mirror:
118
+ amd:
119
+ label: ":amd: (MI300) V1 Executor + Worker"
120
+ dind: false
121
+ device: mi300_1
122
+ timeout_in_minutes: 45
123
+ depends_on:
124
+ - image-build-amd
125
+
126
+- label: ":nvidia: (H200 MIG 35GB) V1 KV Offload"
127
+ device: h200_35gb
128
+ key: v1-kv-offload
129
+ timeout_in_minutes: 45
130
+ source_file_dependencies: *v1-core-kv-metrics-deps
131
+ commands:
132
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
133
+ - pytest -v -s v1/kv_offload
134
+ - pytest -v -s v1/simple_kv_offload
135
+ mirror:
136
+ amd:
137
+ label: ":amd: (MI300) V1 KV Offload"
138
+ dind: false
139
+ device: mi300_1
140
+ timeout_in_minutes: 45
141
+ depends_on:
142
+ - image-build-amd
143
+
144
+- label: ":nvidia: (H200 MIG 35GB) V1 KV Connectors"
145
+ device: h200_35gb
146
+ key: v1-kv-connectors
147
+ timeout_in_minutes: 45
148
+ source_file_dependencies: *v1-core-kv-metrics-deps
149
+ commands:
150
+ - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
151
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
152
- pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
153
- pytest -v -s -m 'not cpu_test' v1/ec_connector/unit
154
+ mirror:
155
+ amd:
156
+ label: ":amd: (MI300) V1 KV Connectors"
157
+ dind: false
158
+ device: mi300_1
159
+ timeout_in_minutes: 45
160
+ depends_on:
161
+ - image-build-amd
162
+
163
+- label: ":nvidia: (H200 MIG 35GB) V1 Metrics + LM Eval"
164
+ device: h200_35gb
165
+ key: v1-metrics-lmeval
166
+ timeout_in_minutes: 45
167
+ env:
168
+ # GitHub intermittently rejects HTTP/2 upload-pack requests from the H200
169
+ # CI image. Force Git's smart protocol to use HTTP/1.1 for source installs.
170
+ GIT_CONFIG_COUNT: "1"
171
+ GIT_CONFIG_KEY_0: http.version
172
+ GIT_CONFIG_VALUE_0: HTTP/1.1
173
+ GIT_TERMINAL_PROMPT: "0"
174
+ source_file_dependencies: *v1-core-kv-metrics-deps
175
+ commands:
176
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
177
- pytest -v -s -m 'not cpu_test' v1/metrics
178
# Integration test for streaming correctness (requires special branch).
179
- pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
180
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
181
mirror:
182
amd:
100
- label: ":amd: (MI300) V1 Core + KV + Metrics"
183
+ label: ":amd: (MI300) V1 Metrics + LM Eval"
184
dind: false
185
device: mi300_1
186
timeout_in_minutes: 65
@@ -336,12 +419,12 @@ steps:
419
depends_on:
420
- image-build-amd
421
339
-- label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config"
340
- key: async-engine-inputs-utils-worker-config-cpu
422
+- label: ":computer: (CPU) Params, Env, Tokenizers, Parser"
423
+ key: cpu-params-env-tokenizers-parser
424
depends_on:
425
- image-build-cpu
343
- timeout_in_minutes: 65
344
- source_file_dependencies:
426
+ timeout_in_minutes: 35
427
+ source_file_dependencies: &cpu-async-engine-deps
428
- vllm/assets/
429
- vllm/config/
430
- vllm/engine/arg_utils.py
@@ -389,15 +472,42 @@ steps:
472
- pytest -v -s test_pooling_params.py
473
- pytest -v -s test_ray_env.py
474
- pytest -v -s test_sampling_params.py
392
- - pytest -v -s -m 'cpu_test' multimodal
393
- - pytest -v -s renderers
394
- - pytest -v -s reasoning
395
- - pytest -v -s tool_parsers
475
- pytest -v -s tokenizers_
476
- pytest -v -s parser
477
- pytest -v -s transformers_utils
478
+
479
+- label: ":computer: (CPU) Multimodal + Config"
480
+ key: cpu-multimodal-config
481
+ depends_on:
482
+ - image-build-cpu
483
+ timeout_in_minutes: 35
484
+ source_file_dependencies: *cpu-async-engine-deps
485
+ device: cpu-small
486
+ commands:
487
+ - pytest -v -s -m 'cpu_test' multimodal
488
- pytest -v -s config
489
490
+- label: ":computer: (CPU) Reasoning + Renderers"
491
+ key: cpu-reasoning-renderers
492
+ depends_on:
493
+ - image-build-cpu
494
+ timeout_in_minutes: 35
495
+ source_file_dependencies: *cpu-async-engine-deps
496
+ device: cpu-small
497
+ commands:
498
+ - pytest -v -s reasoning
499
+ - pytest -v -s renderers
500
+
501
+- label: ":computer: (CPU) Tool Parsers"
502
+ key: cpu-tool-parsers
503
+ depends_on:
504
+ - image-build-cpu
505
+ timeout_in_minutes: 35
506
+ source_file_dependencies: *cpu-async-engine-deps
507
+ device: cpu-small
508
+ commands:
509
+ - pytest -v -s tool_parsers
510
+
511
- label: ":nvidia: (A100) Batch Invariance"
512
key: batch-invariance-a100
513
timeout_in_minutes: 60