b6101d384d · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -1,1524 +1,8 @@

1 -# In this file, you can add more tests to run either by adding a new step or

2 -# adding a new command to an existing step. See different options here for examples.

1 +# This file has been deprecated as of Feb 18, 2026. The content has already been migrated to:

2

4 -# This script will be feed into Jinja template in `test-template-aws.j2` at

5 -# https://github.com/vllm-project/buildkite-ci/blob/main/scripts/test-template-aws.j2

6 -# to generate the final pipeline yaml file.

3 +# .buildkite/test_areas for test jobs

4 +# .buildkite/image_build for image building jobs

5 +# .buildkite/hardware_tests for jobs running on other hardwares (Intel, Ascend NPU, Arm, etc..)

6 +# .buildkite/ci_config.yaml for configuration of CI pipeline

7

8 -# Documentation

9 -# label(str): the name of the test. emojis allowed.

10 -# fast_check(bool): whether to run this on each commit on the fastcheck pipeline.

11 -# torch_nightly(bool): whether to run this on vllm against the torch nightly pipeline.

12 -# fast_check_only(bool): run this test on the fastcheck pipeline only

13 -# optional(bool): never run this test by default (i.e. need to unblock manually) unless it's a scheduled nightly run.

14 -# soft_fail(bool): allow this step to fail without failing the entire pipeline (useful for flaky or experimental tests).

15 -# command(str): the single command to run for tests. incompatible with commands.

16 -# commands(list): the list of commands to run for the test. incompatible with command.

17 -# mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental]

18 -# gpu(str): override the GPU selection for the test. default is L4 GPUs. supports a100, b200, h200

19 -# num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4.

20 -# num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host,

21 -# in this case, commands must be specified. the first command runs on the first host, the second

22 -# command runs on the second host.

23 -# timeout_in_minutes(int): sets a timeout for the step in minutes. if not specified, uses the default timeout.

24 -# parallelism(int): number of parallel jobs to run for this step. enables test sharding using $$BUILDKITE_PARALLEL_JOB

25 -# and $$BUILDKITE_PARALLEL_JOB_COUNT environment variables.

26 -# working_dir(str): specify the place where the command should execute, default to /vllm-workspace/tests

27 -# source_file_dependencies(list): the list of prefixes to opt-in the test for, if empty, the test will always run.

28 -# autorun_on_main (bool): default to false, if true, the test will run automatically when commit is pushed to main branch.

29 -

30 -# When adding a test

31 -# - If the test belongs to an existing group, add it there

32 -# - If the test is short, add to any existing step

33 -# - If the test takes more than 10min, then it is okay to create a new step.

34 -# Note that all steps execute in parallel.

35 -

36 -steps:

37 -##### fast check tests #####

38 -

39 -- label: Pytorch Nightly Dependency Override Check # 2min

40 - # if this test fails, it means the nightly torch version is not compatible with some

41 - # of the dependencies. Please check the error message and add the package to whitelist

42 - # in /vllm/tools/pre_commit/generate_nightly_torch_test.py

43 - soft_fail: true

44 - source_file_dependencies:

45 - - requirements/nightly_torch_test.txt

46 - commands:

47 - - bash standalone_tests/pytorch_nightly_dependency.sh

48 -

49 -- label: Async Engine, Inputs, Utils, Worker Test # 36min

50 - timeout_in_minutes: 50

51 - mirror_hardwares: [amdexperimental]

52 - source_file_dependencies:

53 - - vllm/

54 - - tests/detokenizer

55 - - tests/multimodal

56 - - tests/utils_

57 - commands:

58 - - pytest -v -s detokenizer

59 - - pytest -v -s -m 'not cpu_test' multimodal

60 - - pytest -v -s utils_

61 -

62 -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 20min

63 - timeout_in_minutes: 30

64 - source_file_dependencies:

65 - - vllm/

66 - - tests/test_inputs.py

67 - - tests/test_outputs.py

68 - - tests/test_pooling_params.py

69 - - tests/multimodal

70 - - tests/renderers

71 - - tests/standalone_tests/lazy_imports.py

72 - - tests/tokenizers_

73 - - tests/tool_parsers

74 - - tests/transformers_utils

75 - - tests/config

76 - no_gpu: true

77 - commands:

78 - - python3 standalone_tests/lazy_imports.py

79 - - pytest -v -s test_inputs.py

80 - - pytest -v -s test_outputs.py

81 - - pytest -v -s test_pooling_params.py

82 - - pytest -v -s -m 'cpu_test' multimodal

83 - - pytest -v -s renderers

84 - - pytest -v -s tokenizers_

85 - - pytest -v -s tool_parsers

86 - - pytest -v -s transformers_utils

87 - - pytest -v -s config

88 -

89 -- label: Python-only Installation Test # 10min

90 - timeout_in_minutes: 20

91 - mirror_hardwares: [amdexperimental]

92 - source_file_dependencies:

93 - - tests/standalone_tests/python_only_compile.sh

94 - - setup.py

95 - commands:

96 - - bash standalone_tests/python_only_compile.sh

97 -

98 -- label: Basic Correctness Test # 20min

99 - timeout_in_minutes: 30

100 - mirror_hardwares: [amdexperimental]

101 - fast_check: true

102 - torch_nightly: true

103 - source_file_dependencies:

104 - - vllm/

105 - - tests/basic_correctness/test_basic_correctness

106 - - tests/basic_correctness/test_cpu_offload

107 - - tests/basic_correctness/test_cumem.py

108 - commands:

109 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

110 - - pytest -v -s basic_correctness/test_cumem.py

111 - - pytest -v -s basic_correctness/test_basic_correctness.py

112 - - pytest -v -s basic_correctness/test_cpu_offload.py

113 -

114 -- label: Entrypoints Unit Tests # 5min

115 - timeout_in_minutes: 10

116 - working_dir: "/vllm-workspace/tests"

117 - fast_check: true

118 - source_file_dependencies:

119 - - vllm/entrypoints

120 - - tests/entrypoints/

121 - commands:

122 - - pytest -v -s entrypoints/openai/tool_parsers

123 - - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling

124 -

125 -- label: Entrypoints Integration Test (LLM) # 30min

126 - timeout_in_minutes: 40

127 - mirror_hardwares: [amdexperimental]

128 - working_dir: "/vllm-workspace/tests"

129 - fast_check: true

130 - torch_nightly: true

131 - source_file_dependencies:

132 - - vllm/

133 - - tests/entrypoints/llm

134 - - tests/entrypoints/offline_mode

135 - commands:

136 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

137 - - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py

138 - - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process

139 - - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests

140 -

141 -- label: Entrypoints Integration Test (API Server 1) # 100min

142 - timeout_in_minutes: 130

143 - mirror_hardwares: [amdexperimental]

144 - working_dir: "/vllm-workspace/tests"

145 - fast_check: true

146 - torch_nightly: true

147 - source_file_dependencies:

148 - - vllm/

149 - - tests/entrypoints/openai

150 - - tests/entrypoints/test_chat_utils

151 - commands:

152 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

153 - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/instrumentator --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses

154 - - pytest -v -s entrypoints/test_chat_utils.py

155 -

156 -- label: Entrypoints Integration Test (API Server 2)

157 - timeout_in_minutes: 50

158 - mirror_hardwares: [amdexperimental]

159 - working_dir: "/vllm-workspace/tests"

160 - fast_check: true

161 - torch_nightly: true

162 - source_file_dependencies:

163 - - vllm/

164 - - tests/entrypoints/rpc

165 - - tests/entrypoints/instrumentator

166 - - tests/tool_use

167 - commands:

168 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

169 - - pytest -v -s entrypoints/instrumentator

170 - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc

171 - - pytest -v -s tool_use

172 -

173 -- label: Entrypoints Integration Test (Pooling)

174 - timeout_in_minutes: 50

175 - mirror_hardwares: [amdexperimental]

176 - working_dir: "/vllm-workspace/tests"

177 - fast_check: true

178 - torch_nightly: true

179 - source_file_dependencies:

180 - - vllm/

181 - - tests/entrypoints/pooling

182 - commands:

183 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

184 - - pytest -v -s entrypoints/pooling

185 -

186 -- label: Entrypoints Integration Test (Responses API)

187 - timeout_in_minutes: 50

188 - mirror_hardwares: [amdexperimental]

189 - working_dir: "/vllm-workspace/tests"

190 - fast_check: true

191 - torch_nightly: true

192 - source_file_dependencies:

193 - - vllm/

194 - - tests/entrypoints/openai/responses

195 - commands:

196 - - pytest -v -s entrypoints/openai/responses

197 -

198 -- label: Distributed Tests (4 GPUs) # 35min

199 - timeout_in_minutes: 50

200 - mirror_hardwares: [amdexperimental]

201 - working_dir: "/vllm-workspace/tests"

202 - num_gpus: 4

203 - source_file_dependencies:

204 - - vllm/distributed/

205 - - tests/distributed/test_utils

206 - - tests/distributed/test_pynccl

207 - - tests/distributed/test_events

208 - - tests/compile/fullgraph/test_basic_correctness.py

209 - - examples/offline_inference/rlhf.py

210 - - examples/offline_inference/rlhf_colocate.py

211 - - examples/offline_inference/new_weight_syncing/

212 - - tests/examples/offline_inference/data_parallel.py

213 - - tests/v1/distributed

214 - - tests/v1/engine/test_engine_core_client.py

215 - - tests/distributed/test_symm_mem_allreduce.py

216 - commands:

217 - # https://github.com/NVIDIA/nccl/issues/1838

218 - - export NCCL_CUMEM_HOST_ENABLE=0

219 - # test with torchrun tp=2 and external_dp=2

220 - - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py

221 - # test with torchrun tp=2 and pp=2

222 - - PP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py

223 - # test with torchrun tp=4 and dp=1

224 - - TP_SIZE=4 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py

225 - # test with torchrun tp=2, pp=2 and dp=1

226 - - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py

227 - # test with torchrun tp=1 and dp=4 with ep

228 - - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py

229 - # test with torchrun tp=2 and dp=2 with ep

230 - - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py

231 - # test with internal dp

232 - - python3 ../examples/offline_inference/data_parallel.py --enforce-eager

233 - - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py

234 - - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py

235 - - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py

236 - - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py

237 - - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py

238 - - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp

239 - - pytest -v -s distributed/test_utils.py

240 - - pytest -v -s compile/fullgraph/test_basic_correctness.py

241 - - pytest -v -s distributed/test_pynccl.py

242 - - pytest -v -s distributed/test_events.py

243 - - pytest -v -s distributed/test_symm_mem_allreduce.py

244 - # TODO: create a dedicated test section for multi-GPU example tests

245 - # when we have multiple distributed example tests

246 - # OLD rlhf examples

247 - - pushd ../examples/offline_inference

248 - - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf.py

249 - - VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py

250 - - popd

251 - # NEW rlhf examples

252 - - pushd ../examples/offline_inference/new_weight_syncing

253 - - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf.py

254 - - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_async_new_apis.py

255 - - popd

256 -

257 -- label: Distributed Tests (8 GPUs) # 4min

258 - timeout_in_minutes: 10

259 - gpu: h100

260 - num_gpus: 8

261 - working_dir: "/vllm-workspace/tests"

262 - source_file_dependencies:

263 - - examples/offline_inference/torchrun_dp_example.py

264 - - vllm/config/parallel.py

265 - - vllm/distributed/

266 - - vllm/v1/engine/llm_engine.py

267 - - vllm/v1/executor/uniproc_executor.py

268 - - vllm/v1/worker/gpu_worker.py

269 - commands:

270 - # https://github.com/NVIDIA/nccl/issues/1838

271 - - export NCCL_CUMEM_HOST_ENABLE=0

272 - # test with torchrun tp=2 and dp=4 with ep

273 - - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep

274 -

275 -- label: EPLB Algorithm Test # 5min

276 - timeout_in_minutes: 15

277 - working_dir: "/vllm-workspace/tests"

278 - source_file_dependencies:

279 - - vllm/distributed/eplb

280 - - tests/distributed/test_eplb_algo.py

281 - commands:

282 - - pytest -v -s distributed/test_eplb_algo.py

283 -

284 -- label: EPLB Execution Test # 10min

285 - timeout_in_minutes: 20

286 - working_dir: "/vllm-workspace/tests"

287 - num_gpus: 4

288 - source_file_dependencies:

289 - - vllm/distributed/eplb

290 - - tests/distributed/test_eplb_execute.py

291 - commands:

292 - - pytest -v -s distributed/test_eplb_execute.py

293 - - pytest -v -s distributed/test_eplb_spec_decode.py

294 -

295 -- label: Metrics, Tracing Test # 12min

296 - timeout_in_minutes: 20

297 - mirror_hardwares: [amdexperimental]

298 - num_gpus: 2

299 - source_file_dependencies:

300 - - vllm/

301 - - tests/v1/tracing

302 - commands:

303 - - "pip install \

304 - 'opentelemetry-sdk>=1.26.0' \

305 - 'opentelemetry-api>=1.26.0' \

306 - 'opentelemetry-exporter-otlp>=1.26.0' \

307 - 'opentelemetry-semantic-conventions-ai>=0.4.1'"

308 - - pytest -v -s v1/tracing

309 -

310 -##### fast check tests #####

311 -##### 1 GPU test #####

312 -

313 -- label: Regression Test # 7min

314 - timeout_in_minutes: 20

315 - mirror_hardwares: [amdexperimental]

316 - source_file_dependencies:

317 - - vllm/

318 - - tests/test_regression

319 - commands:

320 - - pip install modelscope

321 - - pytest -v -s test_regression.py

322 - working_dir: "/vllm-workspace/tests" # optional

323 -

324 -- label: Engine Test # 9min

325 - timeout_in_minutes: 15

326 - mirror_hardwares: [amdexperimental]

327 - source_file_dependencies:

328 - - vllm/

329 - - tests/engine

330 - - tests/test_sequence

331 - - tests/test_config

332 - - tests/test_logger

333 - - tests/test_vllm_port

334 - commands:

335 - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py

336 -

337 -- label: V1 Test e2e + engine # 30min

338 - timeout_in_minutes: 45

339 - mirror_hardwares: [amdexperimental]

340 - source_file_dependencies:

341 - - vllm/

342 - - tests/v1

343 - commands:

344 - # TODO: accuracy does not match, whether setting

345 - # VLLM_USE_FLASHINFER_SAMPLER or not on H100.

346 - - pytest -v -s v1/e2e

347 - # Run this test standalone for now;

348 - # need to untangle use (implicit) use of spawn/fork across the tests.

349 - - pytest -v -s v1/engine/test_preprocess_error_handling.py

350 - - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py

351 -

352 -- label: V1 Test entrypoints # 35min

353 - timeout_in_minutes: 50

354 - mirror_hardwares: [amdexperimental]

355 - source_file_dependencies:

356 - - vllm/

357 - - tests/v1

358 - commands:

359 - - pytest -v -s v1/entrypoints

360 -

361 -- label: V1 Test others # 42min

362 - timeout_in_minutes: 60

363 - mirror_hardwares: [amdexperimental]

364 - source_file_dependencies:

365 - - vllm/

366 - - tests/v1

367 - commands:

368 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt

369 - # split the test to avoid interference

370 - - pytest -v -s -m 'not cpu_test' v1/core

371 - - pytest -v -s v1/executor

372 - - pytest -v -s v1/kv_offload

373 - - pytest -v -s v1/sample

374 - - pytest -v -s v1/logits_processors

375 - - pytest -v -s v1/worker

376 - - pytest -v -s -m 'not slow_test' v1/spec_decode

377 - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit

378 - - pytest -v -s -m 'not cpu_test' v1/metrics

379 - - pytest -v -s v1/test_oracle.py

380 - - pytest -v -s v1/test_request.py

381 - - pytest -v -s v1/test_outputs.py

382 - # Integration test for streaming correctness (requires special branch).

383 - - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api

384 - - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine

385 -

386 -- label: V1 Test attention (H100) # 10min

387 - timeout_in_minutes: 30

388 - gpu: h100

389 - source_file_dependencies:

390 - - vllm/config/attention.py

391 - - vllm/model_executor/layers/attention

392 - - vllm/v1/attention

393 - - tests/v1/attention

394 - commands:

395 - - pytest -v -s v1/attention

396 -

397 -- label: Batch Invariance Tests (H100) # 10min

398 - timeout_in_minutes: 25

399 - gpu: h100

400 - source_file_dependencies:

401 - - vllm/v1/attention

402 - - vllm/model_executor/layers

403 - - tests/v1/determinism/

404 - commands:

405 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

406 - - pip install pytest-timeout pytest-forked

407 - - pytest -v -s v1/determinism/test_batch_invariance.py

408 - - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py

409 -

410 -- label: V1 Test attention (B200) # 10min

411 - timeout_in_minutes: 30

412 - gpu: b200

413 - source_file_dependencies:

414 - - vllm/config/attention.py

415 - - vllm/model_executor/layers/attention

416 - - vllm/v1/attention

417 - - tests/v1/attention

418 - commands:

419 - - pytest -v -s v1/attention

420 -

421 -- label: V1 Test others (CPU) # 5 mins

422 - source_file_dependencies:

423 - - vllm/

424 - - tests/v1

425 - no_gpu: true

426 - commands:

427 - # split the test to avoid interference

428 - - pytest -v -s -m 'cpu_test' v1/core

429 - - pytest -v -s v1/structured_output

430 - - pytest -v -s v1/test_serial_utils.py

431 - - pytest -v -s -m 'cpu_test' v1/kv_connector/unit

432 - - pytest -v -s -m 'cpu_test' v1/metrics

433 -

434 -

435 -- label: Examples Test # 30min

436 - timeout_in_minutes: 45

437 - mirror_hardwares: [amdexperimental]

438 - working_dir: "/vllm-workspace/examples"

439 - source_file_dependencies:

440 - - vllm/entrypoints

441 - - vllm/multimodal

442 - - examples/

443 - commands:

444 - - pip install tensorizer # for tensorizer test

445 - # for basic

446 - - python3 offline_inference/basic/chat.py

447 - - python3 offline_inference/basic/generate.py --model facebook/opt-125m

448 - - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10

449 - - python3 offline_inference/basic/classify.py

450 - - python3 offline_inference/basic/embed.py

451 - - python3 offline_inference/basic/score.py

452 - # for multi-modal models

453 - - python3 offline_inference/audio_language.py --seed 0

454 - - python3 offline_inference/vision_language.py --seed 0

455 - - python3 offline_inference/vision_language_multi_image.py --seed 0

456 - - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0

457 - # for pooling models

458 - - python3 pooling/embed/vision_embedding_offline.py --seed 0

459 - # for features demo

460 - - python3 offline_inference/prefix_caching.py

461 - - python3 offline_inference/llm_engine_example.py

462 - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

463 - - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

464 - # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU

465 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

466 -

467 -- label: Platform Tests (CUDA) # 4min

468 - timeout_in_minutes: 15

469 - mirror_hardwares: [amdexperimental]

470 - source_file_dependencies:

471 - - vllm/

472 - - tests/cuda

473 - commands:

474 - - pytest -v -s cuda/test_cuda_context.py

475 -

476 -- label: Samplers Test # 56min

477 - timeout_in_minutes: 75

478 - mirror_hardwares: [amdexperimental]

479 - source_file_dependencies:

480 - - vllm/model_executor/layers

481 - - vllm/sampling_metadata.py

482 - - tests/samplers

483 - - tests/conftest.py

484 - commands:

485 - - pytest -v -s samplers

486 - - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers

487 -

488 -- label: LoRA Test %N # 20min each

489 - timeout_in_minutes: 30

490 - mirror_hardwares: [amdexperimental]

491 - source_file_dependencies:

492 - - vllm/lora

493 - - tests/lora

494 - commands:

495 - - pytest -v -s lora \

496 - --shard-id=$$BUILDKITE_PARALLEL_JOB \

497 - --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \

498 - --ignore=lora/test_chatglm3_tp.py \

499 - --ignore=lora/test_llama_tp.py \

500 - --ignore=lora/test_llm_with_multi_loras.py \

501 - --ignore=lora/test_olmoe_tp.py \

502 - --ignore=lora/test_deepseekv2_tp.py \

503 - --ignore=lora/test_gptoss_tp.py \

504 - --ignore=lora/test_qwen3moe_tp.py

505 -

506 - parallelism: 4

507 -

508 -- label: PyTorch Compilation Unit Tests # 15min

509 - timeout_in_minutes: 30

510 - mirror_hardwares: [amdexperimental]

511 - torch_nightly: true

512 - source_file_dependencies:

513 - - vllm/

514 - - tests/compile

515 - commands:

516 - # Run unit tests defined directly under compile/,

517 - # not including subdirectories, which are usually heavier

518 - # tests covered elsewhere.

519 - # Use `find` to launch multiple instances of pytest so that

520 - # they do not suffer from https://github.com/vllm-project/vllm/issues/28965

521 - # However, find does not normally propagate error codes, so we combine it with xargs

522 - # (using -0 for proper path handling)

523 - - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"

524 - - pytest -s -v compile/passes --ignore compile/passes/distributed

525 -

526 -- label: PyTorch Fullgraph Smoke Test # 15min

527 - timeout_in_minutes: 30

528 - mirror_hardwares: [amdexperimental]

529 - torch_nightly: true

530 - source_file_dependencies:

531 - - vllm/

532 - - tests/compile

533 - commands:

534 - # Run smoke tests under fullgraph directory, except test_full_graph.py

535 - # as it is a heavy test that is covered in other steps.

536 - # Use `find` to launch multiple instances of pytest so that

537 - # they do not suffer from https://github.com/vllm-project/vllm/issues/28965

538 - # However, find does not normally propagate error codes, so we combine it with xargs

539 - # (using -0 for proper path handling)

540 - - "find compile/fullgraph -maxdepth 1 -name 'test_*.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"

541 -

542 -- label: PyTorch Fullgraph Test # 27min

543 - timeout_in_minutes: 40

544 - mirror_hardwares: [amdexperimental]

545 - torch_nightly: true

546 - source_file_dependencies:

547 - - vllm/

548 - - tests/compile

549 - commands:

550 - # fp8 kv scales not supported on sm89, tested on Blackwell instead

551 - - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'

552 - # # Limit to no custom ops to reduce running time

553 - # # Wrap with quotes to escape yaml and avoid starting -k string with a -

554 - # - "pytest -v -s compile/distributed/test_fusions_e2e.py -k 'TRITON and not +quant_fp8 and not Llama-4'"

555 - # Old E2E tests were removed in https://github.com/vllm-project/vllm/pull/33293

556 - # in favor of new tests in fusions_e2e. We avoid replicating the new jobs in this file as it's deprecated.

557 -

558 -- label: Cudagraph test

559 - timeout_in_minutes: 20

560 - mirror_hardwares: [amdexperimental]

561 - source_file_dependencies:

562 - - tests/v1/cudagraph

563 - - vllm/v1/cudagraph_dispatcher.py

564 - - vllm/config/compilation.py

565 - - vllm/compilation

566 - commands:

567 - - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py

568 - - pytest -v -s v1/cudagraph/test_cudagraph_mode.py

569 -

570 -- label: Kernels Core Operation Test # 48min

571 - timeout_in_minutes: 75

572 - mirror_hardwares: [amdexperimental]

573 - source_file_dependencies:

574 - - csrc/

575 - - tests/kernels/core

576 - - tests/kernels/test_top_k_per_row.py

577 - commands:

578 - - pytest -v -s kernels/core kernels/test_top_k_per_row.py

579 -

580 -- label: Kernels Attention Test %N # 23min

581 - timeout_in_minutes: 35

582 - mirror_hardwares: [amdexperimental]

583 - source_file_dependencies:

584 - - csrc/attention/

585 - - vllm/v1/attention

586 - # TODO: remove this dependency (https://github.com/vllm-project/vllm/issues/32267)

587 - - vllm/model_executor/layers/attention

588 - - tests/kernels/attention

589 - commands:

590 - - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

591 - parallelism: 2

592 -

593 -- label: Kernels Quantization Test %N # 64min

594 - timeout_in_minutes: 90

595 - mirror_hardwares: [amdexperimental]

596 - source_file_dependencies:

597 - - csrc/quantization/

598 - - vllm/model_executor/layers/quantization

599 - - tests/kernels/quantization

600 - commands:

601 - - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

602 - parallelism: 2

603 -

604 -- label: Kernels MoE Test %N # 40min

605 - timeout_in_minutes: 60

606 - mirror_hardwares: [amdexperimental]

607 - source_file_dependencies:

608 - - csrc/quantization/cutlass_w8a8/moe/

609 - - csrc/moe/

610 - - tests/kernels/moe

611 - - vllm/model_executor/layers/fused_moe/

612 - - vllm/distributed/device_communicators/

613 - - vllm/envs.py

614 - - vllm/config

615 - commands:

616 - - pytest -v -s kernels/moe --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

617 - parallelism: 2

618 -

619 -- label: Kernels Mamba Test # 31min

620 - timeout_in_minutes: 45

621 - mirror_hardwares: [amdexperimental]

622 - source_file_dependencies:

623 - - csrc/mamba/

624 - - tests/kernels/mamba

625 - - vllm/model_executor/layers/mamba/ops

626 - commands:

627 - - pytest -v -s kernels/mamba

628 -

629 -- label: Kernels DeepGEMM Test (H100)

630 - timeout_in_minutes: 45

631 - gpu: h100

632 - num_gpus: 1

633 - source_file_dependencies:

634 - - tools/install_deepgemm.sh

635 - - vllm/utils/deep_gemm.py

636 - - vllm/model_executor/layers/fused_moe

637 - - vllm/model_executor/layers/quantization

638 - - tests/kernels/quantization/test_block_fp8.py

639 - - tests/kernels/moe/test_deepgemm.py

640 - - tests/kernels/moe/test_batched_deepgemm.py

641 - - tests/kernels/attention/test_deepgemm_attention.py

642 - commands:

643 - - pytest -v -s kernels/quantization/test_block_fp8.py -k deep_gemm

644 - - pytest -v -s kernels/moe/test_deepgemm.py

645 - - pytest -v -s kernels/moe/test_batched_deepgemm.py

646 - - pytest -v -s kernels/attention/test_deepgemm_attention.py

647 -

648 -- label: Kernels Helion Test

649 - timeout_in_minutes: 30

650 - gpu: h100

651 - source_file_dependencies:

652 - - vllm/utils/import_utils.py

653 - - tests/kernels/helion/

654 - commands:

655 - - pip install helion

656 - - pytest -v -s kernels/helion/

657 -

658 -

659 -- label: Kernels FP8 MoE Test (1 H100)

660 - timeout_in_minutes: 90

661 - gpu: h100

662 - num_gpus: 1

663 - optional: true

664 - commands:

665 - - pytest -v -s kernels/moe/test_cutlass_moe.py

666 - - pytest -v -s kernels/moe/test_flashinfer.py

667 - - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py

668 - - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py

669 - - pytest -v -s kernels/moe/test_moe.py

670 - # - pytest -v -s kernels/moe/test_block_fp8.py - failing on main

671 - - pytest -v -s kernels/moe/test_block_int8.py

672 - - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py

673 - - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py

674 -

675 -- label: Kernels FP8 MoE Test (2 H100s)

676 - timeout_in_minutes: 90

677 - gpu: h100

678 - num_gpus: 2

679 - optional: true

680 - commands:

681 - - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py

682 - - pytest -v -s kernels/moe/test_deepep_moe.py

683 - - pytest -v -s kernels/moe/test_pplx_cutlass_moe.py

684 - # - pytest -v -s kernels/moe/test_pplx_moe.py - failing on main

685 -

686 -- label: Kernels Fp4 MoE Test (B200)

687 - timeout_in_minutes: 60

688 - gpu: b200

689 - num_gpus: 1

690 - optional: true

691 - commands:

692 - - pytest -v -s kernels/moe/test_cutedsl_moe.py

693 - - pytest -v -s kernels/moe/test_flashinfer_moe.py

694 - - pytest -v -s kernels/moe/test_nvfp4_moe.py

695 - - pytest -v -s kernels/moe/test_ocp_mx_moe.py

696 -

697 -

698 -- label: Model Executor Test # 23min

699 - timeout_in_minutes: 35

700 - torch_nightly: true

701 - mirror_hardwares: [amdexperimental]

702 - source_file_dependencies:

703 - - vllm/engine/arg_utils.py

704 - - vllm/config/model.py

705 - - vllm/model_executor

706 - - tests/model_executor

707 - - tests/entrypoints/openai/test_tensorizer_entrypoint.py

708 - commands:

709 - - apt-get update && apt-get install -y curl libsodium23

710 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

711 - - pytest -v -s model_executor

712 - - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py

713 -

714 -- label: Benchmarks # 11min

715 - timeout_in_minutes: 20

716 - mirror_hardwares: [amdexperimental]

717 - working_dir: "/vllm-workspace/.buildkite"

718 - source_file_dependencies:

719 - - benchmarks/

720 - commands:

721 - - bash scripts/run-benchmarks.sh

722 -

723 -- label: Benchmarks CLI Test # 7min

724 - timeout_in_minutes: 20

725 - mirror_hardwares: [amdexperimental]

726 - source_file_dependencies:

727 - - vllm/

728 - - tests/benchmarks/

729 - commands:

730 - - pytest -v -s benchmarks/

731 -

732 -- label: Quantization Test # 70min

733 - timeout_in_minutes: 90

734 - mirror_hardwares: [amdexperimental]

735 - source_file_dependencies:

736 - - csrc/

737 - - vllm/model_executor/layers/quantization

738 - - tests/quantization

739 - commands:

740 - # temporary install here since we need nightly, will move to requirements/test.in

741 - # after torchao 0.12 release, and pin a working version of torchao nightly here

742 -

743 - # since torchao nightly is only compatible with torch nightly currently

744 - # https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now

745 - # we can only upgrade after this is resolved

746 - # TODO(jerryzh168): resolve the above comment

747 - - uv pip install --system torchao==0.14.1 --index-url https://download.pytorch.org/whl/cu129

748 - - uv pip install --system conch-triton-kernels

749 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py

750 -

751 -- label: LM Eval Small Models # 53min

752 - timeout_in_minutes: 75

753 - mirror_hardwares: [amdexperimental]

754 - source_file_dependencies:

755 - - csrc/

756 - - vllm/model_executor/layers/quantization

757 - autorun_on_main: true

758 - commands:

759 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt

760 -

761 -- label: OpenAI API correctness # 22min

762 - timeout_in_minutes: 30

763 - mirror_hardwares: [amdexperimental]

764 - source_file_dependencies:

765 - - csrc/

766 - - vllm/entrypoints/openai/

767 - - vllm/model_executor/models/whisper.py

768 - commands: # LMEval+Transcription WER check

769 - - pytest -s entrypoints/openai/correctness/

770 -

771 -##### models test #####

772 -

773 -- label: Basic Models Tests (Initialization)

774 - timeout_in_minutes: 45

775 - mirror_hardwares: [amdexperimental]

776 - torch_nightly: true

777 - source_file_dependencies:

778 - - vllm/

779 - - tests/models/test_initialization.py

780 - - tests/models/registry.py

781 - commands:

782 - # Run a subset of model initialization tests

783 - - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset

784 -

785 -- label: Basic Models Tests (Extra Initialization) %N

786 - timeout_in_minutes: 45

787 - mirror_hardwares: [amdexperimental]

788 - torch_nightly: true

789 - source_file_dependencies:

790 - - vllm/model_executor/models/

791 - - vllm/transformers_utils/

792 - - tests/models/test_initialization.py

793 - - tests/models/registry.py

794 - commands:

795 - # Only when vLLM model source is modified - test initialization of a large

796 - # subset of supported models (the complement of the small subset in the above

797 - # test.) Also run if model initialization test file is modified

798 - - pytest -v -s models/test_initialization.py \

799 - -k 'not test_can_initialize_small_subset' \

800 - --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \

801 - --shard-id=$$BUILDKITE_PARALLEL_JOB

802 - parallelism: 2

803 -

804 -- label: Basic Models Tests (Other)

805 - timeout_in_minutes: 45

806 - mirror_hardwares: [amdexperimental]

807 - torch_nightly: true

808 - source_file_dependencies:

809 - - vllm/

810 - - tests/models/test_terratorch.py

811 - - tests/models/test_transformers.py

812 - - tests/models/test_registry.py

813 - commands:

814 - - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py

815 -

816 -- label: Basic Models Test (Other CPU) # 5min

817 - timeout_in_minutes: 10

818 - torch_nightly: true

819 - source_file_dependencies:

820 - - vllm/

821 - - tests/models/test_utils.py

822 - - tests/models/test_vision.py

823 - no_gpu: true

824 - commands:

825 - - pytest -v -s models/test_utils.py models/test_vision.py

826 -

827 -- label: Language Models Tests (Standard)

828 - timeout_in_minutes: 25

829 - mirror_hardwares: [amdexperimental]

830 - torch_nightly: true

831 - source_file_dependencies:

832 - - vllm/

833 - - tests/models/language

834 - commands:

835 - # Test standard language models, excluding a subset of slow tests

836 - - pip freeze | grep -E 'torch'

837 - - pytest -v -s models/language -m 'core_model and (not slow_test)'

838 -

839 -- label: Language Models Tests (Extra Standard) %N

840 - timeout_in_minutes: 45

841 - mirror_hardwares: [amdexperimental]

842 - torch_nightly: true

843 - source_file_dependencies:

844 - - vllm/model_executor/models/

845 - - tests/models/language/pooling/test_embedding.py

846 - - tests/models/language/generation/test_common.py

847 - - tests/models/language/pooling/test_classification.py

848 - commands:

849 - # Shard slow subset of standard language models tests. Only run when model

850 - # source is modified, or when specified test files are modified

851 - - pip freeze | grep -E 'torch'

852 - - pytest -v -s models/language -m 'core_model and slow_test' \

853 - --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \

854 - --shard-id=$$BUILDKITE_PARALLEL_JOB

855 - parallelism: 2

856 -

857 -- label: Language Models Tests (Hybrid) %N

858 - timeout_in_minutes: 75

859 - mirror_hardwares: [amdexperimental]

860 - torch_nightly: true

861 - source_file_dependencies:

862 - - vllm/

863 - - tests/models/language/generation

864 - commands:

865 - # Install fast path packages for testing against transformers

866 - # Note: also needed to run plamo2 model in vLLM

867 - - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'

868 - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'

869 - # Shard hybrid language model tests

870 - - pytest -v -s models/language/generation \

871 - -m hybrid_model \

872 - --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \

873 - --shard-id=$$BUILDKITE_PARALLEL_JOB

874 - parallelism: 2

875 -

876 -- label: Language Models Test (Extended Generation) # 80min

877 - timeout_in_minutes: 110

878 - mirror_hardwares: [amdexperimental]

879 - optional: true

880 - source_file_dependencies:

881 - - vllm/

882 - - tests/models/language/generation

883 - commands:

884 - # Install fast path packages for testing against transformers

885 - # Note: also needed to run plamo2 model in vLLM

886 - - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'

887 - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'

888 - - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'

889 -

890 -- label: Language Models Test (PPL)

891 - timeout_in_minutes: 110

892 - mirror_hardwares: [amdexperimental]

893 - optional: true

894 - source_file_dependencies:

895 - - vllm/

896 - - tests/models/language/generation_ppl_test

897 - commands:

898 - - pytest -v -s models/language/generation_ppl_test

899 -

900 -- label: Language Models Test (Extended Pooling) # 36min

901 - timeout_in_minutes: 50

902 - mirror_hardwares: [amdexperimental]

903 - optional: true

904 - source_file_dependencies:

905 - - vllm/

906 - - tests/models/language/pooling

907 - commands:

908 - - pytest -v -s models/language/pooling -m 'not core_model'

909 -

910 -- label: Language Models Test (MTEB)

911 - timeout_in_minutes: 110

912 - mirror_hardwares: [amdexperimental]

913 - optional: true

914 - source_file_dependencies:

915 - - vllm/

916 - - tests/models/language/pooling_mteb_test

917 - commands:

918 - - pytest -v -s models/language/pooling_mteb_test

919 -

920 -- label: Multi-Modal Processor Test (CPU)

921 - timeout_in_minutes: 60

922 - source_file_dependencies:

923 - - vllm/

924 - - tests/models/multimodal

925 - no_gpu: true

926 - commands:

927 - - "pip install git+https://github.com/TIGER-AI-Lab/Mantis.git || echo 'Mantis installation skipped (decord not available on CPU-only environment)'"

928 - - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py

929 -

930 -- label: Multi-Modal Processor Test

931 - timeout_in_minutes: 60

932 - source_file_dependencies:

933 - - vllm/

934 - - tests/models/multimodal

935 - commands:

936 - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

937 - - pytest -v -s models/multimodal/processing/test_tensor_schema.py

938 -

939 -- label: Multi-Modal Models Test (Standard) # 60min

940 - timeout_in_minutes: 80

941 - mirror_hardwares: [amdexperimental]

942 - torch_nightly: true

943 - source_file_dependencies:

944 - - vllm/

945 - - tests/models/multimodal

946 - commands:

947 - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

948 - - pip freeze | grep -E 'torch'

949 - - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing

950 - - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work

951 -

952 -- label: Multi-Modal Accuracy Eval (Small Models) # 50min

953 - timeout_in_minutes: 70

954 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"

955 - source_file_dependencies:

956 - - vllm/multimodal/

957 - - vllm/inputs/

958 - - vllm/v1/core/

959 - commands:

960 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1

961 -

962 -- label: Multi-Modal Models Test (Extended) 1

963 - mirror_hardwares: [amdexperimental]

964 - optional: true

965 - source_file_dependencies:

966 - - vllm/

967 - - tests/models/multimodal

968 - commands:

969 - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

970 - - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing

971 -

972 -- label: Multi-Modal Models Test (Extended) 2

973 - mirror_hardwares: [amdexperimental]

974 - optional: true

975 - source_file_dependencies:

976 - - vllm/

977 - - tests/models/multimodal

978 - commands:

979 - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

980 - - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'

981 -

982 -- label: Multi-Modal Models Test (Extended) 3

983 - mirror_hardwares: [amdexperimental]

984 - optional: true

985 - source_file_dependencies:

986 - - vllm/

987 - - tests/models/multimodal

988 - commands:

989 - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

990 - - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'

991 -

992 -- label: Quantized Models Test # 45 min

993 - timeout_in_minutes: 60

994 - mirror_hardwares: [amdexperimental]

995 - source_file_dependencies:

996 - - vllm/model_executor/layers/quantization

997 - - tests/models/quantization

998 - commands:

999 - - pytest -v -s models/quantization

1000 -

1001 -# This test is used only in PR development phase to test individual models and should never run on main

1002 -- label: Custom Models Test

1003 - mirror_hardwares: [amdexperimental]

1004 - optional: true

1005 - commands:

1006 - - echo 'Testing custom models...'

1007 - # PR authors can temporarily add commands below to test individual models

1008 - # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py

1009 - # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*

1010 -

1011 -- label: Transformers Nightly Models Test

1012 - working_dir: "/vllm-workspace/"

1013 - optional: true

1014 - soft_fail: true

1015 - commands:

1016 - - pip install --upgrade git+https://github.com/huggingface/transformers

1017 - - pytest -v -s tests/models/test_initialization.py

1018 - - pytest -v -s tests/models/test_transformers.py

1019 - - pytest -v -s tests/models/multimodal/processing/

1020 - - pytest -v -s tests/models/multimodal/test_mapping.py

1021 - - python3 examples/offline_inference/basic/chat.py

1022 - - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl

1023 - # Whisper needs spawn method to avoid deadlock

1024 - - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper

1025 -

1026 -- label: Blackwell Test # 23 min

1027 - timeout_in_minutes: 30

1028 - working_dir: "/vllm-workspace/"

1029 - gpu: b200

1030 - source_file_dependencies:

1031 - - csrc/quantization/fp4/

1032 - - csrc/attention/mla/

1033 - - csrc/quantization/cutlass_w8a8/moe/

1034 - - vllm/model_executor/layers/fused_moe/cutlass_moe.py

1035 - - vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py

1036 - - vllm/model_executor/layers/fused_moe/flashinfer_a2a_prepare_finalize.py

1037 - - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py

1038 - - vllm/v1/attention/backends/flashinfer.py

1039 - - vllm/v1/attention/backends/mla/cutlass_mla.py

1040 - - vllm/v1/attention/backends/mla/flashinfer_mla.py

1041 - - vllm/v1/attention/selector.py

1042 - - vllm/platforms/cuda.py

1043 - commands:

1044 - - nvidia-smi

1045 - - python3 examples/offline_inference/basic/chat.py

1046 - # Attention

1047 - # num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353

1048 - - pytest -v -s tests/kernels/attention/test_attention_selector.py

1049 - - pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'

1050 - - pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py

1051 - - pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py

1052 - - pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py

1053 - # Quantization

1054 - - pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'

1055 - - pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py

1056 - - pytest -v -s tests/kernels/quantization/test_silu_mul_nvfp4_quant.py

1057 - - pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py

1058 - - pytest -v -s tests/kernels/quantization/test_flashinfer_scaled_mm.py

1059 - - pytest -v -s tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py

1060 - - pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py

1061 - - pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py

1062 - - pytest -v -s tests/kernels/moe/test_nvfp4_moe.py

1063 - - pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py

1064 - - pytest -v -s tests/kernels/moe/test_flashinfer.py

1065 - - pytest -v -s tests/kernels/moe/test_cutedsl_moe.py

1066 - # e2e

1067 - - pytest -v -s tests/models/quantization/test_nvfp4.py

1068 -

1069 -- label: Blackwell Fusion and Compile Tests # 30 min

1070 - timeout_in_minutes: 40

1071 - working_dir: "/vllm-workspace/"

1072 - gpu: b200

1073 - source_file_dependencies:

1074 - - csrc/quantization/fp4/

1075 - - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py

1076 - - vllm/v1/attention/backends/flashinfer.py

1077 - - vllm/v1/worker/

1078 - - vllm/v1/cudagraph_dispatcher.py

1079 - - vllm/compilation/

1080 - # can affect pattern matching

1081 - - vllm/model_executor/layers/layernorm.py

1082 - - vllm/model_executor/layers/activation.py

1083 - - vllm/model_executor/layers/quantization/input_quant_fp8.py

1084 - - tests/compile/test_fusion_attn.py

1085 - - tests/compile/test_silu_mul_quant_fusion.py

1086 - - tests/compile/passes/distributed/test_fusion_all_reduce.py

1087 - - tests/compile/fullgraph/test_full_graph.py

1088 - commands:

1089 - - nvidia-smi

1090 - - pytest -v -s tests/compile/test_fusion_attn.py

1091 - - pytest -v -s tests/compile/test_silu_mul_quant_fusion.py

1092 - # this runner has 2 GPUs available even though num_gpus=2 is not set

1093 - - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py

1094 - # # Limit to Inductor partition, no custom ops, and allreduce & attn fusion to reduce running time

1095 - # # Wrap with quotes to escape yaml

1096 - # - "pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and not +quant_fp8 and not +rms_norm'"

1097 - # Old E2E tests were removed in https://github.com/vllm-project/vllm/pull/33293

1098 - # in favor of new tests in fusions_e2e. We avoid replicating the new jobs in this file as it's deprecated.

1099 -

1100 - # test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)

1101 - - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile

1102 -

1103 -- label: Blackwell GPT-OSS Eval

1104 - timeout_in_minutes: 60

1105 - working_dir: "/vllm-workspace/"

1106 - gpu: b200

1107 - optional: true # run on nightlies

1108 - source_file_dependencies:

1109 - - tests/evals/gpt_oss

1110 - - vllm/model_executor/models/gpt_oss.py

1111 - - vllm/model_executor/layers/quantization/mxfp4.py

1112 - - vllm/v1/attention/backends/flashinfer.py

1113 - commands:

1114 - - uv pip install --system 'gpt-oss[eval]==0.0.5'

1115 - - pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58

1116 -

1117 -- label: Blackwell Quantized MoE Test

1118 - timeout_in_minutes: 60

1119 - working_dir: "/vllm-workspace/"

1120 - gpu: b200

1121 - source_file_dependencies:

1122 - - tests/quantization/test_blackwell_moe.py

1123 - - vllm/model_executor/models/deepseek_v2.py

1124 - - vllm/model_executor/models/gpt_oss.py

1125 - - vllm/model_executor/models/llama4.py

1126 - - vllm/model_executor/layers/fused_moe

1127 - - vllm/model_executor/layers/quantization/compressed_tensors

1128 - - vllm/model_executor/layers/quantization/modelopt.py

1129 - - vllm/model_executor/layers/quantization/mxfp4.py

1130 - - vllm/v1/attention/backends/flashinfer.py

1131 - commands:

1132 - - pytest -s -v tests/quantization/test_blackwell_moe.py

1133 -

1134 -- label: Blackwell LM Eval Small Models

1135 - timeout_in_minutes: 120

1136 - gpu: b200

1137 - optional: true # run on nightlies

1138 - source_file_dependencies:

1139 - - csrc/

1140 - - vllm/model_executor/layers/quantization

1141 - commands:

1142 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt

1143 -

1144 -##### 1 GPU test #####

1145 -##### multi gpus test #####

1146 -

1147 -- label: Distributed Comm Ops Test # 7min

1148 - timeout_in_minutes: 20

1149 - mirror_hardwares: [amdexperimental]

1150 - working_dir: "/vllm-workspace/tests"

1151 - num_gpus: 2

1152 - source_file_dependencies:

1153 - - vllm/distributed

1154 - - tests/distributed

1155 - commands:

1156 - - pytest -v -s distributed/test_comm_ops.py

1157 - - pytest -v -s distributed/test_shm_broadcast.py

1158 - - pytest -v -s distributed/test_shm_buffer.py

1159 - - pytest -v -s distributed/test_shm_storage.py

1160 - - pytest -v -s distributed/test_packed_tensor.py

1161 - - pytest -v -s distributed/test_weight_transfer.py

1162 -

1163 -- label: 2 Node Tests (4 GPUs in total) # 16min

1164 - timeout_in_minutes: 30

1165 - mirror_hardwares: [amdexperimental]

1166 - working_dir: "/vllm-workspace/tests"

1167 - num_gpus: 2

1168 - num_nodes: 2

1169 - source_file_dependencies:

1170 - - vllm/distributed/

1171 - - vllm/engine/

1172 - - vllm/executor/

1173 - - vllm/model_executor/models/

1174 - - tests/distributed/

1175 - - tests/examples/offline_inference/data_parallel.py

1176 - - .buildkite/scripts/run-multi-node-test.sh

1177 - commands:

1178 - - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)

1179 - - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'

1180 - - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'

1181 - - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code

1182 - - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py

1183 - - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py

1184 - - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)

1185 - - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'

1186 - - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'

1187 - - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code

1188 -

1189 -- label: Distributed Tests (2 GPUs) # 68min

1190 - timeout_in_minutes: 90

1191 - mirror_hardwares: [amdexperimental]

1192 - working_dir: "/vllm-workspace/tests"

1193 - num_gpus: 2

1194 - source_file_dependencies:

1195 - - vllm/compilation/

1196 - - vllm/distributed/

1197 - - vllm/engine/

1198 - - vllm/executor/

1199 - - vllm/worker/worker_base.py

1200 - - vllm/v1/engine/

1201 - - vllm/v1/worker/

1202 - - tests/compile/fullgraph/test_basic_correctness.py

1203 - - tests/compile/test_wrapper.py

1204 - - tests/distributed/

1205 - - tests/entrypoints/llm/test_collective_rpc.py

1206 - - tests/v1/distributed

1207 - - tests/v1/entrypoints/openai/test_multi_api_servers.py

1208 - - tests/v1/shutdown

1209 - - tests/v1/worker/test_worker_memory_snapshot.py

1210 - commands:

1211 - # https://github.com/NVIDIA/nccl/issues/1838

1212 - - export NCCL_CUMEM_HOST_ENABLE=0

1213 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py

1214 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py

1215 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py

1216 - - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py

1217 - - pytest -v -s entrypoints/llm/test_collective_rpc.py

1218 - - pytest -v -s ./compile/fullgraph/test_basic_correctness.py

1219 - - pytest -v -s ./compile/test_wrapper.py

1220 - - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'

1221 - - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'

1222 - - pytest -v -s distributed/test_sequence_parallel.py

1223 - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown

1224 - - pytest -v -s v1/worker/test_worker_memory_snapshot.py

1225 -

1226 -- label: Distributed Model Tests (2 GPUs) # 37min

1227 - timeout_in_minutes: 50

1228 - mirror_hardwares: [amdexperimental]

1229 - working_dir: "/vllm-workspace/tests"

1230 - num_gpus: 2

1231 - source_file_dependencies:

1232 - - vllm/model_executor/model_loader/sharded_state_loader.py

1233 - - vllm/model_executor/models/

1234 - - tests/basic_correctness/

1235 - - tests/model_executor/model_loader/test_sharded_state_loader.py

1236 - - tests/models/

1237 - commands:

1238 - - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'

1239 - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py

1240 - # Avoid importing model tests that cause CUDA reinitialization error

1241 - - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'

1242 - - pytest models/language -v -s -m 'distributed(num_gpus=2)'

1243 - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py

1244 - - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'

1245 -

1246 -- label: Plugin Tests (2 GPUs) # 40min

1247 - timeout_in_minutes: 60

1248 - mirror_hardwares: [amdexperimental]

1249 - working_dir: "/vllm-workspace/tests"

1250 - num_gpus: 2

1251 - source_file_dependencies:

1252 - - vllm/plugins/

1253 - - tests/plugins/

1254 - commands:

1255 - # begin platform plugin and general plugin tests, all the code in-between runs on dummy platform

1256 - - pip install -e ./plugins/vllm_add_dummy_platform

1257 - - pytest -v -s plugins_tests/test_platform_plugins.py

1258 - - pip uninstall vllm_add_dummy_platform -y

1259 - # end platform plugin tests

1260 - # begin io_processor plugins test, all the code in between uses the prithvi_io_processor plugin

1261 - - pip install -e ./plugins/prithvi_io_processor_plugin

1262 - - pytest -v -s plugins_tests/test_io_processor_plugins.py

1263 - - pip uninstall prithvi_io_processor_plugin -y

1264 - # end io_processor plugins test

1265 - # begin stat_logger plugins test

1266 - - pip install -e ./plugins/vllm_add_dummy_stat_logger

1267 - - pytest -v -s plugins_tests/test_stats_logger_plugins.py

1268 - - pip uninstall dummy_stat_logger -y

1269 - # end stat_logger plugins test

1270 - # other tests continue here:

1271 - - pytest -v -s plugins_tests/test_scheduler_plugins.py

1272 - - pip install -e ./plugins/vllm_add_dummy_model

1273 - - pytest -v -s distributed/test_distributed_oot.py

1274 - - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process

1275 - - pytest -v -s models/test_oot_registration.py # it needs a clean process

1276 - - pytest -v -s plugins/lora_resolvers # unit tests for lora resolver plugins

1277 -

1278 -- label: Pipeline + Context Parallelism Test # 45min

1279 - timeout_in_minutes: 60

1280 - mirror_hardwares: [amdexperimental]

1281 - working_dir: "/vllm-workspace/tests"

1282 - num_gpus: 4

1283 - source_file_dependencies:

1284 - - vllm/distributed/

1285 - - vllm/engine/

1286 - - vllm/executor/

1287 - - vllm/model_executor/models/

1288 - - tests/distributed/

1289 - commands:

1290 - - pytest -v -s distributed/test_pp_cudagraph.py

1291 - - pytest -v -s distributed/test_pipeline_parallel.py

1292 -

1293 -- label: LoRA TP Test (Distributed) # 17 min

1294 - timeout_in_minutes: 30

1295 - mirror_hardwares: [amdexperimental]

1296 - num_gpus: 4

1297 - source_file_dependencies:

1298 - - vllm/lora

1299 - - tests/lora

1300 - commands:

1301 - # FIXIT: find out which code initialize cuda before running the test

1302 - # before the fix, we need to use spawn to test it

1303 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

1304 - # Alot of these tests are on the edge of OOMing

1305 - - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True

1306 - # There is some Tensor Parallelism related processing logic in LoRA that

1307 - # requires multi-GPU testing for validation.

1308 - - pytest -v -s -x lora/test_chatglm3_tp.py

1309 - - pytest -v -s -x lora/test_llama_tp.py

1310 - - pytest -v -s -x lora/test_llm_with_multi_loras.py

1311 - - pytest -v -s -x lora/test_olmoe_tp.py

1312 - - pytest -v -s -x lora/test_gptoss_tp.py

1313 -

1314 -

1315 -- label: Weight Loading Multiple GPU Test # 33min

1316 - timeout_in_minutes: 45

1317 - mirror_hardwares: [amdexperimental]

1318 - working_dir: "/vllm-workspace/tests"

1319 - num_gpus: 2

1320 - optional: true

1321 - source_file_dependencies:

1322 - - vllm/

1323 - - tests/weight_loading

1324 - commands:

1325 - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt

1326 -

1327 -- label: Weight Loading Multiple GPU Test - Large Models # optional

1328 - mirror_hardwares: [amdexperimental]

1329 - working_dir: "/vllm-workspace/tests"

1330 - num_gpus: 2

1331 - gpu: a100

1332 - optional: true

1333 - source_file_dependencies:

1334 - - vllm/

1335 - - tests/weight_loading

1336 - commands:

1337 - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt

1338 -

1339 -- label: NixlConnector PD accuracy tests (Distributed) # 40min

1340 - timeout_in_minutes: 40

1341 - working_dir: "/vllm-workspace/tests"

1342 - num_gpus: 4

1343 - source_file_dependencies:

1344 - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py

1345 - - tests/v1/kv_connector/nixl_integration/

1346 - commands:

1347 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt

1348 - - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

1349 -

1350 -- label: DP EP NixlConnector PD accuracy tests (Distributed) # 15min

1351 - timeout_in_minutes: 15

1352 - working_dir: "/vllm-workspace/tests"

1353 - num_gpus: 4

1354 - source_file_dependencies:

1355 - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py

1356 - - tests/v1/kv_connector/nixl_integration/

1357 - commands:

1358 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt

1359 - - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

1360 -

1361 -

1362 -##### multi gpus test #####

1363 -##### A100 test #####

1364 -

1365 -- label: Distributed Tests (A100) # optional

1366 - gpu: a100

1367 - optional: true

1368 - num_gpus: 4

1369 - source_file_dependencies:

1370 - - vllm/

1371 - commands:

1372 - # NOTE: don't test llama model here, it seems hf implementation is buggy

1373 - # see https://github.com/vllm-project/vllm/pull/5689 for details

1374 - - pytest -v -s distributed/test_custom_all_reduce.py

1375 - - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py

1376 - - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'

1377 - - pytest -v -s -x lora/test_mixtral.py

1378 -

1379 -- label: Acceptance Length Test (Large Models) # optional

1380 - timeout_in_minutes: 120

1381 - gpu: h100

1382 - optional: true

1383 - num_gpus: 1

1384 - working_dir: "/vllm-workspace/tests"

1385 - source_file_dependencies:

1386 - - vllm/v1/spec_decode/

1387 - - vllm/model_executor/models/mlp_speculator.py

1388 - - tests/v1/spec_decode/test_acceptance_length.py

1389 - commands:

1390 - - export VLLM_ALLOW_INSECURE_SERIALIZATION=1

1391 - - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test

1392 -

1393 -- label: LM Eval Large Models # optional

1394 - gpu: a100

1395 - optional: true

1396 - num_gpus: 4

1397 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"

1398 - source_file_dependencies:

1399 - - csrc/

1400 - - vllm/model_executor/layers/quantization

1401 - commands:

1402 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn

1403 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4

1404 -

1405 -##### H100 test #####

1406 -- label: LM Eval Large Models (H100) # optional

1407 - gpu: h100

1408 - optional: true

1409 - num_gpus: 4

1410 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"

1411 - source_file_dependencies:

1412 - - csrc/

1413 - - vllm/model_executor/layers/quantization

1414 - commands:

1415 - - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100

1416 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4

1417 -

1418 -- label: Sequence Parallel Tests (H100) # 60 min

1419 - timeout_in_minutes: 60

1420 - working_dir: "/vllm-workspace/"

1421 - gpu: h100

1422 - optional: true

1423 - num_gpus: 2

1424 - commands:

1425 - - export VLLM_TEST_CLEAN_GPU_MEMORY=1

1426 - # Run sequence parallel tests

1427 - - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py

1428 - - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py

1429 -

1430 -- label: Distributed Tests (H100) # optional

1431 - gpu: h100

1432 - optional: true

1433 - working_dir: "/vllm-workspace/"

1434 - num_gpus: 2

1435 - commands:

1436 - - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py

1437 - - pytest -v -s tests/distributed/test_context_parallel.py

1438 - - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput

1439 - - pytest -v -s tests/v1/distributed/test_dbo.py

1440 -

1441 -##### H200 test #####

1442 -

1443 -- label: LM Eval Large Models (H200) # optional

1444 - timeout_in_minutes: 60

1445 - gpu: h200

1446 - optional: true

1447 - num_gpus: 8

1448 - commands:

1449 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt

1450 -

1451 -##### B200 test #####

1452 -- label: Distributed Tests (B200) # optional

1453 - gpu: b200

1454 - optional: true

1455 - working_dir: "/vllm-workspace/"

1456 - num_gpus: 2

1457 - commands:

1458 - - pytest -v -s tests/distributed/test_context_parallel.py

1459 - - pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py

1460 - - pytest -v -s tests/v1/distributed/test_dbo.py

1461 -

1462 -##### RL Integration Tests #####

1463 -- label: Prime-RL Integration Test # 15min

1464 - timeout_in_minutes: 30

1465 - optional: true

1466 - soft_fail: true

1467 - num_gpus: 2

1468 - working_dir: "/vllm-workspace"

1469 - source_file_dependencies:

1470 - - vllm/

1471 - - .buildkite/scripts/run-prime-rl-test.sh

1472 - commands:

1473 - - nvidia-smi

1474 - - bash .buildkite/scripts/run-prime-rl-test.sh

1475 -

1476 -- label: DeepSeek V2-Lite Accuracy

1477 - timeout_in_minutes: 60

1478 - gpu: h100

1479 - optional: true

1480 - num_gpus: 4

1481 - working_dir: "/vllm-workspace"

1482 - commands:

1483 - - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010

1484 -

1485 -- label: Qwen3-30B-A3B-FP8-block Accuracy (H100)

1486 - timeout_in_minutes: 60

1487 - gpu: h100

1488 - optional: true

1489 - num_gpus: 4

1490 - working_dir: "/vllm-workspace"

1491 - commands:

1492 - - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020

1493 -

1494 -- label: Qwen3-30B-A3B-FP8-block Accuracy (B200)

1495 - timeout_in_minutes: 60

1496 - gpu: b200

1497 - optional: true

1498 - num_gpus: 2

1499 - working_dir: "/vllm-workspace"

1500 - commands:

1501 - - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1

1502 -

1503 -##### MoE Refactor (Temporary) Tests #####

1504 -

1505 -- label: MoE Refactor Integration Test (H100 - TEMPORARY) # optional

1506 - gpu: h100

1507 - optional: true

1508 - num_gpus: 2

1509 - commands:

1510 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100.txt

1511 -

1512 -- label: MoE Refactor Integration Test (B200 - TEMPORARY) # optional

1513 - gpu: b200

1514 - optional: true

1515 - num_gpus: 2

1516 - commands:

1517 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200.txt

1518 -

1519 -- label: MoE Refactor Integration Test (B200 DP - TEMPORARY) # optional

1520 - gpu: b200

1521 - optional: true

1522 - num_gpus: 2

1523 - commands:

1524 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt

8 +# If you need to make changes to CI, please find the relevant file in these directories and make changes there.

Deprecate test-pipeline.yaml (#34864)