A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Add timeouts to tests (#24260)
Signed-off-by: Rafael Vasquez <rafvasq21@gmail.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Rafael Vasquez committed
Sep 5, 2025 at 20:26 UTC
c954c6629cdf4ebbfc1cab05d7d71af6696c5881
1 file changed
+102
-44
.buildkite/test-pipeline.yaml
+102
-44
@@ -41,7 +41,8 @@ steps:
41
commands:
42
- bash standalone_tests/pytorch_nightly_dependency.sh
43
44
-- label: Async Engine, Inputs, Utils, Worker Test # 24min
44
+- label: Async Engine, Inputs, Utils, Worker Test # 36min
45
+ timeout_in_minutes: 50
46
mirror_hardwares: [amdexperimental]
47
source_file_dependencies:
48
- vllm/
@@ -63,7 +64,8 @@ steps:
64
- pytest -v -s utils_ # Utils
65
- pytest -v -s worker # Worker
66
66
-- label: Python-only Installation Test
67
+- label: Python-only Installation Test # 10min
68
+ timeout_in_minutes: 20
69
mirror_hardwares: [amdexperimental]
70
source_file_dependencies:
71
- tests/standalone_tests/python_only_compile.sh
@@ -71,7 +73,8 @@ steps:
73
commands:
74
- bash standalone_tests/python_only_compile.sh
75
74
-- label: Basic Correctness Test # 30min
76
+- label: Basic Correctness Test # 20min
77
+ timeout_in_minutes: 30
78
mirror_hardwares: [amdexperimental]
79
fast_check: true
80
torch_nightly: true
@@ -88,7 +91,8 @@ steps:
91
- pytest -v -s basic_correctness/test_cpu_offload.py
92
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
93
91
-- label: Core Test # 10min
94
+- label: Core Test # 22min
95
+ timeout_in_minutes: 35
96
mirror_hardwares: [amdexperimental]
97
fast_check: true
98
source_file_dependencies:
@@ -98,7 +102,8 @@ steps:
102
commands:
103
- pytest -v -s core
104
101
-- label: Entrypoints Test (LLM) # 40min
105
+- label: Entrypoints Test (LLM) # 30min
106
+ timeout_in_minutes: 40
107
mirror_hardwares: [amdexperimental]
108
working_dir: "/vllm-workspace/tests"
109
fast_check: true
@@ -114,7 +119,8 @@ steps:
119
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
120
- VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
121
117
-- label: Entrypoints Test (API Server) # 40min
122
+- label: Entrypoints Test (API Server) # 100min
123
+ timeout_in_minutes: 130
124
mirror_hardwares: [amdexperimental]
125
working_dir: "/vllm-workspace/tests"
126
fast_check: true
@@ -129,7 +135,8 @@ steps:
135
- pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_collective_rpc.py
136
- pytest -v -s entrypoints/test_chat_utils.py
137
132
-- label: Distributed Tests (4 GPUs) # 10min
138
+- label: Distributed Tests (4 GPUs) # 35min
139
+ timeout_in_minutes: 50
140
mirror_hardwares: [amdexperimental]
141
working_dir: "/vllm-workspace/tests"
142
num_gpus: 4
@@ -172,7 +179,8 @@ steps:
179
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py
180
- popd
181
175
-- label: EPLB Algorithm Test
182
+- label: EPLB Algorithm Test # 5min
183
+ timeout_in_minutes: 15
184
working_dir: "/vllm-workspace/tests"
185
source_file_dependencies:
186
- vllm/distributed/eplb
@@ -181,6 +189,7 @@ steps:
189
- pytest -v -s distributed/test_eplb_algo.py
190
191
- label: EPLB Execution Test # 5min
192
+ timeout_in_minutes: 15
193
working_dir: "/vllm-workspace/tests"
194
num_gpus: 4
195
source_file_dependencies:
@@ -189,7 +198,8 @@ steps:
198
commands:
199
- pytest -v -s distributed/test_eplb_execute.py
200
192
-- label: Metrics, Tracing Test # 10min
201
+- label: Metrics, Tracing Test # 12min
202
+ timeout_in_minutes: 20
203
mirror_hardwares: [amdexperimental]
204
num_gpus: 2
205
source_file_dependencies:
@@ -208,7 +218,8 @@ steps:
218
##### fast check tests #####
219
##### 1 GPU test #####
220
211
-- label: Regression Test # 5min
221
+- label: Regression Test # 7min
222
+ timeout_in_minutes: 20
223
mirror_hardwares: [amdexperimental]
224
source_file_dependencies:
225
- vllm/
@@ -218,7 +229,8 @@ steps:
229
- pytest -v -s test_regression.py
230
working_dir: "/vllm-workspace/tests" # optional
231
221
-- label: Engine Test # 10min
232
+- label: Engine Test # 25min
233
+ timeout_in_minutes: 40
234
mirror_hardwares: [amdexperimental]
235
source_file_dependencies:
236
- vllm/
@@ -233,7 +245,8 @@ steps:
245
# OOM in the CI unless we run this separately
246
- pytest -v -s tokenization
247
236
-- label: V1 Test e2e + engine
248
+- label: V1 Test e2e + engine # 30min
249
+ timeout_in_minutes: 45
250
mirror_hardwares: [amdexperimental]
251
source_file_dependencies:
252
- vllm/
@@ -244,7 +257,8 @@ steps:
257
- pytest -v -s v1/e2e
258
- pytest -v -s v1/engine
259
247
-- label: V1 Test entrypoints
260
+- label: V1 Test entrypoints # 35min
261
+ timeout_in_minutes: 50
262
mirror_hardwares: [amdexperimental]
263
source_file_dependencies:
264
- vllm/
@@ -252,7 +266,8 @@ steps:
266
commands:
267
- pytest -v -s v1/entrypoints
268
255
-- label: V1 Test others
269
+- label: V1 Test others # 42min
270
+ timeout_in_minutes: 60
271
mirror_hardwares: [amdexperimental]
272
source_file_dependencies:
273
- vllm/
@@ -276,7 +291,8 @@ steps:
291
- pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
292
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
293
279
-- label: Examples Test # 25min
294
+- label: Examples Test # 30min
295
+ timeout_in_minutes: 45
296
mirror_hardwares: [amdexperimental]
297
working_dir: "/vllm-workspace/examples"
298
source_file_dependencies:
@@ -301,7 +317,8 @@ steps:
317
- python3 offline_inference/basic/score.py
318
- VLLM_USE_V1=0 python3 offline_inference/profiling.py --model facebook/opt-125m run_num_steps --num-steps 2
319
304
-- label: Platform Tests (CUDA)
320
+- label: Platform Tests (CUDA) # 4min
321
+ timeout_in_minutes: 15
322
mirror_hardwares: [amdexperimental]
323
source_file_dependencies:
324
- vllm/
@@ -309,7 +326,8 @@ steps:
326
commands:
327
- pytest -v -s cuda/test_cuda_context.py
328
312
-- label: Samplers Test # 36min
329
+- label: Samplers Test # 56min
330
+ timeout_in_minutes: 75
331
mirror_hardwares: [amdexperimental]
332
source_file_dependencies:
333
- vllm/model_executor/layers
@@ -320,15 +338,23 @@ steps:
338
- pytest -v -s samplers
339
- VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
340
323
-- label: LoRA Test %N # 15min each
341
+- label: LoRA Test %N # 20min each
342
+ timeout_in_minutes: 30
343
mirror_hardwares: [amdexperimental]
344
source_file_dependencies:
345
- vllm/lora
346
- tests/lora
328
- command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py
347
+ commands:
348
+ - pytest -v -s lora \
349
+ --shard-id=$$BUILDKITE_PARALLEL_JOB \
350
+ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
351
+ --ignore=lora/test_chatglm3_tp.py \
352
+ --ignore=lora/test_llama_tp.py \
353
+ --ignore=lora/test_llm_with_multi_loras.py
354
parallelism: 4
355
331
-- label: PyTorch Compilation Unit Tests
356
+- label: PyTorch Compilation Unit Tests # 15min
357
+ timeout_in_minutes: 30
358
mirror_hardwares: [amdexperimental]
359
torch_nightly: true
360
source_file_dependencies:
@@ -344,7 +370,8 @@ steps:
370
- pytest -v -s compile/test_fusion_all_reduce.py
371
- pytest -v -s compile/test_decorator.py
372
347
-- label: PyTorch Fullgraph Smoke Test # 9min
373
+- label: PyTorch Fullgraph Smoke Test # 15min
374
+ timeout_in_minutes: 30
375
mirror_hardwares: [amdexperimental]
376
torch_nightly: true
377
source_file_dependencies:
@@ -358,7 +385,8 @@ steps:
385
- pytest -v -s compile/piecewise/test_full_cudagraph.py
386
- pytest -v -s compile/piecewise/test_multiple_graphs.py
387
361
-- label: PyTorch Fullgraph Test # 18min
388
+- label: PyTorch Fullgraph Test # 20min
389
+ timeout_in_minutes: 30
390
mirror_hardwares: [amdexperimental]
391
torch_nightly: true
392
source_file_dependencies:
@@ -367,7 +395,8 @@ steps:
395
commands:
396
- pytest -v -s compile/test_full_graph.py
397
370
-- label: Kernels Core Operation Test
398
+- label: Kernels Core Operation Test # 48min
399
+ timeout_in_minutes: 75
400
mirror_hardwares: [amdexperimental]
401
source_file_dependencies:
402
- csrc/
@@ -375,7 +404,8 @@ steps:
404
commands:
405
- pytest -v -s kernels/core
406
378
-- label: Kernels Attention Test %N
407
+- label: Kernels Attention Test %N # 23min
408
+ timeout_in_minutes: 35
409
mirror_hardwares: [amdexperimental]
410
source_file_dependencies:
411
- csrc/attention/
@@ -386,7 +416,8 @@ steps:
416
- pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
417
parallelism: 2
418
389
-- label: Kernels Quantization Test %N
419
+- label: Kernels Quantization Test %N # 64min
420
+ timeout_in_minutes: 90
421
mirror_hardwares: [amdexperimental]
422
source_file_dependencies:
423
- csrc/quantization/
@@ -396,7 +427,8 @@ steps:
427
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
428
parallelism: 2
429
399
-- label: Kernels MoE Test %N
430
+- label: Kernels MoE Test %N # 40min
431
+ timeout_in_minutes: 60
432
mirror_hardwares: [amdexperimental]
433
source_file_dependencies:
434
- csrc/quantization/cutlass_w8a8/moe/
@@ -408,7 +440,8 @@ steps:
440
- pytest -v -s kernels/moe --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
441
parallelism: 2
442
411
-- label: Kernels Mamba Test
443
+- label: Kernels Mamba Test # 31min
444
+ timeout_in_minutes: 45
445
mirror_hardwares: [amdexperimental]
446
source_file_dependencies:
447
- csrc/mamba/
@@ -416,7 +449,8 @@ steps:
449
commands:
450
- pytest -v -s kernels/mamba
451
419
-- label: Tensorizer Test # 11min
452
+- label: Tensorizer Test # 14min
453
+ timeout_in_minutes: 25
454
mirror_hardwares: [amdexperimental]
455
source_file_dependencies:
456
- vllm/model_executor/model_loader
@@ -428,7 +462,8 @@ steps:
462
- pytest -v -s tensorizer_loader
463
- pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py
464
431
-- label: Model Executor Test
465
+- label: Model Executor Test # 7min
466
+ timeout_in_minutes: 20
467
mirror_hardwares: [amdexperimental]
468
source_file_dependencies:
469
- vllm/model_executor
@@ -438,7 +473,8 @@ steps:
473
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
474
- pytest -v -s model_executor
475
441
-- label: Benchmarks # 9min
476
+- label: Benchmarks # 11min
477
+ timeout_in_minutes: 20
478
mirror_hardwares: [amdexperimental]
479
working_dir: "/vllm-workspace/.buildkite"
480
source_file_dependencies:
@@ -446,7 +482,8 @@ steps:
482
commands:
483
- bash scripts/run-benchmarks.sh
484
449
-- label: Benchmarks CLI Test # 10min
485
+- label: Benchmarks CLI Test # 7min
486
+ timeout_in_minutes: 20
487
mirror_hardwares: [amdexperimental]
488
source_file_dependencies:
489
- vllm/
@@ -454,7 +491,8 @@ steps:
491
commands:
492
- pytest -v -s benchmarks/
493
457
-- label: Quantization Test
494
+- label: Quantization Test # 70min
495
+ timeout_in_minutes: 90
496
mirror_hardwares: [amdexperimental]
497
source_file_dependencies:
498
- csrc/
@@ -467,6 +505,7 @@ steps:
505
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization
506
507
- label: LM Eval Small Models # 53min
508
+ timeout_in_minutes: 75
509
mirror_hardwares: [amdexperimental]
510
source_file_dependencies:
511
- csrc/
@@ -474,7 +513,8 @@ steps:
513
commands:
514
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
515
477
-- label: OpenAI API correctness
516
+- label: OpenAI API correctness # 22min
517
+ timeout_in_minutes: 30
518
mirror_hardwares: [amdexperimental]
519
source_file_dependencies:
520
- csrc/
@@ -483,7 +523,8 @@ steps:
523
commands: # LMEval+Transcription WER check
524
- pytest -s entrypoints/openai/correctness/
525
486
-- label: Encoder Decoder tests # 5min
526
+- label: Encoder Decoder tests # 12min
527
+ timeout_in_minutes: 20
528
mirror_hardwares: [amdexperimental]
529
source_file_dependencies:
530
- vllm/
@@ -491,7 +532,8 @@ steps:
532
commands:
533
- pytest -v -s encoder_decoder
534
494
-- label: OpenAI-Compatible Tool Use # 20 min
535
+- label: OpenAI-Compatible Tool Use # 23 min
536
+ timeout_in_minutes: 35
537
mirror_hardwares: [amdexperimental]
538
fast_check: false
539
source_file_dependencies:
@@ -504,7 +546,8 @@ steps:
546
547
##### models test #####
548
507
-- label: Basic Models Test # 24min
549
+- label: Basic Models Test # 57min
550
+ timeout_in_minutes: 75
551
mirror_hardwares: [amdexperimental]
552
torch_nightly: true
553
source_file_dependencies:
@@ -517,7 +560,8 @@ steps:
560
- pytest -v -s models/test_vision.py
561
- pytest -v -s models/test_initialization.py
562
520
-- label: Language Models Test (Standard)
563
+- label: Language Models Test (Standard) # 35min
564
+ timeout_in_minutes: 45
565
mirror_hardwares: [amdexperimental]
566
torch_nightly: true
567
source_file_dependencies:
@@ -528,6 +572,7 @@ steps:
572
- pytest -v -s models/language -m core_model
573
574
- label: Language Models Test (Hybrid) # 35 min
575
+ timeout_in_minutes: 45
576
mirror_hardwares: [amdexperimental]
577
torch_nightly: true
578
source_file_dependencies:
@@ -540,7 +585,8 @@ steps:
585
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
586
- pytest -v -s models/language/generation -m hybrid_model
587
543
-- label: Language Models Test (Extended Generation) # 1hr20min
588
+- label: Language Models Test (Extended Generation) # 80min
589
+ timeout_in_minutes: 110
590
mirror_hardwares: [amdexperimental]
591
optional: true
592
source_file_dependencies:
@@ -552,6 +598,7 @@ steps:
598
- pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
599
600
- label: Language Models Test (Extended Pooling) # 36min
601
+ timeout_in_minutes: 50
602
mirror_hardwares: [amdexperimental]
603
optional: true
604
source_file_dependencies:
@@ -560,7 +607,8 @@ steps:
607
commands:
608
- pytest -v -s models/language/pooling -m 'not core_model'
609
563
-- label: Multi-Modal Processor Test
610
+- label: Multi-Modal Processor Test # 44min
611
+ timeout_in_minutes: 60
612
source_file_dependencies:
613
- vllm/
614
- tests/models/multimodal
@@ -568,7 +616,8 @@ steps:
616
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
617
- pytest -v -s models/multimodal/processing
618
571
-- label: Multi-Modal Models Test (Standard)
619
+- label: Multi-Modal Models Test (Standard) # 60min
620
+ timeout_in_minutes: 80
621
mirror_hardwares: [amdexperimental]
622
torch_nightly: true
623
source_file_dependencies:
@@ -610,7 +659,8 @@ steps:
659
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
660
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
661
613
-- label: Quantized Models Test
662
+- label: Quantized Models Test # 45 min
663
+ timeout_in_minutes: 60
664
mirror_hardwares: [amdexperimental]
665
source_file_dependencies:
666
- vllm/model_executor/layers/quantization
@@ -640,7 +690,8 @@ steps:
690
- python3 examples/offline_inference/audio_language.py --model-type whisper
691
- python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
692
643
-- label: Blackwell Test
693
+- label: Blackwell Test # 38 min
694
+ timeout_in_minutes: 60
695
working_dir: "/vllm-workspace/"
696
gpu: b200
697
# optional: true
@@ -682,6 +733,7 @@ steps:
733
##### multi gpus test #####
734
735
- label: Distributed Comm Ops Test # 7min
736
+ timeout_in_minutes: 20
737
mirror_hardwares: [amdexperimental]
738
working_dir: "/vllm-workspace/tests"
739
num_gpus: 2
@@ -693,6 +745,7 @@ steps:
745
- pytest -v -s distributed/test_shm_broadcast.py
746
747
- label: 2 Node Tests (4 GPUs in total) # 16min
748
+ timeout_in_minutes: 30
749
mirror_hardwares: [amdexperimental]
750
working_dir: "/vllm-workspace/tests"
751
num_gpus: 2
@@ -716,7 +769,8 @@ steps:
769
- NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
770
- python3 ../examples/offline_inference/data_parallel.py --dp-size=2 --tp-size=1 --node-size=2 --node-rank=1 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code
771
719
-- label: Distributed Tests (2 GPUs) # 40min
772
+- label: Distributed Tests (2 GPUs) # 110min
773
+ timeout_in_minutes: 150
774
mirror_hardwares: [amdexperimental]
775
working_dir: "/vllm-workspace/tests"
776
num_gpus: 2
@@ -757,6 +811,7 @@ steps:
811
- pytest -v -s models/multimodal/generation/test_maverick.py
812
813
- label: Plugin Tests (2 GPUs) # 40min
814
+ timeout_in_minutes: 60
815
mirror_hardwares: [amdexperimental]
816
working_dir: "/vllm-workspace/tests"
817
num_gpus: 2
@@ -783,6 +838,7 @@ steps:
838
- pytest -v -s plugins/lora_resolvers # unit tests for in-tree lora resolver plugins
839
840
- label: Pipeline Parallelism Test # 45min
841
+ timeout_in_minutes: 60
842
mirror_hardwares: [amdexperimental]
843
working_dir: "/vllm-workspace/tests"
844
num_gpus: 4
@@ -796,7 +852,8 @@ steps:
852
- pytest -v -s distributed/test_pp_cudagraph.py
853
- pytest -v -s distributed/test_pipeline_parallel.py
854
799
-- label: LoRA TP Test (Distributed)
855
+- label: LoRA TP Test (Distributed) # 17 min
856
+ timeout_in_minutes: 30
857
mirror_hardwares: [amdexperimental]
858
num_gpus: 4
859
source_file_dependencies:
@@ -814,6 +871,7 @@ steps:
871
872
873
- label: Weight Loading Multiple GPU Test # 33min
874
+ timeout_in_minutes: 45
875
mirror_hardwares: [amdexperimental]
876
working_dir: "/vllm-workspace/tests"
877
num_gpus: 2