@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 73029d4244

[CI][ROCm] Calibrate AMD test timeouts from nightly runtimes (#54695)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Andreas Karatzas committed Sep 1, 2026 at 23:53 UTC 73029d42441321b631779db3475031f5ec26dd6c
1 file changed +193 -195
.buildkite/test-amd.yaml
+193 -195
@@ -112,7 +112,7 @@ steps:
112 #########################################################################################################################################
113
114 - label: ":computer: (CPU) Basic Models Other" # TBD
115 - timeout_in_minutes: 180
115 + timeout_in_minutes: 35
116 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
117 agent_pool: mi250_1
118 no_gpu: true
@@ -128,7 +128,7 @@ steps:
128 - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/
129
130 - label: ":computer: (CPU) Multimodal Processor Shard %N" # TBD
131 - timeout_in_minutes: 180
131 + timeout_in_minutes: 60
132 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
133 agent_pool: mi250_1
134 no_gpu: true
@@ -143,7 +143,7 @@ steps:
143 - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
144
145 - label: ":computer: (CPU) V1 Others" # TBD
146 - timeout_in_minutes: 180
146 + timeout_in_minutes: 45
147 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
148 agent_pool: mi250_1
149 no_gpu: true
@@ -163,7 +163,7 @@ steps:
163 - pytest -v -s -m 'cpu_test' v1/metrics
164
165 - label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config" # TBD
166 - timeout_in_minutes: 180
166 + timeout_in_minutes: 80
167 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
168 agent_pool: mi250_1
169 no_gpu: true
@@ -202,7 +202,7 @@ steps:
202 - pytest -v -s config --ignore=config/test_mp_reducer.py
203
204 - label: ":computer: (CPU) Rust Frontend Cargo Style + Clippy" # TBD
205 - timeout_in_minutes: 180
205 + timeout_in_minutes: 35
206 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
207 agent_pool: mi250_1
208 no_gpu: true
@@ -217,7 +217,7 @@ steps:
217 - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy
218
219 - label: ":computer: (CPU) Rust Frontend Cargo" # TBD
220 - timeout_in_minutes: 180
220 + timeout_in_minutes: 35
221 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
222 agent_pool: mi250_1
223 no_gpu: true
@@ -232,7 +232,7 @@ steps:
232 - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh test
233
234 - label: ":computer: (CPU) Docker Build Metadata" # TBD
235 - timeout_in_minutes: 180
235 + timeout_in_minutes: 35
236 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
237 agent_pool: mi250_1
238 no_gpu: true
@@ -262,7 +262,7 @@ steps:
262 #---------------------------------------------------------- mi250 · compile ----------------------------------------------------------#
263
264 - label: ":amd: (MI250) PyTorch Fullgraph Smoke" # TBD
265 - timeout_in_minutes: 180
265 + timeout_in_minutes: 65
266 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
267 agent_pool: mi250_1
268 optional: true
@@ -281,7 +281,7 @@ steps:
281 #-------------------------------------------------------- mi250 · distributed --------------------------------------------------------#
282
283 - label: ":amd: (MI250) Pipeline + Context Parallelism" # TBD
284 - timeout_in_minutes: 180
284 + timeout_in_minutes: 80
285 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
286 agent_pool: mi250_4
287 num_gpus: 4
@@ -304,7 +304,7 @@ steps:
304 #---------------------------------------------------------- mi250 · kernels ----------------------------------------------------------#
305
306 - label: ":amd: (MI250) Helion Kernels" # TBD
307 - timeout_in_minutes: 180
307 + timeout_in_minutes: 40
308 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
309 agent_pool: mi250_1
310 optional: true
@@ -321,7 +321,7 @@ steps:
321 #----------------------------------------------------- mi250 · models / language -----------------------------------------------------#
322
323 - label: ":amd: (MI250) Language Models (PPL)" # TBD
324 - timeout_in_minutes: 180
324 + timeout_in_minutes: 45
325 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
326 agent_pool: mi250_1
327 working_dir: "/vllm-workspace/tests"
@@ -334,7 +334,7 @@ steps:
334 #---------------------------------------------------- mi250 · models / multimodal ----------------------------------------------------#
335
336 - label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD
337 - timeout_in_minutes: 180
337 + timeout_in_minutes: 45
338 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
339 agent_pool: mi250_1
340 optional: true
@@ -349,7 +349,7 @@ steps:
349 #------------------------------------------------------------ mi250 · v1 -------------------------------------------------------------#
350
351 - label: ":amd: (MI250) Batch Invariance" # TBD
352 - timeout_in_minutes: 180
352 + timeout_in_minutes: 35
353 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
354 agent_pool: mi250_1
355 optional: true
@@ -367,7 +367,7 @@ steps:
367 - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
368
369 - label: ":amd: (MI250) CUDAGraph" # TBD
370 - timeout_in_minutes: 180
370 + timeout_in_minutes: 45
371 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
372 agent_pool: mi250_1
373 optional: true
@@ -383,7 +383,7 @@ steps:
383 - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
384
385 - label: ":amd: (MI250) E2E Core" # TBD
386 - timeout_in_minutes: 180
386 + timeout_in_minutes: 50
387 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
388 agent_pool: mi250_1
389 optional: true
@@ -396,7 +396,7 @@ steps:
396 - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
397
398 - label: ":amd: (MI250) E2E Scheduling" # TBD
399 - timeout_in_minutes: 180
399 + timeout_in_minutes: 55
400 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
401 agent_pool: mi250_1
402 optional: true
@@ -409,7 +409,7 @@ steps:
409 - pytest -v -s v1/e2e/general/test_async_scheduling.py
410
411 - label: ":amd: (MI250) V1 Engine" # TBD
412 - timeout_in_minutes: 180
412 + timeout_in_minutes: 45
413 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
414 agent_pool: mi250_1
415 optional: true
@@ -430,7 +430,7 @@ steps:
430 - pytest -v -s config/test_mp_reducer.py
431
432 - label: ":amd: (MI250) Spec Decode Draft Model" # TBD
433 - timeout_in_minutes: 180
433 + timeout_in_minutes: 60
434 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
435 agent_pool: mi250_1
436 optional: true
@@ -447,7 +447,7 @@ steps:
447 - pytest -v -s v1/e2e/spec_decode/draft_model/
448
449 - label: ":amd: (MI250) Spec Decode Speculators + MTP" # TBD
450 - timeout_in_minutes: 180
450 + timeout_in_minutes: 65
451 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
452 agent_pool: mi250_1
453 optional: true
@@ -468,7 +468,7 @@ steps:
468 #------------------------------------------------------------- mi250 · misc ------------------------------------------------------------#
469
470 - label: ":amd: (MI300) Python-only Installation" # TBD
471 - timeout_in_minutes: 180
471 + timeout_in_minutes: 25
472 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
473 dind: false
474 agent_pool: mi300_1
@@ -490,7 +490,7 @@ steps:
490 #----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------#
491
492 - label: ":amd: (MI300) Basic Correctness" # TBD
493 - timeout_in_minutes: 180
493 + timeout_in_minutes: 60
494 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
495 dind: false
496 agent_pool: mi300_1
@@ -508,7 +508,7 @@ steps:
508 - pytest -v -s basic_correctness/test_cpu_offload.py
509
510 - label: ":amd: (MI300) Distributed Models" # TBD
511 - timeout_in_minutes: 180
511 + timeout_in_minutes: 80
512 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
513 dind: false
514 agent_pool: mi300_2
@@ -539,7 +539,7 @@ steps:
539 #-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------#
540
541 - label: ":amd: (MI300) Benchmarks CLI" # TBD
542 - timeout_in_minutes: 180
542 + timeout_in_minutes: 35
543 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
544 dind: false
545 agent_pool: mi300_1
@@ -554,7 +554,7 @@ steps:
554 #---------------------------------------------------------- mi300 · compile ----------------------------------------------------------#
555
556 - label: ":amd: (MI300) PyTorch Compilation" # TBD
557 - timeout_in_minutes: 180
557 + timeout_in_minutes: 45
558 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
559 dind: false
560 agent_pool: mi300_1
@@ -574,7 +574,7 @@ steps:
574 - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
575
576 - label: ":amd: (MI300) Fusion E2E Config Sweep" # TBD
577 - timeout_in_minutes: 180
577 + timeout_in_minutes: 40
578 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
579 dind: false
580 agent_pool: mi300_1
@@ -595,7 +595,7 @@ steps:
595 - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
596
597 - label: ":amd: (MI300) Fusion E2E Quick" # TBD
598 - timeout_in_minutes: 180
598 + timeout_in_minutes: 30
599 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
600 dind: false
601 agent_pool: mi300_1
@@ -617,7 +617,7 @@ steps:
617 - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
618
619 - label: ":amd: (MI300) PyTorch Compilation Passes" # TBD
620 - timeout_in_minutes: 180
620 + timeout_in_minutes: 40
621 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
622 dind: false
623 agent_pool: mi300_1
@@ -629,7 +629,7 @@ steps:
629 - pytest -s -v compile/passes --ignore compile/passes/distributed
630
631 - label: ":amd: (MI300) PyTorch Fullgraph" # TBD
632 - timeout_in_minutes: 180
632 + timeout_in_minutes: 35
633 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
634 dind: false
635 agent_pool: mi300_1
@@ -647,12 +647,11 @@ steps:
647 - pytest -v -s compile/fullgraph/test_full_graph.py
648
649 - label: ":amd: (MI300) PyTorch Nightly Dependency Override Check" # TBD
650 - timeout_in_minutes: 180
650 + timeout_in_minutes: 25
651 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
652 dind: false
653 agent_pool: mi300_1
654 optional: true
655 - soft_fail: true
655 working_dir: "/vllm-workspace/tests"
656 source_file_dependencies:
657 - requirements/test/nightly-torch.txt
@@ -661,7 +660,7 @@ steps:
660 - bash standalone_tests/pytorch_nightly_dependency.sh
661
662 - label: ":amd: (MI300) Distributed Compile" # TBD
664 - timeout_in_minutes: 180
663 + timeout_in_minutes: 95
664 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
665 dind: false
666 agent_pool: mi300_2
@@ -680,7 +679,7 @@ steps:
679 - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions
680
681 - label: ":amd: (MI300) Distributed Compile + RPC" # TBD
683 - timeout_in_minutes: 180
682 + timeout_in_minutes: 45
683 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
684 dind: false
685 agent_pool: mi300_2
@@ -706,7 +705,7 @@ steps:
705 #----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
706
707 - label: ":amd: (MI300) CUDA Platform" # TBD
709 - timeout_in_minutes: 180
708 + timeout_in_minutes: 25
709 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
710 dind: false
711 agent_pool: mi300_1
@@ -721,7 +720,7 @@ steps:
720 #-------------------------------------------------------- mi300 · detokenizer --------------------------------------------------------#
721
722 - label: ":amd: (MI300) Async Engine, Inputs, Utils, Worker" # TBD
724 - timeout_in_minutes: 180
723 + timeout_in_minutes: 35
724 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
725 dind: false
726 agent_pool: mi300_1
@@ -741,7 +740,7 @@ steps:
740 #-------------------------------------------------------- mi300 · distributed --------------------------------------------------------#
741
742 - label: ":amd: (MI300) Distributed Comm Ops" # TBD
744 - timeout_in_minutes: 180
743 + timeout_in_minutes: 30
744 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
745 dind: false
746 agent_pool: mi300_2
@@ -758,7 +757,7 @@ steps:
757 - pytest -v -s distributed/test_shm_storage.py
758
759 - label: ":amd: (MI300) EPLB Algorithm" # TBD
761 - timeout_in_minutes: 180
760 + timeout_in_minutes: 25
761 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
762 dind: false
763 agent_pool: mi300_1
@@ -774,7 +773,7 @@ steps:
773 - pytest -v -s distributed/test_eplb_utils.py
774
775 - label: ":amd: (MI300) EPLB Execution" # TBD
777 - timeout_in_minutes: 180
776 + timeout_in_minutes: 35
777 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
778 dind: false
779 agent_pool: mi300_4
@@ -790,7 +789,7 @@ steps:
789 - pytest -v -s distributed/test_eplb_spec_decode.py
790
791 - label: ":amd: (MI300) Distributed Features" # TBD
793 - timeout_in_minutes: 180
792 + timeout_in_minutes: 45
793 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
794 dind: false
795 agent_pool: mi300_2
@@ -817,7 +816,7 @@ steps:
816 - pytest -v -s tests/distributed/test_packed_tensor.py
817
818 - label: ":amd: (MI300) Distributed" # TBD
820 - timeout_in_minutes: 180
819 + timeout_in_minutes: 40
820 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
821 dind: false
822 agent_pool: mi300_4
@@ -835,7 +834,7 @@ steps:
834 - pytest -v -s models/test_vision.py -m 'distributed(num_gpus=4)'
835
836 - label: ":amd: (MI300) Distributed Torchrun + Examples" # TBD
838 - timeout_in_minutes: 180
837 + timeout_in_minutes: 35
838 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
839 dind: false
840 agent_pool: mi300_4
@@ -861,7 +860,7 @@ steps:
860 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_http_ipc.py
861
862 - label: ":amd: (MI300) Elastic EP Scaling" # TBD
864 - timeout_in_minutes: 180
863 + timeout_in_minutes: 50
864 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
865 dind: false
866 agent_pool: mi300_4
@@ -879,7 +878,7 @@ steps:
878 - pytest -v -s distributed/test_elastic_ep.py
879
880 - label: ":amd: (MI300) RayExecutorV2" # TBD
882 - timeout_in_minutes: 180
881 + timeout_in_minutes: 60
882 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
883 dind: false
884 agent_pool: mi300_4
@@ -902,7 +901,7 @@ steps:
901 - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
902
903 - label: ":amd: (MI300) Distributed DP + EP" # TBD
905 - timeout_in_minutes: 180
904 + timeout_in_minutes: 30
905 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
906 dind: false
907 agent_pool: mi300_8
@@ -921,7 +920,7 @@ steps:
920 - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
921
922 - label: ":amd: (MI300) Distributed Torchrun + Shutdown" # TBD
924 - timeout_in_minutes: 180
923 + timeout_in_minutes: 35
924 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
925 dind: false
926 agent_pool: mi300_2
@@ -945,7 +944,7 @@ steps:
944 - pytest -v -s v1/worker/test_worker_memory_snapshot.py
945
946 - label: ":amd: (MI300) Distributed Compile + Comm" # TBD
948 - timeout_in_minutes: 180
947 + timeout_in_minutes: 45
948 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
949 dind: false
950 agent_pool: mi300_4
@@ -969,7 +968,7 @@ steps:
968 #---------------------------------------------------------- mi300 · engine -----------------------------------------------------------#
969
970 - label: ":amd: (MI300) Engine Core" # TBD
972 - timeout_in_minutes: 180
971 + timeout_in_minutes: 35
972 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
973 dind: false
974 agent_pool: mi300_1
@@ -990,7 +989,7 @@ steps:
989 #-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------#
990
991 - label: ":amd: (MI300) Entrypoints Unit" # TBD
993 - timeout_in_minutes: 180
992 + timeout_in_minutes: 50
993 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
994 dind: false
995 agent_pool: mi300_1
@@ -1009,7 +1008,7 @@ steps:
1008 - pytest -v -s entrypoints/launchers
1009
1010 - label: ":amd: (MI300) Entrypoints Integration (LLM)" # TBD
1012 - timeout_in_minutes: 180
1011 + timeout_in_minutes: 60
1012 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1013 dind: false
1014 agent_pool: mi300_1
@@ -1026,7 +1025,7 @@ steps:
1025 - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
1026
1027 - label: ":amd: (MI300) Entrypoints Integration (API Server)" # TBD
1029 - timeout_in_minutes: 180
1028 + timeout_in_minutes: 65
1029 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1030 dind: false
1031 agent_pool: mi300_1
@@ -1044,7 +1043,7 @@ steps:
1043 - pytest -v -s entrypoints/scale_out
1044
1045 - label: ":amd: (MI300) Entrypoints Integration (API Server OpenAI - Part 1)" # TBD
1047 - timeout_in_minutes: 180
1046 + timeout_in_minutes: 40
1047 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1048 dind: false
1049 agent_pool: mi300_1
@@ -1059,7 +1058,7 @@ steps:
1058 - pytest -v -s entrypoints/openai/ --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
1059
1060 - label: ":amd: (MI300) Entrypoints Integration (API Server OpenAI - Part 2)" # TBD
1062 - timeout_in_minutes: 180
1061 + timeout_in_minutes: 55
1062 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1063 dind: false
1064 agent_pool: mi300_1
@@ -1075,7 +1074,7 @@ steps:
1074 - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
1075
1076 - label: ":amd: (MI300) Entrypoints Integration (API Server Generate)" # TBD
1078 - timeout_in_minutes: 180
1077 + timeout_in_minutes: 50
1078 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1079 dind: false
1080 agent_pool: mi300_1
@@ -1098,7 +1097,7 @@ steps:
1097 - pytest -v -s entrypoints/cohere
1098
1099 - label: ":amd: (MI300) Entrypoints Integration (Responses API)" # TBD
1101 - timeout_in_minutes: 180
1100 + timeout_in_minutes: 45
1101 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1102 dind: false
1103 agent_pool: mi300_1
@@ -1113,7 +1112,7 @@ steps:
1112 - pytest -v -s entrypoints/openai/responses
1113
1114 - label: ":amd: (MI300) Entrypoints Integration (Speech to Text)" # TBD
1116 - timeout_in_minutes: 180
1115 + timeout_in_minutes: 60
1116 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1117 dind: false
1118 agent_pool: mi300_1
@@ -1127,7 +1126,7 @@ steps:
1126 - pytest -v -s entrypoints/speech_to_text
1127
1128 - label: ":amd: (MI300) Entrypoints Integration (Multimodal)"
1130 - timeout_in_minutes: 180
1129 + timeout_in_minutes: 55
1130 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1131 dind: false
1132 agent_pool: mi300_1
@@ -1142,7 +1141,7 @@ steps:
1141 - pytest -v -s entrypoints/multimodal
1142
1143 - label: ":amd: (MI300) Entrypoints Integration (Pooling)" # TBD
1145 - timeout_in_minutes: 180
1144 + timeout_in_minutes: 65
1145 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1146 dind: false
1147 agent_pool: mi300_1
@@ -1157,7 +1156,7 @@ steps:
1156 - pytest -v -s entrypoints/pooling
1157
1158 - label: ":amd: (MI300) OpenAI API Correctness" # TBD
1160 - timeout_in_minutes: 180
1159 + timeout_in_minutes: 25
1160 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1161 dind: false
1162 agent_pool: mi300_1
@@ -1179,7 +1178,7 @@ steps:
1178 #----------------------------------------------------------- mi300 · evals -----------------------------------------------------------#
1179
1180 - label: ":amd: (MI300) DeepSeek V2-Lite Prefetch Offload Accuracy" # TBD
1182 - timeout_in_minutes: 180
1181 + timeout_in_minutes: 30
1182 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1183 dind: false
1184 agent_pool: mi300_1
@@ -1200,7 +1199,7 @@ steps:
1199 - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
1200
1201 - label: ":amd: (MI300) LM Eval Small Models" # TBD
1203 - timeout_in_minutes: 180
1202 + timeout_in_minutes: 40
1203 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1204 dind: false
1205 agent_pool: mi300_1
@@ -1218,7 +1217,7 @@ steps:
1217 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
1218
1219 - label: ":amd: (MI300) MRCR Eval Small Models" # TBD
1221 - timeout_in_minutes: 180
1220 + timeout_in_minutes: 35
1221 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1222 dind: false
1223 agent_pool: mi300_1
@@ -1236,7 +1235,7 @@ steps:
1235 - pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
1236
1237 - label: ":amd: (MI300) LM Eval Small Models Harness" # TBD
1239 - timeout_in_minutes: 180
1238 + timeout_in_minutes: 35
1239 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1240 dind: false
1241 agent_pool: mi300_1
@@ -1255,7 +1254,7 @@ steps:
1254 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
1255
1256 - label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" # TBD
1258 - timeout_in_minutes: 180
1257 + timeout_in_minutes: 30
1258 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1259 dind: false
1260 agent_pool: mi300_1
@@ -1271,7 +1270,7 @@ steps:
1270 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
1271
1272 - label: ":amd: (MI300) GPQA Eval (GPT-OSS)" # TBD
1274 - timeout_in_minutes: 180
1273 + timeout_in_minutes: 30
1274 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1275 dind: false
1276 agent_pool: mi300_2
@@ -1294,7 +1293,7 @@ steps:
1293 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
1294
1295 - label: ":amd: (MI300) LM Eval Small Models FP8 + Mixed" # TBD
1297 - timeout_in_minutes: 180
1296 + timeout_in_minutes: 55
1297 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1298 dind: false
1299 agent_pool: mi300_2
@@ -1313,7 +1312,7 @@ steps:
1312 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
1313
1314 - label: ":amd: (MI300) DeepSeek V2-Lite Sync EPLB Accuracy" # TBD
1316 - timeout_in_minutes: 180
1315 + timeout_in_minutes: 30
1316 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1317 dind: false
1318 agent_pool: mi300_4
@@ -1336,7 +1335,7 @@ steps:
1335 - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
1336
1337 - label: ":amd: (MI300) LM Eval Large Models Harness" # TBD
1339 - timeout_in_minutes: 180
1338 + timeout_in_minutes: 55
1339 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1340 dind: false
1341 agent_pool: mi300_4
@@ -1357,7 +1356,7 @@ steps:
1356 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1357
1358 - label: ":amd: (MI300) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" # TBD
1360 - timeout_in_minutes: 180
1359 + timeout_in_minutes: 30
1360 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1361 dind: false
1362 agent_pool: mi300_4
@@ -1379,7 +1378,7 @@ steps:
1378 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
1379
1380 - label: ":amd: (MI300) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy" # TBD
1382 - timeout_in_minutes: 180
1381 + timeout_in_minutes: 30
1382 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1383 dind: false
1384 agent_pool: mi300_4
@@ -1401,7 +1400,7 @@ steps:
1400 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
1401
1402 - label: ":amd: (MI300) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy" # TBD
1404 - timeout_in_minutes: 180
1403 + timeout_in_minutes: 35
1404 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1405 dind: false
1406 agent_pool: mi300_4
@@ -1424,7 +1423,7 @@ steps:
1423 - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
1424
1425 - label: ":amd: (MI300) LM Eval Large Models" # TBD
1427 - timeout_in_minutes: 180
1426 + timeout_in_minutes: 35
1427 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1428 dind: false
1429 agent_pool: mi300_8
@@ -1448,7 +1447,7 @@ steps:
1447 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
1448
1449 - label: ":amd: (MI300) LM Eval Large Models ROCm Harness" # TBD
1451 - timeout_in_minutes: 180
1450 + timeout_in_minutes: 55
1451 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1452 dind: false
1453 agent_pool: mi300_8
@@ -1470,7 +1469,7 @@ steps:
1469 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
1470
1471 - label: ":amd: (MI300) LM Eval Large Models FP8" # TBD
1473 - timeout_in_minutes: 180
1472 + timeout_in_minutes: 40
1473 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1474 dind: false
1475 agent_pool: mi300_4
@@ -1493,7 +1492,7 @@ steps:
1492 #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------#
1493
1494 - label: ":amd: (MI300) Examples" # TBD
1496 - timeout_in_minutes: 180
1495 + timeout_in_minutes: 50
1496 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1497 dind: false
1498 agent_pool: mi300_1
@@ -1530,7 +1529,7 @@ steps:
1529 #---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------#
1530
1531 - label: ":amd: (MI300) vLLM IR" # TBD
1533 - timeout_in_minutes: 180
1532 + timeout_in_minutes: 50
1533 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1534 dind: false
1535 agent_pool: mi300_1
@@ -1546,7 +1545,7 @@ steps:
1545 - pytest -v -s tests/kernels/ir
1546
1547 - label: ":amd: (MI300) MLA Kernels" # TBD
1549 - timeout_in_minutes: 180
1548 + timeout_in_minutes: 25
1549 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1550 dind: false
1551 agent_pool: mi300_1
@@ -1568,7 +1567,7 @@ steps:
1567 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
1568
1569 - label: ":amd: (MI300) Attention Kernels Shard %N" # TBD
1571 - timeout_in_minutes: 180
1570 + timeout_in_minutes: 70
1571 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1572 dind: false
1573 agent_pool: mi300_1
@@ -1587,7 +1586,7 @@ steps:
1586 - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1587
1588 - label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD
1590 - timeout_in_minutes: 180
1589 + timeout_in_minutes: 40
1590 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1591 dind: false
1592 agent_pool: mi300_1
@@ -1605,7 +1604,7 @@ steps:
1604 - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1605
1606 - label: ":amd: (MI300) KDA Kernels" # TBD
1608 - timeout_in_minutes: 180
1607 + timeout_in_minutes: 30
1608 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1609 dind: false
1610 agent_pool: mi300_1
@@ -1625,7 +1624,7 @@ steps:
1624 - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py
1625
1626 - label: ":amd: (MI300) Mamba Kernels" # TBD
1628 - timeout_in_minutes: 180
1627 + timeout_in_minutes: 50
1628 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1629 dind: false
1630 agent_pool: mi300_1
@@ -1640,7 +1639,7 @@ steps:
1639 - pytest -v -s kernels/mamba
1640
1641 - label: ":amd: (MI300) MoE Kernels Shard %N" # TBD
1643 - timeout_in_minutes: 180
1642 + timeout_in_minutes: 50
1643 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1644 dind: false
1645 agent_pool: mi300_1
@@ -1666,7 +1665,7 @@ steps:
1665 - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1666
1667 - label: ":amd: (MI300) Quantization Kernels" # TBD
1669 - timeout_in_minutes: 180
1668 + timeout_in_minutes: 55
1669 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1670 dind: false
1671 agent_pool: mi300_1
@@ -1690,7 +1689,7 @@ steps:
1689 - pytest -v -s kernels/quantization
1690
1691 - label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD
1693 - timeout_in_minutes: 180
1692 + timeout_in_minutes: 55
1693 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1694 dind: false
1695 agent_pool: mi300_2
@@ -1711,7 +1710,7 @@ steps:
1710 #----------------------------------------------------------- mi300 · lora ------------------------------------------------------------#
1711
1712 - label: ":amd: (MI300) LoRA Shard %N" # TBD
1714 - timeout_in_minutes: 180
1713 + timeout_in_minutes: 40
1714 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1715 dind: false
1716 agent_pool: mi300_1
@@ -1726,7 +1725,7 @@ steps:
1725 - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py
1726
1727 - label: ":amd: (MI300) LoRA TP (Distributed)" # TBD
1729 - timeout_in_minutes: 180
1728 + timeout_in_minutes: 65
1729 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1730 dind: false
1731 agent_pool: mi300_4
@@ -1749,7 +1748,7 @@ steps:
1748 #------------------------------------------------------ mi300 · model_executor -------------------------------------------------------#
1749
1750 - label: ":amd: (MI300) Model Executor" # TBD
1752 - timeout_in_minutes: 180
1751 + timeout_in_minutes: 60
1752 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1753 dind: false
1754 agent_pool: mi300_1
@@ -1772,7 +1771,7 @@ steps:
1771 #---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------#
1772
1773 - label: ":amd: (MI300) Model Runner V2 Core" # TBD
1775 - timeout_in_minutes: 180
1774 + timeout_in_minutes: 55
1775 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1776 dind: false
1777 agent_pool: mi300_1
@@ -1797,7 +1796,7 @@ steps:
1796 - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
1797
1798 - label: ":amd: (MI300) Model Runner V2 Examples" # TBD
1800 - timeout_in_minutes: 180
1799 + timeout_in_minutes: 40
1800 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1801 dind: false
1802 agent_pool: mi300_1
@@ -1832,7 +1831,7 @@ steps:
1831 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
1832
1833 - label: ":amd: (MI300) Model Runner V2 Distributed" # TBD
1835 - timeout_in_minutes: 180
1834 + timeout_in_minutes: 45
1835 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1836 dind: false
1837 agent_pool: mi300_2
@@ -1854,7 +1853,7 @@ steps:
1853 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
1854
1855 - label: ":amd: (MI300) Model Runner V2 Pipeline Parallelism" # TBD
1857 - timeout_in_minutes: 180
1856 + timeout_in_minutes: 65
1857 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1858 dind: false
1859 agent_pool: mi300_4
@@ -1874,7 +1873,7 @@ steps:
1873 - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
1874
1875 - label: ":amd: (MI300) Model Runner V2 Spec Decode" # TBD
1877 - timeout_in_minutes: 180
1876 + timeout_in_minutes: 90
1877 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1878 dind: false
1879 agent_pool: mi300_1
@@ -1901,7 +1900,7 @@ steps:
1900 #------------------------------------------------------ mi300 · models / basic -------------------------------------------------------#
1901
1902 - label: ":amd: (MI300) Basic Models (Extra Initialization) Shard %N" # TBD
1904 - timeout_in_minutes: 180
1903 + timeout_in_minutes: 60
1904 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1905 dind: false
1906 agent_pool: mi300_1
@@ -1918,7 +1917,7 @@ steps:
1917 - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1918
1919 - label: ":amd: (MI300) Basic Models (Initialization)" # TBD
1921 - timeout_in_minutes: 180
1920 + timeout_in_minutes: 30
1921 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1922 dind: false
1923 agent_pool: mi300_1
@@ -1931,7 +1930,7 @@ steps:
1930 - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
1931
1932 - label: ":amd: (MI300) Basic Models (Other)" # TBD
1934 - timeout_in_minutes: 180
1933 + timeout_in_minutes: 55
1934 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1935 dind: false
1936 agent_pool: mi300_1
@@ -1950,7 +1949,7 @@ steps:
1949 #----------------------------------------------------- mi300 · models / language -----------------------------------------------------#
1950
1951 - label: ":amd: (MI300) Language Models (MTEB)" # TBD
1953 - timeout_in_minutes: 180
1952 + timeout_in_minutes: 60
1953 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1954 dind: false
1955 agent_pool: mi300_1
@@ -1963,7 +1962,7 @@ steps:
1962 - pytest -v -s models/language/pooling_mteb_test
1963
1964 - label: ":amd: (MI300) Language Models (Extended Pooling)" # TBD
1966 - timeout_in_minutes: 180
1965 + timeout_in_minutes: 80
1966 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1967 dind: false
1968 agent_pool: mi300_1
@@ -1976,7 +1975,7 @@ steps:
1975 - pytest -v -s models/language/pooling -m 'not core_model'
1976
1977 - label: ":amd: (MI300) Language Models (Standard)" # TBD
1979 - timeout_in_minutes: 180
1978 + timeout_in_minutes: 45
1979 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1980 dind: false
1981 agent_pool: mi300_1
@@ -1990,7 +1989,7 @@ steps:
1989 - pytest -v -s models/language -m 'core_model and (not slow_test)'
1990
1991 - label: ":amd: (MI300) Language Models (Extra Standard) Shard %N" # TBD
1993 - timeout_in_minutes: 180
1992 + timeout_in_minutes: 40
1993 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1994 dind: false
1995 agent_pool: mi300_1
@@ -2013,7 +2012,7 @@ steps:
2012 - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2013
2014 - label: ":amd: (MI300) Language Models (Extended Generation)" # TBD
2016 - timeout_in_minutes: 180
2015 + timeout_in_minutes: 95
2016 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2017 dind: false
2018 agent_pool: mi300_1
@@ -2028,7 +2027,7 @@ steps:
2027 - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
2028
2029 - label: ":amd: (MI300) Language Models (Hybrid) Shard %N" # TBD
2031 - timeout_in_minutes: 180
2030 + timeout_in_minutes: 65
2031 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2032 dind: false
2033 agent_pool: mi300_1
@@ -2046,7 +2045,7 @@ steps:
2045 #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------#
2046
2047 - label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" # TBD
2049 - timeout_in_minutes: 180
2048 + timeout_in_minutes: 90
2049 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2050 dind: false
2051 agent_pool: mi300_1
@@ -2061,7 +2060,7 @@ steps:
2060 - pytest -v -s models/multimodal/test_mapping.py
2061
2062 - label: ":amd: (MI300) Multimodal Models (Extended Generation 2)" # TBD
2064 - timeout_in_minutes: 180
2063 + timeout_in_minutes: 65
2064 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2065 dind: false
2066 agent_pool: mi300_1
@@ -2075,7 +2074,7 @@ steps:
2074
2075
2076 - label: ":amd: (MI300) Multimodal Models (Extended Generation 3)" # TBD
2078 - timeout_in_minutes: 180
2077 + timeout_in_minutes: 55
2078 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2079 dind: false
2080 agent_pool: mi300_1
@@ -2088,7 +2087,7 @@ steps:
2087 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
2088
2089 - label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" # TBD
2091 - timeout_in_minutes: 180
2090 + timeout_in_minutes: 45
2091 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2092 dind: false
2093 agent_pool: mi300_1
@@ -2102,7 +2101,7 @@ steps:
2101 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
2102
2103 - label: ":amd: (MI300) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD
2105 - timeout_in_minutes: 180
2104 + timeout_in_minutes: 40
2105 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2106 dind: false
2107 agent_pool: mi300_1
@@ -2117,7 +2116,7 @@ steps:
2116 - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
2117
2118 - label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" # TBD
2120 - timeout_in_minutes: 180
2119 + timeout_in_minutes: 70
2120 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2121 dind: false
2122 agent_pool: mi300_1
@@ -2132,7 +2131,7 @@ steps:
2131 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
2132
2133 - label: ":amd: (MI300) Multimodal Processor" # 1h 42m
2135 - timeout_in_minutes: 180
2134 + timeout_in_minutes: 110
2135 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2136 dind: false
2137 agent_pool: mi300_1
@@ -2146,7 +2145,7 @@ steps:
2145 - pytest -v -s models/multimodal/processing/test_tensor_schema.py
2146
2147 - label: ":amd: (MI300) Multimodal Models (Extended Pooling)" # TBD
2149 - timeout_in_minutes: 180
2148 + timeout_in_minutes: 50
2149 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2150 dind: false
2151 agent_pool: mi300_1
@@ -2159,7 +2158,7 @@ steps:
2158 - pytest -v -s models/multimodal/pooling -m 'not core_model'
2159
2160 - label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" # TBD
2162 - timeout_in_minutes: 180
2161 + timeout_in_minutes: 50
2162 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2163 dind: false
2164 agent_pool: mi300_1
@@ -2176,7 +2175,7 @@ steps:
2175 #----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------#
2176
2177 - label: ":amd: (MI300) Quantized Models" # TBD
2179 - timeout_in_minutes: 180
2178 + timeout_in_minutes: 80
2179 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2180 dind: false
2181 agent_pool: mi300_1
@@ -2193,7 +2192,7 @@ steps:
2192 #-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------#
2193
2194 - label: ":amd: (MI300) Transformers Nightly Models (Initialization) Shard %N" # TBD
2196 - timeout_in_minutes: 180
2195 + timeout_in_minutes: 65
2196 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2197 dind: false
2198 agent_pool: mi300_1
@@ -2215,7 +2214,7 @@ steps:
2214 - pytest -v -s tests/models/test_initialization.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2215
2216 - label: ":amd: (MI300) Transformers Nightly Models (Processing) Shard %N" # TBD
2218 - timeout_in_minutes: 180
2217 + timeout_in_minutes: 145
2218 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2219 dind: false
2220 agent_pool: mi300_1
@@ -2237,7 +2236,7 @@ steps:
2236 - pytest -v -s tests/models/multimodal/processing/ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2237
2238 - label: ":amd: (MI300) Transformers Nightly Models (Single)" # TBD
2240 - timeout_in_minutes: 180
2239 + timeout_in_minutes: 50
2240 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2241 dind: false
2242 agent_pool: mi300_1
@@ -2265,7 +2264,7 @@ steps:
2264 #---------------------------------------------------------- mi300 · plugins ----------------------------------------------------------#
2265
2266 - label: ":amd: (MI300) Plugin Integration" # TBD
2268 - timeout_in_minutes: 180
2267 + timeout_in_minutes: 35
2268 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2269 dind: false
2270 agent_pool: mi300_2
@@ -2317,12 +2316,11 @@ steps:
2316 - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins
2317
2318 - label: ":amd: (MI300) GGUF Plugin" # TBD
2320 - timeout_in_minutes: 180
2319 + timeout_in_minutes: 45
2320 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2321 dind: false
2322 agent_pool: mi300_1
2323 optional: true
2325 - soft_fail: true
2324 working_dir: "/vllm-workspace/tests"
2325 source_file_dependencies:
2326 - vllm/model_executor/layers/quantization
@@ -2336,7 +2334,7 @@ steps:
2334 #------------------------------------------------------- mi300 · rust_frontend -------------------------------------------------------#
2335
2336 - label: ":amd: (MI300) Rust Frontend OpenAI Coverage" # TBD
2339 - timeout_in_minutes: 180
2337 + timeout_in_minutes: 35
2338 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2339 dind: false
2340 agent_pool: mi300_1
@@ -2369,7 +2367,7 @@ steps:
2367 - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
2368
2369 - label: ":amd: (MI300) Rust Frontend Serve + Admin Coverage" # TBD
2372 - timeout_in_minutes: 180
2370 + timeout_in_minutes: 30
2371 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2372 dind: false
2373 agent_pool: mi300_1
@@ -2397,7 +2395,7 @@ steps:
2395 - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
2396
2397 - label: ":amd: (MI300) Rust Frontend Core Correctness" # TBD
2400 - timeout_in_minutes: 180
2398 + timeout_in_minutes: 25
2399 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2400 dind: false
2401 agent_pool: mi300_1
@@ -2415,7 +2413,7 @@ steps:
2413 - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
2414
2415 - label: ":amd: (MI300) Rust Frontend Tool Use" # TBD
2418 - timeout_in_minutes: 180
2416 + timeout_in_minutes: 30
2417 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2418 dind: false
2419 agent_pool: mi300_1
@@ -2434,7 +2432,7 @@ steps:
2432 - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
2433
2434 - label: ":amd: (MI300) Rust Frontend Distributed" # TBD
2437 - timeout_in_minutes: 180
2435 + timeout_in_minutes: 30
2436 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2437 dind: false
2438 agent_pool: mi300_4
@@ -2465,7 +2463,7 @@ steps:
2463 #------------------------------------------------------- mi300 · quantization --------------------------------------------------------#
2464
2465 - label: ":amd: (MI300) Quantization" # TBD
2468 - timeout_in_minutes: 180
2466 + timeout_in_minutes: 115
2467 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2468 dind: false
2469 agent_pool: mi300_1
@@ -2492,7 +2490,7 @@ steps:
2490 #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------#
2491
2492 - label: ":amd: (MI300) Samplers" # TBD
2495 - timeout_in_minutes: 180
2493 + timeout_in_minutes: 40
2494 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2495 dind: false
2496 agent_pool: mi300_1
@@ -2513,7 +2511,7 @@ steps:
2511 #------------------------------------------------------------ mi300 · misc ------------------------------------------------------------#
2512
2513 - label: ":amd: (MI300) Regression" # TBD
2516 - timeout_in_minutes: 180
2514 + timeout_in_minutes: 25
2515 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2516 dind: false
2517 agent_pool: mi300_1
@@ -2528,7 +2526,7 @@ steps:
2526 #--------------------------------------------------------- mi300 · ray_compat ---------------------------------------------------------#
2527
2528 - label: ":amd: (MI300) Ray Dependency Compatibility Check" # TBD
2531 - timeout_in_minutes: 180
2529 + timeout_in_minutes: 25
2530 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2531 dind: false
2532 agent_pool: mi300_1
@@ -2543,7 +2541,7 @@ steps:
2541 #------------------------------------------------------------ mi300 · v1 -------------------------------------------------------------#
2542
2543 - label: ":amd: (MI300) Acceptance Length (Large Models)" # TBD
2546 - timeout_in_minutes: 180
2544 + timeout_in_minutes: 45
2545 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2546 dind: false
2547 agent_pool: mi300_1
@@ -2559,7 +2557,7 @@ steps:
2557 - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
2558
2559 - label: ":amd: (MI300) E2E Scheduling" # TBD
2562 - timeout_in_minutes: 180
2560 + timeout_in_minutes: 45
2561 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2562 dind: false
2563 agent_pool: mi300_1
@@ -2573,7 +2571,7 @@ steps:
2571 - pytest -v -s v1/e2e/general/test_async_scheduling.py
2572
2573 - label: ":amd: (MI300) V1 Engine" # TBD
2576 - timeout_in_minutes: 180
2574 + timeout_in_minutes: 40
2575 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2576 dind: false
2577 agent_pool: mi300_1
@@ -2588,7 +2586,7 @@ steps:
2586 - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
2587
2588 - label: ":amd: (MI300) Spec Decode Draft Model" # TBD
2591 - timeout_in_minutes: 180
2589 + timeout_in_minutes: 45
2590 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2591 dind: false
2592 agent_pool: mi300_1
@@ -2606,7 +2604,7 @@ steps:
2604 - pytest -v -s v1/e2e/spec_decode/draft_model/
2605
2606 - label: ":amd: (MI300) Spec Decode Eagle" # TBD
2609 - timeout_in_minutes: 180
2607 + timeout_in_minutes: 60
2608 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2609 dind: false
2610 agent_pool: mi300_1
@@ -2624,7 +2622,7 @@ steps:
2622 - pytest -v -s v1/e2e/spec_decode/eagle/
2623
2624 - label: ":amd: (MI300) Spec Decode N-Gram + Suffix" # TBD
2627 - timeout_in_minutes: 180
2625 + timeout_in_minutes: 35
2626 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2627 dind: false
2628 agent_pool: mi300_1
@@ -2644,7 +2642,7 @@ steps:
2642 - python3 spec_decode/test_custom_proposer.py
2643
2644 - label: ":amd: (MI300) Spec Decode Speculators + MTP" # TBD
2647 - timeout_in_minutes: 180
2645 + timeout_in_minutes: 50
2646 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2647 dind: false
2648 agent_pool: mi300_1
@@ -2664,7 +2662,7 @@ steps:
2662 - pytest -v -s v1/e2e/spec_decode/mtp/
2663
2664 - label: ":amd: (MI300) Speculators Correctness" # TBD
2667 - timeout_in_minutes: 180
2665 + timeout_in_minutes: 35
2666 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2667 dind: false
2668 agent_pool: mi300_1
@@ -2696,7 +2694,7 @@ steps:
2694 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
2695
2696 - label: ":amd: (MI300) V1 Spec Decode" # TBD
2699 - timeout_in_minutes: 180
2697 + timeout_in_minutes: 50
2698 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2699 dind: false
2700 agent_pool: mi300_1
@@ -2709,7 +2707,7 @@ steps:
2707 - pytest -v -s -m 'not slow_test' v1/spec_decode
2708
2709 - label: ":amd: (MI300) Extract Hidden States Integration" # TBD
2712 - timeout_in_minutes: 180
2710 + timeout_in_minutes: 30
2711 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2712 dind: false
2713 agent_pool: mi300_2
@@ -2741,7 +2739,7 @@ steps:
2739 - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration
2740
2741 - label: ":amd: (MI300) V1 Attention Shard %N" # TBD
2744 - timeout_in_minutes: 180
2742 + timeout_in_minutes: 35
2743 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2744 dind: false
2745 agent_pool: mi300_1
@@ -2760,7 +2758,7 @@ steps:
2758 - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
2759
2760 - label: ":amd: (MI300) V1 Core + KV + Metrics" # TBD
2763 - timeout_in_minutes: 180
2761 + timeout_in_minutes: 60
2762 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2763 dind: false
2764 agent_pool: mi300_1
@@ -2794,7 +2792,7 @@ steps:
2792 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
2793
2794 - label: ":amd: (MI300) V1 Sample + Logits" # TBD
2797 - timeout_in_minutes: 180
2795 + timeout_in_minutes: 70
2796 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2797 dind: false
2798 agent_pool: mi300_1
@@ -2816,7 +2814,7 @@ steps:
2814 - pytest -v -s v1/test_outputs.py
2815
2816 - label: ":amd: (MI300) Distributed DP Basic" # TBD
2819 - timeout_in_minutes: 180
2817 + timeout_in_minutes: 45
2818 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2819 dind: false
2820 agent_pool: mi300_2
@@ -2841,7 +2839,7 @@ steps:
2839 - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py
2840
2841 - label: ":amd: (MI300) Metrics, Tracing" # TBD
2844 - timeout_in_minutes: 180
2842 + timeout_in_minutes: 25
2843 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2844 dind: false
2845 agent_pool: mi300_2
@@ -2862,7 +2860,7 @@ steps:
2860 - pytest -v -s tracing
2861
2862 - label: ":amd: (MI300) V1 E2E" # TBD
2865 - timeout_in_minutes: 180
2863 + timeout_in_minutes: 25
2864 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2865 dind: false
2866 agent_pool: mi300_2
@@ -2879,7 +2877,7 @@ steps:
2877 v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism
2878
2879 - label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" # TBD
2882 - timeout_in_minutes: 180
2880 + timeout_in_minutes: 45
2881 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2882 dind: false
2883 agent_pool: mi300_2
@@ -2896,7 +2894,7 @@ steps:
2894 - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
2895
2896 - label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD
2899 - timeout_in_minutes: 180
2897 + timeout_in_minutes: 45
2898 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2899 dind: false
2900 agent_pool: mi300_4
@@ -2912,7 +2910,7 @@ steps:
2910 - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
2911
2912 - label: ":amd: (MI300) Distributed DP Extended" # TBD
2915 - timeout_in_minutes: 180
2913 + timeout_in_minutes: 60
2914 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2915 dind: false
2916 agent_pool: mi300_4
@@ -2936,7 +2934,7 @@ steps:
2934 - pytest -v -s distributed/test_utils.py
2935
2936 - label: ":amd: (MI300) Distributed NixlConnector PD accuracy" # TBD
2939 - timeout_in_minutes: 180
2937 + timeout_in_minutes: 45
2938 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2939 dind: false
2940 agent_pool: mi300_4
@@ -2952,7 +2950,7 @@ steps:
2950 - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
2951
2952 - label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" # TBD
2955 - timeout_in_minutes: 180
2953 + timeout_in_minutes: 30
2954 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2955 dind: false
2956 agent_pool: mi300_4
@@ -2968,7 +2966,7 @@ steps:
2966 - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
2967
2968 - label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD
2971 - timeout_in_minutes: 180
2969 + timeout_in_minutes: 60
2970 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2971 dind: false
2972 agent_pool: mi300_4
@@ -2984,7 +2982,7 @@ steps:
2982 - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
2983
2984 - label: ":amd: (MI300) Hybrid SSM NixlConnector PD prefix cache" # TBD
2987 - timeout_in_minutes: 180
2985 + timeout_in_minutes: 25
2986 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2987 dind: false
2988 agent_pool: mi300_2
@@ -3002,7 +3000,7 @@ steps:
3000 - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
3001
3002 - label: ":amd: (MI300) MultiConnector (Nixl+Offloading) PD accuracy" # TBD
3005 - timeout_in_minutes: 180
3003 + timeout_in_minutes: 30
3004 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3005 dind: false
3006 agent_pool: mi300_2
@@ -3021,7 +3019,7 @@ steps:
3019 - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
3020
3021 - label: ":amd: (MI300) MultiConnector (Nixl+Offloading) PD edge cases" # TBD
3024 - timeout_in_minutes: 180
3022 + timeout_in_minutes: 25
3023 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3024 dind: false
3025 agent_pool: mi300_2
@@ -3040,7 +3038,7 @@ steps:
3038 - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
3039
3040 - label: ":amd: (MI300) V1 E2E Hybrid Chunked Prefill" # TBD
3043 - timeout_in_minutes: 180
3041 + timeout_in_minutes: 30
3042 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3043 dind: false
3044 agent_pool: mi300_4
@@ -3056,7 +3054,7 @@ steps:
3054 #------------------------------------------------------ mi300 · weight_loading -------------------------------------------------------#
3055
3056 - label: ":amd: (MI300) Weight Loading Multi-GPU" # TBD
3059 - timeout_in_minutes: 180
3057 + timeout_in_minutes: 40
3058 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3059 dind: false
3060 agent_pool: mi300_2
@@ -3069,7 +3067,7 @@ steps:
3067 - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
3068
3069 - label: ":amd: (MI300) Weight Loading Multi-GPU (Large Models)" # TBD
3072 - timeout_in_minutes: 180
3070 + timeout_in_minutes: 35
3071 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3072 dind: false
3073 agent_pool: mi300_2
@@ -3091,7 +3089,7 @@ steps:
3089 #-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------#
3090
3091 - label: ":amd: (MI355) Attention Benchmark Smoke" # TBD
3094 - timeout_in_minutes: 180
3092 + timeout_in_minutes: 25
3093 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3094 dind: false
3095 agent_pool: mi355_2
@@ -3108,7 +3106,7 @@ steps:
3106 #-------------------------------------------------------- mi355 · distributed --------------------------------------------------------#
3107
3108 - label: ":amd: (MI355) Distributed Features" # TBD
3111 - timeout_in_minutes: 180
3109 + timeout_in_minutes: 45
3110 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3111 dind: false
3112 agent_pool: mi355_2
@@ -3153,7 +3151,7 @@ steps:
3151 #-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------#
3152
3153 - label: ":amd: (MI355) Entrypoints Integration (API Server)" # TBD
3156 - timeout_in_minutes: 180
3154 + timeout_in_minutes: 55
3155 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3156 dind: false
3157 agent_pool: mi355_1
@@ -3171,7 +3169,7 @@ steps:
3169 - pytest -v -s entrypoints/scale_out
3170
3171 - label: ":amd: (MI355) Entrypoints Integration (API Server OpenAI - Part 1)" # TBD
3174 - timeout_in_minutes: 180
3172 + timeout_in_minutes: 35
3173 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3174 dind: false
3175 agent_pool: mi355_1
@@ -3186,7 +3184,7 @@ steps:
3184 - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
3185
3186 - label: ":amd: (MI355) Entrypoints Integration (API Server OpenAI - Part 2)" # TBD
3189 - timeout_in_minutes: 180
3187 + timeout_in_minutes: 50
3188 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3189 dind: false
3190 agent_pool: mi355_1
@@ -3202,7 +3200,7 @@ steps:
3200 - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
3201
3202 - label: ":amd: (MI355) Entrypoints Integration (API Server Generate)" # TBD
3205 - timeout_in_minutes: 180
3203 + timeout_in_minutes: 40
3204 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3205 dind: false
3206 agent_pool: mi355_1
@@ -3223,7 +3221,7 @@ steps:
3221 - pytest -v -s entrypoints/anthropic
3222
3223 - label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" # TBD
3226 - timeout_in_minutes: 180
3224 + timeout_in_minutes: 50
3225 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
3226 dind: false
3227 agent_pool: mi355_1
@@ -3237,7 +3235,7 @@ steps:
3235 - pytest -v -s entrypoints/speech_to_text
3236
3237 - label: ":amd: (MI355) Entrypoints Integration (Multimodal)"
3240 - timeout_in_minutes: 180
3238 + timeout_in_minutes: 40
3239 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
3240 dind: false
3241 agent_pool: mi355_1
@@ -3251,7 +3249,7 @@ steps:
3249 - pytest -v -s entrypoints/multimodal
3250
3251 - label: ":amd: (MI355) Entrypoints Integration (Pooling)" # TBD
3254 - timeout_in_minutes: 180
3252 + timeout_in_minutes: 55
3253 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3254 dind: false
3255 agent_pool: mi355_1
@@ -3267,7 +3265,7 @@ steps:
3265 #----------------------------------------------------------- mi355 · evals -----------------------------------------------------------#
3266
3267 - label: ":amd: (MI355) GPQA Eval (GPT-OSS)" # TBD
3270 - timeout_in_minutes: 180
3268 + timeout_in_minutes: 40
3269 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3270 dind: false
3271 agent_pool: mi355_2
@@ -3290,7 +3288,7 @@ steps:
3288 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
3289
3290 - label: ":amd: (MI355) LM Eval Qwen3-5 Models" # TBD
3293 - timeout_in_minutes: 180
3291 + timeout_in_minutes: 40
3292 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3293 dind: false
3294 agent_pool: mi355_2
@@ -3313,7 +3311,7 @@ steps:
3311 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
3312
3313 - label: ":amd: (MI355) LM Eval Small Models" # TBD
3316 - timeout_in_minutes: 180
3314 + timeout_in_minutes: 45
3315 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3316 dind: false
3317 agent_pool: mi355_2
@@ -3333,7 +3331,7 @@ steps:
3331 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
3332
3333 - label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" # TBD
3336 - timeout_in_minutes: 180
3334 + timeout_in_minutes: 25
3335 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3336 dind: false
3337 agent_pool: mi355_2
@@ -3354,7 +3352,7 @@ steps:
3352 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
3353
3354 - label: ":amd: (MI355) LM Eval Large Models FP8" # TBD
3357 - timeout_in_minutes: 180
3355 + timeout_in_minutes: 45
3356 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3357 dind: false
3358 agent_pool: mi355_4
@@ -3375,7 +3373,7 @@ steps:
3373 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
3374
3375 - label: ":amd: (MI355) LM Eval Large Models" # TBD
3378 - timeout_in_minutes: 180
3376 + timeout_in_minutes: 35
3377 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3378 dind: false
3379 agent_pool: mi355_8
@@ -3400,7 +3398,7 @@ steps:
3398 #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
3399
3400 - label: ":amd: (MI355) Examples" # TBD
3403 - timeout_in_minutes: 180
3401 + timeout_in_minutes: 40
3402 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3403 dind: false
3404 agent_pool: mi355_1
@@ -3436,7 +3434,7 @@ steps:
3434 #---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------#
3435
3436 - label: ":amd: (MI355) Core Operation Kernels Shard %N" # TBD
3439 - timeout_in_minutes: 180
3437 + timeout_in_minutes: 40
3438 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3439 dind: false
3440 agent_pool: mi355_1
@@ -3454,7 +3452,7 @@ steps:
3452 - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3453
3454 - label: ":amd: (MI355) Kernels" # TBD
3457 - timeout_in_minutes: 180
3455 + timeout_in_minutes: 25
3456 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3457 dind: false
3458 agent_pool: mi355_1
@@ -3480,7 +3478,7 @@ steps:
3478 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
3479
3480 - label: ":amd: (MI355) MLA Kernels" # TBD
3483 - timeout_in_minutes: 180
3481 + timeout_in_minutes: 25
3482 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3483 dind: false
3484 agent_pool: mi355_1
@@ -3502,7 +3500,7 @@ steps:
3500 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
3501
3502 - label: ":amd: (MI355) Attention Kernels Shard %N" # TBD
3505 - timeout_in_minutes: 180
3503 + timeout_in_minutes: 40
3504 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3505 dind: false
3506 agent_pool: mi355_1
@@ -3520,7 +3518,7 @@ steps:
3518 - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3519
3520 - label: ":amd: (MI355) MoE Kernels Shard %N" # TBD
3523 - timeout_in_minutes: 180
3521 + timeout_in_minutes: 45
3522 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3523 dind: false
3524 agent_pool: mi355_1
@@ -3554,7 +3552,7 @@ steps:
3552 - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3553
3554 - label: ":amd: (MI355) FusedMoE Layer Kernels" # TBD
3557 - timeout_in_minutes: 180
3555 + timeout_in_minutes: 35
3556 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3557 dind: false
3558 agent_pool: mi355_2
@@ -3580,7 +3578,7 @@ steps:
3578 - pytest -v -s kernels/moe/test_moe_layer.py
3579
3580 - label: ":amd: (MI355) Quantization Kernels" # TBD
3583 - timeout_in_minutes: 180
3581 + timeout_in_minutes: 50
3582 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3583 dind: false
3584 agent_pool: mi355_1
@@ -3603,7 +3601,7 @@ steps:
3601 - pytest -v -s kernels/quantization
3602
3603 - label: ":amd: (MI355) FP8 MoE Kernels" # TBD
3606 - timeout_in_minutes: 180
3604 + timeout_in_minutes: 40
3605 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3606 dind: false
3607 agent_pool: mi355_1
@@ -3624,7 +3622,7 @@ steps:
3622 - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
3623
3624 - label: ":amd: (MI355) DeepEP FP8 MoE Kernels" # TBD
3627 - timeout_in_minutes: 180
3625 + timeout_in_minutes: 45
3626 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3627 dind: false
3628 agent_pool: mi355_2
@@ -3645,7 +3643,7 @@ steps:
3643 #----------------------------------------------------- mi355 · models / language -----------------------------------------------------#
3644
3645 - label: ":amd: (MI355) Language Models (Extended Generation)" # TBD
3648 - timeout_in_minutes: 180
3646 + timeout_in_minutes: 65
3647 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3648 dind: false
3649 agent_pool: mi355_1
@@ -3659,7 +3657,7 @@ steps:
3657 - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
3658
3659 - label: ":amd: (MI355) Language Models (Extended Pooling)" # TBD
3662 - timeout_in_minutes: 180
3660 + timeout_in_minutes: 65
3661 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3662 dind: false
3663 agent_pool: mi355_1
@@ -3672,7 +3670,7 @@ steps:
3670 - pytest -v -s models/language/pooling -m 'not core_model'
3671
3672 - label: ":amd: (MI355) Language Models (PPL)" # TBD
3675 - timeout_in_minutes: 180
3673 + timeout_in_minutes: 35
3674 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3675 dind: false
3676 agent_pool: mi355_1
@@ -3701,7 +3699,7 @@ steps:
3699 - pytest -v -s models/language/generation_ppl_test
3700
3701 - label: ":amd: (MI355) Language Models (Standard)" # TBD
3704 - timeout_in_minutes: 180
3702 + timeout_in_minutes: 40
3703 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3704 dind: false
3705 agent_pool: mi355_1
@@ -3716,7 +3714,7 @@ steps:
3714 #---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------#
3715
3716 - label: ":amd: (MI355) Multimodal Models (Extended Generation 1)" # TBD
3719 - timeout_in_minutes: 180
3717 + timeout_in_minutes: 60
3718 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3719 dind: false
3720 agent_pool: mi355_1
@@ -3731,7 +3729,7 @@ steps:
3729 - pytest -v -s models/multimodal/test_mapping.py
3730
3731 - label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" # TBD
3734 - timeout_in_minutes: 180
3732 + timeout_in_minutes: 40
3733 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3734 dind: false
3735 agent_pool: mi355_1
@@ -3744,7 +3742,7 @@ steps:
3742 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
3743
3744 - label: ":amd: (MI355) Multimodal Models (Extended Pooling)" # TBD
3747 - timeout_in_minutes: 180
3745 + timeout_in_minutes: 40
3746 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3747 dind: false
3748 agent_pool: mi355_1
@@ -3757,7 +3755,7 @@ steps:
3755 - pytest -v -s models/multimodal/pooling -m 'not core_model'
3756
3757 - label: ":amd: (MI355) Multimodal Models (Standard) 1: qwen2" # TBD
3760 - timeout_in_minutes: 180
3758 + timeout_in_minutes: 35
3759 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3760 dind: false
3761 agent_pool: mi355_1
@@ -3771,7 +3769,7 @@ steps:
3769 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
3770
3771 - label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" # TBD
3774 - timeout_in_minutes: 180
3772 + timeout_in_minutes: 55
3773 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3774 dind: false
3775 agent_pool: mi355_1
@@ -3789,7 +3787,7 @@ steps:
3787 #----------------------------------------------------- mi355 · models / quantized -----------------------------------------------------#
3788
3789 - label: ":amd: (MI355) Quantized Models" # TBD
3792 - timeout_in_minutes: 180
3790 + timeout_in_minutes: 50
3791 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3792 dind: false
3793 agent_pool: mi355_1
@@ -3806,7 +3804,7 @@ steps:
3804 #------------------------------------------------------- mi355 · quantization --------------------------------------------------------#
3805
3806 - label: ":amd: (MI355) Quantization" # TBD
3809 - timeout_in_minutes: 180
3807 + timeout_in_minutes: 80
3808 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3809 dind: false
3810 agent_pool: mi355_1
@@ -3852,7 +3850,7 @@ steps:
3850 #------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------#
3851
3852 - label: ":amd: (MI355) V1 Attention Shard %N" # TBD
3855 - timeout_in_minutes: 180
3853 + timeout_in_minutes: 35
3854 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3855 dind: false
3856 agent_pool: mi355_1
@@ -3870,7 +3868,7 @@ steps:
3868 - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3869
3870 - label: ":amd: (MI355) V1 Core + KV + Metrics" # TBD
3873 - timeout_in_minutes: 180
3871 + timeout_in_minutes: 55
3872 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3873 dind: false
3874 agent_pool: mi355_1
@@ -3899,7 +3897,7 @@ steps:
3897 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
3898
3899 - label: ":amd: (MI355) V1 Sample + Logits" # TBD
3902 - timeout_in_minutes: 180
3900 + timeout_in_minutes: 55
3901 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3902 dind: false
3903 agent_pool: mi355_1
@@ -3920,7 +3918,7 @@ steps:
3918 - pytest -v -s v1/test_outputs.py
3919
3920 - label: ":amd: (MI355) V1 Spec Decode" # TBD
3923 - timeout_in_minutes: 180
3921 + timeout_in_minutes: 45
3922 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3923 dind: false
3924 agent_pool: mi355_1
@@ -3934,7 +3932,7 @@ steps:
3932 #------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------#
3933
3934 - label: ":amd: (MI355) Weight Loading Multi-GPU" # TBD
3937 - timeout_in_minutes: 180
3935 + timeout_in_minutes: 30
3936 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3937 dind: false
3938 agent_pool: mi355_2
@@ -3947,7 +3945,7 @@ steps:
3945 - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
3946
3947 - label: ":amd: (MI355) Weight Loading Multi-GPU (Large Models)" # TBD
3950 - timeout_in_minutes: 180
3948 + timeout_in_minutes: 30
3949 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3950 dind: false
3951 agent_pool: mi355_2
@@ -3963,7 +3961,7 @@ steps:
3961 #----------------------------------------------------------- mi355 · misc ------------------------------------------------------------#
3962
3963 - label: ":amd: (MI355) Regression" # TBD
3966 - timeout_in_minutes: 180
3964 + timeout_in_minutes: 25
3965 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3966 dind: false
3967 agent_pool: mi355_1