[CI][ROCm] Increase timeouts for AMD MI300 jobs near their limits (#59227)
Signed-off-by: Djordje Ramic <djoramic@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
djramic committed
Sep 29, 2026 at 20:25 UTC
df4dbe46e7521e073a63c68247245323d86de9d5
2 files changed
+7
-7
.buildkite/test-amd.yaml
+6
-6
@@ -773,7 +773,7 @@ steps:
773
- bash standalone_tests/pytorch_nightly_dependency.sh
774
775
- label: ":amd: (MI300) Distributed Compile (Inductor Partition)" # TBD
776
- timeout_in_minutes: 65
776
+ timeout_in_minutes: 75
777
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
778
dind: false
779
agent_pool: mi300_2
@@ -1124,7 +1124,7 @@ steps:
1124
- pytest -v -s v1/worker/test_worker_memory_snapshot.py
1125
1126
- label: ":amd: (MI300) Distributed Compile + Comm" # TBD
1127
- timeout_in_minutes: 45
1127
+ timeout_in_minutes: 65
1128
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1129
dind: false
1130
agent_pool: mi300_4
@@ -2035,7 +2035,7 @@ steps:
2035
#----------------------------------------------------- mi300 · models / language -----------------------------------------------------#
2036
2037
- label: ":amd: (MI300) Language Models (MTEB)" # TBD
2038
- timeout_in_minutes: 60
2038
+ timeout_in_minutes: 70
2039
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2040
dind: false
2041
agent_pool: mi300_1
@@ -2062,7 +2062,7 @@ steps:
2062
- pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2063
2064
- label: ":amd: (MI300) Language Models (Extended Generation)" # TBD
2065
- timeout_in_minutes: 95
2065
+ timeout_in_minutes: 105
2066
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2067
dind: false
2068
agent_pool: mi300_1
@@ -2079,7 +2079,7 @@ steps:
2079
#---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------#
2080
2081
- label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" # TBD
2082
- timeout_in_minutes: 90
2082
+ timeout_in_minutes: 100
2083
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2084
dind: false
2085
agent_pool: mi300_1
@@ -2168,7 +2168,7 @@ steps:
2168
#----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------#
2169
2170
- label: ":amd: (MI300) Quantized Models" # TBD
2171
- timeout_in_minutes: 80
2171
+ timeout_in_minutes: 90
2172
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2173
dind: false
2174
agent_pool: mi300_1
.buildkite/test_areas/models_multimodal.yaml
+1
-1
@@ -176,7 +176,7 @@ steps:
176
label: ":amd: (MI300) Multimodal Models (Extended Generation 1)"
177
dind: false
178
device: mi300_1
179
- timeout_in_minutes: 90
179
+ timeout_in_minutes: 100
180
depends_on:
181
- image-build-amd
182