[CI][ROCm] Raise AMD job and server readiness timeouts (#55308)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Sep 4, 2026 at 21:50 UTC
a7f9be9612861d6425738dd0ba1c7fd5805e6c95
4 files changed
+17
-14
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh
+1
-1
@@ -10,7 +10,7 @@ mkdir -p "${OUT_DIR}"
10
11
wait_for_server() {
12
local port=$1
13
- timeout 600 bash -c '
13
+ timeout "${VLLM_ENGINE_READY_TIMEOUT_S:-600}" bash -c '
14
until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
sleep 1
16
done'
.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh
+1
-1
@@ -10,7 +10,7 @@ mkdir -p "${OUT_DIR}"
10
11
wait_for_server() {
12
local port=$1
13
- timeout 600 bash -c '
13
+ timeout "${VLLM_ENGINE_READY_TIMEOUT_S:-600}" bash -c '
14
until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
sleep 1
16
done'
.buildkite/test-amd.yaml
+12
-12
@@ -617,7 +617,7 @@ steps:
617
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
618
619
- label: ":amd: (MI300) PyTorch Compilation Passes" # TBD
620
- timeout_in_minutes: 40
620
+ timeout_in_minutes: 80
621
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
622
dind: false
623
agent_pool: mi300_1
@@ -901,7 +901,7 @@ steps:
901
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
902
903
- label: ":amd: (MI300) Distributed DP + EP" # TBD
904
- timeout_in_minutes: 30
904
+ timeout_in_minutes: 50
905
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
906
dind: false
907
agent_pool: mi300_8
@@ -1008,7 +1008,7 @@ steps:
1008
- pytest -v -s entrypoints/launchers
1009
1010
- label: ":amd: (MI300) Entrypoints Integration (LLM)" # TBD
1011
- timeout_in_minutes: 60
1011
+ timeout_in_minutes: 90
1012
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1013
dind: false
1014
agent_pool: mi300_1
@@ -1312,7 +1312,7 @@ steps:
1312
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
1313
1314
- label: ":amd: (MI300) DeepSeek V2-Lite Sync EPLB Accuracy" # TBD
1315
- timeout_in_minutes: 30
1315
+ timeout_in_minutes: 40
1316
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1317
dind: false
1318
agent_pool: mi300_4
@@ -1332,10 +1332,10 @@ steps:
1332
- vllm/_aiter_ops.py
1333
- vllm/platforms/rocm.py
1334
commands:
1335
- - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
1335
+ - VLLM_ENGINE_READY_TIMEOUT_S=1800 bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
1336
1337
- label: ":amd: (MI300) LM Eval Large Models Harness" # TBD
1338
- timeout_in_minutes: 55
1338
+ timeout_in_minutes: 85
1339
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1340
dind: false
1341
agent_pool: mi300_4
@@ -1400,7 +1400,7 @@ steps:
1400
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
1401
1402
- label: ":amd: (MI300) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy" # TBD
1403
- timeout_in_minutes: 35
1403
+ timeout_in_minutes: 45
1404
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1405
dind: false
1406
agent_pool: mi300_4
@@ -1420,10 +1420,10 @@ steps:
1420
- vllm/_aiter_ops.py
1421
- vllm/platforms/rocm.py
1422
commands:
1423
- - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
1423
+ - VLLM_ENGINE_READY_TIMEOUT_S=1800 bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
1424
1425
- label: ":amd: (MI300) LM Eval Large Models" # TBD
1426
- timeout_in_minutes: 35
1426
+ timeout_in_minutes: 65
1427
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1428
dind: false
1429
agent_pool: mi300_8
@@ -2450,7 +2450,7 @@ steps:
2450
- pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
2451
2452
- label: ":amd: (MI300) Rust Frontend Distributed" # TBD
2453
- timeout_in_minutes: 30
2453
+ timeout_in_minutes: 45
2454
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2455
dind: false
2456
agent_pool: mi300_4
@@ -2474,7 +2474,7 @@ steps:
2474
- export VLLM_USE_RUST_FRONTEND=1
2475
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2476
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py
2477
- - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
2477
+ - VLLM_ENGINE_READY_TIMEOUT_S=1800 TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
2478
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
2479
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
2480
@@ -3076,7 +3076,7 @@ steps:
3076
#------------------------------------------------------ mi300 · weight_loading -------------------------------------------------------#
3077
3078
- label: ":amd: (MI300) Weight Loading Multi-GPU" # TBD
3079
- timeout_in_minutes: 40
3079
+ timeout_in_minutes: 70
3080
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3081
dind: false
3082
agent_pool: mi300_2
tests/v1/distributed/test_internal_lb_dp.py
+3
@@ -184,6 +184,9 @@ class MultinodeInternalLBServerManager:
184
self.model_name,
185
sargs,
186
auto_port=False,
187
+ max_wait_seconds=int(
188
+ os.getenv("VLLM_ENGINE_READY_TIMEOUT_S", "480")
189
+ ),
190
env_dict={
191
"VLLM_SERVER_DEV_MODE": "1",
192
current_platform.device_control_env_var: ",".join(