[CI] Shard distributed model jobs above the 24h P90 threshold (#54752)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Kevin H. Luu committed
Sep 1, 2026 at 15:32 UTC
96031b8623f36522f5e6e70f778d9654341a8d86
1 file changed
+39
-19
.buildkite/test_areas/models_distributed.yaml
+39
-19
@@ -2,9 +2,9 @@ group: Models - Distributed
2
depends_on:
3
- image-build
4
steps:
5
-- label: ":nvidia: (L4) Distributed Models"
5
+- label: ":nvidia: (L4) Distributed Models Shard %N"
6
key: distributed-model-tests-2-gpus
7
- timeout_in_minutes: 60
7
+ timeout_in_minutes: 35
8
working_dir: "/vllm-workspace/tests"
9
device: l4
10
num_devices: 2
@@ -15,20 +15,30 @@ steps:
15
- tests/model_executor/model_loader/test_sharded_state_loader.py
16
- tests/models/
17
commands:
18
- - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
19
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
20
- # Avoid importing model tests that cause CUDA reinitialization error
21
- - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
22
- - pytest models/language -v -s -m 'distributed(num_gpus=2)'
23
- - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
24
- - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
25
- - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
18
+ - |
19
+ case "$$BUILDKITE_PARALLEL_JOB" in
20
+ 0)
21
+ TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
22
+ CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
23
+ ;;
24
+ 1)
25
+ pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
26
+ pytest models/language -v -s -m 'distributed(num_gpus=2)'
27
+ ;;
28
+ 2)
29
+ pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
30
+ pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
31
+ VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
32
+ ;;
33
+ *) exit 2 ;;
34
+ esac
35
+ parallelism: 3
36
mirror:
37
amd:
28
- label: ":amd: (MI300) Distributed Models"
38
+ label: ":amd: (MI300) Distributed Models Shard %N"
39
dind: false
40
device: mi300_2
31
- timeout_in_minutes: 75
41
+ timeout_in_minutes: 35
42
depends_on:
43
- image-build-amd
44
source_file_dependencies:
@@ -43,10 +53,20 @@ steps:
53
- tests/model_executor/model_loader/test_sharded_state_loader.py
54
- tests/models/
55
commands:
46
- - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
47
- - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
48
- - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
49
- - pytest models/language -v -s -m 'distributed(num_gpus=2)'
50
- - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
51
- - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
52
- - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
56
+ - |
57
+ case "$$BUILDKITE_PARALLEL_JOB" in
58
+ 0)
59
+ TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
60
+ HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
61
+ ;;
62
+ 1)
63
+ pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
64
+ pytest models/language -v -s -m 'distributed(num_gpus=2)'
65
+ ;;
66
+ 2)
67
+ pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
68
+ pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
69
+ VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
70
+ ;;
71
+ *) exit 2 ;;
72
+ esac