@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
yaml 251 lines 8.56 KB
Raw
1 group: Models - Multimodal
2 depends_on:
3 - image-build
4 steps:
5 - label: ":nvidia: (H200 MIG 18GB) Multimodal Models (Standard) 1: qwen2"
6 key: multi-modal-models-standard-1-qwen2
7 timeout_in_minutes: 68
8 device: h200_18gb
9 source_file_dependencies:
10 - vllm/
11 - "!vllm/distributed/kv_transfer/"
12 - tests/models/multimodal
13 commands:
14 - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
15 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
16 mirror:
17 amd:
18 label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2"
19 dind: false
20 device: mi355_dpx
21 timeout_in_minutes: 65
22 depends_on:
23 - image-build-amd
24
25 - label: ":nvidia: (H200 MIG 18GB) Multimodal Models (Standard) 2: qwen3 + gemma"
26 key: multi-modal-models-standard-2-qwen3-gemma
27 timeout_in_minutes: 75
28 device: h200_18gb
29 source_file_dependencies:
30 - vllm/
31 - "!vllm/distributed/kv_transfer/"
32 - tests/models/multimodal
33 commands:
34 - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
35 - pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model
36 - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
37 mirror:
38 amd:
39 label: ":amd: (MI355 DPX) Multimodal Models (Standard) 2: qwen3 + gemma"
40 dind: false
41 device: mi355_dpx
42 timeout_in_minutes: 55
43 depends_on:
44 - image-build-amd
45
46 - label: ":nvidia: (H200 MIG 35GB) Multimodal Models (Standard) 3: llava + qwen2_vl"
47 device: h200_35gb
48 key: multi-modal-models-standard-3-llava-qwen2-vl
49 timeout_in_minutes: 40
50 source_file_dependencies:
51 - vllm/
52 - "!vllm/distributed/kv_transfer/"
53 - tests/models/multimodal
54 commands:
55 - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
56 - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
57 mirror:
58 amd:
59 label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl"
60 dind: false
61 device: mi250_1
62 timeout_in_minutes: 55
63 depends_on:
64 - image-build-amd
65
66 - label: ":nvidia: (H200 MIG 35GB) Multimodal Models (Standard) 4: other + whisper"
67 device: h200_35gb
68 key: multi-modal-models-standard-4-other-whisper
69 timeout_in_minutes: 75
70 source_file_dependencies:
71 - vllm/
72 - "!vllm/distributed/kv_transfer/"
73 - tests/models/multimodal
74 commands:
75 - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
76 - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
77 - pytest models/multimodal/generation/test_memory_leak.py -m core_model
78 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
79 mirror:
80 amd:
81 label: ":amd: (MI355 DPX) Multimodal Models (Standard) 4: other + whisper"
82 dind: false
83 device: mi355_dpx
84 timeout_in_minutes: 70
85 depends_on:
86 - image-build-amd
87
88 - label: ":computer: (CPU) Multimodal Processor Shard %N"
89 key: multi-modal-processor-cpu
90 depends_on:
91 - image-build-cpu
92 timeout_in_minutes: 125
93 source_file_dependencies:
94 - vllm/
95 - "!vllm/distributed/kv_transfer/"
96 - tests/models/multimodal
97 - tests/models/registry.py
98 device: cpu-medium
99 commands:
100 - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
101 parallelism: 4
102
103 - label: ":nvidia: (H200 MIG 18GB) Multimodal Processor Shard %N"
104 key: multi-modal-processor
105 timeout_in_minutes: 98
106 device: h200_18gb
107 parallelism: 4
108 source_file_dependencies:
109 - vllm/
110 - "!vllm/distributed/kv_transfer/"
111 - tests/models/multimodal
112 - tests/models/registry.py
113 commands:
114 - pytest -v -s models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
115 mirror:
116 amd:
117 label: ":amd: (MI355 DPX) Multimodal Processor Shard %N"
118 dind: false
119 device: mi355_dpx
120 parallelism: 4
121 timeout_in_minutes: 115
122 depends_on:
123 - image-build-amd
124
125 - label: ":nvidia: (H200 MIG 35GB) Multimodal Accuracy Eval (Small Models)"
126 device: h200_35gb
127 key: multi-modal-accuracy-eval-small-models
128 timeout_in_minutes: 30
129 working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
130 source_file_dependencies:
131 - vllm/multimodal/
132 - vllm/inputs/
133 - vllm/v1/core/
134 commands:
135 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
136 mirror:
137 amd:
138 label: ":amd: (MI355 DPX) Multimodal Accuracy Eval (Small Models)"
139 dind: false
140 device: mi355_dpx
141 timeout_in_minutes: 35
142 depends_on:
143 - image-build-amd
144 source_file_dependencies:
145 - vllm/multimodal/
146 - vllm/inputs/
147 - vllm/v1/core/
148 - vllm/platforms/rocm.py
149 - vllm/model_executor/model_loader/
150
151 - label: ":nvidia: (H200 MIG 35GB) Multimodal Models (Extended Generation 1)"
152 device: h200_35gb
153 key: multi-modal-models-extended-generation-1
154 optional: true
155 source_file_dependencies:
156 - vllm/
157 - "!vllm/distributed/kv_transfer/"
158 - tests/models/multimodal/generation
159 - tests/models/multimodal/test_mapping.py
160 commands:
161 - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
162 - pytest -v -s models/multimodal/test_mapping.py
163 mirror:
164 amd:
165 label: ":amd: (MI300) Multimodal Models (Extended Generation 1)"
166 dind: false
167 device: mi300_1
168 timeout_in_minutes: 100
169 depends_on:
170 - image-build-amd
171
172 - label: ":nvidia: (H200 MIG 35GB) Multimodal Models (PPL)"
173 device: h200_35gb
174 key: multi-modal-models-extended-ppl
175 optional: true
176 source_file_dependencies:
177 - vllm/
178 - "!vllm/distributed/kv_transfer/"
179 - tests/models/multimodal/generation_ppl_test/
180 commands:
181 - pytest -v -s models/multimodal/generation_ppl_test/
182 mirror:
183 amd:
184 label: ":amd: (MI300) Multimodal Models (PPL)"
185 dind: false
186 device: mi300_1
187 timeout_in_minutes: 90
188 depends_on:
189 - image-build-amd
190
191 - label: ":nvidia: (H200 MIG 35GB) Multimodal Models (Extended Generation 2) Shard %N"
192 device: h200_35gb
193 key: multi-modal-models-extended-generation-2
194 parallelism: 4
195 optional: true
196 source_file_dependencies:
197 - vllm/
198 - "!vllm/distributed/kv_transfer/"
199 - tests/models/multimodal/generation
200 commands:
201 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
202 mirror:
203 amd:
204 label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 2) Shard %N"
205 dind: false
206 device: mi355_dpx
207 timeout_in_minutes: 100
208 depends_on:
209 - image-build-amd
210
211 - label: ":nvidia: (H200 MIG 35GB) Multimodal Models (Extended Generation 3)"
212 device: h200_35gb
213 key: multi-modal-models-extended-generation-3
214 optional: true
215 source_file_dependencies:
216 - vllm/
217 - "!vllm/distributed/kv_transfer/"
218 - tests/models/multimodal/generation
219 commands:
220 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
221 mirror:
222 amd:
223 label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 3)"
224 dind: false
225 device: mi355_dpx
226 timeout_in_minutes: 90
227 depends_on:
228 - image-build-amd
229
230 - label: ":nvidia: (H200 MIG 18GB) Multimodal Models (Extended Pooling)"
231 key: multi-modal-models-extended-pooling
232 optional: true
233 device: h200_18gb
234 source_file_dependencies:
235 - vllm/
236 - "!vllm/distributed/kv_transfer/"
237 - tests/models/multimodal/pooling
238 commands:
239 - pytest -v -s models/multimodal/pooling -m 'not core_model'
240 mirror:
241 amd:
242 label: ":amd: (MI300) Multimodal Models (Extended Pooling)"
243 dind: false
244 device: mi300_1
245 timeout_in_minutes: 75
246 depends_on:
247 - image-build-amd
248 source_file_dependencies:
249 - vllm/
250 - "!vllm/distributed/kv_transfer/"
251 - tests/models/multimodal/pooling