A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Build] Add initial conditional testing spec (#6841)
Simon Mo committed
Aug 5, 2024 at 17:39 UTC
e3c664bfcb14a41e43ddb6078ed1464ae9b7852f
1 file changed
+234
-156
.buildkite/test-pipeline.yaml
+234
-156
@@ -5,11 +5,47 @@
5
# https://github.com/vllm-project/buildkite-ci/blob/main/scripts/test-template-aws.j2
6
# to generate the final pipeline yaml file.
7
8
+# Documentation
9
+# label(str): the name of the test. emoji allowed.
10
+# fast_check(bool): whether to run this on each commit on fastcheck pipeline.
11
+# fast_check_only(bool): run this test on fastcheck pipeline only
12
+# command(str): the single command to run for tests. incompatible with commands.
13
+# commands(list): the list of commands to run for test. incompatbile with command.
14
+# mirror_hardwares(list): the list of hardwares to run the test on as well. currently only supports [amd]
15
+# gpu(str): override the GPU selection for the test. default is on L4 GPUs. currently only supports a100
16
+# num_gpus(int): override the number of GPUs for the test. default to 1 GPU. currently support 2,4.
17
+# num_nodes(int): whether to simulate multi-node setup by launch multiple containers on one host,
18
+# in this case, commands must be specified. the first command runs on first host, the second
19
+# command runs on the second host.
20
+# working_dir(str): specify the place where command should execute, default to /vllm-workspace/tests
21
+# source_file_dependencies(list): the list of prefix to opt-in the test for, if empty, the test will always run.
22
+
23
+# When adding a test
24
+# - If the test belong to an existing group, add it there
25
+# - If the test is short, add to any existing step
26
+# - If the test takes more than 10min, then it is okay to create a new step.
27
+# Note that all steps execute in parallel.
28
29
steps:
10
-- label: Async Engine, Inputs, Utils, Worker Test
30
+##### fast check tests #####
31
+
32
+- label: Documentation Build # 2min
33
+ working_dir: "/vllm-workspace/test_docs/docs"
34
fast_check: true
12
- fast_check_only: true
35
+ no_gpu: True
36
+ commands:
37
+ - pip install -r requirements-docs.txt
38
+ - SPHINXOPTS=\"-W\" make html
39
+
40
+- label: Async Engine, Inputs, Utils, Worker Test # 15min
41
+ fast_check: true
42
+ source_file_dependencies:
43
+ - vllm/
44
+ - tests/async_engine
45
+ - tests/test_inputs
46
+ - tests/multimodal
47
+ - tests/test_utils
48
+ - tests/worker
49
commands:
50
- pytest -v -s async_engine # Async Engine
51
- pytest -v -s test_inputs.py
@@ -17,31 +53,12 @@ steps:
53
- pytest -v -s test_utils.py # Utils
54
- pytest -v -s worker # Worker
55
20
-- label: Metrics, Tracing Test
21
- fast_check: true
22
- fast_check_only: true
23
- commands:
24
- - pytest -v -s metrics # Metrics
25
- - "pip install \
26
- opentelemetry-sdk \
27
- opentelemetry-api \
28
- opentelemetry-exporter-otlp \
29
- opentelemetry-semantic-conventions-ai" # Tracing
30
- - pytest -v -s tracing
31
-
32
-- label: Regression Test
33
- mirror_hardwares: [amd]
34
- fast_check: true
35
- command: pytest -v -s test_regression.py
36
- working_dir: "/vllm-workspace/tests" # optional
37
-
38
-- label: AsyncEngine Test
39
- #mirror_hardwares: [amd]
40
- command: pytest -v -s async_engine
41
-
42
-- label: Basic Correctness Test
56
+- label: Basic Correctness Test # 30min
57
mirror_hardwares: [amd]
58
fast_check: true
59
+ source_file_dependencies:
60
+ - vllm/
61
+ - tests/basic_correctness
62
commands:
63
# This flashinfer installation will fail on AMD ROCm, so it is set as optional.
64
- pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl || true
@@ -50,215 +67,264 @@ steps:
67
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
68
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
69
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
53
-
54
-- label: Core Test
70
+
71
+- label: Core Test # 10min
72
mirror_hardwares: [amd]
73
fast_check: true
74
+ source_file_dependencies:
75
+ - vllm/core
76
+ - vllm/distributed
77
+ - tests/core
78
commands:
79
- pytest -v -s core
80
60
-- label: Distributed Comm Ops Test
61
- #mirror_hardwares: [amd]
62
- working_dir: "/vllm-workspace/tests"
63
- num_gpus: 2
64
- commands:
65
- - pytest -v -s distributed/test_comm_ops.py
66
- - pytest -v -s distributed/test_shm_broadcast.py
67
-
68
-- label: 2 Node Tests (4 GPUs in total)
69
- working_dir: "/vllm-workspace/tests"
70
- num_gpus: 2
71
- num_nodes: 2
72
- commands:
73
- - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
74
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
75
- - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
76
- - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
77
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
78
-
79
-- label: Distributed Tests (2 GPUs)
81
+- label: Entrypoints Test # 20min
82
+ fast_check: true
83
mirror_hardwares: [amd]
81
- working_dir: "/vllm-workspace/tests"
82
- num_gpus: 2
84
+ source_file_dependencies:
85
+ - vllm/entrypoints
86
+ - tests/entrypoints
87
commands:
84
- - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85
- - TARGET_TEST_SUITE=L4 pytest -v -s distributed/test_basic_distributed_correctness.py
86
- - pytest -v -s distributed/test_chunked_prefill_distributed.py
87
- - pytest -v -s distributed/test_multimodal_broadcast.py
88
- - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
89
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
90
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
88
+ - pytest -v -s entrypoints/llm
89
+ - pytest -v -s entrypoints/openai
90
92
-- label: Distributed Tests (4 GPUs)
93
- #mirror_hardwares: [amd]
91
+- label: Distributed Tests (4 GPUs) # 10min
92
working_dir: "/vllm-workspace/tests"
93
num_gpus: 4
94
fast_check: true
95
+ source_file_dependencies:
96
+ - vllm/
97
+ - tests/distributed
98
+ - tests/spec_decode/e2e/test_integration_dist_tp4
99
commands:
100
- pytest -v -s distributed/test_pynccl.py
101
- pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
102
101
-- label: Pipeline Parallelism Test
102
- working_dir: "/vllm-workspace/tests"
103
- num_gpus: 4
103
+##### fast check tests #####
104
+##### 1 GPU test #####
105
+
106
+- label: Metrics, Tracing Test # 10min
107
+ source_file_dependencies:
108
+ - vllm/
109
+ - tests/metrics
110
+ - tests/tracing
111
commands:
105
- - pytest -v -s distributed/test_pipeline_parallel.py
112
+ - pytest -v -s metrics
113
+ - "pip install \
114
+ opentelemetry-sdk \
115
+ opentelemetry-api \
116
+ opentelemetry-exporter-otlp \
117
+ opentelemetry-semantic-conventions-ai"
118
+ - pytest -v -s tracing
119
107
-- label: Engine Test
120
+- label: Regression Test # 5min
121
mirror_hardwares: [amd]
122
+ source_file_dependencies:
123
+ - vllm/
124
+ - tests/test_regression
125
+ command: pytest -v -s test_regression.py
126
+ working_dir: "/vllm-workspace/tests" # optional
127
+
128
+- label: Engine Test # 10min
129
+ mirror_hardwares: [amd]
130
+ source_file_dependencies:
131
+ - vllm/
132
+ - tests/engine
133
+ - tests/tokenization
134
commands:
135
- pytest -v -s engine test_sequence.py test_config.py test_logger.py
136
# OOM in the CI unless we run this separately
137
- pytest -v -s tokenization
138
114
-- label: Entrypoints Test
115
- fast_check: true
116
- mirror_hardwares: [amd]
117
-
118
- commands:
119
- - pytest -v -s entrypoints/llm
120
- - pytest -v -s entrypoints/openai
121
-
122
-- label: Examples Test
139
+- label: Examples Test # 12min
140
working_dir: "/vllm-workspace/examples"
141
mirror_hardwares: [amd]
142
+ source_file_dependencies:
143
+ - vllm/entrypoints
144
+ - examples/
145
commands:
126
- # install tensorizer for tensorize_vllm_model.py
127
- - pip install awscli tensorizer
146
+ - pip install awscli tensorizer # for llava example and tensorizer test
147
- python3 offline_inference.py
148
- python3 cpu_offload.py
149
- python3 offline_inference_with_prefix.py
150
- python3 llm_engine_example.py
132
- - python3 offline_inference_vision_language.py
151
+ - python3 llava_example.py
152
- python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
153
135
-- label: Inputs Test
136
- #mirror_hardwares: [amd]
137
- commands:
138
- - pytest -v -s test_inputs.py
139
- - pytest -v -s multimodal
140
-
141
-# - label: Kernels Test %N
142
-# #mirror_hardwares: [amd]
143
-# commands:
144
-# - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
145
-# - pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
146
-# parallelism: 4
147
-
148
-- label: Models Test
149
- #mirror_hardwares: [amd]
154
+- label: Models Test # 1hr10min
155
+ source_file_dependencies:
156
+ - vllm/
157
+ - tests/models
158
commands:
159
- pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
160
- pytest -v -s models -m \"not vlm\"
161
154
-- label: Vision Language Models Test
162
+- label: Vision Language Models Test # 42min
163
mirror_hardwares: [amd]
164
+ source_file_dependencies:
165
+ - vllm/
166
commands:
167
- pytest -v -s models -m vlm
168
159
-- label: Prefix Caching Test
169
+- label: Prefix Caching Test # 7min
170
mirror_hardwares: [amd]
171
+ source_file_dependencies:
172
+ - vllm/
173
+ - tests/prefix_caching
174
commands:
175
- pytest -v -s prefix_caching
176
164
-- label: Samplers Test
165
- #mirror_hardwares: [amd]
177
+- label: Samplers Test # 18min
178
+ source_file_dependencies:
179
+ - vllm/model_executor/layers
180
+ - vllm/sampling_metadata.py
181
+ - tests/samplers
182
command: pytest -v -s samplers
183
168
-- label: LogitsProcessor Test
184
+- label: LogitsProcessor Test # 5min
185
mirror_hardwares: [amd]
186
+ source_file_dependencies:
187
+ - vllm/model_executor/layers
188
+ - tests/test_logits_processor
189
command: pytest -v -s test_logits_processor.py
190
172
-- label: Utils Test
173
- commands:
174
- - pytest -v -s test_utils.py
175
- - pytest -v -s test_embedded_commit.py
176
-
177
-- label: Worker Test
178
- mirror_hardwares: [amd]
179
- command: pytest -v -s worker
180
-
181
-- label: Speculative decoding tests
182
- #mirror_hardwares: [amd]
191
+- label: Speculative decoding tests # 22min
192
+ source_file_dependencies:
193
+ - vllm/spec_decode
194
+ - tests/spec_decode
195
commands:
196
# See https://github.com/vllm-project/vllm/issues/5152
197
- export VLLM_ATTENTION_BACKEND=XFORMERS
198
- pytest -v -s spec_decode
199
188
-# - label: LoRA Test %N
189
-# #mirror_hardwares: [amd]
190
-# command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
191
-# parallelism: 4
192
-
193
-# - label: LoRA Long Context (Distributed)
194
-# #mirror_hardwares: [amd]
195
-# num_gpus: 4
196
-# # This test runs llama 13B, so it is required to run on 4 GPUs.
197
-# commands:
198
-# # FIXIT: find out which code initialize cuda before running the test
199
-# # before the fix, we need to use spawn to test it
200
-# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
201
-# - pytest -v -s -x lora/test_long_context.py
202
-
203
-- label: Tensorizer Test
204
- #mirror_hardwares: [amd]
205
- fast_check: true
200
+- label: LoRA Test %N # 30min each
201
+ source_file_dependencies:
202
+ - vllm/lora
203
+ - csrc/punica
204
+ - tests/lora
205
+ command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
206
+ parallelism: 4
207
+
208
+- label: Kernels Test %N # 30min each
209
+ source_file_dependencies:
210
+ - csrc/
211
+ - vllm/attention
212
+ - tests/kernels
213
+ commands:
214
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
215
+ - pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
216
+ parallelism: 4
217
+
218
+- label: Tensorizer Test # 11min
219
+ soft_fail: true
220
+ source_file_dependencies:
221
+ - vllm/model_executor/model_loader
222
+ - tests/tensorizer_loader
223
commands:
224
- apt-get install -y curl libsodium23
225
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
226
- pytest -v -s tensorizer_loader
227
211
-- label: Metrics Test
212
- mirror_hardwares: [amd]
213
- command: pytest -v -s metrics
214
-
215
-- label: Quantization Test
216
- #mirror_hardwares: [amd]
217
- command: pytest -v -s quantization
218
-
219
-- label: Tracing Test
220
- commands:
221
- - "pip install \
222
- opentelemetry-sdk \
223
- opentelemetry-api \
224
- opentelemetry-exporter-otlp \
225
- opentelemetry-semantic-conventions-ai"
226
- - pytest -v -s tracing
227
-
228
-- label: Benchmarks
228
+- label: Benchmarks # 9min
229
working_dir: "/vllm-workspace/.buildkite"
230
mirror_hardwares: [amd]
231
+ source_file_dependencies:
232
+ - benchmarks/
233
commands:
234
- pip install aiohttp
235
- bash run-benchmarks.sh
236
235
-- label: LM Eval Small Models
237
+- label: Quantization Test # 15min
238
+ source_file_dependencies:
239
+ - csrc/
240
+ - vllm/model_executor/layers/quantization
241
+ - tests/quantization
242
+ command: pytest -v -s quantization
243
+
244
+- label: LM Eval Small Models # 53min
245
working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
246
+ source_file_dependencies:
247
+ - csrc/
248
+ - vllm/model_executor/layers/quantization
249
commands:
250
- pip install lm-eval
251
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
252
- bash ./run-tests.sh -c configs/models-small.txt -t 1
253
242
-- label: LM Eval Large Models
243
- gpu: a100
254
+
255
+##### 1 GPU test #####
256
+##### multi gpus test #####
257
+
258
+- label: Distributed Comm Ops Test # 7min
259
+ working_dir: "/vllm-workspace/tests"
260
+ num_gpus: 2
261
+ source_file_dependencies:
262
+ - vllm/distributed
263
+ - tests/distributed
264
+ commands:
265
+ - pytest -v -s distributed/test_comm_ops.py
266
+ - pytest -v -s distributed/test_shm_broadcast.py
267
+
268
+- label: 2 Node Tests (4 GPUs in total) # 16min
269
+ working_dir: "/vllm-workspace/tests"
270
+ num_gpus: 2
271
+ num_nodes: 2
272
+ source_file_dependencies:
273
+ - vllm/
274
+ - tests/distributed/test_same_node
275
+ commands:
276
+ - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
277
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
278
+ - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
279
+ - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
280
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
281
+
282
+- label: Distributed Tests (2 GPUs) # 28min
283
+ mirror_hardwares: [amd]
284
+ working_dir: "/vllm-workspace/tests"
285
+ num_gpus: 2
286
+ source_file_dependencies:
287
+ - vllm/
288
+ - tests/distributed
289
+ commands:
290
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
291
+ - TARGET_TEST_SUITE=L4 pytest -v -s distributed/test_basic_distributed_correctness.py
292
+ - pytest -v -s distributed/test_chunked_prefill_distributed.py
293
+ - pytest -v -s distributed/test_multimodal_broadcast.py
294
+ - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
295
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
296
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
297
+
298
+- label: Pipeline Parallelism Test # 23min
299
+ working_dir: "/vllm-workspace/tests"
300
num_gpus: 4
245
- working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
301
+ source_file_dependencies:
302
+ - vllm/
303
+ - tests/distributed/test_pipeline_parallel
304
commands:
247
- - pip install lm-eval
248
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
249
- - bash ./run-tests.sh -c configs/models-large.txt -t 4
305
+ - pytest -v -s distributed/test_pipeline_parallel.py
306
251
-- label: Documentation Build
252
- working_dir: "/vllm-workspace/test_docs/docs"
253
- fast_check: true
254
- no_gpu: True
307
+- label: LoRA Long Context (Distributed) # 11min
308
+ # This test runs llama 13B, so it is required to run on 4 GPUs.
309
+ num_gpus: 4
310
+ source_file_dependencies:
311
+ - vllm/lora
312
+ - csrc/punica
313
+ - tests/lora/test_long_context
314
commands:
256
- - pip install -r requirements-docs.txt
257
- - SPHINXOPTS=\"-W\" make html
315
+ # FIXIT: find out which code initialize cuda before running the test
316
+ # before the fix, we need to use spawn to test it
317
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
318
+ - pytest -v -s -x lora/test_long_context.py
319
+
320
+##### multi gpus test #####
321
+##### A100 test #####
322
259
-- label: Distributed Tests (A100)
323
+- label: Distributed Tests (A100) # optional
324
gpu: a100
325
num_gpus: 4
326
+ source_file_dependencies:
327
+ - vllm/
328
commands:
329
# NOTE: don't test llama model here, it seems hf implementation is buggy
330
# see https://github.com/vllm-project/vllm/pull/5689 for details
@@ -266,3 +332,15 @@ steps:
332
- pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
333
- TARGET_TEST_SUITE=A100 pytest -v -s distributed/test_basic_distributed_correctness.py
334
- pytest -v -s -x lora/test_mixtral.py
335
+
336
+- label: LM Eval Large Models # optional
337
+ gpu: a100
338
+ num_gpus: 4
339
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
340
+ source_file_dependencies:
341
+ - csrc/
342
+ - vllm/model_executor/layers/quantization
343
+ commands:
344
+ - pip install lm-eval
345
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
346
+ - bash ./run-tests.sh -c configs/models-large.txt -t 4