A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[CI/Build] Update models tests & examples (#8874)
Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Sep 29, 2024 at 00:54 UTC
e1a3f5e831a467b2867a66e0e56ac0f70ed44394
13 files changed
+239
-184
.buildkite/test-pipeline.yaml
+32
-19
@@ -9,6 +9,7 @@
9
# label(str): the name of the test. emoji allowed.
10
# fast_check(bool): whether to run this on each commit on fastcheck pipeline.
11
# fast_check_only(bool): run this test on fastcheck pipeline only
12
+# optional(bool): never run this test by default (i.e. need to unblock manually)
13
# command(str): the single command to run for tests. incompatible with commands.
14
# commands(list): the list of commands to run for test. incompatbile with command.
15
# mirror_hardwares(list): the list of hardwares to run the test on as well. currently only supports [amd]
@@ -39,7 +40,7 @@ steps:
40
# Check API reference (if it fails, you may have missing mock imports)
41
- grep \"sig sig-object py\" build/html/dev/sampling_params.html
42
42
-- label: Async Engine, Inputs, Utils, Worker Test # 15min
43
+- label: Async Engine, Inputs, Utils, Worker Test # 24min
44
fast_check: true
45
source_file_dependencies:
46
- vllm/
@@ -81,7 +82,7 @@ steps:
82
commands:
83
- pytest -v -s core
84
84
-- label: Entrypoints Test # 20min
85
+- label: Entrypoints Test # 40min
86
working_dir: "/vllm-workspace/tests"
87
fast_check: true
88
mirror_hardwares: [amd]
@@ -151,7 +152,7 @@ steps:
152
# OOM in the CI unless we run this separately
153
- pytest -v -s tokenization
154
154
-- label: Examples Test # 12min
155
+- label: Examples Test # 15min
156
working_dir: "/vllm-workspace/examples"
157
#mirror_hardwares: [amd]
158
source_file_dependencies:
@@ -169,7 +170,7 @@ steps:
170
- python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
171
- python3 offline_inference_encoder_decoder.py
172
172
-- label: Prefix Caching Test # 7min
173
+- label: Prefix Caching Test # 9min
174
#mirror_hardwares: [amd]
175
source_file_dependencies:
176
- vllm/
@@ -177,7 +178,7 @@ steps:
178
commands:
179
- pytest -v -s prefix_caching
180
180
-- label: Samplers Test # 18min
181
+- label: Samplers Test # 36min
182
source_file_dependencies:
183
- vllm/model_executor/layers
184
- vllm/sampling_metadata.py
@@ -193,7 +194,7 @@ steps:
194
- tests/test_logits_processor
195
command: pytest -v -s test_logits_processor.py
196
196
-- label: Speculative decoding tests # 22min
197
+- label: Speculative decoding tests # 30min
198
source_file_dependencies:
199
- vllm/spec_decode
200
- tests/spec_decode
@@ -203,7 +204,7 @@ steps:
204
- pytest -v -s spec_decode/e2e/test_multistep_correctness.py
205
- pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
206
206
-- label: LoRA Test %N # 30min each
207
+- label: LoRA Test %N # 15min each
208
mirror_hardwares: [amd]
209
source_file_dependencies:
210
- vllm/lora
@@ -211,7 +212,7 @@ steps:
212
command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
213
parallelism: 4
214
214
-- label: "PyTorch Fullgraph Smoke Test"
215
+- label: "PyTorch Fullgraph Smoke Test" # 9min
216
fast_check: true
217
source_file_dependencies:
218
- vllm/
@@ -219,14 +220,14 @@ steps:
220
commands:
221
- pytest -v -s compile/test_full_graph_smoke.py
222
222
-- label: "PyTorch Fullgraph Test"
223
+- label: "PyTorch Fullgraph Test" # 18min
224
source_file_dependencies:
225
- vllm/
226
- tests/compile
227
commands:
228
- pytest -v -s compile/test_full_graph.py
229
229
-- label: Kernels Test %N # 30min each
230
+- label: Kernels Test %N # 1h each
231
mirror_hardwares: [amd]
232
source_file_dependencies:
233
- csrc/
@@ -256,7 +257,7 @@ steps:
257
- pip install aiohttp
258
- bash run-benchmarks.sh
259
259
-- label: Quantization Test # 15min
260
+- label: Quantization Test # 33min
261
source_file_dependencies:
262
- csrc/
263
- vllm/model_executor/layers/quantization
@@ -300,7 +301,7 @@ steps:
301
- pytest -v -s models/test_oot_registration.py # it needs a clean process
302
- pytest -v -s models/*.py --ignore=models/test_oot_registration.py
303
303
-- label: Decoder-only Language Models Test # 1h3min
304
+- label: Decoder-only Language Models Test # 1h36min
305
#mirror_hardwares: [amd]
306
source_file_dependencies:
307
- vllm/
@@ -308,7 +309,7 @@ steps:
309
commands:
310
- pytest -v -s models/decoder_only/language
311
311
-- label: Decoder-only Multi-Modal Models Test # 56min
312
+- label: Decoder-only Multi-Modal Models Test # 1h31min
313
#mirror_hardwares: [amd]
314
source_file_dependencies:
315
- vllm/
@@ -318,15 +319,25 @@ steps:
319
- pytest -v -s models/decoder_only/audio_language
320
- pytest -v -s models/decoder_only/vision_language
321
321
-- label: Other Models Test # 5min
322
+- label: Other Models Test # 6min
323
#mirror_hardwares: [amd]
324
source_file_dependencies:
325
- vllm/
326
- tests/models/embedding/language
327
- tests/models/encoder_decoder/language
328
+ - tests/models/encoder_decoder/vision_language
329
commands:
330
- pytest -v -s models/embedding/language
331
- pytest -v -s models/encoder_decoder/language
332
+ - pytest -v -s models/encoder_decoder/vision_language
333
+
334
+- label: Custom Models Test
335
+ #mirror_hardwares: [amd]
336
+ optional: true
337
+ commands:
338
+ # PR authors can temporarily add commands below to test individual models
339
+ # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
340
+ # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
341
342
##### 1 GPU test #####
343
##### multi gpus test #####
@@ -359,7 +370,7 @@ steps:
370
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
371
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
372
362
-- label: Distributed Tests (2 GPUs) # 28min
373
+- label: Distributed Tests (2 GPUs) # 40min
374
#mirror_hardwares: [amd]
375
working_dir: "/vllm-workspace/tests"
376
num_gpus: 2
@@ -376,14 +387,16 @@ steps:
387
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
388
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
389
# Avoid importing model tests that cause CUDA reinitialization error
379
- - pytest models/encoder_decoder/language/test_bart.py models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
390
+ - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
391
+ - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
392
+ - pytest models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
393
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
394
- pip install -e ./plugins/vllm_add_dummy_model
395
- pytest -v -s distributed/test_distributed_oot.py
396
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
397
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
398
386
-- label: Multi-step Tests (4 GPUs) # 21min
399
+- label: Multi-step Tests (4 GPUs) # 36min
400
working_dir: "/vllm-workspace/tests"
401
num_gpus: 4
402
source_file_dependencies:
@@ -401,7 +414,7 @@ steps:
414
- pytest -v -s multi_step/test_correctness_async_llm.py
415
- pytest -v -s multi_step/test_correctness_llm.py
416
404
-- label: Pipeline Parallelism Test # 23min
417
+- label: Pipeline Parallelism Test # 45min
418
working_dir: "/vllm-workspace/tests"
419
num_gpus: 4
420
source_file_dependencies:
@@ -427,7 +440,7 @@ steps:
440
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
441
- pytest -v -s -x lora/test_long_context.py
442
430
-- label: Weight Loading Multiple GPU Test
443
+- label: Weight Loading Multiple GPU Test # 33min
444
working_dir: "/vllm-workspace/tests"
445
num_gpus: 2
446
source_file_dependencies:
examples/offline_inference_vision_language.py
+19
-9
@@ -12,6 +12,10 @@ from vllm.assets.image import ImageAsset
12
from vllm.assets.video import VideoAsset
13
from vllm.utils import FlexibleArgumentParser
14
15
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
16
+# lower-end GPUs.
17
+# Unless specified, these settings have been tested to work on a single L4.
18
+
19
20
# LLaVA-1.5
21
def run_llava(question, modality):
@@ -19,7 +23,7 @@ def run_llava(question, modality):
23
24
prompt = f"USER: <image>\n{question}\nASSISTANT:"
25
22
- llm = LLM(model="llava-hf/llava-1.5-7b-hf")
26
+ llm = LLM(model="llava-hf/llava-1.5-7b-hf", max_model_len=4096)
27
stop_token_ids = None
28
return llm, prompt, stop_token_ids
29
@@ -57,7 +61,7 @@ def run_llava_onevision(question, modality):
61
<|im_start|>assistant\n"
62
63
llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
60
- max_model_len=32768)
64
+ max_model_len=16384)
65
stop_token_ids = None
66
return llm, prompt, stop_token_ids
67
@@ -67,7 +71,7 @@ def run_fuyu(question, modality):
71
assert modality == "image"
72
73
prompt = f"{question}\n"
70
- llm = LLM(model="adept/fuyu-8b")
74
+ llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2)
75
stop_token_ids = None
76
return llm, prompt, stop_token_ids
77
@@ -99,7 +103,8 @@ def run_phi3v(question, modality):
103
llm = LLM(
104
model="microsoft/Phi-3-vision-128k-instruct",
105
trust_remote_code=True,
102
- max_num_seqs=5,
106
+ max_model_len=4096,
107
+ max_num_seqs=2,
108
mm_processor_kwargs={"num_crops": 16},
109
)
110
stop_token_ids = None
@@ -122,7 +127,7 @@ def run_chameleon(question, modality):
127
assert modality == "image"
128
129
prompt = f"{question}<image>"
125
- llm = LLM(model="facebook/chameleon-7b")
130
+ llm = LLM(model="facebook/chameleon-7b", max_model_len=4096)
131
stop_token_ids = None
132
return llm, prompt, stop_token_ids
133
@@ -145,6 +150,8 @@ def run_minicpmv(question, modality):
150
trust_remote_code=True)
151
llm = LLM(
152
model=model_name,
153
+ max_model_len=4096,
154
+ max_num_seqs=2,
155
trust_remote_code=True,
156
)
157
# NOTE The stop_token_ids are different for various versions of MiniCPM-V
@@ -177,7 +184,7 @@ def run_internvl(question, modality):
184
llm = LLM(
185
model=model_name,
186
trust_remote_code=True,
180
- max_num_seqs=5,
187
+ max_model_len=4096,
188
)
189
190
tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -215,7 +222,8 @@ def run_qwen_vl(question, modality):
222
llm = LLM(
223
model="Qwen/Qwen-VL",
224
trust_remote_code=True,
218
- max_num_seqs=5,
225
+ max_model_len=1024,
226
+ max_num_seqs=2,
227
)
228
229
prompt = f"{question}Picture 1: <img></img>\n"
@@ -229,8 +237,10 @@ def run_qwen2_vl(question, modality):
237
238
model_name = "Qwen/Qwen2-VL-7B-Instruct"
239
240
+ # Tested on L40
241
llm = LLM(
242
model=model_name,
243
+ max_model_len=8192,
244
max_num_seqs=5,
245
)
246
@@ -252,10 +262,10 @@ def run_mllama(question, modality):
262
# max_model_len (131072) for this model may cause OOM.
263
# You may lower either to run this example on lower-end GPUs.
264
255
- # The configuration below has been confirmed to launch on a
256
- # single H100 GPU.
265
+ # The configuration below has been confirmed to launch on a single L40 GPU.
266
llm = LLM(
267
model=model_name,
268
+ max_model_len=4096,
269
max_num_seqs=16,
270
enforce_eager=True,
271
)
examples/offline_inference_vision_language_multi_image.py
+10
-3
@@ -28,12 +28,18 @@ class ModelRequestData(NamedTuple):
28
chat_template: Optional[str]
29
30
31
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
32
+# lower-end GPUs.
33
+# Unless specified, these settings have been tested to work on a single L4.
34
+
35
+
36
def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
37
model_name = "Qwen/Qwen-VL-Chat"
38
llm = LLM(
39
model=model_name,
40
trust_remote_code=True,
36
- max_num_seqs=5,
41
+ max_model_len=1024,
42
+ max_num_seqs=2,
43
limit_mm_per_prompt={"image": len(image_urls)},
44
)
45
placeholders = "".join(f"Picture {i}: <img></img>\n"
@@ -83,6 +89,7 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
89
model="microsoft/Phi-3.5-vision-instruct",
90
trust_remote_code=True,
91
max_model_len=4096,
92
+ max_num_seqs=2,
93
limit_mm_per_prompt={"image": len(image_urls)},
94
mm_processor_kwargs={"num_crops": 4},
95
)
@@ -106,7 +113,6 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
113
llm = LLM(
114
model=model_name,
115
trust_remote_code=True,
109
- max_num_seqs=5,
116
max_model_len=4096,
117
limit_mm_per_prompt={"image": len(image_urls)},
118
)
@@ -148,10 +154,11 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
154
155
model_name = "Qwen/Qwen2-VL-7B-Instruct"
156
157
+ # Tested on L40
158
llm = LLM(
159
model=model_name,
153
- max_num_seqs=5,
160
max_model_len=32768 if process_vision_info is None else 4096,
161
+ max_num_seqs=5,
162
limit_mm_per_prompt={"image": len(image_urls)},
163
)
164
tests/conftest.py
+44
-40
@@ -246,17 +246,14 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
246
247
class HfRunner:
248
249
- def wrap_device(self, input: _T) -> _T:
250
- if not is_cpu():
251
- # Check if the input is already on the GPU
252
- if hasattr(input, 'device') and input.device.type == "cuda":
253
- return input # Already on GPU, no need to move
254
- return input.to("cuda")
255
- else:
256
- # Check if the input is already on the CPU
257
- if hasattr(input, 'device') and input.device.type == "cpu":
258
- return input # Already on CPU, no need to move
259
- return input.to("cpu")
249
+ def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
250
+ if device is None:
251
+ return self.wrap_device(input, "cpu" if is_cpu() else "cuda")
252
+
253
+ if hasattr(input, "device") and input.device.type == device:
254
+ return input
255
+
256
+ return input.to(device)
257
258
def __init__(
259
self,
@@ -333,7 +330,7 @@ class HfRunner:
330
inputs = self.postprocess_inputs(inputs)
331
332
output_ids = self.model.generate(
336
- **self.wrap_device(inputs),
333
+ **self.wrap_device(inputs, device=self.model.device.type),
334
use_cache=True,
335
**kwargs,
336
)
@@ -406,7 +403,7 @@ class HfRunner:
403
inputs = self.postprocess_inputs(inputs)
404
405
output = self.model.generate(
409
- **self.wrap_device(inputs),
406
+ **self.wrap_device(inputs, device=self.model.device.type),
407
use_cache=True,
408
do_sample=False,
409
max_new_tokens=max_tokens,
@@ -414,40 +411,39 @@ class HfRunner:
411
return_dict_in_generate=True,
412
**kwargs,
413
)
417
- seq_logprobs: List[torch.Tensor] = []
418
- for hidden_states in output.hidden_states:
419
- last_hidden_states = hidden_states[-1][0]
420
- logits = torch.matmul(
421
- last_hidden_states,
422
- self.model.get_output_embeddings().weight.t(),
423
- )
424
- if self.model.get_output_embeddings().bias is not None:
425
- logits += self.model.get_output_embeddings(
426
- ).bias.unsqueeze(0)
427
- logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
428
- seq_logprobs.append(logprobs)
414
+ seq_logprobs = self._hidden_states_to_seq_logprobs(
415
+ output.hidden_states)
416
all_logprobs.append(seq_logprobs)
417
return all_logprobs
418
432
- def _hidden_states_to_logprobs(
419
+ def _hidden_states_to_seq_logprobs(
420
self,
434
- hidden_states,
435
- num_logprobs,
436
- ) -> Tuple[List[Dict[int, float]], int]:
421
+ hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],
422
+ ) -> List[torch.Tensor]:
423
+ output_embeddings = self.model.get_output_embeddings()
424
+
425
seq_logprobs: List[torch.Tensor] = []
438
- output_len = len(hidden_states)
426
for _, hidden_state in enumerate(hidden_states):
427
last_hidden_states = hidden_state[-1][0]
428
logits = torch.matmul(
442
- last_hidden_states,
443
- self.model.get_output_embeddings().weight.t(),
429
+ last_hidden_states.to(output_embeddings.weight.device),
430
+ output_embeddings.weight.t(),
431
)
445
- if getattr(self.model.get_output_embeddings(), "bias",
446
- None) is not None:
447
- logits += self.model.get_output_embeddings().bias.unsqueeze(0)
432
+ if getattr(output_embeddings, "bias", None) is not None:
433
+ logits += output_embeddings.bias.unsqueeze(0)
434
logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
435
seq_logprobs.append(logprobs)
436
437
+ return seq_logprobs
438
+
439
+ def _hidden_states_to_logprobs(
440
+ self,
441
+ hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],
442
+ num_logprobs: int,
443
+ ) -> Tuple[List[Dict[int, float]], int]:
444
+ seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
445
+ output_len = len(hidden_states)
446
+
447
# convert to dict
448
seq_logprobs_lst: List[Dict[int, float]] = []
449
for tok_idx, tok_logprobs in enumerate(seq_logprobs):
@@ -500,7 +496,7 @@ class HfRunner:
496
inputs = self.postprocess_inputs(inputs)
497
498
output = self.model.generate(
503
- **self.wrap_device(inputs),
499
+ **self.wrap_device(inputs, device=self.model.device.type),
500
use_cache=True,
501
do_sample=False,
502
max_new_tokens=max_tokens,
@@ -543,12 +539,20 @@ class HfRunner:
539
540
for (encoder_prompt,
541
decoder_prompt) in to_enc_dec_tuple_list(encoder_decoder_prompts):
542
+
543
encoder_input_ids = self.wrap_device(
547
- self.tokenizer(encoder_prompt, return_tensors="pt").input_ids)
548
- decoder_input_ids = (
549
- None if decoder_prompt is None else self.wrap_device(
544
+ self.tokenizer(encoder_prompt, return_tensors="pt").input_ids,
545
+ device=self.model.device.type,
546
+ )
547
+
548
+ if decoder_prompt is None:
549
+ decoder_input_ids = None
550
+ else:
551
+ decoder_input_ids = self.wrap_device(
552
self.tokenizer(decoder_prompt,
551
- return_tensors="pt").input_ids))
553
+ return_tensors="pt").input_ids,
554
+ device=self.model.device.type,
555
+ )
556
557
output = self.model.generate(
558
encoder_input_ids,
tests/models/decoder_only/vision_language/test_llava_onevision.py
+15
-14
@@ -16,8 +16,7 @@ from ...utils import check_logprobs_close
16
# Video test
17
HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
18
"sample_demo_1":
19
- "<|im_start|>user <video>\nwhy is this video funny? \
20
- <|im_end|><|im_start|>assistant\n"
19
+ "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n" # noqa: E501
20
})
21
22
models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
@@ -165,6 +164,9 @@ def run_video_test(
164
)
165
166
167
+@pytest.mark.skip(
168
+ reason=
169
+ "Model is too big, test passed on L40 locally but will OOM on CI machine.")
170
@pytest.mark.parametrize("model", models)
171
@pytest.mark.parametrize(
172
"size_factors",
@@ -208,6 +210,9 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
210
)
211
212
213
+@pytest.mark.skip(
214
+ reason=
215
+ "Model is too big, test passed on L40 locally but will OOM on CI machine.")
216
@pytest.mark.parametrize("model", models)
217
@pytest.mark.parametrize(
218
"sizes",
@@ -254,9 +259,8 @@ def run_image_test(
259
# max_model_len should be greater than image_feature_size
260
with vllm_runner(model,
261
dtype=dtype,
257
- max_num_seqs=1,
262
max_model_len=16384,
259
- gpu_memory_utilization=0.98,
263
+ max_num_seqs=2,
264
tensor_parallel_size=tensor_parallel_size,
265
distributed_executor_backend=distributed_executor_backend,
266
enforce_eager=True,
@@ -302,8 +306,9 @@ def run_image_test(
306
)
307
308
305
-# FIXME: Swap to a smaller model for this architecture
306
-@pytest.mark.skip(reason="Model OOMing on CI")
309
+@pytest.mark.skip(
310
+ reason=
311
+ "Model is too big, test passed on L40 locally but will OOM on CI machine.")
312
@pytest.mark.parametrize("model", models)
313
@pytest.mark.parametrize("dtype", ["half"])
314
@pytest.mark.parametrize("max_tokens", [128])
@@ -316,14 +321,10 @@ def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
321
322
inputs = [(
323
[
319
- "<|im_start|>user <image><image>\nDescribe 2 images. \
320
- <|im_end|><|im_start|>assistant\n",
321
- "<|im_start|>user <image><image>\nDescribe 2 images. \
322
- <|im_end|><|im_start|>assistant\n",
323
- "<|im_start|>user <image><image><image><image>\nDescribe 4 images. \
324
- <|im_end|><|im_start|>assistant\n",
325
- "<|im_start|>user <image>\nWhat is the season? \
326
- <|im_end|><|im_start|>assistant\n",
324
+ "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
325
+ "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
326
+ "<|im_start|>user\n<image><image><image><image>\nDescribe 4 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
327
+ "<|im_start|>user\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
328
],
329
[
330
[stop_sign, cherry_blossom],
tests/models/decoder_only/vision_language/test_minicpmv.py
+1
-1
@@ -79,7 +79,7 @@ def run_test(
79
# max_model_len should be greater than image_feature_size
80
with vllm_runner(model,
81
max_model_len=4096,
82
- max_num_seqs=1,
82
+ max_num_seqs=2,
83
dtype=dtype,
84
limit_mm_per_prompt={"image": mm_limit},
85
tensor_parallel_size=tensor_parallel_size,
tests/models/decoder_only/vision_language/test_phi3v.py
+1
-1
@@ -90,7 +90,7 @@ def run_test(
90
# max_model_len should be greater than image_feature_size
91
with vllm_runner(model,
92
max_model_len=4096,
93
- max_num_seqs=1,
93
+ max_num_seqs=2,
94
dtype=dtype,
95
limit_mm_per_prompt={"image": mm_limit},
96
tensor_parallel_size=tensor_parallel_size,
tests/models/decoder_only/vision_language/test_qwen.py
+1
-1
@@ -221,7 +221,7 @@ def run_test(
221
# Qwen encodes each image into a fixed content size of 256
222
with vllm_runner(model,
223
max_model_len=1024,
224
- max_num_seqs=1,
224
+ max_num_seqs=2,
225
dtype=dtype,
226
limit_mm_per_prompt={"image": mm_limit},
227
tensor_parallel_size=tensor_parallel_size,
tests/models/encoder_decoder/vision_language/test_broadcast.py
new
+35
@@ -0,0 +1,35 @@
1
+import pytest
2
+
3
+from ....utils import multi_gpu_test
4
+
5
+
6
+@multi_gpu_test(num_gpus=2)
7
+@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
8
+@pytest.mark.parametrize("model", [
9
+ "meta-llama/Llama-3.2-11B-Vision-Instruct",
10
+])
11
+def test_models(hf_runner, vllm_runner, image_assets,
12
+ distributed_executor_backend, model) -> None:
13
+
14
+ dtype = "half"
15
+ max_tokens = 5
16
+ num_logprobs = 5
17
+ tensor_parallel_size = 2
18
+
19
+ if model.startswith("meta-llama/Llama-3.2-11B-Vision-Instruct"):
20
+ from .test_mllama import models, run_test
21
+ else:
22
+ raise NotImplementedError(f"Unsupported model: {model}")
23
+
24
+ run_test(
25
+ hf_runner,
26
+ vllm_runner,
27
+ image_assets,
28
+ model=models[0],
29
+ size_factors=[0.25, 0.5, 1.0],
30
+ dtype=dtype,
31
+ max_tokens=max_tokens,
32
+ num_logprobs=num_logprobs,
33
+ tensor_parallel_size=tensor_parallel_size,
34
+ distributed_executor_backend=distributed_executor_backend,
35
+ )
tests/models/encoder_decoder/vision_language/test_mllama.py
+69
-84
@@ -9,7 +9,6 @@ from vllm.sequence import SampleLogprobs
9
10
from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11
_ImageAssets)
12
-from ....utils import multi_gpu_test
12
from ...utils import check_logprobs_close
13
14
_LIMIT_IMAGE_PER_PROMPT = 1
@@ -47,14 +46,46 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
46
if token_id != image_token_id or output_ids[idx - 1] != image_token_id
47
]
48
50
- assert output_str[0] == " "
51
- hf_output_str = output_str[1:]
49
+ hf_output_str = output_str
50
if hf_output_ids[-1] == eos_token_id:
51
hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
52
53
return hf_output_ids, hf_output_str, out_logprobs
54
55
56
+def _get_inputs(
57
+ image_assets: _ImageAssets,
58
+ *,
59
+ size_factors: Optional[List[float]] = None,
60
+ sizes: Optional[List[Tuple[int, int]]] = None,
61
+) -> List[Tuple[List[str], PromptImageInput]]:
62
+ images = [asset.pil_image for asset in image_assets]
63
+
64
+ if size_factors is not None:
65
+ inputs_per_image = [(
66
+ [prompt for _ in size_factors],
67
+ [rescale_image_size(image, factor) for factor in size_factors],
68
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
69
+ elif sizes is not None:
70
+ inputs_per_image = [(
71
+ [
72
+ prompt if size is not None else text_only_prompts[0]
73
+ for size in sizes
74
+ ],
75
+ [
76
+ image.resize(size) if size is not None else None
77
+ for size in sizes
78
+ ],
79
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
80
+ if len(sizes) == 0:
81
+ inputs_per_image.append(
82
+ (text_only_prompts, [None] * len(text_only_prompts)))
83
+ else:
84
+ raise ValueError("You must provide either `size_factors` or `sizes`")
85
+
86
+ return inputs_per_image
87
+
88
+
89
@overload
90
def run_test(
91
hf_runner: Type[HfRunner],
@@ -103,39 +134,17 @@ def run_test(
134
tensor_parallel_size: int,
135
distributed_executor_backend: Optional[str] = None,
136
):
106
- images = [asset.pil_image for asset in image_assets]
107
-
108
- if size_factors is not None:
109
- inputs_per_image = [(
110
- [prompt for _ in size_factors],
111
- [rescale_image_size(image, factor) for factor in size_factors],
112
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113
- elif sizes is not None:
114
- inputs_per_image = [(
115
- [
116
- prompt if size is not None else text_only_prompts[0]
117
- for size in sizes
118
- ],
119
- [
120
- image.resize(size) if size is not None else None
121
- for size in sizes
122
- ],
123
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
124
- if len(sizes) == 0:
125
- inputs_per_image.append(
126
- (text_only_prompts, [None] * len(text_only_prompts)))
127
- else:
128
- raise ValueError("You must provide either `size_factors` or `sizes`")
129
-
130
- _run_test(hf_runner,
131
- vllm_runner,
132
- inputs_per_image,
133
- model,
134
- dtype=dtype,
135
- max_tokens=max_tokens,
136
- num_logprobs=num_logprobs,
137
- tensor_parallel_size=tensor_parallel_size,
138
- distributed_executor_backend=distributed_executor_backend)
137
+ _run_test(
138
+ hf_runner,
139
+ vllm_runner,
140
+ _get_inputs(image_assets, size_factors=size_factors, sizes=sizes),
141
+ model,
142
+ dtype=dtype,
143
+ max_tokens=max_tokens,
144
+ num_logprobs=num_logprobs,
145
+ tensor_parallel_size=tensor_parallel_size,
146
+ distributed_executor_backend=distributed_executor_backend,
147
+ )
148
149
150
def _run_test(
@@ -167,8 +176,8 @@ def _run_test(
176
# max_model_len should be greater than image_feature_size
177
with vllm_runner(model,
178
dtype=dtype,
170
- max_num_seqs=16,
179
max_model_len=4096,
180
+ max_num_seqs=2,
181
tensor_parallel_size=tensor_parallel_size,
182
distributed_executor_backend=distributed_executor_backend,
183
enforce_eager=True,
@@ -185,7 +194,6 @@ def _run_test(
194
def process(hf_inputs: BatchEncoding):
195
return hf_inputs
196
188
- from transformers import AutoConfig
197
from transformers.models.mllama import MllamaConfig as MllamaConfigHf
198
199
# use transformer's MllamaConfig for hf_runner
@@ -193,6 +201,7 @@ def _run_test(
201
AutoConfig.register("mllama", MllamaConfigHf, exist_ok=True)
202
with hf_runner(model,
203
dtype=dtype,
204
+ model_kwargs={"device_map": "auto"},
205
postprocess_inputs=process,
206
auto_cls=AutoModelForVision2Seq) as hf_model:
207
hf_outputs_per_image = [
@@ -218,26 +227,29 @@ def _run_test(
227
)
228
229
230
+SIZES = [
231
+ # Text only
232
+ [],
233
+ # Single-size
234
+ [(512, 512)],
235
+ # Single-size, batched
236
+ [(512, 512), (512, 512), (512, 512)],
237
+ # Multi-size, batched
238
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
239
+ (1024, 1024), (512, 1536), (512, 2028)],
240
+ # Multi-size, batched, including text only
241
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
242
+ (1024, 1024), (512, 1536), (512, 2028), None],
243
+ # mllama has 8 possible aspect ratios, carefully set the sizes
244
+ # to cover all of them
245
+]
246
+
247
+
248
+@pytest.mark.skip(
249
+ reason=
250
+ "Model is too big, test passed on L40 locally but will OOM on CI machine.")
251
@pytest.mark.parametrize("model", models)
222
-@pytest.mark.parametrize(
223
- "sizes",
224
- [
225
- # Text only
226
- [],
227
- # Single-size
228
- [(512, 512)],
229
- # Single-size, batched
230
- [(512, 512), (512, 512), (512, 512)],
231
- # Multi-size, batched
232
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
233
- (1024, 1024), (512, 1536), (512, 2028)],
234
- # Multi-size, batched, including text only
235
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
236
- (1024, 1024), (512, 1536), (512, 2028), None],
237
- # mllama has 8 possible aspect ratios, carefully set the sizes
238
- # to cover all of them
239
- ],
240
-)
252
+@pytest.mark.parametrize("sizes", SIZES)
253
@pytest.mark.parametrize("dtype", ["bfloat16"])
254
@pytest.mark.parametrize("max_tokens", [128])
255
@pytest.mark.parametrize("num_logprobs", [5])
@@ -254,30 +266,3 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
266
num_logprobs=num_logprobs,
267
tensor_parallel_size=1,
268
)
257
-
258
-
259
-@multi_gpu_test(num_gpus=2)
260
-@pytest.mark.parametrize("model", models)
261
-@pytest.mark.parametrize(
262
- "sizes",
263
- [
264
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
265
- (1024, 1024), (512, 1536), (512, 2028), None],
266
- ],
267
-)
268
-@pytest.mark.parametrize("dtype", ["bfloat16"])
269
-@pytest.mark.parametrize("max_tokens", [128])
270
-@pytest.mark.parametrize("num_logprobs", [5])
271
-def test_models_distributed(hf_runner, vllm_runner, image_assets, model, sizes,
272
- dtype, max_tokens, num_logprobs) -> None:
273
- run_test(
274
- hf_runner,
275
- vllm_runner,
276
- image_assets,
277
- model,
278
- sizes=sizes,
279
- dtype=dtype,
280
- max_tokens=max_tokens,
281
- num_logprobs=num_logprobs,
282
- tensor_parallel_size=2,
283
- )
tests/models/utils.py
+8
-1
@@ -1,9 +1,12 @@
1
import warnings
2
from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4
+import torch
5
+
6
from vllm.config import ModelConfig
7
from vllm.inputs import InputContext
8
from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9
+from vllm.utils import is_cpu
10
11
TokensText = Tuple[List[int], str]
12
@@ -247,6 +250,7 @@ def check_logprobs_close(
250
def build_model_context(model_name: str,
251
tokenizer_name: Optional[str] = None,
252
trust_remote_code: bool = False,
253
+ dtype: Optional[Union[str, torch.dtype]] = None,
254
mm_processor_kwargs: Optional[Dict] = None,
255
limit_mm_per_prompt: Optional[Dict] = None):
256
"""Creates an InputContext for a given model.
@@ -264,12 +268,15 @@ def build_model_context(model_name: str,
268
"""
269
if tokenizer_name is None:
270
tokenizer_name = model_name
271
+ if dtype is None:
272
+ dtype = "bfloat16" if is_cpu() else "half"
273
+
274
model_config = ModelConfig(
275
model_name,
276
tokenizer_name,
277
tokenizer_mode="auto",
278
trust_remote_code=trust_remote_code,
272
- dtype="float32",
279
+ dtype=dtype,
280
seed=0,
281
mm_processor_kwargs=mm_processor_kwargs,
282
limit_mm_per_prompt=limit_mm_per_prompt,
vllm/inputs/registry.py
+2
-10
@@ -185,16 +185,8 @@ class InputRegistry:
185
return wrapper
186
187
def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
188
- if model_cls in self._dummy_encoder_factories_by_model_type:
189
- dummy_factory = self._dummy_encoder_factories_by_model_type[
190
- model_cls]
191
- else:
192
- logger.warning(
193
- "No dummy encoder data factory registered to %s. "
194
- "Using the dummy data factory for the model instead.",
195
- model_cls)
196
- dummy_factory = self._get_dummy_data_factory(model_cls)
197
- return dummy_factory
188
+ return self._dummy_encoder_factories_by_model_type \
189
+ .get(model_cls, self._default_dummy_data_factory)
190
191
def dummy_data_for_profiling(
192
self,
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
-1
@@ -159,7 +159,8 @@ def apply_fp8_linear(
159
160
# Making sure the dummy tensor is on the same device as the weight
161
global TORCH_DEVICE_IDENTITY
162
- if TORCH_DEVICE_IDENTITY.device != weight.device:
162
+ if (TORCH_DEVICE_IDENTITY is not None
163
+ and TORCH_DEVICE_IDENTITY.device != weight.device):
164
TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
165
166
# GEMM