@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1a3f5e831
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[CI/Build] Update models tests & examples (#8874)

Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Sep 29, 2024 at 00:54 UTC e1a3f5e831a467b2867a66e0e56ac0f70ed44394
13 files changed +239 -184
.buildkite/test-pipeline.yaml
+32 -19
@@ -9,6 +9,7 @@
9 # label(str): the name of the test. emoji allowed.
10 # fast_check(bool): whether to run this on each commit on fastcheck pipeline.
11 # fast_check_only(bool): run this test on fastcheck pipeline only
12 +# optional(bool): never run this test by default (i.e. need to unblock manually)
13 # command(str): the single command to run for tests. incompatible with commands.
14 # commands(list): the list of commands to run for test. incompatbile with command.
15 # mirror_hardwares(list): the list of hardwares to run the test on as well. currently only supports [amd]
@@ -39,7 +40,7 @@ steps:
40 # Check API reference (if it fails, you may have missing mock imports)
41 - grep \"sig sig-object py\" build/html/dev/sampling_params.html
42
42 -- label: Async Engine, Inputs, Utils, Worker Test # 15min
43 +- label: Async Engine, Inputs, Utils, Worker Test # 24min
44 fast_check: true
45 source_file_dependencies:
46 - vllm/
@@ -81,7 +82,7 @@ steps:
82 commands:
83 - pytest -v -s core
84
84 -- label: Entrypoints Test # 20min
85 +- label: Entrypoints Test # 40min
86 working_dir: "/vllm-workspace/tests"
87 fast_check: true
88 mirror_hardwares: [amd]
@@ -151,7 +152,7 @@ steps:
152 # OOM in the CI unless we run this separately
153 - pytest -v -s tokenization
154
154 -- label: Examples Test # 12min
155 +- label: Examples Test # 15min
156 working_dir: "/vllm-workspace/examples"
157 #mirror_hardwares: [amd]
158 source_file_dependencies:
@@ -169,7 +170,7 @@ steps:
170 - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
171 - python3 offline_inference_encoder_decoder.py
172
172 -- label: Prefix Caching Test # 7min
173 +- label: Prefix Caching Test # 9min
174 #mirror_hardwares: [amd]
175 source_file_dependencies:
176 - vllm/
@@ -177,7 +178,7 @@ steps:
178 commands:
179 - pytest -v -s prefix_caching
180
180 -- label: Samplers Test # 18min
181 +- label: Samplers Test # 36min
182 source_file_dependencies:
183 - vllm/model_executor/layers
184 - vllm/sampling_metadata.py
@@ -193,7 +194,7 @@ steps:
194 - tests/test_logits_processor
195 command: pytest -v -s test_logits_processor.py
196
196 -- label: Speculative decoding tests # 22min
197 +- label: Speculative decoding tests # 30min
198 source_file_dependencies:
199 - vllm/spec_decode
200 - tests/spec_decode
@@ -203,7 +204,7 @@ steps:
204 - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
205 - pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
206
206 -- label: LoRA Test %N # 30min each
207 +- label: LoRA Test %N # 15min each
208 mirror_hardwares: [amd]
209 source_file_dependencies:
210 - vllm/lora
@@ -211,7 +212,7 @@ steps:
212 command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
213 parallelism: 4
214
214 -- label: "PyTorch Fullgraph Smoke Test"
215 +- label: "PyTorch Fullgraph Smoke Test" # 9min
216 fast_check: true
217 source_file_dependencies:
218 - vllm/
@@ -219,14 +220,14 @@ steps:
220 commands:
221 - pytest -v -s compile/test_full_graph_smoke.py
222
222 -- label: "PyTorch Fullgraph Test"
223 +- label: "PyTorch Fullgraph Test" # 18min
224 source_file_dependencies:
225 - vllm/
226 - tests/compile
227 commands:
228 - pytest -v -s compile/test_full_graph.py
229
229 -- label: Kernels Test %N # 30min each
230 +- label: Kernels Test %N # 1h each
231 mirror_hardwares: [amd]
232 source_file_dependencies:
233 - csrc/
@@ -256,7 +257,7 @@ steps:
257 - pip install aiohttp
258 - bash run-benchmarks.sh
259
259 -- label: Quantization Test # 15min
260 +- label: Quantization Test # 33min
261 source_file_dependencies:
262 - csrc/
263 - vllm/model_executor/layers/quantization
@@ -300,7 +301,7 @@ steps:
301 - pytest -v -s models/test_oot_registration.py # it needs a clean process
302 - pytest -v -s models/*.py --ignore=models/test_oot_registration.py
303
303 -- label: Decoder-only Language Models Test # 1h3min
304 +- label: Decoder-only Language Models Test # 1h36min
305 #mirror_hardwares: [amd]
306 source_file_dependencies:
307 - vllm/
@@ -308,7 +309,7 @@ steps:
309 commands:
310 - pytest -v -s models/decoder_only/language
311
311 -- label: Decoder-only Multi-Modal Models Test # 56min
312 +- label: Decoder-only Multi-Modal Models Test # 1h31min
313 #mirror_hardwares: [amd]
314 source_file_dependencies:
315 - vllm/
@@ -318,15 +319,25 @@ steps:
319 - pytest -v -s models/decoder_only/audio_language
320 - pytest -v -s models/decoder_only/vision_language
321
321 -- label: Other Models Test # 5min
322 +- label: Other Models Test # 6min
323 #mirror_hardwares: [amd]
324 source_file_dependencies:
325 - vllm/
326 - tests/models/embedding/language
327 - tests/models/encoder_decoder/language
328 + - tests/models/encoder_decoder/vision_language
329 commands:
330 - pytest -v -s models/embedding/language
331 - pytest -v -s models/encoder_decoder/language
332 + - pytest -v -s models/encoder_decoder/vision_language
333 +
334 +- label: Custom Models Test
335 + #mirror_hardwares: [amd]
336 + optional: true
337 + commands:
338 + # PR authors can temporarily add commands below to test individual models
339 + # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
340 + # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
341
342 ##### 1 GPU test #####
343 ##### multi gpus test #####
@@ -359,7 +370,7 @@ steps:
370 - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
371 - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
372
362 -- label: Distributed Tests (2 GPUs) # 28min
373 +- label: Distributed Tests (2 GPUs) # 40min
374 #mirror_hardwares: [amd]
375 working_dir: "/vllm-workspace/tests"
376 num_gpus: 2
@@ -376,14 +387,16 @@ steps:
387 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
388 - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
389 # Avoid importing model tests that cause CUDA reinitialization error
379 - - pytest models/encoder_decoder/language/test_bart.py models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
390 + - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
391 + - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
392 + - pytest models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
393 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
394 - pip install -e ./plugins/vllm_add_dummy_model
395 - pytest -v -s distributed/test_distributed_oot.py
396 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
397 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
398
386 -- label: Multi-step Tests (4 GPUs) # 21min
399 +- label: Multi-step Tests (4 GPUs) # 36min
400 working_dir: "/vllm-workspace/tests"
401 num_gpus: 4
402 source_file_dependencies:
@@ -401,7 +414,7 @@ steps:
414 - pytest -v -s multi_step/test_correctness_async_llm.py
415 - pytest -v -s multi_step/test_correctness_llm.py
416
404 -- label: Pipeline Parallelism Test # 23min
417 +- label: Pipeline Parallelism Test # 45min
418 working_dir: "/vllm-workspace/tests"
419 num_gpus: 4
420 source_file_dependencies:
@@ -427,7 +440,7 @@ steps:
440 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
441 - pytest -v -s -x lora/test_long_context.py
442
430 -- label: Weight Loading Multiple GPU Test
443 +- label: Weight Loading Multiple GPU Test # 33min
444 working_dir: "/vllm-workspace/tests"
445 num_gpus: 2
446 source_file_dependencies:
examples/offline_inference_vision_language.py
+19 -9
@@ -12,6 +12,10 @@ from vllm.assets.image import ImageAsset
12 from vllm.assets.video import VideoAsset
13 from vllm.utils import FlexibleArgumentParser
14
15 +# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
16 +# lower-end GPUs.
17 +# Unless specified, these settings have been tested to work on a single L4.
18 +
19
20 # LLaVA-1.5
21 def run_llava(question, modality):
@@ -19,7 +23,7 @@ def run_llava(question, modality):
23
24 prompt = f"USER: <image>\n{question}\nASSISTANT:"
25
22 - llm = LLM(model="llava-hf/llava-1.5-7b-hf")
26 + llm = LLM(model="llava-hf/llava-1.5-7b-hf", max_model_len=4096)
27 stop_token_ids = None
28 return llm, prompt, stop_token_ids
29
@@ -57,7 +61,7 @@ def run_llava_onevision(question, modality):
61 <|im_start|>assistant\n"
62
63 llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
60 - max_model_len=32768)
64 + max_model_len=16384)
65 stop_token_ids = None
66 return llm, prompt, stop_token_ids
67
@@ -67,7 +71,7 @@ def run_fuyu(question, modality):
71 assert modality == "image"
72
73 prompt = f"{question}\n"
70 - llm = LLM(model="adept/fuyu-8b")
74 + llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2)
75 stop_token_ids = None
76 return llm, prompt, stop_token_ids
77
@@ -99,7 +103,8 @@ def run_phi3v(question, modality):
103 llm = LLM(
104 model="microsoft/Phi-3-vision-128k-instruct",
105 trust_remote_code=True,
102 - max_num_seqs=5,
106 + max_model_len=4096,
107 + max_num_seqs=2,
108 mm_processor_kwargs={"num_crops": 16},
109 )
110 stop_token_ids = None
@@ -122,7 +127,7 @@ def run_chameleon(question, modality):
127 assert modality == "image"
128
129 prompt = f"{question}<image>"
125 - llm = LLM(model="facebook/chameleon-7b")
130 + llm = LLM(model="facebook/chameleon-7b", max_model_len=4096)
131 stop_token_ids = None
132 return llm, prompt, stop_token_ids
133
@@ -145,6 +150,8 @@ def run_minicpmv(question, modality):
150 trust_remote_code=True)
151 llm = LLM(
152 model=model_name,
153 + max_model_len=4096,
154 + max_num_seqs=2,
155 trust_remote_code=True,
156 )
157 # NOTE The stop_token_ids are different for various versions of MiniCPM-V
@@ -177,7 +184,7 @@ def run_internvl(question, modality):
184 llm = LLM(
185 model=model_name,
186 trust_remote_code=True,
180 - max_num_seqs=5,
187 + max_model_len=4096,
188 )
189
190 tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -215,7 +222,8 @@ def run_qwen_vl(question, modality):
222 llm = LLM(
223 model="Qwen/Qwen-VL",
224 trust_remote_code=True,
218 - max_num_seqs=5,
225 + max_model_len=1024,
226 + max_num_seqs=2,
227 )
228
229 prompt = f"{question}Picture 1: <img></img>\n"
@@ -229,8 +237,10 @@ def run_qwen2_vl(question, modality):
237
238 model_name = "Qwen/Qwen2-VL-7B-Instruct"
239
240 + # Tested on L40
241 llm = LLM(
242 model=model_name,
243 + max_model_len=8192,
244 max_num_seqs=5,
245 )
246
@@ -252,10 +262,10 @@ def run_mllama(question, modality):
262 # max_model_len (131072) for this model may cause OOM.
263 # You may lower either to run this example on lower-end GPUs.
264
255 - # The configuration below has been confirmed to launch on a
256 - # single H100 GPU.
265 + # The configuration below has been confirmed to launch on a single L40 GPU.
266 llm = LLM(
267 model=model_name,
268 + max_model_len=4096,
269 max_num_seqs=16,
270 enforce_eager=True,
271 )
examples/offline_inference_vision_language_multi_image.py
+10 -3
@@ -28,12 +28,18 @@ class ModelRequestData(NamedTuple):
28 chat_template: Optional[str]
29
30
31 +# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
32 +# lower-end GPUs.
33 +# Unless specified, these settings have been tested to work on a single L4.
34 +
35 +
36 def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
37 model_name = "Qwen/Qwen-VL-Chat"
38 llm = LLM(
39 model=model_name,
40 trust_remote_code=True,
36 - max_num_seqs=5,
41 + max_model_len=1024,
42 + max_num_seqs=2,
43 limit_mm_per_prompt={"image": len(image_urls)},
44 )
45 placeholders = "".join(f"Picture {i}: <img></img>\n"
@@ -83,6 +89,7 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
89 model="microsoft/Phi-3.5-vision-instruct",
90 trust_remote_code=True,
91 max_model_len=4096,
92 + max_num_seqs=2,
93 limit_mm_per_prompt={"image": len(image_urls)},
94 mm_processor_kwargs={"num_crops": 4},
95 )
@@ -106,7 +113,6 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
113 llm = LLM(
114 model=model_name,
115 trust_remote_code=True,
109 - max_num_seqs=5,
116 max_model_len=4096,
117 limit_mm_per_prompt={"image": len(image_urls)},
118 )
@@ -148,10 +154,11 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
154
155 model_name = "Qwen/Qwen2-VL-7B-Instruct"
156
157 + # Tested on L40
158 llm = LLM(
159 model=model_name,
153 - max_num_seqs=5,
160 max_model_len=32768 if process_vision_info is None else 4096,
161 + max_num_seqs=5,
162 limit_mm_per_prompt={"image": len(image_urls)},
163 )
164
tests/conftest.py
+44 -40
@@ -246,17 +246,14 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
246
247 class HfRunner:
248
249 - def wrap_device(self, input: _T) -> _T:
250 - if not is_cpu():
251 - # Check if the input is already on the GPU
252 - if hasattr(input, 'device') and input.device.type == "cuda":
253 - return input # Already on GPU, no need to move
254 - return input.to("cuda")
255 - else:
256 - # Check if the input is already on the CPU
257 - if hasattr(input, 'device') and input.device.type == "cpu":
258 - return input # Already on CPU, no need to move
259 - return input.to("cpu")
249 + def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
250 + if device is None:
251 + return self.wrap_device(input, "cpu" if is_cpu() else "cuda")
252 +
253 + if hasattr(input, "device") and input.device.type == device:
254 + return input
255 +
256 + return input.to(device)
257
258 def __init__(
259 self,
@@ -333,7 +330,7 @@ class HfRunner:
330 inputs = self.postprocess_inputs(inputs)
331
332 output_ids = self.model.generate(
336 - **self.wrap_device(inputs),
333 + **self.wrap_device(inputs, device=self.model.device.type),
334 use_cache=True,
335 **kwargs,
336 )
@@ -406,7 +403,7 @@ class HfRunner:
403 inputs = self.postprocess_inputs(inputs)
404
405 output = self.model.generate(
409 - **self.wrap_device(inputs),
406 + **self.wrap_device(inputs, device=self.model.device.type),
407 use_cache=True,
408 do_sample=False,
409 max_new_tokens=max_tokens,
@@ -414,40 +411,39 @@ class HfRunner:
411 return_dict_in_generate=True,
412 **kwargs,
413 )
417 - seq_logprobs: List[torch.Tensor] = []
418 - for hidden_states in output.hidden_states:
419 - last_hidden_states = hidden_states[-1][0]
420 - logits = torch.matmul(
421 - last_hidden_states,
422 - self.model.get_output_embeddings().weight.t(),
423 - )
424 - if self.model.get_output_embeddings().bias is not None:
425 - logits += self.model.get_output_embeddings(
426 - ).bias.unsqueeze(0)
427 - logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
428 - seq_logprobs.append(logprobs)
414 + seq_logprobs = self._hidden_states_to_seq_logprobs(
415 + output.hidden_states)
416 all_logprobs.append(seq_logprobs)
417 return all_logprobs
418
432 - def _hidden_states_to_logprobs(
419 + def _hidden_states_to_seq_logprobs(
420 self,
434 - hidden_states,
435 - num_logprobs,
436 - ) -> Tuple[List[Dict[int, float]], int]:
421 + hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],
422 + ) -> List[torch.Tensor]:
423 + output_embeddings = self.model.get_output_embeddings()
424 +
425 seq_logprobs: List[torch.Tensor] = []
438 - output_len = len(hidden_states)
426 for _, hidden_state in enumerate(hidden_states):
427 last_hidden_states = hidden_state[-1][0]
428 logits = torch.matmul(
442 - last_hidden_states,
443 - self.model.get_output_embeddings().weight.t(),
429 + last_hidden_states.to(output_embeddings.weight.device),
430 + output_embeddings.weight.t(),
431 )
445 - if getattr(self.model.get_output_embeddings(), "bias",
446 - None) is not None:
447 - logits += self.model.get_output_embeddings().bias.unsqueeze(0)
432 + if getattr(output_embeddings, "bias", None) is not None:
433 + logits += output_embeddings.bias.unsqueeze(0)
434 logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
435 seq_logprobs.append(logprobs)
436
437 + return seq_logprobs
438 +
439 + def _hidden_states_to_logprobs(
440 + self,
441 + hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],
442 + num_logprobs: int,
443 + ) -> Tuple[List[Dict[int, float]], int]:
444 + seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
445 + output_len = len(hidden_states)
446 +
447 # convert to dict
448 seq_logprobs_lst: List[Dict[int, float]] = []
449 for tok_idx, tok_logprobs in enumerate(seq_logprobs):
@@ -500,7 +496,7 @@ class HfRunner:
496 inputs = self.postprocess_inputs(inputs)
497
498 output = self.model.generate(
503 - **self.wrap_device(inputs),
499 + **self.wrap_device(inputs, device=self.model.device.type),
500 use_cache=True,
501 do_sample=False,
502 max_new_tokens=max_tokens,
@@ -543,12 +539,20 @@ class HfRunner:
539
540 for (encoder_prompt,
541 decoder_prompt) in to_enc_dec_tuple_list(encoder_decoder_prompts):
542 +
543 encoder_input_ids = self.wrap_device(
547 - self.tokenizer(encoder_prompt, return_tensors="pt").input_ids)
548 - decoder_input_ids = (
549 - None if decoder_prompt is None else self.wrap_device(
544 + self.tokenizer(encoder_prompt, return_tensors="pt").input_ids,
545 + device=self.model.device.type,
546 + )
547 +
548 + if decoder_prompt is None:
549 + decoder_input_ids = None
550 + else:
551 + decoder_input_ids = self.wrap_device(
552 self.tokenizer(decoder_prompt,
551 - return_tensors="pt").input_ids))
553 + return_tensors="pt").input_ids,
554 + device=self.model.device.type,
555 + )
556
557 output = self.model.generate(
558 encoder_input_ids,
tests/models/decoder_only/vision_language/test_llava_onevision.py
+15 -14
@@ -16,8 +16,7 @@ from ...utils import check_logprobs_close
16 # Video test
17 HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
18 "sample_demo_1":
19 - "<|im_start|>user <video>\nwhy is this video funny? \
20 - <|im_end|><|im_start|>assistant\n"
19 + "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n" # noqa: E501
20 })
21
22 models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
@@ -165,6 +164,9 @@ def run_video_test(
164 )
165
166
167 +@pytest.mark.skip(
168 + reason=
169 + "Model is too big, test passed on L40 locally but will OOM on CI machine.")
170 @pytest.mark.parametrize("model", models)
171 @pytest.mark.parametrize(
172 "size_factors",
@@ -208,6 +210,9 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
210 )
211
212
213 +@pytest.mark.skip(
214 + reason=
215 + "Model is too big, test passed on L40 locally but will OOM on CI machine.")
216 @pytest.mark.parametrize("model", models)
217 @pytest.mark.parametrize(
218 "sizes",
@@ -254,9 +259,8 @@ def run_image_test(
259 # max_model_len should be greater than image_feature_size
260 with vllm_runner(model,
261 dtype=dtype,
257 - max_num_seqs=1,
262 max_model_len=16384,
259 - gpu_memory_utilization=0.98,
263 + max_num_seqs=2,
264 tensor_parallel_size=tensor_parallel_size,
265 distributed_executor_backend=distributed_executor_backend,
266 enforce_eager=True,
@@ -302,8 +306,9 @@ def run_image_test(
306 )
307
308
305 -# FIXME: Swap to a smaller model for this architecture
306 -@pytest.mark.skip(reason="Model OOMing on CI")
309 +@pytest.mark.skip(
310 + reason=
311 + "Model is too big, test passed on L40 locally but will OOM on CI machine.")
312 @pytest.mark.parametrize("model", models)
313 @pytest.mark.parametrize("dtype", ["half"])
314 @pytest.mark.parametrize("max_tokens", [128])
@@ -316,14 +321,10 @@ def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
321
322 inputs = [(
323 [
319 - "<|im_start|>user <image><image>\nDescribe 2 images. \
320 - <|im_end|><|im_start|>assistant\n",
321 - "<|im_start|>user <image><image>\nDescribe 2 images. \
322 - <|im_end|><|im_start|>assistant\n",
323 - "<|im_start|>user <image><image><image><image>\nDescribe 4 images. \
324 - <|im_end|><|im_start|>assistant\n",
325 - "<|im_start|>user <image>\nWhat is the season? \
326 - <|im_end|><|im_start|>assistant\n",
324 + "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
325 + "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
326 + "<|im_start|>user\n<image><image><image><image>\nDescribe 4 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
327 + "<|im_start|>user\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
328 ],
329 [
330 [stop_sign, cherry_blossom],
tests/models/decoder_only/vision_language/test_minicpmv.py
+1 -1
@@ -79,7 +79,7 @@ def run_test(
79 # max_model_len should be greater than image_feature_size
80 with vllm_runner(model,
81 max_model_len=4096,
82 - max_num_seqs=1,
82 + max_num_seqs=2,
83 dtype=dtype,
84 limit_mm_per_prompt={"image": mm_limit},
85 tensor_parallel_size=tensor_parallel_size,
tests/models/decoder_only/vision_language/test_phi3v.py
+1 -1
@@ -90,7 +90,7 @@ def run_test(
90 # max_model_len should be greater than image_feature_size
91 with vllm_runner(model,
92 max_model_len=4096,
93 - max_num_seqs=1,
93 + max_num_seqs=2,
94 dtype=dtype,
95 limit_mm_per_prompt={"image": mm_limit},
96 tensor_parallel_size=tensor_parallel_size,
tests/models/decoder_only/vision_language/test_qwen.py
+1 -1
@@ -221,7 +221,7 @@ def run_test(
221 # Qwen encodes each image into a fixed content size of 256
222 with vllm_runner(model,
223 max_model_len=1024,
224 - max_num_seqs=1,
224 + max_num_seqs=2,
225 dtype=dtype,
226 limit_mm_per_prompt={"image": mm_limit},
227 tensor_parallel_size=tensor_parallel_size,
tests/models/encoder_decoder/vision_language/test_broadcast.py new
+35
@@ -0,0 +1,35 @@
1 +import pytest
2 +
3 +from ....utils import multi_gpu_test
4 +
5 +
6 +@multi_gpu_test(num_gpus=2)
7 +@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
8 +@pytest.mark.parametrize("model", [
9 + "meta-llama/Llama-3.2-11B-Vision-Instruct",
10 +])
11 +def test_models(hf_runner, vllm_runner, image_assets,
12 + distributed_executor_backend, model) -> None:
13 +
14 + dtype = "half"
15 + max_tokens = 5
16 + num_logprobs = 5
17 + tensor_parallel_size = 2
18 +
19 + if model.startswith("meta-llama/Llama-3.2-11B-Vision-Instruct"):
20 + from .test_mllama import models, run_test
21 + else:
22 + raise NotImplementedError(f"Unsupported model: {model}")
23 +
24 + run_test(
25 + hf_runner,
26 + vllm_runner,
27 + image_assets,
28 + model=models[0],
29 + size_factors=[0.25, 0.5, 1.0],
30 + dtype=dtype,
31 + max_tokens=max_tokens,
32 + num_logprobs=num_logprobs,
33 + tensor_parallel_size=tensor_parallel_size,
34 + distributed_executor_backend=distributed_executor_backend,
35 + )
tests/models/encoder_decoder/vision_language/test_mllama.py
+69 -84
@@ -9,7 +9,6 @@ from vllm.sequence import SampleLogprobs
9
10 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11 _ImageAssets)
12 -from ....utils import multi_gpu_test
12 from ...utils import check_logprobs_close
13
14 _LIMIT_IMAGE_PER_PROMPT = 1
@@ -47,14 +46,46 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
46 if token_id != image_token_id or output_ids[idx - 1] != image_token_id
47 ]
48
50 - assert output_str[0] == " "
51 - hf_output_str = output_str[1:]
49 + hf_output_str = output_str
50 if hf_output_ids[-1] == eos_token_id:
51 hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
52
53 return hf_output_ids, hf_output_str, out_logprobs
54
55
56 +def _get_inputs(
57 + image_assets: _ImageAssets,
58 + *,
59 + size_factors: Optional[List[float]] = None,
60 + sizes: Optional[List[Tuple[int, int]]] = None,
61 +) -> List[Tuple[List[str], PromptImageInput]]:
62 + images = [asset.pil_image for asset in image_assets]
63 +
64 + if size_factors is not None:
65 + inputs_per_image = [(
66 + [prompt for _ in size_factors],
67 + [rescale_image_size(image, factor) for factor in size_factors],
68 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
69 + elif sizes is not None:
70 + inputs_per_image = [(
71 + [
72 + prompt if size is not None else text_only_prompts[0]
73 + for size in sizes
74 + ],
75 + [
76 + image.resize(size) if size is not None else None
77 + for size in sizes
78 + ],
79 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
80 + if len(sizes) == 0:
81 + inputs_per_image.append(
82 + (text_only_prompts, [None] * len(text_only_prompts)))
83 + else:
84 + raise ValueError("You must provide either `size_factors` or `sizes`")
85 +
86 + return inputs_per_image
87 +
88 +
89 @overload
90 def run_test(
91 hf_runner: Type[HfRunner],
@@ -103,39 +134,17 @@ def run_test(
134 tensor_parallel_size: int,
135 distributed_executor_backend: Optional[str] = None,
136 ):
106 - images = [asset.pil_image for asset in image_assets]
107 -
108 - if size_factors is not None:
109 - inputs_per_image = [(
110 - [prompt for _ in size_factors],
111 - [rescale_image_size(image, factor) for factor in size_factors],
112 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113 - elif sizes is not None:
114 - inputs_per_image = [(
115 - [
116 - prompt if size is not None else text_only_prompts[0]
117 - for size in sizes
118 - ],
119 - [
120 - image.resize(size) if size is not None else None
121 - for size in sizes
122 - ],
123 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
124 - if len(sizes) == 0:
125 - inputs_per_image.append(
126 - (text_only_prompts, [None] * len(text_only_prompts)))
127 - else:
128 - raise ValueError("You must provide either `size_factors` or `sizes`")
129 -
130 - _run_test(hf_runner,
131 - vllm_runner,
132 - inputs_per_image,
133 - model,
134 - dtype=dtype,
135 - max_tokens=max_tokens,
136 - num_logprobs=num_logprobs,
137 - tensor_parallel_size=tensor_parallel_size,
138 - distributed_executor_backend=distributed_executor_backend)
137 + _run_test(
138 + hf_runner,
139 + vllm_runner,
140 + _get_inputs(image_assets, size_factors=size_factors, sizes=sizes),
141 + model,
142 + dtype=dtype,
143 + max_tokens=max_tokens,
144 + num_logprobs=num_logprobs,
145 + tensor_parallel_size=tensor_parallel_size,
146 + distributed_executor_backend=distributed_executor_backend,
147 + )
148
149
150 def _run_test(
@@ -167,8 +176,8 @@ def _run_test(
176 # max_model_len should be greater than image_feature_size
177 with vllm_runner(model,
178 dtype=dtype,
170 - max_num_seqs=16,
179 max_model_len=4096,
180 + max_num_seqs=2,
181 tensor_parallel_size=tensor_parallel_size,
182 distributed_executor_backend=distributed_executor_backend,
183 enforce_eager=True,
@@ -185,7 +194,6 @@ def _run_test(
194 def process(hf_inputs: BatchEncoding):
195 return hf_inputs
196
188 - from transformers import AutoConfig
197 from transformers.models.mllama import MllamaConfig as MllamaConfigHf
198
199 # use transformer's MllamaConfig for hf_runner
@@ -193,6 +201,7 @@ def _run_test(
201 AutoConfig.register("mllama", MllamaConfigHf, exist_ok=True)
202 with hf_runner(model,
203 dtype=dtype,
204 + model_kwargs={"device_map": "auto"},
205 postprocess_inputs=process,
206 auto_cls=AutoModelForVision2Seq) as hf_model:
207 hf_outputs_per_image = [
@@ -218,26 +227,29 @@ def _run_test(
227 )
228
229
230 +SIZES = [
231 + # Text only
232 + [],
233 + # Single-size
234 + [(512, 512)],
235 + # Single-size, batched
236 + [(512, 512), (512, 512), (512, 512)],
237 + # Multi-size, batched
238 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
239 + (1024, 1024), (512, 1536), (512, 2028)],
240 + # Multi-size, batched, including text only
241 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
242 + (1024, 1024), (512, 1536), (512, 2028), None],
243 + # mllama has 8 possible aspect ratios, carefully set the sizes
244 + # to cover all of them
245 +]
246 +
247 +
248 +@pytest.mark.skip(
249 + reason=
250 + "Model is too big, test passed on L40 locally but will OOM on CI machine.")
251 @pytest.mark.parametrize("model", models)
222 -@pytest.mark.parametrize(
223 - "sizes",
224 - [
225 - # Text only
226 - [],
227 - # Single-size
228 - [(512, 512)],
229 - # Single-size, batched
230 - [(512, 512), (512, 512), (512, 512)],
231 - # Multi-size, batched
232 - [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
233 - (1024, 1024), (512, 1536), (512, 2028)],
234 - # Multi-size, batched, including text only
235 - [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
236 - (1024, 1024), (512, 1536), (512, 2028), None],
237 - # mllama has 8 possible aspect ratios, carefully set the sizes
238 - # to cover all of them
239 - ],
240 -)
252 +@pytest.mark.parametrize("sizes", SIZES)
253 @pytest.mark.parametrize("dtype", ["bfloat16"])
254 @pytest.mark.parametrize("max_tokens", [128])
255 @pytest.mark.parametrize("num_logprobs", [5])
@@ -254,30 +266,3 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
266 num_logprobs=num_logprobs,
267 tensor_parallel_size=1,
268 )
257 -
258 -
259 -@multi_gpu_test(num_gpus=2)
260 -@pytest.mark.parametrize("model", models)
261 -@pytest.mark.parametrize(
262 - "sizes",
263 - [
264 - [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
265 - (1024, 1024), (512, 1536), (512, 2028), None],
266 - ],
267 -)
268 -@pytest.mark.parametrize("dtype", ["bfloat16"])
269 -@pytest.mark.parametrize("max_tokens", [128])
270 -@pytest.mark.parametrize("num_logprobs", [5])
271 -def test_models_distributed(hf_runner, vllm_runner, image_assets, model, sizes,
272 - dtype, max_tokens, num_logprobs) -> None:
273 - run_test(
274 - hf_runner,
275 - vllm_runner,
276 - image_assets,
277 - model,
278 - sizes=sizes,
279 - dtype=dtype,
280 - max_tokens=max_tokens,
281 - num_logprobs=num_logprobs,
282 - tensor_parallel_size=2,
283 - )
tests/models/utils.py
+8 -1
@@ -1,9 +1,12 @@
1 import warnings
2 from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4 +import torch
5 +
6 from vllm.config import ModelConfig
7 from vllm.inputs import InputContext
8 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9 +from vllm.utils import is_cpu
10
11 TokensText = Tuple[List[int], str]
12
@@ -247,6 +250,7 @@ def check_logprobs_close(
250 def build_model_context(model_name: str,
251 tokenizer_name: Optional[str] = None,
252 trust_remote_code: bool = False,
253 + dtype: Optional[Union[str, torch.dtype]] = None,
254 mm_processor_kwargs: Optional[Dict] = None,
255 limit_mm_per_prompt: Optional[Dict] = None):
256 """Creates an InputContext for a given model.
@@ -264,12 +268,15 @@ def build_model_context(model_name: str,
268 """
269 if tokenizer_name is None:
270 tokenizer_name = model_name
271 + if dtype is None:
272 + dtype = "bfloat16" if is_cpu() else "half"
273 +
274 model_config = ModelConfig(
275 model_name,
276 tokenizer_name,
277 tokenizer_mode="auto",
278 trust_remote_code=trust_remote_code,
272 - dtype="float32",
279 + dtype=dtype,
280 seed=0,
281 mm_processor_kwargs=mm_processor_kwargs,
282 limit_mm_per_prompt=limit_mm_per_prompt,
vllm/inputs/registry.py
+2 -10
@@ -185,16 +185,8 @@ class InputRegistry:
185 return wrapper
186
187 def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
188 - if model_cls in self._dummy_encoder_factories_by_model_type:
189 - dummy_factory = self._dummy_encoder_factories_by_model_type[
190 - model_cls]
191 - else:
192 - logger.warning(
193 - "No dummy encoder data factory registered to %s. "
194 - "Using the dummy data factory for the model instead.",
195 - model_cls)
196 - dummy_factory = self._get_dummy_data_factory(model_cls)
197 - return dummy_factory
188 + return self._dummy_encoder_factories_by_model_type \
189 + .get(model_cls, self._default_dummy_data_factory)
190
191 def dummy_data_for_profiling(
192 self,
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2 -1
@@ -159,7 +159,8 @@ def apply_fp8_linear(
159
160 # Making sure the dummy tensor is on the same device as the weight
161 global TORCH_DEVICE_IDENTITY
162 - if TORCH_DEVICE_IDENTITY.device != weight.device:
162 + if (TORCH_DEVICE_IDENTITY is not None
163 + and TORCH_DEVICE_IDENTITY.device != weight.device):
164 TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
165
166 # GEMM