@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f690372b68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Core] Update dtype detection and defaults (#14858)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 19, 2025 at 13:49 UTC f690372b6803c43870aefa631ab207ea3d163b1d
22 files changed +175 -227
tests/compile/test_basic_correctness.py
+1 -1
@@ -60,7 +60,7 @@ class TestSetting:
60 # embedding model
61 TestSetting(
62 model="BAAI/bge-multilingual-gemma2",
63 - model_args=["--task", "embed"],
63 + model_args=["--task", "embed", "--dtype", "bfloat16"],
64 pp_size=1,
65 tp_size=1,
66 attn_backend="FLASH_ATTN",
tests/conftest.py
+63 -53
@@ -14,8 +14,8 @@ import torch.nn as nn
14 import torch.nn.functional as F
15 from huggingface_hub import snapshot_download
16 from PIL import Image
17 -from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
18 - BatchFeature)
17 +from transformers import (AutoConfig, AutoModelForCausalLM, AutoTokenizer,
18 + BatchEncoding, BatchFeature)
19 from transformers.models.auto.auto_factory import _BaseAutoModelClass
20
21 from tests.models.utils import (TokensTextLogprobs,
@@ -23,7 +23,7 @@ from tests.models.utils import (TokensTextLogprobs,
23 from vllm import LLM, SamplingParams
24 from vllm.assets.image import ImageAsset
25 from vllm.assets.video import VideoAsset
26 -from vllm.config import TaskOption, TokenizerPoolConfig
26 +from vllm.config import TaskOption, TokenizerPoolConfig, _get_and_verify_dtype
27 from vllm.connections import global_http_connection
28 from vllm.distributed import (cleanup_dist_env_and_memory,
29 init_distributed_environment,
@@ -34,8 +34,7 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
34 from vllm.logger import init_logger
35 from vllm.outputs import RequestOutput
36 from vllm.sampling_params import BeamSearchParams
37 -from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
38 - identity, is_list_of)
37 +from vllm.utils import cuda_device_count_stateless, is_list_of
38
39 logger = init_logger(__name__)
40
@@ -271,14 +270,18 @@ _R = TypeVar("_R")
270
271 class HfRunner:
272
274 - def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
273 + def get_default_device(self):
274 from vllm.platforms import current_platform
275 +
276 + return ("cpu" if current_platform.is_cpu()
277 + or current_platform.is_openvino() else "cuda")
278 +
279 + def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
280 if x is None or isinstance(x, (bool, )):
281 return x
282
283 if device is None:
280 - device = "cpu" if current_platform.is_cpu(
281 - ) or current_platform.is_openvino() else "cuda"
284 + device = self.device
285
286 if isinstance(x, dict):
287 return {k: self.wrap_device(v, device) for k, v in x.items()}
@@ -291,45 +294,59 @@ class HfRunner:
294 def __init__(
295 self,
296 model_name: str,
294 - dtype: str = "half",
297 + dtype: str = "auto",
298 *,
299 model_kwargs: Optional[dict[str, Any]] = None,
300 is_sentence_transformer: bool = False,
301 is_cross_encoder: bool = False,
302 skip_tokenizer_init: bool = False,
303 auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
301 - postprocess_inputs: Callable[..., BatchEncoding] = identity,
304 ) -> None:
303 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
304 -
305 self.model_name = model_name
306
307 + self.config = AutoConfig.from_pretrained(
308 + model_name,
309 + trust_remote_code=True,
310 + )
311 + self.device = self.get_default_device()
312 + self.dtype = torch_dtype = _get_and_verify_dtype(self.config, dtype)
313 +
314 + model_kwargs = model_kwargs if model_kwargs is not None else {}
315 + model_kwargs.setdefault("torch_dtype", torch_dtype)
316 +
317 if is_sentence_transformer:
318 # Lazy init required for AMD CI
319 from sentence_transformers import SentenceTransformer
310 - self.model = self.wrap_device(
311 - SentenceTransformer(
312 - model_name,
313 - device="cpu",
314 - trust_remote_code=True,
315 - ).to(dtype=torch_dtype))
320 +
321 + self.model = SentenceTransformer(
322 + model_name,
323 + device=self.device,
324 + model_kwargs=model_kwargs,
325 + trust_remote_code=True,
326 + )
327 elif is_cross_encoder:
328 # Lazy init required for AMD CI
329 from sentence_transformers import CrossEncoder
319 - self.model = CrossEncoder(model_name,
320 - device="cpu",
321 - trust_remote_code=True)
322 - self.model.model = self.wrap_device(self.model.model)\
323 - .to(dtype=torch_dtype)
330 +
331 + self.model = CrossEncoder(
332 + model_name,
333 + device=self.device,
334 + automodel_args=model_kwargs,
335 + trust_remote_code=True,
336 + )
337 else:
325 - model_kwargs = model_kwargs if model_kwargs is not None else {}
326 - self.model = self.wrap_device(
327 - auto_cls.from_pretrained(
328 - model_name,
329 - torch_dtype=torch_dtype,
330 - trust_remote_code=True,
331 - **model_kwargs,
332 - ))
338 + model = auto_cls.from_pretrained(
339 + model_name,
340 + trust_remote_code=True,
341 + **model_kwargs,
342 + )
343 +
344 + if (getattr(model, "quantization_method", None) != "bitsandbytes"
345 + and len({p.device
346 + for p in model.parameters()}) < 2):
347 + model = model.to(self.device)
348 +
349 + self.model = model
350
351 if not skip_tokenizer_init:
352 self.tokenizer = AutoTokenizer.from_pretrained(
@@ -349,16 +366,13 @@ class HfRunner:
366 if skip_tokenizer_init:
367 self.tokenizer = self.processor.tokenizer
368
352 - self.dtype = dtype
353 - self.postprocess_inputs = postprocess_inputs
354 -
369 def get_inputs(
370 self,
371 prompts: list[str],
372 images: Optional[PromptImageInput] = None,
373 videos: Optional[PromptVideoInput] = None,
374 audios: Optional[PromptAudioInput] = None,
361 - ) -> list[BatchEncoding]:
375 + ) -> list[Union[BatchFeature, BatchEncoding]]:
376 if images is not None:
377 assert len(prompts) == len(images)
378
@@ -368,7 +382,7 @@ class HfRunner:
382 if audios is not None:
383 assert len(prompts) == len(audios)
384
371 - all_inputs: list[BatchEncoding] = []
385 + all_inputs: list[Union[BatchFeature, BatchEncoding]] = []
386 for i, prompt in enumerate(prompts):
387 processor_kwargs: dict[str, Any] = {
388 "text": prompt,
@@ -384,7 +398,8 @@ class HfRunner:
398 processor_kwargs["sampling_rate"] = sr
399
400 inputs = self.processor(**processor_kwargs)
387 - inputs = self.postprocess_inputs(inputs, dtype=self.dtype)
401 + if isinstance(inputs, BatchFeature):
402 + inputs = inputs.to(dtype=self.dtype)
403
404 all_inputs.append(inputs)
405
@@ -417,7 +432,7 @@ class HfRunner:
432 outputs: list[tuple[list[list[int]], list[str]]] = []
433 for inputs in all_inputs:
434 output_ids = self.model.generate(
420 - **self.wrap_device(inputs, device=self.model.device.type),
435 + **self.wrap_device(inputs),
436 use_cache=True,
437 **kwargs,
438 )
@@ -488,7 +503,7 @@ class HfRunner:
503 all_logprobs: list[list[torch.Tensor]] = []
504 for inputs in all_inputs:
505 output = self.model.generate(
491 - **self.wrap_device(inputs, device=self.model.device.type),
506 + **self.wrap_device(inputs),
507 use_cache=True,
508 do_sample=False,
509 max_new_tokens=max_tokens,
@@ -569,7 +584,7 @@ class HfRunner:
584
585 for inputs in all_inputs:
586 output = self.model.generate(
572 - **self.wrap_device(inputs, device=self.model.device.type),
587 + **self.wrap_device(inputs),
588 use_cache=True,
589 do_sample=False,
590 max_new_tokens=max_tokens,
@@ -620,19 +635,15 @@ class HfRunner:
635 if images is not None and images[i] is not None:
636 processor_kwargs["images"] = images[i]
637
623 - encoder_inputs = self.wrap_device(
624 - self.processor(**processor_kwargs),
625 - device=self.model.device.type,
626 - )
638 + encoder_inputs = self.processor(**processor_kwargs)
639 + encoder_inputs = self.wrap_device(encoder_inputs)
640
641 if decoder_prompt is None:
642 decoder_input_ids = None
643 else:
631 - decoder_input_ids = self.wrap_device(
632 - self.tokenizer(decoder_prompt,
633 - return_tensors="pt").input_ids,
634 - device=self.model.device.type,
635 - )
644 + decoder_inputs = self.tokenizer(decoder_prompt,
645 + return_tensors="pt")
646 + decoder_input_ids = self.wrap_device(decoder_inputs.input_ids)
647
648 output = self.model.generate(
649 decoder_input_ids=decoder_input_ids,
@@ -684,6 +695,7 @@ class VllmRunner:
695 """
696 The default value of some arguments have been modified from
697 :class:`~vllm.LLM` as follows:
698 +
699 - `trust_remote_code`: Set to `True` instead of `False` for convenience.
700 - `seed`: Set to `0` instead of `None` for test reproducibility.
701 - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.
@@ -701,10 +713,8 @@ class VllmRunner:
713 tokenizer_mode: str = "auto",
714 trust_remote_code: bool = True,
715 seed: Optional[int] = 0,
704 - # Use smaller max model length, otherwise bigger model cannot run due
705 - # to kv cache size limit.
716 max_model_len: int = 1024,
707 - dtype: str = "half",
717 + dtype: str = "auto",
718 disable_log_stats: bool = True,
719 tensor_parallel_size: int = 1,
720 block_size: int = 16,
@@ -1110,4 +1120,4 @@ def pytest_collection_modifyitems(config, items):
1120 skip_optional = pytest.mark.skip(reason="need --optional option to run")
1121 for item in items:
1122 if "optional" in item.keywords:
1113 - item.add_marker(skip_optional)
\ No newline at end of file
1123 + item.add_marker(skip_optional)
tests/entrypoints/llm/test_chat.py
-1
@@ -64,7 +64,6 @@ def test_multi_chat():
64 def test_chat_multi_image(image_urls: list[str]):
65 llm = LLM(
66 model="microsoft/Phi-3.5-vision-instruct",
67 - dtype="bfloat16",
67 max_model_len=4096,
68 max_num_seqs=5,
69 enforce_eager=True,
tests/entrypoints/openai/test_audio.py
-2
@@ -18,8 +18,6 @@ TEST_AUDIO_URLS = [
18 @pytest.fixture(scope="module")
19 def server():
20 args = [
21 - "--dtype",
22 - "bfloat16",
21 "--max-model-len",
22 "2048",
23 "--max-num-seqs",
tests/entrypoints/openai/test_video.py
-2
@@ -24,8 +24,6 @@ def server():
24 args = [
25 "--task",
26 "generate",
27 - "--dtype",
28 - "bfloat16",
27 "--max-model-len",
28 "32768",
29 "--max-num-seqs",
tests/entrypoints/openai/test_vision.py
-2
@@ -25,8 +25,6 @@ def server():
25 args = [
26 "--task",
27 "generate",
28 - "--dtype",
29 - "bfloat16",
28 "--max-model-len",
29 "2048",
30 "--max-num-seqs",
tests/entrypoints/openai/test_vision_embedding.py
-2
@@ -28,8 +28,6 @@ def server():
28 args = [
29 "--task",
30 "embed",
31 - "--dtype",
32 - "bfloat16",
31 "--max-model-len",
32 "2048",
33 "--max-num-seqs",
tests/entrypoints/test_chat_utils.py
+3 -3
@@ -34,7 +34,7 @@ def phi3v_model_config():
34 tokenizer=PHI3V_MODEL_ID,
35 tokenizer_mode="auto",
36 trust_remote_code=True,
37 - dtype="bfloat16",
37 + dtype="auto",
38 seed=0,
39 limit_mm_per_prompt={
40 "image": 2,
@@ -58,7 +58,7 @@ def mllama_model_config():
58 tokenizer=MLLAMA_MODEL_ID,
59 tokenizer_mode="auto",
60 trust_remote_code=True,
61 - dtype="bfloat16",
61 + dtype="auto",
62 seed=0,
63 limit_mm_per_prompt={
64 "image": 2,
@@ -669,7 +669,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
669 tokenizer=MLLAMA_MODEL_ID,
670 tokenizer_mode="auto",
671 trust_remote_code=True,
672 - dtype="bfloat16",
672 + dtype="auto",
673 seed=0,
674 limit_mm_per_prompt={
675 "image": 2,
tests/models/decoder_only/audio_language/test_ultravox.py
+2 -13
@@ -5,11 +5,10 @@ from typing import Optional
5 import numpy as np
6 import pytest
7 import pytest_asyncio
8 -from transformers import AutoModel, AutoTokenizer, BatchEncoding
8 +from transformers import AutoModel, AutoTokenizer
9
10 from vllm.multimodal.audio import resample_audio
11 from vllm.sequence import SampleLogprobs
12 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
12
13 from ....conftest import HfRunner, VllmRunner
14 from ....utils import RemoteOpenAIServer
@@ -107,8 +106,6 @@ def run_test(
106 **kwargs,
107 ):
108 """Inference result should be the same between hf and vllm."""
110 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
111 -
109 # NOTE: take care of the order. run vLLM first, and then run HF.
110 # vLLM needs a fresh new process without cuda initialization.
111 # if we run HF first, the cuda initialization will be done and it
@@ -124,15 +121,7 @@ def run_test(
121 for vllm_prompt, _, audio in prompts_and_audios
122 ]
123
127 - def process(hf_inputs: BatchEncoding, **kwargs):
128 - hf_inputs["audio_values"] = hf_inputs["audio_values"] \
129 - .to(torch_dtype) # type: ignore
130 - return hf_inputs
131 -
132 - with hf_runner(model,
133 - dtype=dtype,
134 - postprocess_inputs=process,
135 - auto_cls=AutoModel) as hf_model:
124 + with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
125 hf_outputs_per_audio = [
126 hf_model.generate_greedy_logprobs_limit(
127 [hf_prompt],
tests/models/decoder_only/vision_language/test_models.py
+3 -36
@@ -122,9 +122,6 @@ VLM_TEST_SETTINGS = {
122 "cherry_blossom": "What is in the picture?",
123 }),
124 auto_cls=AutoModelForImageTextToText,
125 - postprocess_inputs=model_utils.cast_dtype_post_processor(
126 - "pixel_values"
127 - ),
125 vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
126 dtype="bfloat16",
127 marks=[pytest.mark.skip(reason="vLLM does not support PrefixLM attention mask")], # noqa: E501
@@ -179,7 +176,6 @@ VLM_TEST_SETTINGS = {
176 # "cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
177 # }),
178 # multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
182 - # postprocess_inputs=model_utils.cast_dtype_post_processor("pixel_values"), # noqa: E501
179 # stop_str=["<|im_end|>"],
180 # image_size_factors=[(0.10, 0.15)],
181 # max_tokens=64,
@@ -200,9 +196,6 @@ VLM_TEST_SETTINGS = {
196 max_model_len=4096,
197 max_num_seqs=2,
198 auto_cls=AutoModelForImageTextToText,
203 - postprocess_inputs=model_utils.cast_dtype_post_processor(
204 - "pixel_values"
205 - ),
199 # For chameleon, we only compare the sequences
200 vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
201 hf_output_post_proc = lambda hf_output, model: hf_output[:2],
@@ -222,7 +215,6 @@ VLM_TEST_SETTINGS = {
215 }),
216 multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
217 patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
225 - postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
218 hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
219 stop_str=["<｜end▁of▁sentence｜>", "<｜begin▁of▁sentence｜>"], # noqa: E501
220 image_size_factors=[(), (1.0, ), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
@@ -258,7 +250,6 @@ VLM_TEST_SETTINGS = {
250 max_model_len=4096,
251 max_num_seqs=2,
252 auto_cls=AutoModelForImageTextToText,
261 - dtype="bfloat16",
253 vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
254 patch_hf_runner=model_utils.gemma3_patch_hf_runner,
255 ),
@@ -272,7 +263,6 @@ VLM_TEST_SETTINGS = {
263 }),
264 max_model_len=2048,
265 max_num_seqs=2,
275 - dtype="bfloat16",
266 get_stop_token_ids=lambda tok: [151329, 151336, 151338],
267 patch_hf_runner=model_utils.glm4v_patch_hf_runner,
268 # The image embeddings match with HF but the outputs of the language
@@ -295,7 +285,6 @@ VLM_TEST_SETTINGS = {
285 }),
286 multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
287 max_model_len=8192,
298 - dtype="bfloat16",
288 use_tokenizer_eos=True,
289 num_logprobs=10,
290 patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
@@ -324,10 +313,6 @@ VLM_TEST_SETTINGS = {
313 }),
314 multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
315 max_model_len=4096,
327 - # NOTE: Mono-InternVL-2B doesn't work with fp16,
328 - # it will result NaN during inference.
329 - # See: https://huggingface.co/OpenGVLab/Mono-InternVL-2B/discussions/9
330 - dtype="bfloat16",
316 use_tokenizer_eos=True,
317 patch_hf_runner=model_utils.internvl_patch_hf_runner,
318 ),
@@ -351,9 +336,6 @@ VLM_TEST_SETTINGS = {
336 prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
337 num_video_frames=16,
338 max_model_len=16384,
354 - postprocess_inputs=model_utils.cast_dtype_post_processor(
355 - "pixel_values_videos"
356 - ),
339 auto_cls=AutoModelForVision2Seq,
340 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
341 custom_test_opts=[CustomTestOptions(
@@ -378,9 +360,6 @@ VLM_TEST_SETTINGS = {
360 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
361 prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
362 max_model_len=4096,
381 - postprocess_inputs=model_utils.cast_dtype_post_processor(
382 - "pixel_values"
383 - ),
363 get_stop_token_ids=lambda tok: [128009],
364 auto_cls=AutoModelForImageTextToText,
365 vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
@@ -400,8 +379,8 @@ VLM_TEST_SETTINGS = {
379 max_model_len=4096,
380 max_num_seqs=2,
381 get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
403 - postprocess_inputs=model_utils.wrap_inputs_post_processor,
382 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
383 + patch_hf_runner=model_utils.minicpmv_25_patch_hf_runner,
384 ),
385 "minicpmo_26": VLMTestInfo(
386 models=["openbmb/MiniCPM-o-2_6"],
@@ -411,11 +390,8 @@ VLM_TEST_SETTINGS = {
390 max_model_len=4096,
391 max_num_seqs=2,
392 get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
414 - postprocess_inputs=model_utils.ignore_inputs_post_processor(
415 - "image_sizes"
416 - ),
393 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
418 - patch_hf_runner=model_utils.minicpmo_patch_hf_runner
394 + patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
395 ),
396 "minicpmv_26": VLMTestInfo(
397 models=["openbmb/MiniCPM-V-2_6"],
@@ -425,10 +401,8 @@ VLM_TEST_SETTINGS = {
401 max_model_len=4096,
402 max_num_seqs=2,
403 get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
428 - postprocess_inputs=model_utils.ignore_inputs_post_processor(
429 - "image_sizes"
430 - ),
404 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
405 + patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
406 ),
407 "molmo": VLMTestInfo(
408 models=["allenai/Molmo-7B-D-0924"],
@@ -437,7 +411,6 @@ VLM_TEST_SETTINGS = {
411 max_model_len=4096,
412 max_num_seqs=2,
413 patch_hf_runner=model_utils.molmo_patch_hf_runner,
440 - postprocess_inputs=model_utils.molmo_post_processor,
414 ),
415 # Tests for phi3v currently live in another file because of a bug in
416 # transformers. Once this issue is fixed, we can enable them here instead.
@@ -482,9 +455,6 @@ VLM_TEST_SETTINGS = {
455 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
456 max_model_len=4096,
457 auto_cls=AutoModelForImageTextToText,
485 - postprocess_inputs=model_utils.cast_dtype_post_processor(
486 - "pixel_values"
487 - ),
458 vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
459 hf_output_post_proc = lambda hf_output, model: hf_output[:2],
460 comparator=check_outputs_equal,
@@ -529,9 +499,6 @@ VLM_TEST_SETTINGS = {
499 test_type=VLMTestType.CUSTOM_INPUTS,
500 max_model_len=16384,
501 max_num_seqs=2,
532 - postprocess_inputs=model_utils.cast_dtype_post_processor(
533 - "pixel_values"
534 - ),
502 auto_cls=AutoModelForVision2Seq,
503 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
504 custom_test_opts=[CustomTestOptions(
tests/models/decoder_only/vision_language/vlm_utils/core.py
-3
@@ -4,7 +4,6 @@ from typing import Any, Callable, Optional, Union
4
5 import torch
6 from PIL.Image import Image
7 -from transformers import BatchEncoding
7 from transformers.models.auto.auto_factory import _BaseAutoModelClass
8
9 from vllm.config import TaskOption
@@ -31,7 +30,6 @@ def run_test(
30 vllm_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]],
31 auto_cls: type[_BaseAutoModelClass],
32 use_tokenizer_eos: bool,
34 - postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
33 comparator: Callable[..., None],
34 get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]],
35 stop_str: Optional[list[str]],
@@ -101,7 +99,6 @@ def run_test(
99 hf_model = hf_runner(model,
100 dtype=dtype,
101 auto_cls=auto_cls,
104 - postprocess_inputs=postprocess_inputs,
102 model_kwargs=hf_model_kwargs)
103
104 # Some models need to patch things like the model processor, e.g., internvl
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+49 -42
@@ -6,16 +6,15 @@ typically specific to a small subset of models.
6 import re
7 import types
8 from pathlib import PosixPath
9 -from typing import Callable, Optional, Union
9 +from typing import Optional, Union
10
11 import torch
12 from PIL.Image import Image
13 -from transformers import (AutoConfig, AutoTokenizer, BatchEncoding,
13 +from transformers import (AutoConfig, AutoTokenizer, BatchFeature,
14 GenerationConfig)
15
16 from vllm.sequence import SampleLogprobs
17 from vllm.transformers_utils.tokenizer import patch_padding_side
18 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
19 from .....conftest import HfRunner, ImageAsset, _ImageAssets
20 from .types import RunnerOutput
@@ -211,40 +210,6 @@ def get_llava_embeddings(image_assets: _ImageAssets):
210 return [asset.image_embeds for asset in image_assets]
211
212
214 -####### postprocessors to run on HF BatchEncoding
215 -def cast_dtype_post_processor(
216 - hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
217 - """Gets a handle to a post processor which converts a given key into a
218 - target data type."""
219 -
220 - def process(hf_inputs: BatchEncoding, dtype: str):
221 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
222 - hf_inputs[hf_inp_key] = hf_inputs[hf_inp_key].to(torch_dtype)
223 - return hf_inputs
224 -
225 - return process
226 -
227 -
228 -def ignore_inputs_post_processor(
229 - hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
230 - """Gets a handle to a post processor which ignores a given key."""
231 -
232 - def process(hf_inputs: BatchEncoding, dtype: str):
233 - del hf_inputs[hf_inp_key]
234 - return hf_inputs
235 -
236 - return process
237 -
238 -
239 -def wrap_inputs_post_processor(hf_inputs: BatchEncoding, dtype: str):
240 - return {"model_inputs": hf_inputs}
241 -
242 -
243 -def molmo_post_processor(hf_inputs: BatchEncoding, dtype: str):
244 - hf_inputs = cast_dtype_post_processor("images")(hf_inputs, dtype)
245 - return {k: v.unsqueeze(0) for k, v in hf_inputs.items()}
246 -
247 -
213 ####### Prompt path encoders for models that need models on disk
214 def qwen_prompt_path_encoder(
215 tmp_path: PosixPath, prompt: str, assets: Union[list[ImageAsset],
@@ -295,8 +260,7 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
260 for k in inputs.keys() # noqa
261 if k not in ("seq_lens", "sft_format")
262 }
298 - inputs = BatchEncoding(data=inputs, tensor_type="pt")
299 - return inputs
263 + return BatchFeature(data=inputs, tensor_type="pt")
264
265 hf_model.processor = processor
266 hf_model.model.get_output_embeddings = lambda: \
@@ -529,10 +493,52 @@ def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
493 return hf_model
494
495
532 -def minicpmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
496 +def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
497 orig_generate = hf_model.model.generate
498
535 - def _generate(self, *args, **kwargs):
499 + def _generate(
500 + self,
501 + *args,
502 + input_ids=None,
503 + pixel_values=None,
504 + image_sizes=None,
505 + image_bound=None,
506 + tgt_sizes=None,
507 + **kwargs,
508 + ):
509 + model_inputs = {
510 + "input_ids": input_ids,
511 + "pixel_values": pixel_values,
512 + "image_sizes": image_sizes,
513 + "image_bound": image_bound,
514 + "tgt_sizes": tgt_sizes,
515 + }
516 + for k in list(model_inputs.keys()):
517 + if model_inputs[k] is None:
518 + model_inputs.pop(k)
519 +
520 + return orig_generate(model_inputs, *args, decode_text=False, **kwargs)
521 +
522 + hf_model.model.generate = types.MethodType(_generate, hf_model.model)
523 +
524 + return hf_model
525 +
526 +
527 +def minicpmo_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
528 + orig_generate = hf_model.model.generate
529 +
530 + def _generate(self, *args, image_sizes=None, **kwargs):
531 + return orig_generate(*args, decode_text=False, **kwargs)
532 +
533 + hf_model.model.generate = types.MethodType(_generate, hf_model.model)
534 +
535 + return hf_model
536 +
537 +
538 +def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
539 + orig_generate = hf_model.model.generate
540 +
541 + def _generate(self, *args, image_sizes=None, **kwargs):
542 return orig_generate(*args, decode_text=False, **kwargs)
543
544 hf_model.model.generate = types.MethodType(_generate, hf_model.model)
@@ -551,10 +557,11 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
557
558 def _generate(self, max_new_tokens=None, do_sample=None, **kwargs):
559 batch = {
554 - k: kwargs.pop(k)
560 + k: kwargs.pop(k).unsqueeze(0)
561 for k in ("input_ids", "images", "image_input_idx", "image_masks")
562 if k in kwargs
563 }
564 + batch = BatchFeature(batch).to(dtype=self.dtype)
565
566 return self.generate_from_batch(
567 batch,
tests/models/decoder_only/vision_language/vlm_utils/types.py
+2 -9
@@ -8,13 +8,12 @@ from typing import Any, Callable, NamedTuple, Optional, Union
8 import torch
9 from PIL.Image import Image
10 from pytest import MarkDecorator
11 -from transformers import AutoModelForCausalLM, BatchEncoding
11 +from transformers import AutoModelForCausalLM
12 from transformers.models.auto.auto_factory import _BaseAutoModelClass
13
14 from vllm.config import TaskOption
15 from vllm.sequence import SampleLogprobs
16 from vllm.transformers_utils.tokenizer import AnyTokenizer
17 -from vllm.utils import identity
17
18 from .....conftest import IMAGE_ASSETS, HfRunner, ImageAsset, _ImageAssets
19 from ....utils import check_logprobs_close
@@ -110,11 +109,6 @@ class VLMTestInfo(NamedTuple):
109 # Indicates we should explicitly pass the EOS from the tokenizer
110 use_tokenizer_eos: bool = False
111 auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
113 - # Callable to pass to the HF runner to run on inputs; for now, we also pass
114 - # the data type to input post processing, because almost all of the uses of
115 - # postprocess_inputs are to fix the data types of BatchEncoding values.
116 - postprocess_inputs: Callable[[BatchEncoding, str],
117 - BatchEncoding] = identity
112 patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]] = None
113
114 # Post processors that if defined, will run oun the outputs of the
@@ -130,7 +124,7 @@ class VLMTestInfo(NamedTuple):
124 # is all combinations of .models + all fields below
125 max_tokens: Union[int, tuple[int]] = 128
126 num_logprobs: Union[int, tuple[int]] = 5
133 - dtype: Union[str, Iterable[str]] = "half"
127 + dtype: Union[str, Union[list[str], tuple[str, ...]]] = "auto"
128 distributed_executor_backend: Optional[Union[str, Iterable[str]]] = None
129 # Only expanded in video tests
130 num_video_frames: Union[int, tuple[int]] = 16
@@ -171,7 +165,6 @@ class VLMTestInfo(NamedTuple):
165 "vllm_output_post_proc": self.vllm_output_post_proc,
166 "auto_cls": self.auto_cls,
167 "use_tokenizer_eos": self.use_tokenizer_eos,
174 - "postprocess_inputs": self.postprocess_inputs,
168 "comparator": self.comparator,
169 "get_stop_token_ids": self.get_stop_token_ids,
170 "hf_model_kwargs": self.hf_model_kwargs,
tests/models/embedding/vision_language/test_dse_qwen2_vl.py
+28 -24
@@ -1,12 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from functools import partial
3 from typing import Callable
4
5 import pytest
6 import torch
7 +import torch.nn.functional as F
8 from PIL import Image
9 -from transformers import BatchEncoding, Qwen2VLForConditionalGeneration
9 +from transformers import Qwen2VLForConditionalGeneration
10
11 from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
12 from ....utils import large_gpu_test
@@ -75,10 +75,6 @@ def apply_chat_template_and_add_eos(
75 return prompt
76
77
78 -def postprocess_inputs(hf_model: HfRunner, inputs: BatchEncoding, **kwargs):
79 - return hf_model.model.prepare_inputs_for_generation(**inputs, **kwargs)
80 -
81 -
78 def _run_test(
79 hf_runner: type[HfRunner],
80 vllm_runner: type[VllmRunner],
@@ -118,14 +114,8 @@ def _run_test(
114 with hf_runner(model,
115 dtype=dtype,
116 auto_cls=Qwen2VLForConditionalGeneration) as hf_model:
121 - hf_model.postprocess_inputs = partial(
122 - postprocess_inputs,
123 - hf_model,
124 - cache_position=torch.arange(
125 - 0,
126 - 1, # 1 for batch size
127 - requires_grad=False),
128 - use_cache=False)
117 +
118 + prompts = []
119 for text, image, embed_text in zip(input_texts, input_images,
120 embed_texts):
121 # dse requires non-standard input processing
@@ -133,20 +123,34 @@ def _run_test(
123 messages = get_messages(image, text, embed_text)
124 prompt = apply_chat_template_and_add_eos(
125 messages, hf_model.processor.apply_chat_template)
136 - inputs = hf_model.get_inputs(
137 - prompts=[[prompt]],
138 - images=[[image]],
139 - )
140 - with torch.no_grad():
126 +
127 + prompts.append(prompt)
128 +
129 + all_inputs = hf_model.get_inputs(
130 + prompts=prompts,
131 + images=input_images,
132 + )
133 +
134 + with torch.no_grad():
135 + all_outputs = []
136 + for inputs in all_inputs:
137 + inputs = hf_model.model.prepare_inputs_for_generation(
138 + **inputs,
139 + cache_position=torch.arange(1), # 1 for batch size
140 + use_cache=False,
141 + )
142 outputs = hf_model.model(
142 - **hf_model.wrap_device(inputs[0],
143 - device=hf_model.model.device.type),
143 + **hf_model.wrap_device(inputs),
144 return_dict=True,
145 output_hidden_states=True,
146 )
147 - pooled_output = torch.nn.functional.normalize(
148 - outputs.hidden_states[-1][0, -1], p=2, dim=-1)
149 - hf_outputs.append(pooled_output.tolist())
147 + pooled_output = F.normalize(outputs.hidden_states[-1][0, -1],
148 + p=2,
149 + dim=-1)
150 +
151 + all_outputs.append(pooled_output.tolist())
152 +
153 + hf_outputs = all_outputs
154
155 check_embeddings_close(
156 embeddings_0_lst=hf_outputs,
tests/models/embedding/vision_language/test_llava_next.py
+1 -2
@@ -86,8 +86,7 @@ def _run_test(
86 for inputs in all_inputs:
87 # Based on: https://huggingface.co/royokong/e5-v
88 outputs = hf_model.model(
89 - **hf_model.wrap_device(inputs,
90 - device=hf_model.model.device.type),
89 + **hf_model.wrap_device(inputs),
90 return_dict=True,
91 output_hidden_states=True,
92 )
tests/models/embedding/vision_language/test_phi3v.py
+1 -2
@@ -53,8 +53,7 @@ def _run_test(
53 for inputs in all_inputs:
54 # Based on: https://github.com/TIGER-AI-Lab/VLM2Vec/blob/db3b951bccabba220c1f53ab46a734e50dd2fc08/src/model.py
55 outputs = hf_model.model(
56 - **hf_model.wrap_device(inputs,
57 - device=hf_model.model.device.type),
56 + **hf_model.wrap_device(inputs),
57 return_dict=True,
58 output_hidden_states=True,
59 )
tests/models/encoder_decoder/vision_language/test_mllama.py
+1 -6
@@ -4,8 +4,7 @@ from typing import Optional, overload
4
5 import pytest
6 import torch
7 -from transformers import (AutoConfig, AutoModelForImageTextToText,
8 - AutoTokenizer, BatchEncoding)
7 +from transformers import AutoConfig, AutoModelForImageTextToText, AutoTokenizer
8
9 from vllm import LLM, SamplingParams
10 from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -227,13 +226,9 @@ def _run_test(
226 for prompts, images in inputs
227 ]
228
230 - def process(hf_inputs: BatchEncoding, **kwargs):
231 - return hf_inputs
232 -
229 with hf_runner(model,
230 dtype=dtype,
231 model_kwargs={"device_map": "auto"},
236 - postprocess_inputs=process,
232 auto_cls=AutoModelForImageTextToText) as hf_model:
233 hf_outputs_per_image = [
234 hf_model.generate_greedy_logprobs_limit(prompts,
tests/models/utils.py
+4 -7
@@ -2,7 +2,7 @@
2
3 import warnings
4 from collections.abc import Sequence
5 -from typing import Optional, Union
5 +from typing import Any, Optional, Union
6
7 import torch
8
@@ -254,9 +254,9 @@ def check_logprobs_close(
254 def build_model_context(
255 model_id: str,
256 task: TaskOption = "auto",
257 - dtype: Optional[Union[str, torch.dtype]] = None,
258 - mm_processor_kwargs: Optional[dict] = None,
259 - limit_mm_per_prompt: Optional[dict] = None,
257 + dtype: Union[str, torch.dtype] = "auto",
258 + mm_processor_kwargs: Optional[dict[str, Any]] = None,
259 + limit_mm_per_prompt: Optional[dict[str, int]] = None,
260 disable_mm_preprocessor_cache: bool = True,
261 ):
262 """Creates an InputContext for a given model.
@@ -274,9 +274,6 @@ def build_model_context(
274 model_info.check_available_online(on_fail="skip")
275 model_info.check_transformers_version(on_fail="skip")
276
277 - if dtype is None:
278 - dtype = "half"
279 -
277 model_config = ModelConfig(
278 model_id,
279 task=task,
tests/multimodal/test_processing.py
+3 -3
@@ -853,7 +853,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
853 tokenizer_mode="auto",
854 trust_remote_code=False,
855 seed=0,
856 - dtype="half",
856 + dtype="auto",
857 revision=None,
858 limit_mm_per_prompt=limit_mm_per_prompt,
859 )
@@ -892,7 +892,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
892 tokenizer_mode="auto",
893 trust_remote_code=False,
894 seed=0,
895 - dtype="half",
895 + dtype="auto",
896 revision=None,
897 limit_mm_per_prompt=limit_mm_per_prompt,
898 )
@@ -965,7 +965,7 @@ def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):
965 tokenizer_mode="auto",
966 trust_remote_code=False,
967 seed=0,
968 - dtype="half",
968 + dtype="auto",
969 revision=None,
970 )
971
tests/tensorizer_loader/test_tensorizer.py
+2 -2
@@ -166,7 +166,7 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
166 test_prompts = multilora_inference.create_test_prompts(lora_path)
167
168 # Serialize model before deserializing and binding LoRA adapters
169 - with vllm_runner(model_ref, ) as vllm_model:
169 + with vllm_runner(model_ref) as vllm_model:
170 model_path = tmp_path / (model_ref + ".tensors")
171
172 vllm_model.apply_model(
@@ -208,7 +208,7 @@ def test_load_without_tensorizer_load_format(vllm_runner):
208 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
209 def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
210 ## Serialize model
211 - with vllm_runner(model_ref, ) as vllm_model:
211 + with vllm_runner(model_ref) as vllm_model:
212 model_path = tmp_path / (model_ref + ".tensors")
213
214 vllm_model.apply_model(
tests/v1/engine/test_llm_engine.py
+1 -1
@@ -50,7 +50,7 @@ def _get_test_sampling_params(
50 """Generate random sampling params for a batch."""
51
52 def get_mostly_n_gt1() -> int:
53 - """Mostly n \in [2,20], ~1/3 n=1"""
53 + r"""Mostly n \in [2,20], ~1/3 n=1"""
54 x = random.randint(0, 28)
55 if x < 10:
56 return 1
vllm/config.py
+11 -11
@@ -347,7 +347,7 @@ class ModelConfig:
347 self.encoder_config = self._get_encoder_config()
348 self.hf_image_processor_config = get_hf_image_processor_config(
349 self.model, revision)
350 - self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
350 + self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
351 self.use_async_output_proc = use_async_output_proc
352 self.mm_processor_kwargs = mm_processor_kwargs
353 self.disable_mm_preprocessor_cache = disable_mm_preprocessor_cache
@@ -2526,6 +2526,14 @@ def _get_and_verify_dtype(
2526 # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
2527 # because config.torch_dtype can be None.
2528 config_dtype = getattr(config, "torch_dtype", None)
2529 +
2530 + # Fallbacks for multi-modal models if the root config
2531 + # does not define torch_dtype
2532 + if config_dtype is None and hasattr(config, "text_config"):
2533 + config_dtype = getattr(config.text_config, "torch_dtype", None)
2534 + if config_dtype is None and hasattr(config, "vision_config"):
2535 + config_dtype = getattr(config.vision_config, "torch_dtype", None)
2536 +
2537 if config_dtype is None:
2538 config_dtype = torch.float32
2539
@@ -2533,16 +2541,8 @@ def _get_and_verify_dtype(
2541 dtype = dtype.lower()
2542 if dtype == "auto":
2543 if config_dtype == torch.float32:
2536 - if config.model_type in ("gemma2", "gemma3", "gemma3_text"):
2537 - logger.info(
2538 - "For Gemma 2 and 3, we downcast float32 to bfloat16 "
2539 - "instead of float16 by default. Please specify `dtype` "
2540 - "if you want to use float16.")
2541 - torch_dtype = torch.bfloat16
2542 - else:
2543 - # Following the common practice, we use float16 for float32
2544 - # models.
2545 - torch_dtype = torch.float16
2544 + # Following common practice, we use float16 for float32 models
2545 + torch_dtype = torch.float16
2546 else:
2547 torch_dtype = config_dtype
2548