@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ebb3930d28

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 12:15 UTC

[Misc] Move config fields to MultiModalConfig (#17343)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 29, 2025 at 14:37 UTC ebb3930d28927da0e432ba8923ef9f83c6fb12f5

8 files changed +62 -36

vllm/config.py

+42 -15

@@ -263,6 +263,10 @@ class ModelConfig:

263 the model name will be the same as `model`.

264 limit_mm_per_prompt: Maximum number of data items per modality

265 per prompt. Only applicable for multimodal models.

266 + mm_processor_kwargs: Overrides for the multi-modal processor obtained

267 + from `AutoProcessor.from_pretrained`.

268 + disable_mm_preprocessor_cache: If True, disable caching of the

269 + processed multi-modal inputs.

270 use_async_output_proc: Whether to use async output processor.

271 Defaults to True.

272 config_format: The config format which shall be loaded.

@@ -273,10 +277,6 @@ class ModelConfig:

277 hf_overrides: If a dictionary, contains arguments to be forwarded to the

278 HuggingFace config. If a callable, it is called to update the

279 HuggingFace config.

276 - mm_processor_kwargs: Arguments to be forwarded to the model's processor

277 - for multi-modal data, e.g., image processor.

278 - disable_mm_preprocessor_cache: If true, then disables caching of the

279 - multi-modal preprocessor/mapper. (not recommended)

280 override_neuron_config: Initialize non default neuron config or

281 override default neuron config that are specific to Neuron devices,

282 this argument will be used to configure the neuron config that

@@ -320,7 +320,6 @@ class ModelConfig:

320 factors.append(self.max_logprobs)

321 factors.append(self.disable_sliding_window)

322 factors.append(self.trust_remote_code)

323 - factors.append(self.mm_processor_kwargs)

323 factors.append(self.generation_config)

324 factors.append(self.model_impl)

325 factors.append(self.override_generation_config)

@@ -359,12 +358,12 @@ class ModelConfig:

358 skip_tokenizer_init: bool = False,

359 served_model_name: Optional[Union[str, list[str]]] = None,

360 limit_mm_per_prompt: Optional[dict[str, int]] = None,

361 + mm_processor_kwargs: Optional[dict[str, Any]] = None,

362 + disable_mm_preprocessor_cache: bool = False,

363 use_async_output_proc: bool = True,

364 config_format: ConfigFormat = ConfigFormat.AUTO,

365 hf_token: Optional[Union[bool, str]] = None,

366 hf_overrides: Optional[HfOverrides] = None,

366 - mm_processor_kwargs: Optional[dict[str, Any]] = None,

367 - disable_mm_preprocessor_cache: bool = False,

367 override_neuron_config: Optional[dict[str, Any]] = None,

368 override_pooler_config: Optional["PoolerConfig"] = None,

369 logits_processor_pattern: Optional[str] = None,

@@ -469,8 +468,6 @@ class ModelConfig:

468 self.model, hf_token=hf_token, revision=revision)

469 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)

470 self.use_async_output_proc = use_async_output_proc

472 - self.mm_processor_kwargs = mm_processor_kwargs

473 - self.disable_mm_preprocessor_cache = disable_mm_preprocessor_cache

471

472 # Set enforce_eager to False if the value is unset.

473 if self.enforce_eager is None:

@@ -515,7 +512,10 @@ class ModelConfig:

512 self.served_model_name = get_served_model_name(model,

513 served_model_name)

514 self.multimodal_config = self._init_multimodal_config(

518 - limit_mm_per_prompt)

515 + limit_mm_per_prompt=limit_mm_per_prompt,

516 + mm_processor_kwargs=mm_processor_kwargs,

517 + disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,

518 + )

519 if not self.skip_tokenizer_init:

520 self._verify_tokenizer_mode()

521

@@ -581,14 +581,27 @@ class ModelConfig:

581 self.tokenizer = s3_tokenizer.dir

582

583 def _init_multimodal_config(

584 - self, limit_mm_per_prompt: Optional[dict[str, int]]

584 + self,

585 + limit_mm_per_prompt: Optional[dict[str, int]],

586 + mm_processor_kwargs: Optional[dict[str, Any]],

587 + disable_mm_preprocessor_cache: bool,

588 ) -> Optional["MultiModalConfig"]:

589 if self.registry.is_multimodal_model(self.architectures):

587 - return MultiModalConfig(limit_per_prompt=limit_mm_per_prompt or {})

590 + return MultiModalConfig(

591 + limit_per_prompt=limit_mm_per_prompt or {},

592 + mm_processor_kwargs=mm_processor_kwargs or {},

593 + disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,

594 + )

595

596 if limit_mm_per_prompt:

597 raise ValueError("`limit_mm_per_prompt` is only supported for "

598 "multimodal models.")

599 + if mm_processor_kwargs:

600 + raise ValueError("`mm_processor_kwargs` is only supported for "

601 + "multimodal models.")

602 + if disable_mm_preprocessor_cache:

603 + raise ValueError("`disable_mm_preprocessor_cache` is only "

604 + "supported for multimodal models.")

605

606 return None

607

@@ -2776,7 +2789,23 @@ class MultiModalConfig:

2789 Defaults to 1 (V0) or 999 (V1) for each modality.

2790

2791 For example, to allow up to 16 images and 2 videos per prompt:

2779 - ``{"images": 16, "videos": 2}``

2792 + :code:`{"images": 16, "videos": 2}`

2793 + """

2794 +

2795 + mm_processor_kwargs: Optional[dict[str, object]] = None

2796 + """

2797 + Overrides for the multi-modal processor obtained from

2798 + :meth:`transformers.AutoProcessor.from_pretrained`.

2799 +

2800 + The available overrides depend on the model that is being run.

2801 +

2802 + For example, for Phi-3-Vision:

2803 + :code:`{"num_crops": 4}`.

2804 + """

2805 +

2806 + disable_mm_preprocessor_cache: bool = False

2807 + """

2808 + If :code:`True`, disable caching of the processed multi-modal inputs.

2809 """

2810

2811 def compute_hash(self) -> str:

@@ -4080,8 +4109,6 @@ class VllmConfig:

4109 f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "

4110 f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa

4111 f"use_async_output_proc={self.model_config.use_async_output_proc}, "

4083 - f"disable_mm_preprocessor_cache={self.model_config.disable_mm_preprocessor_cache!r}, " # noqa

4084 - f"mm_processor_kwargs={self.model_config.mm_processor_kwargs}, "

4112 f"pooler_config={self.model_config.pooler_config!r}, "

4113 f"compilation_config={self.compilation_config!r}")

4114

vllm/engine/arg_utils.py

+4 -12

@@ -672,20 +672,12 @@ class EngineArgs:

672 )

673 multimodal_group.add_argument('--limit-mm-per-prompt',

674 **multimodal_kwargs["limit_per_prompt"])

675 -

676 - parser.add_argument(

675 + multimodal_group.add_argument(

676 '--mm-processor-kwargs',

678 - default=None,

679 - type=json.loads,

680 - help=('Overrides for the multi-modal processor obtained from '

681 - '``AutoProcessor.from_pretrained``. The available overrides '

682 - 'depend on the model that is being run.'

683 - 'For example, for Phi-3-Vision: ``{"num_crops": 4}``.'))

684 - parser.add_argument(

677 + **multimodal_kwargs["mm_processor_kwargs"])

678 + multimodal_group.add_argument(

679 '--disable-mm-preprocessor-cache',

686 - action='store_true',

687 - help='If True, disable caching of the processed multi-modal '

688 - 'inputs.')

680 + **multimodal_kwargs["disable_mm_preprocessor_cache"])

681

682 # LoRA related configs

683 lora_kwargs = get_kwargs(LoRAConfig)

vllm/inputs/registry.py

+4 -2

@@ -101,7 +101,8 @@ class InputContext:

101 Initialize a HuggingFace-like processor class, merging the

102 keyword arguments with those in the model's configuration.

103 """

104 - base_kwargs = self.model_config.mm_processor_kwargs

104 + mm_config = self.model_config.get_multimodal_config()

105 + base_kwargs = mm_config.mm_processor_kwargs

106 if base_kwargs is None:

107 base_kwargs = {}

108

@@ -139,7 +140,8 @@ class InputProcessingContext(InputContext):

140 """

141 assert callable(hf_processor)

142

142 - base_kwargs = self.model_config.mm_processor_kwargs

143 + mm_config = self.model_config.get_multimodal_config()

144 + base_kwargs = mm_config.mm_processor_kwargs

145 if base_kwargs is None:

146 base_kwargs = {}

147

vllm/model_executor/models/qwen2_vl.py

+3 -2

@@ -774,8 +774,9 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):

774 size: Optional[dict[str, int]] = None,

775 **kwargs: object,

776 ):

777 - if self.ctx.model_config.mm_processor_kwargs:

778 - kwargs.update(self.ctx.model_config.mm_processor_kwargs)

777 + mm_config = self.ctx.model_config.get_multimodal_config()

778 + if mm_config.mm_processor_kwargs:

779 + kwargs.update(mm_config.mm_processor_kwargs)

780

781 if min_pixels is not None:

782 kwargs["min_pixels"] = min_pixels

vllm/multimodal/registry.py

+2 -1

@@ -262,7 +262,8 @@ class MultiModalRegistry:

262 if tokenizer is None:

263 tokenizer = cached_tokenizer_from_config(model_config)

264 if disable_cache is None:

265 - disable_cache = model_config.disable_mm_preprocessor_cache

265 + mm_config = model_config.get_multimodal_config()

266 + disable_cache = mm_config.disable_mm_preprocessor_cache

267

268 model_cls = self._get_model_cls(model_config)

269 factories = self._processor_factories[model_cls]

vllm/transformers_utils/processor.py

+2 -1

@@ -33,7 +33,8 @@ class HashableList(list):

35 def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):

36 - base_kwargs = model_config.mm_processor_kwargs

36 + mm_config = model_config.get_multimodal_config()

37 + base_kwargs = mm_config.mm_processor_kwargs

38 if base_kwargs is None:

39 base_kwargs = {}

vllm/v1/engine/mm_input_cache.py

+4 -1

@@ -33,7 +33,10 @@ from vllm.utils import is_list_of

33 class MirroredProcessingCache:

35 def __init__(self, model_config):

36 - self.use_cache = not model_config.disable_mm_preprocessor_cache

36 + mm_config = model_config.multimodal_config

37 + disable_mm_preprocessor_cache = mm_config is not None and \

38 + not mm_config.disable_mm_preprocessor_cache

39 + self.use_cache = not disable_mm_preprocessor_cache

40 self.mm_cache = ProcessingCache.get_lru_cache(VLLM_MM_INPUT_CACHE_GIB,

41 MultiModalKwargs)

vllm/v1/engine/processor.py

+1 -2

@@ -51,8 +51,7 @@ class Processor:

51 self.mm_input_cache_client = MirroredProcessingCache(self.model_config)

53 # Multi-modal hasher (for images)

54 - self.use_hash = (

55 - not self.model_config.disable_mm_preprocessor_cache) or \

54 + self.use_hash = self.mm_input_cache_client.use_cache or \

55 self.cache_config.enable_prefix_caching

57 def _validate_logprobs(