@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bbf55c4805
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[VLM] Refactor `MultiModalConfig` initialization and profiling (#7530)

Roger Wang committed Aug 17, 2024 at 13:30 UTC bbf55c4805efba5f1d7094f5e2888b3ef26c0fd7
29 files changed +143 -190
tests/entrypoints/openai/test_audio.py
+6 -2
@@ -86,8 +86,12 @@ def server_function(port):
86
87 ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
88
89 - with patch("vllm.entrypoints.chat_utils._mm_token_str",
90 - lambda *_, **__: "_"):
89 + with patch(
90 + "vllm.entrypoints.chat_utils._mm_token_str",
91 + lambda *_, **__: "_"), patch(
92 + "vllm.model_executor.models.ModelRegistry.is_multimodal_model"
93 + ) as mock:
94 + mock.return_value = True
95 sys.argv = ["placeholder.py"] + \
96 (f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
97 "--dtype bfloat16 --enforce-eager --api-key token-abc123 "
tests/multimodal/test_mapper.py
+9 -9
@@ -4,7 +4,7 @@ import numpy as np
4 import pytest
5 from transformers import CLIPImageProcessor, LlavaNextImageProcessor
6
7 -from vllm.config import ModelConfig, MultiModalConfig
7 +from vllm.config import ModelConfig
8 from vllm.multimodal import MultiModalRegistry
9 from vllm.multimodal.utils import rescale_image_size
10
@@ -30,10 +30,10 @@ def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
30 seed=0,
31 dtype=dtype,
32 revision=None,
33 + limit_mm_per_prompt={"image": 1},
34 )
34 - mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
35
36 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
36 + mm_registry.init_mm_limits_per_prompt(model_config)
37
38 for asset in image_assets:
39 image = rescale_image_size(asset.pil_image, size_factor)
@@ -73,10 +73,10 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
73 seed=0,
74 dtype=dtype,
75 revision=None,
76 + limit_mm_per_prompt={"image": 1},
77 )
77 - mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
78
79 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
79 + mm_registry.init_mm_limits_per_prompt(model_config)
80
81 for asset in image_assets:
82 image = rescale_image_size(asset.pil_image, size_factor)
@@ -115,10 +115,10 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
115 seed=0,
116 dtype="half",
117 revision=None,
118 + limit_mm_per_prompt={"image": limit},
119 )
119 - mm_config = MultiModalConfig(limit_per_prompt={"image": limit})
120
121 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
121 + mm_registry.init_mm_limits_per_prompt(model_config)
122
123 image = image_assets[0].pil_image
124 if num_images == 0:
@@ -145,10 +145,10 @@ def test_image_mapper_multi(image_assets, mm_registry, num_images):
145 seed=0,
146 dtype="half",
147 revision=None,
148 + limit_mm_per_prompt={"image": num_images},
149 )
149 - mm_config = MultiModalConfig(limit_per_prompt={"image": num_images})
150
151 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
151 + mm_registry.init_mm_limits_per_prompt(model_config)
152
153 image = image_assets[0].pil_image
154 mm_inputs = {"image": [image] * num_images}
vllm/config.py
+33 -5
@@ -109,6 +109,8 @@ class ModelConfig:
109 matches the model name exposed via the APIs. If multiple model
110 names provided, the first name will be used. If not specified,
111 the model name will be the same as `model`.
112 + limit_mm_per_prompt: Maximum number of data instances per modality
113 + per prompt. Only applicable for multimodal models.
114 """
115
116 def __init__(
@@ -134,7 +136,7 @@ class ModelConfig:
136 disable_sliding_window: bool = False,
137 skip_tokenizer_init: bool = False,
138 served_model_name: Optional[Union[str, List[str]]] = None,
137 - multimodal_config: Optional["MultiModalConfig"] = None,
139 + limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
140 ) -> None:
141 self.model = model
142 self.tokenizer = tokenizer
@@ -211,14 +213,29 @@ class ModelConfig:
213 sliding_window_len=self.get_hf_config_sliding_window())
214 self.served_model_name = get_served_model_name(model,
215 served_model_name)
214 - self.multimodal_config = multimodal_config
215 -
216 + self.multimodal_config = self._init_multimodal_config(
217 + limit_mm_per_prompt)
218 if not self.skip_tokenizer_init:
219 self._verify_tokenizer_mode()
220 self._verify_embedding_mode()
221 self._verify_quantization()
222 self._verify_cuda_graph()
223
224 + def _init_multimodal_config(
225 + self, limit_mm_per_prompt: Optional[Mapping[str, int]]
226 + ) -> Optional["MultiModalConfig"]:
227 + architectures = getattr(self.hf_config, "architectures", [])
228 + if any(
229 + ModelRegistry.is_multimodal_model(arch)
230 + for arch in architectures):
231 + return MultiModalConfig(limit_per_prompt=limit_mm_per_prompt or {})
232 + else:
233 + if limit_mm_per_prompt:
234 + raise ValueError(
235 + "limit_mm_per_prompt is only supported for multimodal "
236 + "models.")
237 + return None
238 +
239 def _verify_tokenizer_mode(self) -> None:
240 tokenizer_mode = self.tokenizer_mode.lower()
241 if tokenizer_mode not in ["auto", "slow"]:
@@ -467,6 +484,18 @@ class ModelConfig:
484 if t != "attention"
485 ])
486
487 + def get_multimodal_config(self) -> "MultiModalConfig":
488 + """
489 + Get the multimodal configuration of the model.
490 +
491 + Raises:
492 + ValueError: If the model is not multimodal.
493 + """
494 + if self.multimodal_config is None:
495 + raise ValueError("The model is not multimodal.")
496 +
497 + return self.multimodal_config
498 +
499 @property
500 def is_encoder_decoder_model(self) -> bool:
501 """Extract the HF encoder/decoder model flag."""
@@ -1450,7 +1479,7 @@ class PromptAdapterConfig:
1479 class MultiModalConfig:
1480 """Controls the behavior of multimodal models."""
1481
1453 - limit_per_prompt: Mapping[str, int]
1482 + limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
1483 """
1484 The maximum number of multi-modal input instances allowed per prompt
1485 for each :class:`~vllm.multimodal.MultiModalPlugin`.
@@ -1710,7 +1739,6 @@ class EngineConfig:
1739 device_config: DeviceConfig
1740 load_config: LoadConfig
1741 lora_config: Optional[LoRAConfig]
1713 - multimodal_config: Optional[MultiModalConfig]
1742 speculative_config: Optional[SpeculativeConfig]
1743 decoding_config: Optional[DecodingConfig]
1744 observability_config: Optional[ObservabilityConfig]
vllm/engine/arg_utils.py
+3 -6
@@ -7,7 +7,7 @@ from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
7
8 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
10 - MultiModalConfig, ObservabilityConfig, ParallelConfig,
10 + ObservabilityConfig, ParallelConfig,
11 PromptAdapterConfig, SchedulerConfig,
12 SpeculativeConfig, TokenizerPoolConfig)
13 from vllm.executor.executor_base import ExecutorBase
@@ -765,9 +765,6 @@ class EngineArgs:
765 "CPU offload space must be non-negative"
766 f", but got {self.cpu_offload_gb}")
767
768 - multimodal_config = MultiModalConfig(
769 - limit_per_prompt=self.limit_mm_per_prompt or {})
770 -
768 device_config = DeviceConfig(device=self.device)
769 model_config = ModelConfig(
770 model=self.model,
@@ -791,7 +788,8 @@ class EngineArgs:
788 disable_sliding_window=self.disable_sliding_window,
789 skip_tokenizer_init=self.skip_tokenizer_init,
790 served_model_name=self.served_model_name,
794 - multimodal_config=multimodal_config)
791 + limit_mm_per_prompt=self.limit_mm_per_prompt,
792 + )
793 cache_config = CacheConfig(
794 block_size=self.block_size,
795 gpu_memory_utilization=self.gpu_memory_utilization,
@@ -970,7 +968,6 @@ class EngineArgs:
968 scheduler_config=scheduler_config,
969 device_config=device_config,
970 lora_config=lora_config,
973 - multimodal_config=multimodal_config,
971 speculative_config=speculative_config,
972 load_config=load_config,
973 decoding_config=decoding_config,
vllm/engine/llm_engine.py
+1 -6
@@ -10,7 +10,7 @@ from typing_extensions import assert_never
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
13 - MultiModalConfig, ObservabilityConfig, ParallelConfig,
13 + ObservabilityConfig, ParallelConfig,
14 PromptAdapterConfig, SchedulerConfig,
15 SpeculativeConfig)
16 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
@@ -100,8 +100,6 @@ class LLMEngine:
100 scheduler_config: The configuration related to the request scheduler.
101 device_config: The configuration related to the device.
102 lora_config (Optional): The configuration related to serving multi-LoRA.
103 - multimodal_config (Optional): The configuration related to multimodal
104 - models.
103 speculative_config (Optional): The configuration related to speculative
104 decoding.
105 executor_class: The model executor class for managing distributed
@@ -172,7 +170,6 @@ class LLMEngine:
170 device_config: DeviceConfig,
171 load_config: LoadConfig,
172 lora_config: Optional[LoRAConfig],
175 - multimodal_config: Optional[MultiModalConfig],
173 speculative_config: Optional[SpeculativeConfig],
174 decoding_config: Optional[DecodingConfig],
175 observability_config: Optional[ObservabilityConfig],
@@ -235,7 +232,6 @@ class LLMEngine:
232 self.model_config = model_config
233 self.cache_config = cache_config
234 self.lora_config = lora_config
238 - self.multimodal_config = multimodal_config
235 self.parallel_config = parallel_config
236 self.scheduler_config = scheduler_config
237 self.device_config = device_config
@@ -278,7 +274,6 @@ class LLMEngine:
274 scheduler_config=scheduler_config,
275 device_config=device_config,
276 lora_config=lora_config,
281 - multimodal_config=multimodal_config,
277 speculative_config=speculative_config,
278 load_config=load_config,
279 prompt_adapter_config=prompt_adapter_config,
vllm/executor/cpu_executor.py
-1
@@ -141,7 +141,6 @@ class CPUExecutor(ExecutorBase):
141 rank=rank,
142 distributed_init_method=self.distributed_init_method,
143 lora_config=self.lora_config,
144 - multimodal_config=self.multimodal_config,
144 kv_cache_dtype=self.cache_config.cache_dtype,
145 prompt_adapter_config=self.prompt_adapter_config,
146 is_driver_worker=rank == 0,
vllm/executor/executor_base.py
+2 -4
@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
2 from typing import List, Optional, Set, Tuple
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5 - ModelConfig, MultiModalConfig, ObservabilityConfig,
6 - ParallelConfig, PromptAdapterConfig, SchedulerConfig,
5 + ModelConfig, ObservabilityConfig, ParallelConfig,
6 + PromptAdapterConfig, SchedulerConfig,
7 SpeculativeConfig)
8 from vllm.lora.request import LoRARequest
9 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -29,7 +29,6 @@ class ExecutorBase(ABC):
29 device_config: DeviceConfig,
30 load_config: LoadConfig,
31 lora_config: Optional[LoRAConfig],
32 - multimodal_config: Optional[MultiModalConfig],
32 speculative_config: Optional[SpeculativeConfig],
33 prompt_adapter_config: Optional[PromptAdapterConfig],
34 observability_config: Optional[ObservabilityConfig],
@@ -41,7 +40,6 @@ class ExecutorBase(ABC):
40 self.parallel_config = parallel_config
41 self.scheduler_config = scheduler_config
42 self.device_config = device_config
44 - self.multimodal_config = multimodal_config
43 self.speculative_config = speculative_config
44 self.prompt_adapter_config = prompt_adapter_config
45 self.observability_config = observability_config
vllm/executor/gpu_executor.py
-1
@@ -55,7 +55,6 @@ class GPUExecutor(ExecutorBase):
55 rank=rank,
56 distributed_init_method=distributed_init_method,
57 lora_config=self.lora_config,
58 - multimodal_config=self.multimodal_config,
58 speculative_config=self.speculative_config,
59 prompt_adapter_config=self.prompt_adapter_config,
60 is_driver_worker=(not self.parallel_config)
vllm/executor/openvino_executor.py
-1
@@ -49,7 +49,6 @@ class OpenVINOExecutor(ExecutorBase):
49 rank=0,
50 distributed_init_method=distributed_init_method,
51 lora_config=self.lora_config,
52 - multimodal_config=self.multimodal_config,
52 kv_cache_dtype=self.cache_config.cache_dtype,
53 is_driver_worker=True,
54 )
vllm/executor/ray_xpu_executor.py
+2 -6
@@ -7,9 +7,8 @@ from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
7
8 import vllm.envs as envs
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, MultiModalConfig, ParallelConfig,
11 - PromptAdapterConfig, SchedulerConfig,
12 - SpeculativeConfig)
10 + ModelConfig, ParallelConfig, PromptAdapterConfig,
11 + SchedulerConfig, SpeculativeConfig)
12 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13 DistributedGPUExecutor, DistributedGPUExecutorAsync)
14 from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -46,7 +45,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
45 device_config: DeviceConfig,
46 load_config: LoadConfig,
47 lora_config: Optional[LoRAConfig],
49 - multimodal_config: Optional[MultiModalConfig],
48 prompt_adapter_config: Optional[PromptAdapterConfig],
49 speculative_config: Optional[SpeculativeConfig],
50 ) -> None:
@@ -61,7 +59,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
59 self.parallel_config = parallel_config
60 self.scheduler_config = scheduler_config
61 self.device_config = device_config
64 - self.multimodal_config = multimodal_config
62 self.prompt_adapter_config = prompt_adapter_config
63
64 placement_group = self.parallel_config.placement_group
@@ -203,7 +200,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
200 rank=rank,
201 distributed_init_method=distributed_init_method,
202 lora_config=self.lora_config,
206 - multimodal_config=self.multimodal_config,
203 is_driver_worker=rank == 0,
204 ))
205 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
vllm/executor/tpu_executor.py
-1
@@ -52,7 +52,6 @@ class TPUExecutor(ExecutorBase):
52 local_rank=local_rank,
53 rank=rank,
54 distributed_init_method=distributed_init_method,
55 - multimodal_config=self.multimodal_config,
55 is_driver_worker=rank == 0,
56 )
57
vllm/executor/xpu_executor.py
+2 -5
@@ -3,9 +3,8 @@ from typing import List, Optional
3 import torch
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, MultiModalConfig, ParallelConfig,
7 - PromptAdapterConfig, SchedulerConfig,
8 - SpeculativeConfig)
6 + ModelConfig, ParallelConfig, PromptAdapterConfig,
7 + SchedulerConfig, SpeculativeConfig)
8 from vllm.executor.executor_base import ExecutorAsyncBase
9 from vllm.executor.gpu_executor import GPUExecutor
10 from vllm.logger import init_logger
@@ -29,7 +28,6 @@ class XPUExecutor(GPUExecutor):
28 device_config: DeviceConfig,
29 load_config: LoadConfig,
30 lora_config: Optional[LoRAConfig],
32 - multimodal_config: Optional[MultiModalConfig],
31 prompt_adapter_config: Optional[PromptAdapterConfig],
32 speculative_config: Optional[SpeculativeConfig],
33 ) -> None:
@@ -46,7 +44,6 @@ class XPUExecutor(GPUExecutor):
44 self.parallel_config = parallel_config
45 self.scheduler_config = scheduler_config
46 self.device_config = device_config
49 - self.multimodal_config = multimodal_config
47 self.prompt_adapter_config = prompt_adapter_config
48 self.speculative_config = None
49
vllm/inputs/registry.py
+1 -15
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13 from .data import LLMInputs
14
15 if TYPE_CHECKING:
16 - from vllm.config import ModelConfig, MultiModalConfig
16 + from vllm.config import ModelConfig
17 from vllm.multimodal import MultiModalDataDict, MultiModalRegistry
18 from vllm.sequence import SequenceData
19
@@ -32,20 +32,6 @@ class InputContext:
32 model_config: "ModelConfig"
33 """The configuration of the model."""
34
35 - def get_multimodal_config(self) -> "MultiModalConfig":
36 - """
37 - Get the multimodal configuration of the model.
38 -
39 - Raises:
40 - ValueError: If the model is not multimodal.
41 - """
42 -
43 - multimodal_config = self.model_config.multimodal_config
44 - if multimodal_config is None:
45 - raise ValueError("No multimodal config found")
46 -
47 - return multimodal_config
48 -
35 def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C:
36 """
37 Get the HuggingFace configuration
vllm/model_executor/model_loader/__init__.py
+1 -4
@@ -3,8 +3,7 @@ from typing import Optional
3 from torch import nn
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, MultiModalConfig, ParallelConfig,
7 - SchedulerConfig)
6 + ModelConfig, ParallelConfig, SchedulerConfig)
7 from vllm.model_executor.model_loader.loader import (BaseModelLoader,
8 get_model_loader)
9 from vllm.model_executor.model_loader.utils import (
@@ -15,13 +14,11 @@ def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
14 device_config: DeviceConfig, parallel_config: ParallelConfig,
15 scheduler_config: SchedulerConfig,
16 lora_config: Optional[LoRAConfig],
18 - multimodal_config: Optional[MultiModalConfig],
17 cache_config: CacheConfig) -> nn.Module:
18 loader = get_model_loader(load_config)
19 return loader.load_model(model_config=model_config,
20 device_config=device_config,
21 lora_config=lora_config,
24 - multimodal_config=multimodal_config,
22 parallel_config=parallel_config,
23 scheduler_config=scheduler_config,
24 cache_config=cache_config)
vllm/model_executor/model_loader/loader.py
+14 -32
@@ -132,9 +132,7 @@ def _get_model_initialization_kwargs(
132 "please open an issue on github.")
133
134 if supports_multimodal(model_class):
135 - if multimodal_config is None:
136 - raise ValueError("Provide multi-modal related configurations "
137 - "through LLM entrypoint or engine arguments.")
135 + assert multimodal_config is not None
136
137 extra_kwargs["multimodal_config"] = multimodal_config
138
@@ -164,7 +162,6 @@ def _initialize_model(
162 model_config: ModelConfig,
163 load_config: LoadConfig,
164 lora_config: Optional[LoRAConfig],
167 - multimodal_config: Optional[MultiModalConfig],
165 cache_config: CacheConfig,
166 scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
167 """Initialize a model with the given configurations."""
@@ -173,10 +170,10 @@ def _initialize_model(
170 return build_model(
171 model_class,
172 model_config.hf_config,
173 + cache_config=cache_config,
174 quant_config=_get_quantization_config(model_config, load_config),
175 lora_config=lora_config,
178 - multimodal_config=multimodal_config,
179 - cache_config=cache_config,
176 + multimodal_config=model_config.multimodal_config,
177 scheduler_config=scheduler_config,
178 )
179
@@ -191,7 +188,6 @@ class BaseModelLoader(ABC):
188 def load_model(self, *, model_config: ModelConfig,
189 device_config: DeviceConfig,
190 lora_config: Optional[LoRAConfig],
194 - multimodal_config: Optional[MultiModalConfig],
191 parallel_config: ParallelConfig,
192 scheduler_config: SchedulerConfig,
193 cache_config: CacheConfig) -> nn.Module:
@@ -336,7 +332,6 @@ class DefaultModelLoader(BaseModelLoader):
332 def load_model(self, *, model_config: ModelConfig,
333 device_config: DeviceConfig,
334 lora_config: Optional[LoRAConfig],
339 - multimodal_config: Optional[MultiModalConfig],
335 parallel_config: ParallelConfig,
336 scheduler_config: SchedulerConfig,
337 cache_config: CacheConfig) -> nn.Module:
@@ -344,8 +339,8 @@ class DefaultModelLoader(BaseModelLoader):
339 with set_default_torch_dtype(model_config.dtype):
340 with target_device:
341 model = _initialize_model(model_config, self.load_config,
347 - lora_config, multimodal_config,
348 - cache_config, scheduler_config)
342 + lora_config, cache_config,
343 + scheduler_config)
344 model.load_weights(
345 self._get_weights_iterator(model_config.model,
346 model_config.revision,
@@ -379,15 +374,14 @@ class DummyModelLoader(BaseModelLoader):
374 def load_model(self, *, model_config: ModelConfig,
375 device_config: DeviceConfig,
376 lora_config: Optional[LoRAConfig],
382 - multimodal_config: Optional[MultiModalConfig],
377 parallel_config: ParallelConfig,
378 scheduler_config: SchedulerConfig,
379 cache_config: CacheConfig) -> nn.Module:
380 with set_default_torch_dtype(model_config.dtype):
381 with torch.device(device_config.device):
382 model = _initialize_model(model_config, self.load_config,
389 - lora_config, multimodal_config,
390 - cache_config, scheduler_config)
383 + lora_config, cache_config,
384 + scheduler_config)
385 # NOTE(woosuk): For accurate performance evaluation, we assign
386 # random values to the weights.
387 initialize_dummy_weights(model)
@@ -420,7 +414,6 @@ class TensorizerLoader(BaseModelLoader):
414 model_config: ModelConfig,
415 device_config: DeviceConfig,
416 lora_config: Optional[LoRAConfig],
423 - multimodal_config: Optional[MultiModalConfig],
417 cache_config: CacheConfig,
418 ) -> nn.Module:
419 """Load a serialized model with tensorizer to the CPU.
@@ -433,8 +426,7 @@ class TensorizerLoader(BaseModelLoader):
426 with set_default_torch_dtype(model_config.dtype):
427 with torch.device(device_config.device):
428 model = _initialize_model(model_config, self.load_config,
436 - lora_config, multimodal_config,
437 - cache_config)
429 + lora_config, cache_config)
430
431 model.load_weights(self._get_weights_iterator())
432 return model.eval()
@@ -444,7 +436,6 @@ class TensorizerLoader(BaseModelLoader):
436 model_config: ModelConfig,
437 device_config: DeviceConfig,
438 lora_config: Optional[LoRAConfig],
447 - multimodal_config: Optional[MultiModalConfig],
439 cache_config: CacheConfig,
440 ) -> nn.Module:
441 """Load a serialized model with tensorizer.
@@ -458,7 +449,7 @@ class TensorizerLoader(BaseModelLoader):
449 quant_config = _get_quantization_config(
450 model_config, self.load_config)
451 extra_kwargs = _get_model_initialization_kwargs(
461 - model_class, lora_config, multimodal_config)
452 + model_class, lora_config, model_config.multimodal_config)
453 extra_kwargs["quant_config"] = quant_config
454 extra_kwargs["cache_config"] = cache_config
455
@@ -473,7 +464,6 @@ class TensorizerLoader(BaseModelLoader):
464 def load_model(self, *, model_config: ModelConfig,
465 device_config: DeviceConfig,
466 lora_config: Optional[LoRAConfig],
476 - multimodal_config: Optional[MultiModalConfig],
467 parallel_config: ParallelConfig,
468 scheduler_config: SchedulerConfig,
469 cache_config: CacheConfig) -> nn.Module:
@@ -487,11 +477,9 @@ class TensorizerLoader(BaseModelLoader):
477
478 if is_vllm_tensorized(self.tensorizer_config):
479 return self._load_model_serialized(model_config, device_config,
490 - lora_config, multimodal_config,
491 - cache_config)
480 + lora_config, cache_config)
481 return self._load_model_serialized_cpu(model_config, device_config,
493 - lora_config, multimodal_config,
494 - cache_config)
482 + lora_config, cache_config)
483
484 @staticmethod
485 def save_model(
@@ -577,7 +565,6 @@ class ShardedStateLoader(BaseModelLoader):
565 def load_model(self, *, model_config: ModelConfig,
566 device_config: DeviceConfig,
567 lora_config: Optional[LoRAConfig],
580 - multimodal_config: Optional[MultiModalConfig],
568 parallel_config: ParallelConfig,
569 scheduler_config: SchedulerConfig,
570 cache_config: CacheConfig) -> nn.Module:
@@ -591,8 +578,7 @@ class ShardedStateLoader(BaseModelLoader):
578 with set_default_torch_dtype(model_config.dtype):
579 with torch.device(device_config.device):
580 model = _initialize_model(model_config, self.load_config,
594 - lora_config, multimodal_config,
595 - cache_config)
581 + lora_config, cache_config)
582 rank = get_tensor_model_parallel_rank()
583 pattern = os.path.join(
584 local_model_path,
@@ -955,15 +941,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
941 def load_model(self, *, model_config: ModelConfig,
942 device_config: DeviceConfig,
943 lora_config: Optional[LoRAConfig],
958 - multimodal_config: Optional[MultiModalConfig],
944 parallel_config: ParallelConfig,
945 scheduler_config: SchedulerConfig,
946 cache_config: CacheConfig) -> nn.Module:
947 with set_default_torch_dtype(model_config.dtype):
948 with torch.device(device_config.device):
949 model = _initialize_model(model_config, self.load_config,
965 - lora_config, multimodal_config,
966 - cache_config)
950 + lora_config, cache_config)
951
952 self._load_weights(model_config, model)
953
@@ -1032,7 +1016,6 @@ class GGUFModelLoader(BaseModelLoader):
1016 def load_model(self, *, model_config: ModelConfig,
1017 device_config: DeviceConfig,
1018 lora_config: Optional[LoRAConfig],
1035 - multimodal_config: Optional[MultiModalConfig],
1019 parallel_config: ParallelConfig,
1020 scheduler_config: SchedulerConfig,
1021 cache_config: CacheConfig) -> nn.Module:
@@ -1047,8 +1030,7 @@ class GGUFModelLoader(BaseModelLoader):
1030 with set_default_torch_dtype(model_config.dtype):
1031 with torch.device(device_config.device):
1032 model = _initialize_model(model_config, self.load_config,
1050 - lora_config, multimodal_config,
1051 - cache_config)
1033 + lora_config, cache_config)
1034 model.load_weights(
1035 self._get_weights_iterator(local_model_path, gguf_weights_map))
1036 return model
vllm/model_executor/models/__init__.py
+27 -16
@@ -9,17 +9,12 @@ from vllm.utils import is_hip
9
10 logger = init_logger(__name__)
11
12 -# Architecture -> (module, class).
12 _GENERATION_MODELS = {
13 "AquilaModel": ("llama", "LlamaForCausalLM"),
14 "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19 - "Blip2ForConditionalGeneration":
20 - ("blip2", "Blip2ForConditionalGeneration"),
21 - "ChameleonForConditionalGeneration":
22 - ("chameleon", "ChameleonForConditionalGeneration"),
18 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20 "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
@@ -28,7 +23,6 @@ _GENERATION_MODELS = {
23 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24 "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
31 - "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
26 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
27 "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
28 "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
@@ -37,13 +31,8 @@ _GENERATION_MODELS = {
31 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
32 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
33 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
40 - "InternVLChatModel": ("internvl", "InternVLChatModel"),
34 "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
35 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
43 - "LlavaForConditionalGeneration":
44 - ("llava", "LlavaForConditionalGeneration"),
45 - "LlavaNextForConditionalGeneration":
46 - ("llava_next", "LlavaNextForConditionalGeneration"),
36 # For decapoda-research/llama-*
37 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
38 "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
@@ -53,17 +42,13 @@ _GENERATION_MODELS = {
42 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
43 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
44 "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
56 - "MiniCPMV": ("minicpmv", "MiniCPMV"),
45 "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
46 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
47 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
48 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
49 "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
62 - "PaliGemmaForConditionalGeneration": ("paligemma",
63 - "PaliGemmaForConditionalGeneration"),
50 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
51 "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
66 - "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
52 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
53 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
54 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
@@ -83,6 +68,22 @@ _EMBEDDING_MODELS = {
68 "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
69 }
70
71 +_MULTIMODAL_MODELS = {
72 + "Blip2ForConditionalGeneration":
73 + ("blip2", "Blip2ForConditionalGeneration"),
74 + "ChameleonForConditionalGeneration":
75 + ("chameleon", "ChameleonForConditionalGeneration"),
76 + "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
77 + "InternVLChatModel": ("internvl", "InternVLChatModel"),
78 + "LlavaForConditionalGeneration":
79 + ("llava", "LlavaForConditionalGeneration"),
80 + "LlavaNextForConditionalGeneration":
81 + ("llava_next", "LlavaNextForConditionalGeneration"),
82 + "MiniCPMV": ("minicpmv", "MiniCPMV"),
83 + "PaliGemmaForConditionalGeneration": ("paligemma",
84 + "PaliGemmaForConditionalGeneration"),
85 + "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
86 +}
87 _CONDITIONAL_GENERATION_MODELS = {
88 "BartModel": ("bart", "BartForConditionalGeneration"),
89 "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
@@ -91,7 +92,8 @@ _CONDITIONAL_GENERATION_MODELS = {
92 _MODELS = {
93 **_GENERATION_MODELS,
94 **_EMBEDDING_MODELS,
94 - **_CONDITIONAL_GENERATION_MODELS
95 + **_MULTIMODAL_MODELS,
96 + **_CONDITIONAL_GENERATION_MODELS,
97 }
98
99 # Architecture -> type.
@@ -182,6 +184,15 @@ class ModelRegistry:
184 def is_embedding_model(model_arch: str) -> bool:
185 return model_arch in _EMBEDDING_MODELS
186
187 + @staticmethod
188 + def is_multimodal_model(model_arch: str) -> bool:
189 +
190 + # TODO: find a way to avoid initializing CUDA prematurely to
191 + # use `supports_multimodal` to determine if a model is multimodal
192 + # model_cls = ModelRegistry._try_load_model_cls(model_arch)
193 + # from vllm.model_executor.models.interfaces import supports_multimodal
194 + return model_arch in _MULTIMODAL_MODELS
195 +
196
197 __all__ = [
198 "ModelRegistry",
vllm/multimodal/registry.py
+2 -2
@@ -2,7 +2,7 @@ import functools
2 from collections import UserDict
3 from typing import Dict, Mapping, Optional, Sequence
4
5 -from vllm.config import ModelConfig, MultiModalConfig
5 +from vllm.config import ModelConfig
6 from vllm.logger import init_logger
7
8 from .audio import AudioPlugin
@@ -181,7 +181,6 @@ class MultiModalRegistry:
181 def init_mm_limits_per_prompt(
182 self,
183 model_config: ModelConfig,
184 - multimodal_config: Optional[MultiModalConfig],
184 ) -> None:
185 """
186 Initialize the maximum number of multi-modal input instances for each
@@ -192,6 +191,7 @@ class MultiModalRegistry:
191 "`mm_limits` has already been set for model=%s, and will "
192 "be overwritten by the new values.", model_config.model)
193
194 + multimodal_config = model_config.multimodal_config
195 if multimodal_config is None:
196 limits_per_plugin = self._disabled_limits_per_plugin
197 else:
vllm/spec_decode/draft_model_runner.py
+2 -4
@@ -23,8 +23,8 @@ except ImportError:
23 FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24
25 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26 - ModelConfig, MultiModalConfig, ObservabilityConfig,
27 - ParallelConfig, PromptAdapterConfig, SchedulerConfig)
26 + ModelConfig, ObservabilityConfig, ParallelConfig,
27 + PromptAdapterConfig, SchedulerConfig)
28 from vllm.logger import init_logger
29 from vllm.multimodal import MultiModalInputs
30 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
@@ -66,7 +66,6 @@ class TP1DraftModelRunner(ModelRunner):
66 lora_config: Optional[LoRAConfig],
67 kv_cache_dtype: Optional[str] = "auto",
68 is_driver_worker: bool = False,
69 - multimodal_config: Optional[MultiModalConfig] = None,
69 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
70 return_hidden_states: bool = False,
71 observability_config: Optional[ObservabilityConfig] = None,
@@ -86,7 +85,6 @@ class TP1DraftModelRunner(ModelRunner):
85 lora_config=lora_config,
86 kv_cache_dtype=kv_cache_dtype,
87 is_driver_worker=is_driver_worker,
89 - multimodal_config=multimodal_config,
88 prompt_adapter_config=prompt_adapter_config,
89 return_hidden_states=return_hidden_states,
90 observability_config=observability_config,
vllm/spec_decode/target_model_runner.py
+2 -4
@@ -1,8 +1,8 @@
1 from typing import List, Optional
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4 - ModelConfig, MultiModalConfig, ObservabilityConfig,
5 - ParallelConfig, PromptAdapterConfig, SchedulerConfig)
4 + ModelConfig, ObservabilityConfig, ParallelConfig,
5 + PromptAdapterConfig, SchedulerConfig)
6 from vllm.sequence import SequenceGroupMetadata
7 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
8 ModelRunner)
@@ -31,7 +31,6 @@ class TargetModelRunner(ModelRunner):
31 kv_cache_dtype: Optional[str] = "auto",
32 is_driver_worker: bool = False,
33 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34 - multimodal_config: Optional[MultiModalConfig] = None,
34 return_hidden_states: bool = False,
35 observability_config: Optional[ObservabilityConfig] = None):
36 # An internal boolean member variable to indicate if token log
@@ -47,7 +46,6 @@ class TargetModelRunner(ModelRunner):
46 lora_config=lora_config,
47 kv_cache_dtype=kv_cache_dtype,
48 is_driver_worker=is_driver_worker,
50 - multimodal_config=multimodal_config,
49 prompt_adapter_config=prompt_adapter_config,
50 return_hidden_states=return_hidden_states,
51 observability_config=observability_config,
vllm/worker/cpu_model_runner.py
+2 -5
@@ -6,8 +6,8 @@ from torch import nn
6
7 from vllm.attention import AttentionMetadata, get_attn_backend
8 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 - ModelConfig, MultiModalConfig, ParallelConfig,
10 - PromptAdapterConfig, SchedulerConfig)
9 + ModelConfig, ParallelConfig, PromptAdapterConfig,
10 + SchedulerConfig)
11 from vllm.logger import init_logger
12 from vllm.model_executor import SamplingMetadata
13 from vllm.model_executor.model_loader import get_model
@@ -79,7 +79,6 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
79 cache_config: CacheConfig,
80 load_config: LoadConfig,
81 lora_config: Optional[LoRAConfig],
82 - multimodal_config: Optional[MultiModalConfig],
82 kv_cache_dtype: Optional[str] = "auto",
83 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
84 is_driver_worker: bool = False,
@@ -94,7 +93,6 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
93 self.device_config = device_config
94 self.cache_config = cache_config
95 self.lora_config = lora_config
97 - self.multimodal_config = multimodal_config
96 self.prompt_adapter_config = prompt_adapter_config
97 self.load_config = load_config
98 self.is_driver_worker = is_driver_worker
@@ -125,7 +123,6 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
123 self.model = get_model(model_config=self.model_config,
124 load_config=self.load_config,
125 device_config=self.device_config,
128 - multimodal_config=self.multimodal_config,
126 lora_config=self.lora_config,
127 parallel_config=self.parallel_config,
128 scheduler_config=self.scheduler_config,
vllm/worker/cpu_worker.py
+2 -5
@@ -7,8 +7,8 @@ import torch.distributed
7 import vllm.envs as envs
8 from vllm.attention import get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, MultiModalConfig, ParallelConfig,
11 - PromptAdapterConfig, SchedulerConfig)
10 + ModelConfig, ParallelConfig, PromptAdapterConfig,
11 + SchedulerConfig)
12 from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment)
14 from vllm.logger import init_logger
@@ -132,7 +132,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
132 rank: int,
133 distributed_init_method: str,
134 lora_config: Optional[LoRAConfig] = None,
135 - multimodal_config: Optional[MultiModalConfig] = None,
135 kv_cache_dtype: Optional[str] = "auto",
136 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
137 is_driver_worker: bool = False,
@@ -148,7 +147,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
147 self.distributed_init_method = distributed_init_method
148 self.lora_config = lora_config
149 self.prompt_adapter_config = prompt_adapter_config
151 - self.multimodal_config = multimodal_config
150 self.is_driver_worker = is_driver_worker
151 if self.is_driver_worker:
152 assert self.rank == 0, "The driver worker must have rank 0."
@@ -173,7 +171,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
171 cache_config,
172 load_config=self.load_config,
173 lora_config=self.lora_config,
176 - multimodal_config=self.multimodal_config,
174 kv_cache_dtype=kv_cache_dtype,
175 prompt_adapter_config=self.prompt_adapter_config,
176 is_driver_worker=is_driver_worker)
vllm/worker/embedding_model_runner.py
+2 -4
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4 import torch
5
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 - ModelConfig, MultiModalConfig, ObservabilityConfig,
8 - ParallelConfig, PromptAdapterConfig, SchedulerConfig)
7 + ModelConfig, ObservabilityConfig, ParallelConfig,
8 + PromptAdapterConfig, SchedulerConfig)
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.multimodal import MultiModalInputs
@@ -44,7 +44,6 @@ class EmbeddingModelRunner(
44 kv_cache_dtype: Optional[str] = "auto",
45 is_driver_worker: bool = False,
46 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
47 - multimodal_config: Optional[MultiModalConfig] = None,
47 observability_config: Optional[ObservabilityConfig] = None,
48 ):
49 super().__init__(model_config,
@@ -57,7 +56,6 @@ class EmbeddingModelRunner(
56 kv_cache_dtype=kv_cache_dtype,
57 is_driver_worker=is_driver_worker,
58 prompt_adapter_config=prompt_adapter_config,
60 - multimodal_config=multimodal_config,
59 observability_config=observability_config)
60
61 @torch.inference_mode()
vllm/worker/enc_dec_model_runner.py
+9 -14
@@ -10,8 +10,8 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
10 get_global_forced_attn_backend,
11 global_force_attn_backend)
12 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 - ModelConfig, MultiModalConfig, ObservabilityConfig,
14 - ParallelConfig, PromptAdapterConfig, SchedulerConfig)
13 + ModelConfig, ObservabilityConfig, ParallelConfig,
14 + PromptAdapterConfig, SchedulerConfig)
15 from vllm.inputs import INPUT_REGISTRY, InputRegistry
16 from vllm.logger import init_logger
17 from vllm.model_executor import SamplingMetadata
@@ -82,7 +82,6 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
82 kv_cache_dtype: Optional[str] = "auto",
83 is_driver_worker: bool = False,
84 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
85 - multimodal_config: Optional[MultiModalConfig] = None,
85 observability_config: Optional[ObservabilityConfig] = None,
86 input_registry: InputRegistry = INPUT_REGISTRY,
87 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
@@ -90,7 +89,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
89 '''
90 EncoderDecoderModelRunner constructor.
91
93 - `lora_config`, `multimodal_config`, and prompt_adapter_config are
92 + `lora_config` and `prompt_adapter_config` are
93 unused (since these features are not yet supported for encoder/decoder
94 models) but these arguments are present here for compatibility with
95 the base-class constructor.
@@ -273,14 +272,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
272 # number of tokens equal to max_num_batched_tokens.
273 seqs: List[SequenceGroupMetadata] = []
274
276 - model_config = self.model_config
277 - mm_config = self.multimodal_config
278 -
279 - input_registry = self.input_registry
280 - mm_registry = self.mm_registry
281 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
282 -
283 - max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
275 + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
276 + self.model_config)
277 if max_mm_tokens > 0:
278 raise NotImplementedError(
279 "Multi-modal encoder-decoder models are not supported yet")
@@ -291,8 +284,10 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
284 (group_id < max_num_batched_tokens % max_num_seqs))
285 batch_size += seq_len
286
294 - seq_data, _ = input_registry \
295 - .dummy_data_for_profiling(model_config, seq_len, mm_registry)
287 + seq_data, _ = self.input_registry \
288 + .dummy_data_for_profiling(self.model_config,
289 + seq_len,
290 + self.mm_registry)
291
292 # Having more tokens is over-conservative but otherwise fine
293 assert len(seq_data.prompt_token_ids) >= seq_len, (
vllm/worker/model_runner.py
+9 -14
@@ -27,8 +27,8 @@ except ImportError:
27 import vllm.envs as envs
28 from vllm.attention import AttentionMetadata, get_attn_backend
29 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
30 - ModelConfig, MultiModalConfig, ObservabilityConfig,
31 - ParallelConfig, PromptAdapterConfig, SchedulerConfig)
30 + ModelConfig, ObservabilityConfig, ParallelConfig,
31 + PromptAdapterConfig, SchedulerConfig)
32 from vllm.distributed import get_pp_group
33 from vllm.distributed.parallel_state import graph_capture
34 from vllm.inputs import INPUT_REGISTRY, InputRegistry
@@ -804,7 +804,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
804 kv_cache_dtype: Optional[str] = "auto",
805 is_driver_worker: bool = False,
806 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
807 - multimodal_config: Optional[MultiModalConfig] = None,
807 return_hidden_states: bool = False,
808 observability_config: Optional[ObservabilityConfig] = None,
809 input_registry: InputRegistry = INPUT_REGISTRY,
@@ -819,7 +818,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
818 self.load_config = load_config
819 self.is_driver_worker = is_driver_worker
820 self.prompt_adapter_config = prompt_adapter_config
822 - self.multimodal_config = multimodal_config
821 self.return_hidden_states = return_hidden_states
822 self.observability_config = observability_config
823
@@ -866,6 +864,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
864 self.mm_registry = mm_registry
865 self.multi_modal_input_mapper = mm_registry \
866 .create_input_mapper(model_config)
867 + self.mm_registry.init_mm_limits_per_prompt(self.model_config)
868
869 # Lazy initialization
870 self.model: nn.Module # Set after load_model
@@ -893,7 +892,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
892 device_config=self.device_config,
893 load_config=self.load_config,
894 lora_config=self.lora_config,
896 - multimodal_config=self.multimodal_config,
895 parallel_config=self.parallel_config,
896 scheduler_config=self.scheduler_config,
897 cache_config=self.cache_config)
@@ -1056,14 +1054,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1054 # To exercise the worst scenario for GPU memory consumption,
1055 # the number of seqs (batch_size) is chosen to maximize the number
1056 # of images processed.
1059 - model_config = self.model_config
1060 - mm_config = self.multimodal_config
1057
1062 - input_registry = self.input_registry
1063 - mm_registry = self.mm_registry
1064 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
1065 -
1066 - max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
1058 + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
1059 + self.model_config)
1060 if max_mm_tokens > 0:
1061 max_num_seqs_orig = max_num_seqs
1062 max_num_seqs = min(max_num_seqs,
@@ -1082,8 +1075,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1075 (group_id < max_num_batched_tokens % max_num_seqs))
1076 batch_size += seq_len
1077
1085 - seq_data, dummy_multi_modal_data = input_registry \
1086 - .dummy_data_for_profiling(model_config, seq_len, mm_registry)
1078 + seq_data, dummy_multi_modal_data = self.input_registry \
1079 + .dummy_data_for_profiling(self.model_config,
1080 + seq_len,
1081 + self.mm_registry)
1082
1083 seq = SequenceGroupMetadata(
1084 request_id=str(group_id),
vllm/worker/tpu_model_runner.py
+1 -4
@@ -11,7 +11,7 @@ import torch_xla.runtime as xr
11
12 from vllm.attention import AttentionMetadata, get_attn_backend
13 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
14 - MultiModalConfig, ParallelConfig, SchedulerConfig)
14 + ParallelConfig, SchedulerConfig)
15 from vllm.logger import init_logger
16 from vllm.model_executor.model_loader import get_model
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -89,7 +89,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
89 device_config: DeviceConfig,
90 cache_config: CacheConfig,
91 load_config: LoadConfig,
92 - multimodal_config: Optional[MultiModalConfig] = None,
92 is_driver_worker: bool = False,
93 ):
94 self.model_config = model_config
@@ -98,7 +97,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
97 self.device_config = device_config
98 self.cache_config = cache_config
99 self.load_config = load_config
101 - self.multimodal_config = multimodal_config
100 self.is_driver_worker = is_driver_worker
101
102 self.block_size = self.cache_config.block_size
@@ -142,7 +140,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
140 parallel_config=self.parallel_config,
141 cache_config=self.cache_config,
142 scheduler_config=self.scheduler_config,
145 - multimodal_config=self.multimodal_config,
143 lora_config=None,
144 )
145 model = model.eval()
vllm/worker/tpu_worker.py
+1 -4
@@ -7,7 +7,7 @@ import torch_xla.runtime as xr
7
8 import vllm.envs as envs
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
10 - MultiModalConfig, ParallelConfig, SchedulerConfig)
10 + ParallelConfig, SchedulerConfig)
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment)
13 from vllm.logger import init_logger
@@ -31,7 +31,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
31 device_config: DeviceConfig,
32 cache_config: CacheConfig,
33 load_config: LoadConfig,
34 - multimodal_config: Optional[MultiModalConfig],
34 local_rank: int,
35 rank: int,
36 distributed_init_method: str,
@@ -44,7 +43,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
43 self.device_config = device_config
44 self.cache_config = cache_config
45 self.load_config = load_config
47 - self.multimodal_config = multimodal_config
46 self.local_rank = local_rank
47 self.rank = rank
48 self.distributed_init_method = distributed_init_method
@@ -64,7 +62,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
62 device_config,
63 cache_config,
64 load_config,
67 - multimodal_config,
65 is_driver_worker=is_driver_worker)
66
67 def init_device(self) -> None:
vllm/worker/utils.py
+1 -1
@@ -39,7 +39,7 @@ def assert_enc_dec_mr_supported_scenario(
39 raise NotImplementedError(
40 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41
42 - if enc_dec_mr.multimodal_config is not None:
42 + if enc_dec_mr.model_config.multimodal_config is not None:
43 raise NotImplementedError(
44 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45
vllm/worker/worker.py
+2 -5
@@ -7,8 +7,8 @@ import torch
7 import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, MultiModalConfig, ObservabilityConfig,
11 - ParallelConfig, PromptAdapterConfig, SchedulerConfig,
10 + ModelConfig, ObservabilityConfig, ParallelConfig,
11 + PromptAdapterConfig, SchedulerConfig,
12 SpeculativeConfig)
13 from vllm.distributed import (ensure_model_parallel_initialized,
14 init_distributed_environment,
@@ -46,7 +46,6 @@ class Worker(LocalOrDistributedWorkerBase):
46 rank: int,
47 distributed_init_method: str,
48 lora_config: Optional[LoRAConfig] = None,
49 - multimodal_config: Optional[MultiModalConfig] = None,
49 speculative_config: Optional[SpeculativeConfig] = None,
50 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
51 is_driver_worker: bool = False,
@@ -73,7 +72,6 @@ class Worker(LocalOrDistributedWorkerBase):
72 # note: lazy import to avoid importing torch before initializing
73 from vllm.utils import init_cached_hf_modules
74 init_cached_hf_modules()
76 - self.multimodal_config = multimodal_config
75 self.observability_config = observability_config
76
77 # Return hidden states from target model if the draft model is an
@@ -103,7 +101,6 @@ class Worker(LocalOrDistributedWorkerBase):
101 kv_cache_dtype=self.cache_config.cache_dtype,
102 is_driver_worker=is_driver_worker,
103 prompt_adapter_config=prompt_adapter_config,
106 - multimodal_config=multimodal_config,
104 observability_config=observability_config,
105 **speculative_args,
106 )
vllm/worker/xpu_model_runner.py
+7 -10
@@ -125,6 +125,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
125 self.mm_registry = mm_registry
126 self.multi_modal_input_mapper = mm_registry \
127 .create_input_mapper(model_config)
128 + self.mm_registry.init_mm_limits_per_prompt(self.model_config)
129
130 # Lazy initialization.
131 self.model: nn.Module # Set after init_Model
@@ -166,14 +167,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
167 # To exercise the worst scenario for GPU memory consumption,
168 # the number of seqs (batch_size) is chosen to maximize the number
169 # of images processed.
169 - model_config = self.model_config
170 - mm_config = self.multimodal_config
171 -
172 - input_registry = self.input_registry
173 - mm_registry = self.mm_registry
174 - mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
175 -
176 - max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
170 + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
171 + self.model_config)
172 if max_mm_tokens > 0:
173 max_num_seqs_orig = max_num_seqs
174 max_num_seqs = min(max_num_seqs,
@@ -190,8 +185,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
185 seq_len = (max_num_batched_tokens // max_num_seqs +
186 (group_id < max_num_batched_tokens % max_num_seqs))
187
193 - seq_data, dummy_multi_modal_data = input_registry \
194 - .dummy_data_for_profiling(model_config, seq_len, mm_registry)
188 + seq_data, dummy_multi_modal_data = self.input_registry \
189 + .dummy_data_for_profiling(self.model_config,
190 + seq_len,
191 + self.mm_registry)
192
193 seq = SequenceGroupMetadata(
194 request_id=str(group_id),