A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[VLM] Refactor `MultiModalConfig` initialization and profiling (#7530)
Roger Wang committed
Aug 17, 2024 at 13:30 UTC
bbf55c4805efba5f1d7094f5e2888b3ef26c0fd7
29 files changed
+143
-190
tests/entrypoints/openai/test_audio.py
+6
-2
@@ -86,8 +86,12 @@ def server_function(port):
86
87
ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
88
89
- with patch("vllm.entrypoints.chat_utils._mm_token_str",
90
- lambda *_, **__: "_"):
89
+ with patch(
90
+ "vllm.entrypoints.chat_utils._mm_token_str",
91
+ lambda *_, **__: "_"), patch(
92
+ "vllm.model_executor.models.ModelRegistry.is_multimodal_model"
93
+ ) as mock:
94
+ mock.return_value = True
95
sys.argv = ["placeholder.py"] + \
96
(f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
97
"--dtype bfloat16 --enforce-eager --api-key token-abc123 "
tests/multimodal/test_mapper.py
+9
-9
@@ -4,7 +4,7 @@ import numpy as np
4
import pytest
5
from transformers import CLIPImageProcessor, LlavaNextImageProcessor
6
7
-from vllm.config import ModelConfig, MultiModalConfig
7
+from vllm.config import ModelConfig
8
from vllm.multimodal import MultiModalRegistry
9
from vllm.multimodal.utils import rescale_image_size
10
@@ -30,10 +30,10 @@ def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
30
seed=0,
31
dtype=dtype,
32
revision=None,
33
+ limit_mm_per_prompt={"image": 1},
34
)
34
- mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
35
36
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
36
+ mm_registry.init_mm_limits_per_prompt(model_config)
37
38
for asset in image_assets:
39
image = rescale_image_size(asset.pil_image, size_factor)
@@ -73,10 +73,10 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
73
seed=0,
74
dtype=dtype,
75
revision=None,
76
+ limit_mm_per_prompt={"image": 1},
77
)
77
- mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
78
79
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
79
+ mm_registry.init_mm_limits_per_prompt(model_config)
80
81
for asset in image_assets:
82
image = rescale_image_size(asset.pil_image, size_factor)
@@ -115,10 +115,10 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
115
seed=0,
116
dtype="half",
117
revision=None,
118
+ limit_mm_per_prompt={"image": limit},
119
)
119
- mm_config = MultiModalConfig(limit_per_prompt={"image": limit})
120
121
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
121
+ mm_registry.init_mm_limits_per_prompt(model_config)
122
123
image = image_assets[0].pil_image
124
if num_images == 0:
@@ -145,10 +145,10 @@ def test_image_mapper_multi(image_assets, mm_registry, num_images):
145
seed=0,
146
dtype="half",
147
revision=None,
148
+ limit_mm_per_prompt={"image": num_images},
149
)
149
- mm_config = MultiModalConfig(limit_per_prompt={"image": num_images})
150
151
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
151
+ mm_registry.init_mm_limits_per_prompt(model_config)
152
153
image = image_assets[0].pil_image
154
mm_inputs = {"image": [image] * num_images}
vllm/config.py
+33
-5
@@ -109,6 +109,8 @@ class ModelConfig:
109
matches the model name exposed via the APIs. If multiple model
110
names provided, the first name will be used. If not specified,
111
the model name will be the same as `model`.
112
+ limit_mm_per_prompt: Maximum number of data instances per modality
113
+ per prompt. Only applicable for multimodal models.
114
"""
115
116
def __init__(
@@ -134,7 +136,7 @@ class ModelConfig:
136
disable_sliding_window: bool = False,
137
skip_tokenizer_init: bool = False,
138
served_model_name: Optional[Union[str, List[str]]] = None,
137
- multimodal_config: Optional["MultiModalConfig"] = None,
139
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
140
) -> None:
141
self.model = model
142
self.tokenizer = tokenizer
@@ -211,14 +213,29 @@ class ModelConfig:
213
sliding_window_len=self.get_hf_config_sliding_window())
214
self.served_model_name = get_served_model_name(model,
215
served_model_name)
214
- self.multimodal_config = multimodal_config
215
-
216
+ self.multimodal_config = self._init_multimodal_config(
217
+ limit_mm_per_prompt)
218
if not self.skip_tokenizer_init:
219
self._verify_tokenizer_mode()
220
self._verify_embedding_mode()
221
self._verify_quantization()
222
self._verify_cuda_graph()
223
224
+ def _init_multimodal_config(
225
+ self, limit_mm_per_prompt: Optional[Mapping[str, int]]
226
+ ) -> Optional["MultiModalConfig"]:
227
+ architectures = getattr(self.hf_config, "architectures", [])
228
+ if any(
229
+ ModelRegistry.is_multimodal_model(arch)
230
+ for arch in architectures):
231
+ return MultiModalConfig(limit_per_prompt=limit_mm_per_prompt or {})
232
+ else:
233
+ if limit_mm_per_prompt:
234
+ raise ValueError(
235
+ "limit_mm_per_prompt is only supported for multimodal "
236
+ "models.")
237
+ return None
238
+
239
def _verify_tokenizer_mode(self) -> None:
240
tokenizer_mode = self.tokenizer_mode.lower()
241
if tokenizer_mode not in ["auto", "slow"]:
@@ -467,6 +484,18 @@ class ModelConfig:
484
if t != "attention"
485
])
486
487
+ def get_multimodal_config(self) -> "MultiModalConfig":
488
+ """
489
+ Get the multimodal configuration of the model.
490
+
491
+ Raises:
492
+ ValueError: If the model is not multimodal.
493
+ """
494
+ if self.multimodal_config is None:
495
+ raise ValueError("The model is not multimodal.")
496
+
497
+ return self.multimodal_config
498
+
499
@property
500
def is_encoder_decoder_model(self) -> bool:
501
"""Extract the HF encoder/decoder model flag."""
@@ -1450,7 +1479,7 @@ class PromptAdapterConfig:
1479
class MultiModalConfig:
1480
"""Controls the behavior of multimodal models."""
1481
1453
- limit_per_prompt: Mapping[str, int]
1482
+ limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
1483
"""
1484
The maximum number of multi-modal input instances allowed per prompt
1485
for each :class:`~vllm.multimodal.MultiModalPlugin`.
@@ -1710,7 +1739,6 @@ class EngineConfig:
1739
device_config: DeviceConfig
1740
load_config: LoadConfig
1741
lora_config: Optional[LoRAConfig]
1713
- multimodal_config: Optional[MultiModalConfig]
1742
speculative_config: Optional[SpeculativeConfig]
1743
decoding_config: Optional[DecodingConfig]
1744
observability_config: Optional[ObservabilityConfig]
vllm/engine/arg_utils.py
+3
-6
@@ -7,7 +7,7 @@ from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
7
8
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
10
- MultiModalConfig, ObservabilityConfig, ParallelConfig,
10
+ ObservabilityConfig, ParallelConfig,
11
PromptAdapterConfig, SchedulerConfig,
12
SpeculativeConfig, TokenizerPoolConfig)
13
from vllm.executor.executor_base import ExecutorBase
@@ -765,9 +765,6 @@ class EngineArgs:
765
"CPU offload space must be non-negative"
766
f", but got {self.cpu_offload_gb}")
767
768
- multimodal_config = MultiModalConfig(
769
- limit_per_prompt=self.limit_mm_per_prompt or {})
770
-
768
device_config = DeviceConfig(device=self.device)
769
model_config = ModelConfig(
770
model=self.model,
@@ -791,7 +788,8 @@ class EngineArgs:
788
disable_sliding_window=self.disable_sliding_window,
789
skip_tokenizer_init=self.skip_tokenizer_init,
790
served_model_name=self.served_model_name,
794
- multimodal_config=multimodal_config)
791
+ limit_mm_per_prompt=self.limit_mm_per_prompt,
792
+ )
793
cache_config = CacheConfig(
794
block_size=self.block_size,
795
gpu_memory_utilization=self.gpu_memory_utilization,
@@ -970,7 +968,6 @@ class EngineArgs:
968
scheduler_config=scheduler_config,
969
device_config=device_config,
970
lora_config=lora_config,
973
- multimodal_config=multimodal_config,
971
speculative_config=speculative_config,
972
load_config=load_config,
973
decoding_config=decoding_config,
vllm/engine/llm_engine.py
+1
-6
@@ -10,7 +10,7 @@ from typing_extensions import assert_never
10
import vllm.envs as envs
11
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
13
- MultiModalConfig, ObservabilityConfig, ParallelConfig,
13
+ ObservabilityConfig, ParallelConfig,
14
PromptAdapterConfig, SchedulerConfig,
15
SpeculativeConfig)
16
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
@@ -100,8 +100,6 @@ class LLMEngine:
100
scheduler_config: The configuration related to the request scheduler.
101
device_config: The configuration related to the device.
102
lora_config (Optional): The configuration related to serving multi-LoRA.
103
- multimodal_config (Optional): The configuration related to multimodal
104
- models.
103
speculative_config (Optional): The configuration related to speculative
104
decoding.
105
executor_class: The model executor class for managing distributed
@@ -172,7 +170,6 @@ class LLMEngine:
170
device_config: DeviceConfig,
171
load_config: LoadConfig,
172
lora_config: Optional[LoRAConfig],
175
- multimodal_config: Optional[MultiModalConfig],
173
speculative_config: Optional[SpeculativeConfig],
174
decoding_config: Optional[DecodingConfig],
175
observability_config: Optional[ObservabilityConfig],
@@ -235,7 +232,6 @@ class LLMEngine:
232
self.model_config = model_config
233
self.cache_config = cache_config
234
self.lora_config = lora_config
238
- self.multimodal_config = multimodal_config
235
self.parallel_config = parallel_config
236
self.scheduler_config = scheduler_config
237
self.device_config = device_config
@@ -278,7 +274,6 @@ class LLMEngine:
274
scheduler_config=scheduler_config,
275
device_config=device_config,
276
lora_config=lora_config,
281
- multimodal_config=multimodal_config,
277
speculative_config=speculative_config,
278
load_config=load_config,
279
prompt_adapter_config=prompt_adapter_config,
vllm/executor/cpu_executor.py
-1
@@ -141,7 +141,6 @@ class CPUExecutor(ExecutorBase):
141
rank=rank,
142
distributed_init_method=self.distributed_init_method,
143
lora_config=self.lora_config,
144
- multimodal_config=self.multimodal_config,
144
kv_cache_dtype=self.cache_config.cache_dtype,
145
prompt_adapter_config=self.prompt_adapter_config,
146
is_driver_worker=rank == 0,
vllm/executor/executor_base.py
+2
-4
@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
2
from typing import List, Optional, Set, Tuple
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5
- ModelConfig, MultiModalConfig, ObservabilityConfig,
6
- ParallelConfig, PromptAdapterConfig, SchedulerConfig,
5
+ ModelConfig, ObservabilityConfig, ParallelConfig,
6
+ PromptAdapterConfig, SchedulerConfig,
7
SpeculativeConfig)
8
from vllm.lora.request import LoRARequest
9
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -29,7 +29,6 @@ class ExecutorBase(ABC):
29
device_config: DeviceConfig,
30
load_config: LoadConfig,
31
lora_config: Optional[LoRAConfig],
32
- multimodal_config: Optional[MultiModalConfig],
32
speculative_config: Optional[SpeculativeConfig],
33
prompt_adapter_config: Optional[PromptAdapterConfig],
34
observability_config: Optional[ObservabilityConfig],
@@ -41,7 +40,6 @@ class ExecutorBase(ABC):
40
self.parallel_config = parallel_config
41
self.scheduler_config = scheduler_config
42
self.device_config = device_config
44
- self.multimodal_config = multimodal_config
43
self.speculative_config = speculative_config
44
self.prompt_adapter_config = prompt_adapter_config
45
self.observability_config = observability_config
vllm/executor/gpu_executor.py
-1
@@ -55,7 +55,6 @@ class GPUExecutor(ExecutorBase):
55
rank=rank,
56
distributed_init_method=distributed_init_method,
57
lora_config=self.lora_config,
58
- multimodal_config=self.multimodal_config,
58
speculative_config=self.speculative_config,
59
prompt_adapter_config=self.prompt_adapter_config,
60
is_driver_worker=(not self.parallel_config)
vllm/executor/openvino_executor.py
-1
@@ -49,7 +49,6 @@ class OpenVINOExecutor(ExecutorBase):
49
rank=0,
50
distributed_init_method=distributed_init_method,
51
lora_config=self.lora_config,
52
- multimodal_config=self.multimodal_config,
52
kv_cache_dtype=self.cache_config.cache_dtype,
53
is_driver_worker=True,
54
)
vllm/executor/ray_xpu_executor.py
+2
-6
@@ -7,9 +7,8 @@ from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
7
8
import vllm.envs as envs
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, MultiModalConfig, ParallelConfig,
11
- PromptAdapterConfig, SchedulerConfig,
12
- SpeculativeConfig)
10
+ ModelConfig, ParallelConfig, PromptAdapterConfig,
11
+ SchedulerConfig, SpeculativeConfig)
12
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13
DistributedGPUExecutor, DistributedGPUExecutorAsync)
14
from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -46,7 +45,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
45
device_config: DeviceConfig,
46
load_config: LoadConfig,
47
lora_config: Optional[LoRAConfig],
49
- multimodal_config: Optional[MultiModalConfig],
48
prompt_adapter_config: Optional[PromptAdapterConfig],
49
speculative_config: Optional[SpeculativeConfig],
50
) -> None:
@@ -61,7 +59,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
59
self.parallel_config = parallel_config
60
self.scheduler_config = scheduler_config
61
self.device_config = device_config
64
- self.multimodal_config = multimodal_config
62
self.prompt_adapter_config = prompt_adapter_config
63
64
placement_group = self.parallel_config.placement_group
@@ -203,7 +200,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
200
rank=rank,
201
distributed_init_method=distributed_init_method,
202
lora_config=self.lora_config,
206
- multimodal_config=self.multimodal_config,
203
is_driver_worker=rank == 0,
204
))
205
self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
vllm/executor/tpu_executor.py
-1
@@ -52,7 +52,6 @@ class TPUExecutor(ExecutorBase):
52
local_rank=local_rank,
53
rank=rank,
54
distributed_init_method=distributed_init_method,
55
- multimodal_config=self.multimodal_config,
55
is_driver_worker=rank == 0,
56
)
57
vllm/executor/xpu_executor.py
+2
-5
@@ -3,9 +3,8 @@ from typing import List, Optional
3
import torch
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, MultiModalConfig, ParallelConfig,
7
- PromptAdapterConfig, SchedulerConfig,
8
- SpeculativeConfig)
6
+ ModelConfig, ParallelConfig, PromptAdapterConfig,
7
+ SchedulerConfig, SpeculativeConfig)
8
from vllm.executor.executor_base import ExecutorAsyncBase
9
from vllm.executor.gpu_executor import GPUExecutor
10
from vllm.logger import init_logger
@@ -29,7 +28,6 @@ class XPUExecutor(GPUExecutor):
28
device_config: DeviceConfig,
29
load_config: LoadConfig,
30
lora_config: Optional[LoRAConfig],
32
- multimodal_config: Optional[MultiModalConfig],
31
prompt_adapter_config: Optional[PromptAdapterConfig],
32
speculative_config: Optional[SpeculativeConfig],
33
) -> None:
@@ -46,7 +44,6 @@ class XPUExecutor(GPUExecutor):
44
self.parallel_config = parallel_config
45
self.scheduler_config = scheduler_config
46
self.device_config = device_config
49
- self.multimodal_config = multimodal_config
47
self.prompt_adapter_config = prompt_adapter_config
48
self.speculative_config = None
49
vllm/inputs/registry.py
+1
-15
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13
from .data import LLMInputs
14
15
if TYPE_CHECKING:
16
- from vllm.config import ModelConfig, MultiModalConfig
16
+ from vllm.config import ModelConfig
17
from vllm.multimodal import MultiModalDataDict, MultiModalRegistry
18
from vllm.sequence import SequenceData
19
@@ -32,20 +32,6 @@ class InputContext:
32
model_config: "ModelConfig"
33
"""The configuration of the model."""
34
35
- def get_multimodal_config(self) -> "MultiModalConfig":
36
- """
37
- Get the multimodal configuration of the model.
38
-
39
- Raises:
40
- ValueError: If the model is not multimodal.
41
- """
42
-
43
- multimodal_config = self.model_config.multimodal_config
44
- if multimodal_config is None:
45
- raise ValueError("No multimodal config found")
46
-
47
- return multimodal_config
48
-
35
def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C:
36
"""
37
Get the HuggingFace configuration
vllm/model_executor/model_loader/__init__.py
+1
-4
@@ -3,8 +3,7 @@ from typing import Optional
3
from torch import nn
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, MultiModalConfig, ParallelConfig,
7
- SchedulerConfig)
6
+ ModelConfig, ParallelConfig, SchedulerConfig)
7
from vllm.model_executor.model_loader.loader import (BaseModelLoader,
8
get_model_loader)
9
from vllm.model_executor.model_loader.utils import (
@@ -15,13 +14,11 @@ def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
14
device_config: DeviceConfig, parallel_config: ParallelConfig,
15
scheduler_config: SchedulerConfig,
16
lora_config: Optional[LoRAConfig],
18
- multimodal_config: Optional[MultiModalConfig],
17
cache_config: CacheConfig) -> nn.Module:
18
loader = get_model_loader(load_config)
19
return loader.load_model(model_config=model_config,
20
device_config=device_config,
21
lora_config=lora_config,
24
- multimodal_config=multimodal_config,
22
parallel_config=parallel_config,
23
scheduler_config=scheduler_config,
24
cache_config=cache_config)
vllm/model_executor/model_loader/loader.py
+14
-32
@@ -132,9 +132,7 @@ def _get_model_initialization_kwargs(
132
"please open an issue on github.")
133
134
if supports_multimodal(model_class):
135
- if multimodal_config is None:
136
- raise ValueError("Provide multi-modal related configurations "
137
- "through LLM entrypoint or engine arguments.")
135
+ assert multimodal_config is not None
136
137
extra_kwargs["multimodal_config"] = multimodal_config
138
@@ -164,7 +162,6 @@ def _initialize_model(
162
model_config: ModelConfig,
163
load_config: LoadConfig,
164
lora_config: Optional[LoRAConfig],
167
- multimodal_config: Optional[MultiModalConfig],
165
cache_config: CacheConfig,
166
scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
167
"""Initialize a model with the given configurations."""
@@ -173,10 +170,10 @@ def _initialize_model(
170
return build_model(
171
model_class,
172
model_config.hf_config,
173
+ cache_config=cache_config,
174
quant_config=_get_quantization_config(model_config, load_config),
175
lora_config=lora_config,
178
- multimodal_config=multimodal_config,
179
- cache_config=cache_config,
176
+ multimodal_config=model_config.multimodal_config,
177
scheduler_config=scheduler_config,
178
)
179
@@ -191,7 +188,6 @@ class BaseModelLoader(ABC):
188
def load_model(self, *, model_config: ModelConfig,
189
device_config: DeviceConfig,
190
lora_config: Optional[LoRAConfig],
194
- multimodal_config: Optional[MultiModalConfig],
191
parallel_config: ParallelConfig,
192
scheduler_config: SchedulerConfig,
193
cache_config: CacheConfig) -> nn.Module:
@@ -336,7 +332,6 @@ class DefaultModelLoader(BaseModelLoader):
332
def load_model(self, *, model_config: ModelConfig,
333
device_config: DeviceConfig,
334
lora_config: Optional[LoRAConfig],
339
- multimodal_config: Optional[MultiModalConfig],
335
parallel_config: ParallelConfig,
336
scheduler_config: SchedulerConfig,
337
cache_config: CacheConfig) -> nn.Module:
@@ -344,8 +339,8 @@ class DefaultModelLoader(BaseModelLoader):
339
with set_default_torch_dtype(model_config.dtype):
340
with target_device:
341
model = _initialize_model(model_config, self.load_config,
347
- lora_config, multimodal_config,
348
- cache_config, scheduler_config)
342
+ lora_config, cache_config,
343
+ scheduler_config)
344
model.load_weights(
345
self._get_weights_iterator(model_config.model,
346
model_config.revision,
@@ -379,15 +374,14 @@ class DummyModelLoader(BaseModelLoader):
374
def load_model(self, *, model_config: ModelConfig,
375
device_config: DeviceConfig,
376
lora_config: Optional[LoRAConfig],
382
- multimodal_config: Optional[MultiModalConfig],
377
parallel_config: ParallelConfig,
378
scheduler_config: SchedulerConfig,
379
cache_config: CacheConfig) -> nn.Module:
380
with set_default_torch_dtype(model_config.dtype):
381
with torch.device(device_config.device):
382
model = _initialize_model(model_config, self.load_config,
389
- lora_config, multimodal_config,
390
- cache_config, scheduler_config)
383
+ lora_config, cache_config,
384
+ scheduler_config)
385
# NOTE(woosuk): For accurate performance evaluation, we assign
386
# random values to the weights.
387
initialize_dummy_weights(model)
@@ -420,7 +414,6 @@ class TensorizerLoader(BaseModelLoader):
414
model_config: ModelConfig,
415
device_config: DeviceConfig,
416
lora_config: Optional[LoRAConfig],
423
- multimodal_config: Optional[MultiModalConfig],
417
cache_config: CacheConfig,
418
) -> nn.Module:
419
"""Load a serialized model with tensorizer to the CPU.
@@ -433,8 +426,7 @@ class TensorizerLoader(BaseModelLoader):
426
with set_default_torch_dtype(model_config.dtype):
427
with torch.device(device_config.device):
428
model = _initialize_model(model_config, self.load_config,
436
- lora_config, multimodal_config,
437
- cache_config)
429
+ lora_config, cache_config)
430
431
model.load_weights(self._get_weights_iterator())
432
return model.eval()
@@ -444,7 +436,6 @@ class TensorizerLoader(BaseModelLoader):
436
model_config: ModelConfig,
437
device_config: DeviceConfig,
438
lora_config: Optional[LoRAConfig],
447
- multimodal_config: Optional[MultiModalConfig],
439
cache_config: CacheConfig,
440
) -> nn.Module:
441
"""Load a serialized model with tensorizer.
@@ -458,7 +449,7 @@ class TensorizerLoader(BaseModelLoader):
449
quant_config = _get_quantization_config(
450
model_config, self.load_config)
451
extra_kwargs = _get_model_initialization_kwargs(
461
- model_class, lora_config, multimodal_config)
452
+ model_class, lora_config, model_config.multimodal_config)
453
extra_kwargs["quant_config"] = quant_config
454
extra_kwargs["cache_config"] = cache_config
455
@@ -473,7 +464,6 @@ class TensorizerLoader(BaseModelLoader):
464
def load_model(self, *, model_config: ModelConfig,
465
device_config: DeviceConfig,
466
lora_config: Optional[LoRAConfig],
476
- multimodal_config: Optional[MultiModalConfig],
467
parallel_config: ParallelConfig,
468
scheduler_config: SchedulerConfig,
469
cache_config: CacheConfig) -> nn.Module:
@@ -487,11 +477,9 @@ class TensorizerLoader(BaseModelLoader):
477
478
if is_vllm_tensorized(self.tensorizer_config):
479
return self._load_model_serialized(model_config, device_config,
490
- lora_config, multimodal_config,
491
- cache_config)
480
+ lora_config, cache_config)
481
return self._load_model_serialized_cpu(model_config, device_config,
493
- lora_config, multimodal_config,
494
- cache_config)
482
+ lora_config, cache_config)
483
484
@staticmethod
485
def save_model(
@@ -577,7 +565,6 @@ class ShardedStateLoader(BaseModelLoader):
565
def load_model(self, *, model_config: ModelConfig,
566
device_config: DeviceConfig,
567
lora_config: Optional[LoRAConfig],
580
- multimodal_config: Optional[MultiModalConfig],
568
parallel_config: ParallelConfig,
569
scheduler_config: SchedulerConfig,
570
cache_config: CacheConfig) -> nn.Module:
@@ -591,8 +578,7 @@ class ShardedStateLoader(BaseModelLoader):
578
with set_default_torch_dtype(model_config.dtype):
579
with torch.device(device_config.device):
580
model = _initialize_model(model_config, self.load_config,
594
- lora_config, multimodal_config,
595
- cache_config)
581
+ lora_config, cache_config)
582
rank = get_tensor_model_parallel_rank()
583
pattern = os.path.join(
584
local_model_path,
@@ -955,15 +941,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
941
def load_model(self, *, model_config: ModelConfig,
942
device_config: DeviceConfig,
943
lora_config: Optional[LoRAConfig],
958
- multimodal_config: Optional[MultiModalConfig],
944
parallel_config: ParallelConfig,
945
scheduler_config: SchedulerConfig,
946
cache_config: CacheConfig) -> nn.Module:
947
with set_default_torch_dtype(model_config.dtype):
948
with torch.device(device_config.device):
949
model = _initialize_model(model_config, self.load_config,
965
- lora_config, multimodal_config,
966
- cache_config)
950
+ lora_config, cache_config)
951
952
self._load_weights(model_config, model)
953
@@ -1032,7 +1016,6 @@ class GGUFModelLoader(BaseModelLoader):
1016
def load_model(self, *, model_config: ModelConfig,
1017
device_config: DeviceConfig,
1018
lora_config: Optional[LoRAConfig],
1035
- multimodal_config: Optional[MultiModalConfig],
1019
parallel_config: ParallelConfig,
1020
scheduler_config: SchedulerConfig,
1021
cache_config: CacheConfig) -> nn.Module:
@@ -1047,8 +1030,7 @@ class GGUFModelLoader(BaseModelLoader):
1030
with set_default_torch_dtype(model_config.dtype):
1031
with torch.device(device_config.device):
1032
model = _initialize_model(model_config, self.load_config,
1050
- lora_config, multimodal_config,
1051
- cache_config)
1033
+ lora_config, cache_config)
1034
model.load_weights(
1035
self._get_weights_iterator(local_model_path, gguf_weights_map))
1036
return model
vllm/model_executor/models/__init__.py
+27
-16
@@ -9,17 +9,12 @@ from vllm.utils import is_hip
9
10
logger = init_logger(__name__)
11
12
-# Architecture -> (module, class).
12
_GENERATION_MODELS = {
13
"AquilaModel": ("llama", "LlamaForCausalLM"),
14
"AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19
- "Blip2ForConditionalGeneration":
20
- ("blip2", "Blip2ForConditionalGeneration"),
21
- "ChameleonForConditionalGeneration":
22
- ("chameleon", "ChameleonForConditionalGeneration"),
18
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
@@ -28,7 +23,6 @@ _GENERATION_MODELS = {
23
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
31
- "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
26
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
27
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
28
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
@@ -37,13 +31,8 @@ _GENERATION_MODELS = {
31
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
32
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
33
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
40
- "InternVLChatModel": ("internvl", "InternVLChatModel"),
34
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
35
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
43
- "LlavaForConditionalGeneration":
44
- ("llava", "LlavaForConditionalGeneration"),
45
- "LlavaNextForConditionalGeneration":
46
- ("llava_next", "LlavaNextForConditionalGeneration"),
36
# For decapoda-research/llama-*
37
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
38
"MistralForCausalLM": ("llama", "LlamaForCausalLM"),
@@ -53,17 +42,13 @@ _GENERATION_MODELS = {
42
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
43
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
44
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
56
- "MiniCPMV": ("minicpmv", "MiniCPMV"),
45
"NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
46
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
47
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
48
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
49
"PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
62
- "PaliGemmaForConditionalGeneration": ("paligemma",
63
- "PaliGemmaForConditionalGeneration"),
50
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
51
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
66
- "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
52
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
53
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
54
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
@@ -83,6 +68,22 @@ _EMBEDDING_MODELS = {
68
"MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
69
}
70
71
+_MULTIMODAL_MODELS = {
72
+ "Blip2ForConditionalGeneration":
73
+ ("blip2", "Blip2ForConditionalGeneration"),
74
+ "ChameleonForConditionalGeneration":
75
+ ("chameleon", "ChameleonForConditionalGeneration"),
76
+ "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
77
+ "InternVLChatModel": ("internvl", "InternVLChatModel"),
78
+ "LlavaForConditionalGeneration":
79
+ ("llava", "LlavaForConditionalGeneration"),
80
+ "LlavaNextForConditionalGeneration":
81
+ ("llava_next", "LlavaNextForConditionalGeneration"),
82
+ "MiniCPMV": ("minicpmv", "MiniCPMV"),
83
+ "PaliGemmaForConditionalGeneration": ("paligemma",
84
+ "PaliGemmaForConditionalGeneration"),
85
+ "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
86
+}
87
_CONDITIONAL_GENERATION_MODELS = {
88
"BartModel": ("bart", "BartForConditionalGeneration"),
89
"BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
@@ -91,7 +92,8 @@ _CONDITIONAL_GENERATION_MODELS = {
92
_MODELS = {
93
**_GENERATION_MODELS,
94
**_EMBEDDING_MODELS,
94
- **_CONDITIONAL_GENERATION_MODELS
95
+ **_MULTIMODAL_MODELS,
96
+ **_CONDITIONAL_GENERATION_MODELS,
97
}
98
99
# Architecture -> type.
@@ -182,6 +184,15 @@ class ModelRegistry:
184
def is_embedding_model(model_arch: str) -> bool:
185
return model_arch in _EMBEDDING_MODELS
186
187
+ @staticmethod
188
+ def is_multimodal_model(model_arch: str) -> bool:
189
+
190
+ # TODO: find a way to avoid initializing CUDA prematurely to
191
+ # use `supports_multimodal` to determine if a model is multimodal
192
+ # model_cls = ModelRegistry._try_load_model_cls(model_arch)
193
+ # from vllm.model_executor.models.interfaces import supports_multimodal
194
+ return model_arch in _MULTIMODAL_MODELS
195
+
196
197
__all__ = [
198
"ModelRegistry",
vllm/multimodal/registry.py
+2
-2
@@ -2,7 +2,7 @@ import functools
2
from collections import UserDict
3
from typing import Dict, Mapping, Optional, Sequence
4
5
-from vllm.config import ModelConfig, MultiModalConfig
5
+from vllm.config import ModelConfig
6
from vllm.logger import init_logger
7
8
from .audio import AudioPlugin
@@ -181,7 +181,6 @@ class MultiModalRegistry:
181
def init_mm_limits_per_prompt(
182
self,
183
model_config: ModelConfig,
184
- multimodal_config: Optional[MultiModalConfig],
184
) -> None:
185
"""
186
Initialize the maximum number of multi-modal input instances for each
@@ -192,6 +191,7 @@ class MultiModalRegistry:
191
"`mm_limits` has already been set for model=%s, and will "
192
"be overwritten by the new values.", model_config.model)
193
194
+ multimodal_config = model_config.multimodal_config
195
if multimodal_config is None:
196
limits_per_plugin = self._disabled_limits_per_plugin
197
else:
vllm/spec_decode/draft_model_runner.py
+2
-4
@@ -23,8 +23,8 @@ except ImportError:
23
FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24
25
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26
- ModelConfig, MultiModalConfig, ObservabilityConfig,
27
- ParallelConfig, PromptAdapterConfig, SchedulerConfig)
26
+ ModelConfig, ObservabilityConfig, ParallelConfig,
27
+ PromptAdapterConfig, SchedulerConfig)
28
from vllm.logger import init_logger
29
from vllm.multimodal import MultiModalInputs
30
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
@@ -66,7 +66,6 @@ class TP1DraftModelRunner(ModelRunner):
66
lora_config: Optional[LoRAConfig],
67
kv_cache_dtype: Optional[str] = "auto",
68
is_driver_worker: bool = False,
69
- multimodal_config: Optional[MultiModalConfig] = None,
69
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
70
return_hidden_states: bool = False,
71
observability_config: Optional[ObservabilityConfig] = None,
@@ -86,7 +85,6 @@ class TP1DraftModelRunner(ModelRunner):
85
lora_config=lora_config,
86
kv_cache_dtype=kv_cache_dtype,
87
is_driver_worker=is_driver_worker,
89
- multimodal_config=multimodal_config,
88
prompt_adapter_config=prompt_adapter_config,
89
return_hidden_states=return_hidden_states,
90
observability_config=observability_config,
vllm/spec_decode/target_model_runner.py
+2
-4
@@ -1,8 +1,8 @@
1
from typing import List, Optional
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4
- ModelConfig, MultiModalConfig, ObservabilityConfig,
5
- ParallelConfig, PromptAdapterConfig, SchedulerConfig)
4
+ ModelConfig, ObservabilityConfig, ParallelConfig,
5
+ PromptAdapterConfig, SchedulerConfig)
6
from vllm.sequence import SequenceGroupMetadata
7
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
8
ModelRunner)
@@ -31,7 +31,6 @@ class TargetModelRunner(ModelRunner):
31
kv_cache_dtype: Optional[str] = "auto",
32
is_driver_worker: bool = False,
33
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34
- multimodal_config: Optional[MultiModalConfig] = None,
34
return_hidden_states: bool = False,
35
observability_config: Optional[ObservabilityConfig] = None):
36
# An internal boolean member variable to indicate if token log
@@ -47,7 +46,6 @@ class TargetModelRunner(ModelRunner):
46
lora_config=lora_config,
47
kv_cache_dtype=kv_cache_dtype,
48
is_driver_worker=is_driver_worker,
50
- multimodal_config=multimodal_config,
49
prompt_adapter_config=prompt_adapter_config,
50
return_hidden_states=return_hidden_states,
51
observability_config=observability_config,
vllm/worker/cpu_model_runner.py
+2
-5
@@ -6,8 +6,8 @@ from torch import nn
6
7
from vllm.attention import AttentionMetadata, get_attn_backend
8
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
- ModelConfig, MultiModalConfig, ParallelConfig,
10
- PromptAdapterConfig, SchedulerConfig)
9
+ ModelConfig, ParallelConfig, PromptAdapterConfig,
10
+ SchedulerConfig)
11
from vllm.logger import init_logger
12
from vllm.model_executor import SamplingMetadata
13
from vllm.model_executor.model_loader import get_model
@@ -79,7 +79,6 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
79
cache_config: CacheConfig,
80
load_config: LoadConfig,
81
lora_config: Optional[LoRAConfig],
82
- multimodal_config: Optional[MultiModalConfig],
82
kv_cache_dtype: Optional[str] = "auto",
83
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
84
is_driver_worker: bool = False,
@@ -94,7 +93,6 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
93
self.device_config = device_config
94
self.cache_config = cache_config
95
self.lora_config = lora_config
97
- self.multimodal_config = multimodal_config
96
self.prompt_adapter_config = prompt_adapter_config
97
self.load_config = load_config
98
self.is_driver_worker = is_driver_worker
@@ -125,7 +123,6 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
123
self.model = get_model(model_config=self.model_config,
124
load_config=self.load_config,
125
device_config=self.device_config,
128
- multimodal_config=self.multimodal_config,
126
lora_config=self.lora_config,
127
parallel_config=self.parallel_config,
128
scheduler_config=self.scheduler_config,
vllm/worker/cpu_worker.py
+2
-5
@@ -7,8 +7,8 @@ import torch.distributed
7
import vllm.envs as envs
8
from vllm.attention import get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, MultiModalConfig, ParallelConfig,
11
- PromptAdapterConfig, SchedulerConfig)
10
+ ModelConfig, ParallelConfig, PromptAdapterConfig,
11
+ SchedulerConfig)
12
from vllm.distributed import (ensure_model_parallel_initialized,
13
init_distributed_environment)
14
from vllm.logger import init_logger
@@ -132,7 +132,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
132
rank: int,
133
distributed_init_method: str,
134
lora_config: Optional[LoRAConfig] = None,
135
- multimodal_config: Optional[MultiModalConfig] = None,
135
kv_cache_dtype: Optional[str] = "auto",
136
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
137
is_driver_worker: bool = False,
@@ -148,7 +147,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
147
self.distributed_init_method = distributed_init_method
148
self.lora_config = lora_config
149
self.prompt_adapter_config = prompt_adapter_config
151
- self.multimodal_config = multimodal_config
150
self.is_driver_worker = is_driver_worker
151
if self.is_driver_worker:
152
assert self.rank == 0, "The driver worker must have rank 0."
@@ -173,7 +171,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
171
cache_config,
172
load_config=self.load_config,
173
lora_config=self.lora_config,
176
- multimodal_config=self.multimodal_config,
174
kv_cache_dtype=kv_cache_dtype,
175
prompt_adapter_config=self.prompt_adapter_config,
176
is_driver_worker=is_driver_worker)
vllm/worker/embedding_model_runner.py
+2
-4
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4
import torch
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
- ModelConfig, MultiModalConfig, ObservabilityConfig,
8
- ParallelConfig, PromptAdapterConfig, SchedulerConfig)
7
+ ModelConfig, ObservabilityConfig, ParallelConfig,
8
+ PromptAdapterConfig, SchedulerConfig)
9
from vllm.logger import init_logger
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.multimodal import MultiModalInputs
@@ -44,7 +44,6 @@ class EmbeddingModelRunner(
44
kv_cache_dtype: Optional[str] = "auto",
45
is_driver_worker: bool = False,
46
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
47
- multimodal_config: Optional[MultiModalConfig] = None,
47
observability_config: Optional[ObservabilityConfig] = None,
48
):
49
super().__init__(model_config,
@@ -57,7 +56,6 @@ class EmbeddingModelRunner(
56
kv_cache_dtype=kv_cache_dtype,
57
is_driver_worker=is_driver_worker,
58
prompt_adapter_config=prompt_adapter_config,
60
- multimodal_config=multimodal_config,
59
observability_config=observability_config)
60
61
@torch.inference_mode()
vllm/worker/enc_dec_model_runner.py
+9
-14
@@ -10,8 +10,8 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
10
get_global_forced_attn_backend,
11
global_force_attn_backend)
12
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
- ModelConfig, MultiModalConfig, ObservabilityConfig,
14
- ParallelConfig, PromptAdapterConfig, SchedulerConfig)
13
+ ModelConfig, ObservabilityConfig, ParallelConfig,
14
+ PromptAdapterConfig, SchedulerConfig)
15
from vllm.inputs import INPUT_REGISTRY, InputRegistry
16
from vllm.logger import init_logger
17
from vllm.model_executor import SamplingMetadata
@@ -82,7 +82,6 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
82
kv_cache_dtype: Optional[str] = "auto",
83
is_driver_worker: bool = False,
84
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
85
- multimodal_config: Optional[MultiModalConfig] = None,
85
observability_config: Optional[ObservabilityConfig] = None,
86
input_registry: InputRegistry = INPUT_REGISTRY,
87
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
@@ -90,7 +89,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
89
'''
90
EncoderDecoderModelRunner constructor.
91
93
- `lora_config`, `multimodal_config`, and prompt_adapter_config are
92
+ `lora_config` and `prompt_adapter_config` are
93
unused (since these features are not yet supported for encoder/decoder
94
models) but these arguments are present here for compatibility with
95
the base-class constructor.
@@ -273,14 +272,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
272
# number of tokens equal to max_num_batched_tokens.
273
seqs: List[SequenceGroupMetadata] = []
274
276
- model_config = self.model_config
277
- mm_config = self.multimodal_config
278
-
279
- input_registry = self.input_registry
280
- mm_registry = self.mm_registry
281
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
282
-
283
- max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
275
+ max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
276
+ self.model_config)
277
if max_mm_tokens > 0:
278
raise NotImplementedError(
279
"Multi-modal encoder-decoder models are not supported yet")
@@ -291,8 +284,10 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
284
(group_id < max_num_batched_tokens % max_num_seqs))
285
batch_size += seq_len
286
294
- seq_data, _ = input_registry \
295
- .dummy_data_for_profiling(model_config, seq_len, mm_registry)
287
+ seq_data, _ = self.input_registry \
288
+ .dummy_data_for_profiling(self.model_config,
289
+ seq_len,
290
+ self.mm_registry)
291
292
# Having more tokens is over-conservative but otherwise fine
293
assert len(seq_data.prompt_token_ids) >= seq_len, (
vllm/worker/model_runner.py
+9
-14
@@ -27,8 +27,8 @@ except ImportError:
27
import vllm.envs as envs
28
from vllm.attention import AttentionMetadata, get_attn_backend
29
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
30
- ModelConfig, MultiModalConfig, ObservabilityConfig,
31
- ParallelConfig, PromptAdapterConfig, SchedulerConfig)
30
+ ModelConfig, ObservabilityConfig, ParallelConfig,
31
+ PromptAdapterConfig, SchedulerConfig)
32
from vllm.distributed import get_pp_group
33
from vllm.distributed.parallel_state import graph_capture
34
from vllm.inputs import INPUT_REGISTRY, InputRegistry
@@ -804,7 +804,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
804
kv_cache_dtype: Optional[str] = "auto",
805
is_driver_worker: bool = False,
806
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
807
- multimodal_config: Optional[MultiModalConfig] = None,
807
return_hidden_states: bool = False,
808
observability_config: Optional[ObservabilityConfig] = None,
809
input_registry: InputRegistry = INPUT_REGISTRY,
@@ -819,7 +818,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
818
self.load_config = load_config
819
self.is_driver_worker = is_driver_worker
820
self.prompt_adapter_config = prompt_adapter_config
822
- self.multimodal_config = multimodal_config
821
self.return_hidden_states = return_hidden_states
822
self.observability_config = observability_config
823
@@ -866,6 +864,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
864
self.mm_registry = mm_registry
865
self.multi_modal_input_mapper = mm_registry \
866
.create_input_mapper(model_config)
867
+ self.mm_registry.init_mm_limits_per_prompt(self.model_config)
868
869
# Lazy initialization
870
self.model: nn.Module # Set after load_model
@@ -893,7 +892,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
892
device_config=self.device_config,
893
load_config=self.load_config,
894
lora_config=self.lora_config,
896
- multimodal_config=self.multimodal_config,
895
parallel_config=self.parallel_config,
896
scheduler_config=self.scheduler_config,
897
cache_config=self.cache_config)
@@ -1056,14 +1054,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1054
# To exercise the worst scenario for GPU memory consumption,
1055
# the number of seqs (batch_size) is chosen to maximize the number
1056
# of images processed.
1059
- model_config = self.model_config
1060
- mm_config = self.multimodal_config
1057
1062
- input_registry = self.input_registry
1063
- mm_registry = self.mm_registry
1064
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
1065
-
1066
- max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
1058
+ max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
1059
+ self.model_config)
1060
if max_mm_tokens > 0:
1061
max_num_seqs_orig = max_num_seqs
1062
max_num_seqs = min(max_num_seqs,
@@ -1082,8 +1075,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1075
(group_id < max_num_batched_tokens % max_num_seqs))
1076
batch_size += seq_len
1077
1085
- seq_data, dummy_multi_modal_data = input_registry \
1086
- .dummy_data_for_profiling(model_config, seq_len, mm_registry)
1078
+ seq_data, dummy_multi_modal_data = self.input_registry \
1079
+ .dummy_data_for_profiling(self.model_config,
1080
+ seq_len,
1081
+ self.mm_registry)
1082
1083
seq = SequenceGroupMetadata(
1084
request_id=str(group_id),
vllm/worker/tpu_model_runner.py
+1
-4
@@ -11,7 +11,7 @@ import torch_xla.runtime as xr
11
12
from vllm.attention import AttentionMetadata, get_attn_backend
13
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
14
- MultiModalConfig, ParallelConfig, SchedulerConfig)
14
+ ParallelConfig, SchedulerConfig)
15
from vllm.logger import init_logger
16
from vllm.model_executor.model_loader import get_model
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -89,7 +89,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
89
device_config: DeviceConfig,
90
cache_config: CacheConfig,
91
load_config: LoadConfig,
92
- multimodal_config: Optional[MultiModalConfig] = None,
92
is_driver_worker: bool = False,
93
):
94
self.model_config = model_config
@@ -98,7 +97,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
97
self.device_config = device_config
98
self.cache_config = cache_config
99
self.load_config = load_config
101
- self.multimodal_config = multimodal_config
100
self.is_driver_worker = is_driver_worker
101
102
self.block_size = self.cache_config.block_size
@@ -142,7 +140,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
140
parallel_config=self.parallel_config,
141
cache_config=self.cache_config,
142
scheduler_config=self.scheduler_config,
145
- multimodal_config=self.multimodal_config,
143
lora_config=None,
144
)
145
model = model.eval()
vllm/worker/tpu_worker.py
+1
-4
@@ -7,7 +7,7 @@ import torch_xla.runtime as xr
7
8
import vllm.envs as envs
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
10
- MultiModalConfig, ParallelConfig, SchedulerConfig)
10
+ ParallelConfig, SchedulerConfig)
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment)
13
from vllm.logger import init_logger
@@ -31,7 +31,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
31
device_config: DeviceConfig,
32
cache_config: CacheConfig,
33
load_config: LoadConfig,
34
- multimodal_config: Optional[MultiModalConfig],
34
local_rank: int,
35
rank: int,
36
distributed_init_method: str,
@@ -44,7 +43,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
43
self.device_config = device_config
44
self.cache_config = cache_config
45
self.load_config = load_config
47
- self.multimodal_config = multimodal_config
46
self.local_rank = local_rank
47
self.rank = rank
48
self.distributed_init_method = distributed_init_method
@@ -64,7 +62,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
62
device_config,
63
cache_config,
64
load_config,
67
- multimodal_config,
65
is_driver_worker=is_driver_worker)
66
67
def init_device(self) -> None:
vllm/worker/utils.py
+1
-1
@@ -39,7 +39,7 @@ def assert_enc_dec_mr_supported_scenario(
39
raise NotImplementedError(
40
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41
42
- if enc_dec_mr.multimodal_config is not None:
42
+ if enc_dec_mr.model_config.multimodal_config is not None:
43
raise NotImplementedError(
44
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45
vllm/worker/worker.py
+2
-5
@@ -7,8 +7,8 @@ import torch
7
import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, MultiModalConfig, ObservabilityConfig,
11
- ParallelConfig, PromptAdapterConfig, SchedulerConfig,
10
+ ModelConfig, ObservabilityConfig, ParallelConfig,
11
+ PromptAdapterConfig, SchedulerConfig,
12
SpeculativeConfig)
13
from vllm.distributed import (ensure_model_parallel_initialized,
14
init_distributed_environment,
@@ -46,7 +46,6 @@ class Worker(LocalOrDistributedWorkerBase):
46
rank: int,
47
distributed_init_method: str,
48
lora_config: Optional[LoRAConfig] = None,
49
- multimodal_config: Optional[MultiModalConfig] = None,
49
speculative_config: Optional[SpeculativeConfig] = None,
50
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
51
is_driver_worker: bool = False,
@@ -73,7 +72,6 @@ class Worker(LocalOrDistributedWorkerBase):
72
# note: lazy import to avoid importing torch before initializing
73
from vllm.utils import init_cached_hf_modules
74
init_cached_hf_modules()
76
- self.multimodal_config = multimodal_config
75
self.observability_config = observability_config
76
77
# Return hidden states from target model if the draft model is an
@@ -103,7 +101,6 @@ class Worker(LocalOrDistributedWorkerBase):
101
kv_cache_dtype=self.cache_config.cache_dtype,
102
is_driver_worker=is_driver_worker,
103
prompt_adapter_config=prompt_adapter_config,
106
- multimodal_config=multimodal_config,
104
observability_config=observability_config,
105
**speculative_args,
106
)
vllm/worker/xpu_model_runner.py
+7
-10
@@ -125,6 +125,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
125
self.mm_registry = mm_registry
126
self.multi_modal_input_mapper = mm_registry \
127
.create_input_mapper(model_config)
128
+ self.mm_registry.init_mm_limits_per_prompt(self.model_config)
129
130
# Lazy initialization.
131
self.model: nn.Module # Set after init_Model
@@ -166,14 +167,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
167
# To exercise the worst scenario for GPU memory consumption,
168
# the number of seqs (batch_size) is chosen to maximize the number
169
# of images processed.
169
- model_config = self.model_config
170
- mm_config = self.multimodal_config
171
-
172
- input_registry = self.input_registry
173
- mm_registry = self.mm_registry
174
- mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
175
-
176
- max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
170
+ max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
171
+ self.model_config)
172
if max_mm_tokens > 0:
173
max_num_seqs_orig = max_num_seqs
174
max_num_seqs = min(max_num_seqs,
@@ -190,8 +185,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
185
seq_len = (max_num_batched_tokens // max_num_seqs +
186
(group_id < max_num_batched_tokens % max_num_seqs))
187
193
- seq_data, dummy_multi_modal_data = input_registry \
194
- .dummy_data_for_profiling(model_config, seq_len, mm_registry)
188
+ seq_data, dummy_multi_modal_data = self.input_registry \
189
+ .dummy_data_for_profiling(self.model_config,
190
+ seq_len,
191
+ self.mm_registry)
192
193
seq = SequenceGroupMetadata(
194
request_id=str(group_id),