@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 133707123e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Replace embedding models with pooling adapter (#10769)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 1, 2024 at 08:02 UTC 133707123e730a3544875d432a9435bdfe5e34cf
32 files changed +387 -323
.buildkite/test-pipeline.yaml
+2 -2
@@ -334,7 +334,6 @@ steps:
334 commands:
335 - pytest -v -s models/decoder_only/language -m 'core_model or quant_model'
336 - pytest -v -s models/embedding/language -m core_model
337 - - pytest -v -s models/embedding/vision_language -m core_model
337
338 - label: Language Models Test (Extended) # 50min
339 optional: true
@@ -346,7 +345,6 @@ steps:
345 commands:
346 - pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
347 - pytest -v -s models/embedding/language -m 'not core_model'
349 - - pytest -v -s models/embedding/vision_language -m 'not core_model'
348
349 - label: Multi-Modal Models Test (Standard) # 26min
350 #mirror_hardwares: [amd]
@@ -359,6 +357,7 @@ steps:
357 commands:
358 - pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
359 - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
360 + - pytest -v -s models/embedding/vision_language -m core_model
361 - pytest -v -s models/encoder_decoder/language -m core_model
362 - pytest -v -s models/encoder_decoder/vision_language -m core_model
363
@@ -376,6 +375,7 @@ steps:
375 # https://github.com/huggingface/transformers/issues/34307
376 - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
377 - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
378 + - pytest -v -s models/embedding/vision_language -m 'not core_model'
379 - pytest -v -s models/encoder_decoder/language -m 'not core_model'
380 - pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
381
docs/source/models/supported_models.rst
+14 -1
@@ -357,7 +357,7 @@ Text Embedding
357 - ✅︎
358 * - :code:`Qwen2Model`, :code:`Qwen2ForCausalLM`
359 - Qwen2-based
360 - - :code:`ssmits/Qwen2-7B-Instruct-embed-base`, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc.
360 + - :code:`ssmits/Qwen2-7B-Instruct-embed-base` (see note), :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc.
361 - ✅︎
362 - ✅︎
363 * - :code:`RobertaModel`, :code:`RobertaForMaskedLM`
@@ -378,6 +378,10 @@ Text Embedding
378 .. tip::
379 You can override the model's pooling method by passing :code:`--override-pooler-config`.
380
381 +.. note::
382 + :code:`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
383 + You should manually set mean pooling by passing :code:`--override-pooler-config '{"pooling_type": "MEAN"}'`.
384 +
385 .. note::
386 Unlike base Qwen2, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` uses bi-directional attention.
387 You can set :code:`--hf-overrides '{"is_causal": false}'` to change the attention mask accordingly.
@@ -397,12 +401,21 @@ Reward Modeling
401 - Example HF Models
402 - :ref:`LoRA <lora>`
403 - :ref:`PP <distributed_serving>`
404 + * - :code:`LlamaForCausalLM`
405 + - Llama-based
406 + - :code:`peiyi9979/math-shepherd-mistral-7b-prm`, etc.
407 + - ✅︎
408 + - ✅︎
409 * - :code:`Qwen2ForRewardModel`
410 - Qwen2-based
411 - :code:`Qwen/Qwen2.5-Math-RM-72B`, etc.
412 - ✅︎
413 - ✅︎
414
415 +.. important::
416 + For process-supervised reward models such as :code:`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
417 + e.g.: :code:`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
418 +
419 .. note::
420 As an interim measure, these models are supported in both offline and online inference via Embeddings API.
421
tests/conftest.py
-1
@@ -263,7 +263,6 @@ class HfRunner:
263 dtype: str = "half",
264 *,
265 model_kwargs: Optional[Dict[str, Any]] = None,
266 - is_embedding_model: bool = False,
266 is_sentence_transformer: bool = False,
267 is_cross_encoder: bool = False,
268 skip_tokenizer_init: bool = False,
tests/models/embedding/language/test_embedding.py
+5
@@ -4,6 +4,8 @@ Run `pytest tests/models/embedding/language/test_embedding.py`.
4 """
5 import pytest
6
7 +from vllm.config import PoolerConfig
8 +
9 from ..utils import check_embeddings_close
10
11
@@ -33,6 +35,9 @@ def test_models(
35 dtype: str,
36 ) -> None:
37 vllm_extra_kwargs = {}
38 + if model == "ssmits/Qwen2-7B-Instruct-embed-base":
39 + vllm_extra_kwargs["override_pooler_config"] = \
40 + PoolerConfig(pooling_type="MEAN")
41 if model == "Alibaba-NLP/gte-Qwen2-7B-instruct":
42 vllm_extra_kwargs["hf_overrides"] = {"is_causal": False}
43
tests/models/test_registry.py
+14 -17
@@ -6,11 +6,8 @@ import torch.cuda
6 from vllm.model_executor.models import (is_embedding_model,
7 is_text_generation_model,
8 supports_multimodal)
9 -# yapf conflicts with isort for this block
10 -# yapf: disable
11 -from vllm.model_executor.models.registry import (_CROSS_ENCODER_MODELS,
12 - _EMBEDDING_MODELS,
13 - _MULTIMODAL_MODELS,
9 +from vllm.model_executor.models.adapters import as_embedding_model
10 +from vllm.model_executor.models.registry import (_MULTIMODAL_MODELS,
11 _SPECULATIVE_DECODING_MODELS,
12 _TEXT_GENERATION_MODELS,
13 ModelRegistry)
@@ -26,18 +23,18 @@ def test_registry_imports(model_arch):
23 model_cls, _ = ModelRegistry.resolve_model_cls(model_arch)
24
25 if model_arch in _SPECULATIVE_DECODING_MODELS:
29 - pass # Ignore these models which do not have a unified format
30 - else:
31 - assert is_text_generation_model(model_cls) is (
32 - model_arch in _TEXT_GENERATION_MODELS
33 - or model_arch in _MULTIMODAL_MODELS)
34 -
35 - embedding_models = {**_EMBEDDING_MODELS, **_CROSS_ENCODER_MODELS}
36 - assert is_embedding_model(model_cls) is (model_arch
37 - in embedding_models)
38 -
39 - assert supports_multimodal(model_cls) is (model_arch
40 - in _MULTIMODAL_MODELS)
26 + return # Ignore these models which do not have a unified format
27 +
28 + if (model_arch in _TEXT_GENERATION_MODELS
29 + or model_arch in _MULTIMODAL_MODELS):
30 + assert is_text_generation_model(model_cls)
31 +
32 + # All vLLM models should be convertible to an embedding model
33 + embed_model = as_embedding_model(model_cls)
34 + assert is_embedding_model(embed_model)
35 +
36 + if model_arch in _MULTIMODAL_MODELS:
37 + assert supports_multimodal(model_cls)
38
39
40 @fork_new_process_for_each_test
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py
+40 -5
@@ -1,13 +1,34 @@
1 -from typing import List, Optional, Union
1 +from typing import Iterable, List, Optional, Tuple, Union
2
3 import torch
4 +import torch.nn as nn
5
6 from vllm.attention import AttentionMetadata
6 -from vllm.model_executor.models.gemma2 import Gemma2EmbeddingModel
7 -from vllm.sequence import IntermediateTensors
7 +from vllm.config import VllmConfig
8 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
9 +from vllm.model_executor.models.gemma2 import Gemma2Model
10 +from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
11 +from vllm.model_executor.pooling_metadata import PoolingMetadata
12 +from vllm.sequence import IntermediateTensors, PoolerOutput
13
14
10 -class MyGemma2Embedding(Gemma2EmbeddingModel):
15 +class MyGemma2Embedding(nn.Module):
16 +
17 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
18 + super().__init__()
19 +
20 + self.model = Gemma2Model(vllm_config=vllm_config,
21 + prefix=maybe_prefix(prefix, "model"))
22 +
23 + self._pooler = Pooler.from_config_with_defaults(
24 + vllm_config.model_config.pooler_config,
25 + pooling_type=PoolingType.LAST,
26 + normalize=True,
27 + softmax=False,
28 + )
29 +
30 + self.make_empty_intermediate_tensors = (
31 + self.model.make_empty_intermediate_tensors)
32
33 def forward(
34 self,
@@ -18,7 +39,7 @@ class MyGemma2Embedding(Gemma2EmbeddingModel):
39 intermediate_tensors: Optional[IntermediateTensors] = None,
40 inputs_embeds: Optional[torch.Tensor] = None,
41 ) -> Union[torch.Tensor, IntermediateTensors]:
21 - hidden_states = super().forward(
42 + hidden_states = self.model(
43 input_ids,
44 positions,
45 kv_caches,
@@ -32,3 +53,17 @@ class MyGemma2Embedding(Gemma2EmbeddingModel):
53
54 # Return all-zero embeddings
55 return torch.zeros_like(hidden_states)
56 +
57 + def pooler(
58 + self,
59 + hidden_states: torch.Tensor,
60 + pooling_metadata: PoolingMetadata,
61 + ) -> Optional[PoolerOutput]:
62 + return self._pooler(hidden_states, pooling_metadata)
63 +
64 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
65 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
66 + weights = hf_to_vllm_mapper.apply(weights)
67 + weights = ((name, data) for name, data in weights
68 + if not name.startswith("lm_head."))
69 + return self.model.load_weights(weights)
tests/test_config.py
+1 -2
@@ -26,8 +26,7 @@ def test_auto_task(model_id, expected_task):
26
27
28 @pytest.mark.parametrize(("model_id", "bad_task"), [
29 - ("facebook/opt-125m", "embedding"),
30 - ("intfloat/e5-mistral-7b-instruct", "generate"),
29 + ("Qwen/Qwen2.5-Math-RM-72B", "generate"),
30 ])
31 def test_incorrect_task(model_id, bad_task):
32 with pytest.raises(ValueError, match=r"does not support the .* task"):
vllm/config.py
+25
@@ -370,6 +370,31 @@ class ModelConfig:
370 selected_task = next(iter(supported_tasks_lst))
371
372 if len(supported_tasks) > 1:
373 + suffix_to_preferred_task: List[Tuple[str, _Task]] = [
374 + # Hardcode the models that are exceptions
375 + ("AquilaModel", "generate"),
376 + ("ChatGLMModel", "generate"),
377 + # Other models follow this pattern
378 + ("ForCausalLM", "generate"),
379 + ("ForConditionalGeneration", "generate"),
380 + ("ChatModel", "generate"),
381 + ("LMHeadModel", "generate"),
382 + ("EmbeddingModel", "embedding"),
383 + ("RewardModel", "embedding"),
384 + ("ForSequenceClassification", "embedding"),
385 + ]
386 + info, arch = ModelRegistry.inspect_model_cls(architectures)
387 +
388 + for suffix, pref_task in suffix_to_preferred_task:
389 + if arch.endswith(suffix) and pref_task in supported_tasks:
390 + selected_task = pref_task
391 + break
392 + else:
393 + if (arch.endswith("Model")
394 + and info.architecture.endswith("ForCausalLM")
395 + and "embedding" in supported_tasks):
396 + selected_task = "embedding"
397 +
398 logger.info(
399 "This model supports multiple tasks: %s. "
400 "Defaulting to '%s'.", supported_tasks, selected_task)
vllm/inputs/registry.py
+8 -8
@@ -11,8 +11,8 @@ from typing_extensions import TypeVar, assert_never
11 from vllm.logger import init_logger
12 from vllm.transformers_utils.processor import cached_get_processor
13 from vllm.transformers_utils.tokenizer import AnyTokenizer
14 -from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
15 - resolve_mm_processor_kwargs)
14 +from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
15 + print_warning_once, resolve_mm_processor_kwargs)
16
17 from .data import ProcessorInputs, SingletonInputs
18 from .parse import is_encoder_decoder_inputs
@@ -136,12 +136,12 @@ class InputRegistry:
136 """
137
138 def __init__(self) -> None:
139 - self._dummy_factories_by_model_type: Dict[Type[nn.Module],
140 - DummyDataFactory] = {}
141 - self._dummy_encoder_factories_by_model_type: Dict[
142 - Type[nn.Module], DummyDataFactory] = {}
143 - self._input_processors_by_model_type: Dict[Type[nn.Module],
144 - InputProcessor] = {}
139 + self._dummy_factories_by_model_type = \
140 + ClassRegistry[nn.Module, DummyDataFactory]()
141 + self._dummy_encoder_factories_by_model_type = \
142 + ClassRegistry[nn.Module, DummyDataFactory]()
143 + self._input_processors_by_model_type = \
144 + ClassRegistry[nn.Module, InputProcessor]()
145
146 def _default_dummy_data_factory(
147 self,
vllm/model_executor/layers/pooler.py
+1 -3
@@ -60,9 +60,7 @@ class Pooler(nn.Module):
60 softmax: bool,
61 step_tag_id: Optional[int] = None,
62 returned_token_ids: Optional[List[int]] = None,
63 - ) -> Optional["Pooler"]:
64 - if pooler_config is None:
65 - return None
63 + ) -> "Pooler":
64 return cls(
65 pooling_type=PoolingType[pooler_config.pooling_type]
66 if pooler_config.pooling_type is not None else pooling_type,
vllm/model_executor/model_loader/loader.py
+14 -4
@@ -9,6 +9,7 @@ import itertools
9 import json
10 import math
11 import os
12 +import warnings
13 from abc import ABC, abstractmethod
14 from contextlib import contextmanager
15 from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, cast
@@ -97,22 +98,31 @@ def device_loading_context(module: torch.nn.Module,
98 logger = init_logger(__name__)
99
100
100 -def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
101 +def _initialize_model(
102 + vllm_config: VllmConfig,
103 + *,
104 + prefix: str = "",
105 + architectures: Optional[list[str]] = None,
106 +) -> nn.Module:
107 """Initialize a model with the given configurations."""
108 model_config = vllm_config.model_config
103 - model_class, _ = get_model_architecture(model_config)
109 + model_class, _ = get_model_architecture(model_config,
110 + architectures=architectures)
111 +
112 signatures = inspect.signature(model_class.__init__)
113 all_params = [param.name for param in signatures.parameters.values()]
114 if "vllm_config" in all_params and "prefix" in all_params:
115 # new-style model class
116 with set_current_vllm_config(vllm_config):
117 return model_class(vllm_config=vllm_config, prefix=prefix)
118 +
119 msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
120 "input arguments. Possibly you have an old-style model class"
121 " registered from out of tree and it is used for new vLLM version. "
122 "Check https://docs.vllm.ai/en/latest/design/arch_overview.html "
123 "for the design and update the model class accordingly.")
115 - logger.warning(msg)
124 + warnings.warn(msg, DeprecationWarning, stacklevel=2)
125 +
126 logger.warning(
127 "Trying to guess the arguments for old-style model class %s",
128 model_class,
@@ -356,7 +366,7 @@ class DefaultModelLoader(BaseModelLoader):
366 weights_to_load = {name for name, _ in model.named_parameters()}
367 loaded_weights = model.load_weights(
368 self._get_all_weights(model_config, model))
359 - # We only enable strict check for non-quantiized models
369 + # We only enable strict check for non-quantized models
370 # that have loaded weights tracking currently.
371 if model_config.quantization is None and loaded_weights is not None:
372 weights_not_loaded = weights_to_load - loaded_weights
vllm/model_executor/model_loader/utils.py
+14 -4
@@ -1,12 +1,13 @@
1 """Utilities for selecting and loading models."""
2 import contextlib
3 -from typing import Tuple, Type
3 +from typing import Optional, Tuple, Type
4
5 import torch
6 from torch import nn
7
8 from vllm.config import ModelConfig
9 from vllm.model_executor.models import ModelRegistry
10 +from vllm.model_executor.models.adapters import as_embedding_model
11
12
13 @contextlib.contextmanager
@@ -19,8 +20,13 @@ def set_default_torch_dtype(dtype: torch.dtype):
20
21
22 def get_model_architecture(
22 - model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
23 - architectures = getattr(model_config.hf_config, "architectures", [])
23 + model_config: ModelConfig,
24 + *,
25 + architectures: Optional[list[str]] = None,
26 +) -> Tuple[Type[nn.Module], str]:
27 + if architectures is None:
28 + architectures = getattr(model_config.hf_config, "architectures", [])
29 +
30 # Special handling for quantized Mixtral.
31 # FIXME(woosuk): This is a temporary hack.
32 mixtral_supported = [
@@ -32,7 +38,11 @@ def get_model_architecture(
38 and "MixtralForCausalLM" in architectures):
39 architectures = ["QuantMixtralForCausalLM"]
40
35 - return ModelRegistry.resolve_model_cls(architectures)
41 + model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
42 + if model_config.task == "embedding":
43 + model_cls = as_embedding_model(model_cls)
44 +
45 + return model_cls, arch
46
47
48 def get_architecture_class_name(model_config: ModelConfig) -> str:
vllm/model_executor/models/adapters.py new
+98
@@ -0,0 +1,98 @@
1 +from collections.abc import Iterable
2 +from typing import Any, TypeVar
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from .interfaces_base import VllmModelForEmbedding, is_embedding_model
8 +
9 +_T = TypeVar("_T", bound=type[nn.Module])
10 +
11 +
12 +def as_embedding_model(cls: _T) -> _T:
13 + """Subclass an existing vLLM model to support embeddings."""
14 + # Avoid modifying existing embedding models
15 + if is_embedding_model(cls):
16 + return cls
17 +
18 + # Lazy import
19 + from vllm.config import VllmConfig
20 + from vllm.model_executor.layers.pooler import (Pooler, PoolerOutput,
21 + PoolingType)
22 + from vllm.model_executor.pooling_metadata import PoolingMetadata
23 +
24 + from .utils import AutoWeightsLoader, WeightsMapper
25 +
26 + class ModelForEmbedding(cls, VllmModelForEmbedding):
27 +
28 + def __init__(
29 + self,
30 + *,
31 + vllm_config: "VllmConfig",
32 + prefix: str = "",
33 + **kwargs: Any,
34 + ) -> None:
35 + super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
36 +
37 + # These are not used in embedding models
38 + for attr in ("lm_head", "logits_processor"):
39 + if hasattr(self, attr):
40 + delattr(self, attr)
41 +
42 + pooler_config = vllm_config.model_config.pooler_config
43 + assert pooler_config is not None
44 +
45 + # If the model already defines a pooler instance, don't overwrite it
46 + if not getattr(self, "_pooler", None):
47 + self._pooler = Pooler.from_config_with_defaults(
48 + pooler_config,
49 + pooling_type=PoolingType.LAST,
50 + normalize=True,
51 + softmax=False,
52 + )
53 +
54 + def pooler(
55 + self,
56 + hidden_states: torch.Tensor,
57 + pooling_metadata: PoolingMetadata,
58 + ) -> PoolerOutput:
59 + return self._pooler(hidden_states, pooling_metadata)
60 +
61 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
62 + # TODO: Support uninitialized params tracking
63 +
64 + # We have deleted this attribute, so don't load it
65 + weights = ((name, data) for name, data in weights
66 + if not name.startswith("lm_head."))
67 +
68 + # If `*ForCausalLM` defines `load_weights` on the inner model
69 + # and there are no other inner modules with parameters,
70 + # we support loading from both `*Model` and `*ForCausalLM`
71 + if hasattr(self, "model") and hasattr(self.model, "load_weights"):
72 + # Whether only `self.model` contains parameters
73 + model_is_only_param = all(
74 + name == "model" or next(child.parameters(), None) is None
75 + for name, child in self.named_children())
76 +
77 + if model_is_only_param:
78 + mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
79 + weights = mapper.apply(weights)
80 +
81 + self.model.load_weights(weights)
82 + return
83 +
84 + # For most other models
85 + if hasattr(cls, "load_weights"):
86 + cls.load_weights(self, weights) # type: ignore
87 + # Fallback
88 + else:
89 + loader = AutoWeightsLoader(self)
90 + loader.load_weights(weights)
91 +
92 + ModelForEmbedding.__name__ = cls.__name__ \
93 + .removesuffix("ForCausalLM") \
94 + .removesuffix("ForConditionalGeneration") \
95 + .removesuffix("ChatModel") \
96 + .removesuffix("LMHeadModel") + "ForEmbedding"
97 +
98 + return ModelForEmbedding # type: ignore
vllm/model_executor/models/blip2.py
+3 -2
@@ -512,9 +512,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
512 )
513
514 self.language_model = init_vllm_registered_model(
515 - config.text_config,
515 vllm_config=vllm_config,
517 - prefix=maybe_prefix(prefix, "language_model"))
516 + hf_config=config.text_config,
517 + prefix=maybe_prefix(prefix, "language_model"),
518 + )
519
520 self.make_empty_intermediate_tensors = (
521 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/gemma2.py
+2 -56
@@ -30,19 +30,17 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
30 QKVParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 -from vllm.model_executor.pooling_metadata import PoolingMetadata
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, PoolerOutput
40 +from vllm.sequence import IntermediateTensors
41
42 from .interfaces import SupportsLoRA, SupportsPP
45 -from .utils import (AutoWeightsLoader, WeightsMapper, extract_layer_index,
43 +from .utils import (AutoWeightsLoader, extract_layer_index,
44 is_pp_missing_parameter,
45 make_empty_intermediate_tensors_factory, make_layers,
46 maybe_prefix)
@@ -455,55 +453,3 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
453 if self.config.tie_word_embeddings else None),
454 )
455 return loader.load_weights(weights)
458 -
459 -
460 -class Gemma2EmbeddingModel(nn.Module, SupportsPP):
461 - """
462 - A model that uses Gemma2 with additional embedding functionalities.
463 -
464 - This class encapsulates the Gemma2Model and provides an interface for
465 - embedding operations and customized pooling functions.
466 -
467 - Attributes:
468 - model: An instance of Gemma2Model used for forward operations.
469 - _pooler: An instance of Pooler used for pooling operations.
470 - """
471 -
472 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
473 - super().__init__()
474 -
475 - self.model = Gemma2Model(vllm_config=vllm_config,
476 - prefix=maybe_prefix(prefix, "model"))
477 - self._pooler = Pooler.from_config_with_defaults(
478 - vllm_config.model_config.pooler_config,
479 - pooling_type=PoolingType.LAST,
480 - normalize=True,
481 - softmax=False)
482 - self.make_empty_intermediate_tensors = (
483 - self.model.make_empty_intermediate_tensors)
484 -
485 - def forward(
486 - self,
487 - input_ids: Optional[torch.Tensor],
488 - positions: torch.Tensor,
489 - kv_caches: List[torch.Tensor],
490 - attn_metadata: AttentionMetadata,
491 - intermediate_tensors: Optional[IntermediateTensors] = None,
492 - inputs_embeds: Optional[torch.Tensor] = None,
493 - ) -> Union[torch.Tensor, IntermediateTensors]:
494 - return self.model(input_ids, positions, kv_caches, attn_metadata,
495 - intermediate_tensors, inputs_embeds)
496 -
497 - def pooler(
498 - self,
499 - hidden_states: torch.Tensor,
500 - pooling_metadata: PoolingMetadata,
501 - ) -> Optional[PoolerOutput]:
502 - return self._pooler(hidden_states, pooling_metadata)
503 -
504 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
505 - hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
506 - weights = hf_to_vllm_mapper.apply(weights)
507 - weights = ((name, data) for name, data in weights
508 - if not name.startswith("lm_head."))
509 - self.model.load_weights(weights)
vllm/model_executor/models/internvl.py
+3 -2
@@ -474,9 +474,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
474 )
475
476 self.language_model = init_vllm_registered_model(
477 - config.text_config,
477 vllm_config=vllm_config,
479 - prefix=maybe_prefix(prefix, "language_model"))
478 + hf_config=config.text_config,
479 + prefix=maybe_prefix(prefix, "language_model"),
480 + )
481
482 self.mlp1 = self._init_mlp1(config)
483
vllm/model_executor/models/llama.py
+7 -95
@@ -37,7 +37,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 QKVParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
40 from vllm.model_executor.layers.quantization import QuantizationConfig
41 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
42 get_compressed_tensors_cache_scale)
@@ -47,14 +46,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
46 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
47 from vllm.model_executor.model_loader.weight_utils import (
48 default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
50 -from vllm.model_executor.pooling_metadata import PoolingMetadata
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.platforms import current_platform
53 -from vllm.sequence import IntermediateTensors, PoolerOutput
51 +from vllm.sequence import IntermediateTensors
52
53 from .interfaces import SupportsLoRA, SupportsPP
56 -from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
57 - extract_layer_index, is_pp_missing_parameter,
54 +from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,
55 + is_pp_missing_parameter,
56 make_empty_intermediate_tensors_factory, make_layers,
57 maybe_prefix)
58
@@ -511,11 +509,12 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
509 config = vllm_config.model_config.hf_config
510 quant_config = vllm_config.quant_config
511 lora_config = vllm_config.lora_config
514 - pooler_config = vllm_config.model_config.pooler_config
512 self.config = config
513 self.lora_config = lora_config
514
518 - self.model = self._init_model(vllm_config=vllm_config, prefix=prefix)
515 + self.model = self._init_model(vllm_config=vllm_config,
516 + prefix=maybe_prefix(prefix, "model"))
517 +
518 if get_pp_group().is_last_rank:
519 self.unpadded_vocab_size = config.vocab_size
520 if lora_config:
@@ -544,13 +543,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
543 self.sampler = get_sampler()
544 else:
545 self.lm_head = PPMissingLayer()
546 +
547 self.make_empty_intermediate_tensors = (
548 self.model.make_empty_intermediate_tensors)
549 - self._pooler = Pooler.from_config_with_defaults(
550 - pooler_config,
551 - pooling_type=PoolingType.STEP,
552 - normalize=False,
553 - softmax=False)
549
550 def _init_model(self, vllm_config: VllmConfig, prefix: str = ""):
551 return LlamaModel(vllm_config=vllm_config, prefix=prefix)
@@ -581,14 +576,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
576 sampling_metadata)
577 return logits
578
584 - def pooler(
585 - self,
586 - hidden_states: torch.Tensor,
587 - pooling_metadata: PoolingMetadata,
588 - ) -> Optional[PoolerOutput]:
589 - logits = self.compute_logits(hidden_states, None)
590 - return self._pooler(logits, pooling_metadata)
591 -
579 def sample(self, logits: torch.Tensor,
580 sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:
581 next_tokens = self.sampler(logits, sampling_metadata)
@@ -639,78 +626,3 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
626 name = name.replace(item, mapping[item])
627
628 return name, loaded_weight
642 -
643 -
644 -class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
645 - """
646 - A model that uses Llama with additional embedding functionalities.
647 -
648 - This class encapsulates the LlamaModel and provides an interface for
649 - embedding operations and customized pooling functions.
650 -
651 - Attributes:
652 - model: An instance of LlamaModel used for forward operations.
653 - _pooler: An instance of Pooler used for pooling operations.
654 - """
655 - packed_modules_mapping = {
656 - "qkv_proj": ["q_proj", "k_proj", "v_proj"],
657 - "gate_up_proj": ["gate_proj", "up_proj"]
658 - }
659 -
660 - # LoRA specific attributes
661 - supported_lora_modules = [
662 - "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
663 - ]
664 - embedding_modules = {
665 - "embed_tokens": "input_embeddings",
666 - }
667 - embedding_padding_modules = []
668 -
669 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
670 - super().__init__()
671 -
672 - pooler_config = vllm_config.model_config.pooler_config
673 -
674 - self.model = LlamaModel(vllm_config=vllm_config,
675 - prefix=maybe_prefix(prefix, "model"))
676 - self._pooler = Pooler.from_config_with_defaults(
677 - pooler_config,
678 - pooling_type=PoolingType.LAST,
679 - normalize=True,
680 - softmax=False)
681 - self.make_empty_intermediate_tensors = (
682 - self.model.make_empty_intermediate_tensors)
683 -
684 - def forward(
685 - self,
686 - input_ids: Optional[torch.Tensor],
687 - positions: torch.Tensor,
688 - kv_caches: List[torch.Tensor],
689 - attn_metadata: AttentionMetadata,
690 - intermediate_tensors: Optional[IntermediateTensors] = None,
691 - inputs_embeds: Optional[torch.Tensor] = None,
692 - ) -> Union[torch.Tensor, IntermediateTensors]:
693 - return self.model(input_ids, positions, kv_caches, attn_metadata,
694 - intermediate_tensors, inputs_embeds)
695 -
696 - def pooler(
697 - self,
698 - hidden_states: torch.Tensor,
699 - pooling_metadata: PoolingMetadata,
700 - ) -> Optional[PoolerOutput]:
701 - return self._pooler(hidden_states, pooling_metadata)
702 -
703 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
704 - hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
705 - weights = hf_to_vllm_mapper.apply(weights)
706 - weights = ((name, data) for name, data in weights
707 - if not name.startswith("lm_head."))
708 - self.model.load_weights(weights)
709 -
710 - def load_kv_cache_scales(self, quantization_param_path: str) -> None:
711 - self.model.load_kv_cache_scales(quantization_param_path)
712 -
713 - # LRUCacheWorkerLoRAManager instantiation requires model config.
714 - @property
715 - def config(self):
716 - return self.model.config
vllm/model_executor/models/llava.py
+3 -2
@@ -319,9 +319,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
319 projector_hidden_act=config.projector_hidden_act)
320
321 self.language_model = init_vllm_registered_model(
322 - config.text_config,
322 vllm_config=vllm_config,
324 - prefix=maybe_prefix(prefix, "language_model"))
323 + hf_config=config.text_config,
324 + prefix=maybe_prefix(prefix, "language_model"),
325 + )
326
327 self.make_empty_intermediate_tensors = (
328 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_next.py
+5 -21
@@ -14,13 +14,11 @@ from vllm.attention import AttentionMetadata
14 from vllm.config import VllmConfig
15 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16 InputContext)
17 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
17 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19 -from vllm.model_executor.pooling_metadata import PoolingMetadata
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.multimodal.inputs import NestedTensors
23 -from vllm.sequence import IntermediateTensors, PoolerOutput
21 +from vllm.sequence import IntermediateTensors
22 from vllm.utils import is_list_of
23
24 from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -286,7 +284,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
284 super().__init__()
285 config = vllm_config.model_config.hf_config
286 quant_config = vllm_config.quant_config
289 - pooler_config = vllm_config.model_config.pooler_config
287 multimodal_config = vllm_config.model_config.multimodal_config
288
289 vision_feature_layer = config.vision_feature_layer
@@ -321,17 +318,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
318 projector_hidden_act=config.projector_hidden_act)
319
320 self.language_model = init_vllm_registered_model(
324 - config.text_config,
321 vllm_config=vllm_config,
326 - prefix=maybe_prefix(prefix, "language_model"))
327 -
328 - # The same model class supports both language generation and embedding
329 - # because the architecture name is the same
330 - self._pooler = Pooler.from_config_with_defaults(
331 - pooler_config,
332 - pooling_type=PoolingType.LAST,
333 - normalize=True,
334 - softmax=False)
322 + hf_config=config.text_config,
323 + prefix=maybe_prefix(prefix, "language_model"),
324 + )
325 +
326 self.make_empty_intermediate_tensors = (
327 self.language_model.make_empty_intermediate_tensors)
328
@@ -678,13 +669,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
669 ) -> Optional[SamplerOutput]:
670 return self.language_model.sample(logits, sampling_metadata)
671
681 - def pooler(
682 - self,
683 - hidden_states: torch.Tensor,
684 - pooling_metadata: PoolingMetadata,
685 - ) -> Optional[PoolerOutput]:
686 - return self._pooler(hidden_states, pooling_metadata)
687 -
672 def load_weights(self, weights: Iterable[Tuple[str,
673 torch.Tensor]]) -> Set[str]:
674 loader = AutoWeightsLoader(self)
vllm/model_executor/models/llava_next_video.py
+3 -2
@@ -275,9 +275,10 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
275 text_hidden_size=config.text_config.hidden_size,
276 projector_hidden_act=config.projector_hidden_act)
277 self.language_model = init_vllm_registered_model(
278 - config.text_config,
278 vllm_config=vllm_config,
280 - prefix=maybe_prefix(prefix, "language_model"))
279 + hf_config=config.text_config,
280 + prefix=maybe_prefix(prefix, "language_model"),
281 + )
282
283 self.make_empty_intermediate_tensors = (
284 self.language_model.model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_onevision.py
+3 -2
@@ -422,9 +422,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
422 prefix=maybe_prefix(prefix, "vision_tower"))
423 self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
424 self.language_model = init_vllm_registered_model(
425 - config.text_config,
425 vllm_config=vllm_config,
427 - prefix=maybe_prefix(prefix, "language_model"))
426 + hf_config=config.text_config,
427 + prefix=maybe_prefix(prefix, "language_model"),
428 + )
429 self.image_newline = nn.Parameter(
430 torch.empty(config.text_config.hidden_size))
431
vllm/model_executor/models/paligemma.py
+3 -2
@@ -151,9 +151,10 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
151 self.quant_config = quant_config
152 config.text_config.architectures = ["GemmaForCausalLM"]
153 self.language_model = init_vllm_registered_model(
154 - config.text_config,
154 vllm_config=vllm_config,
156 - prefix=maybe_prefix(prefix, "language_model"))
155 + hf_config=config.text_config,
156 + prefix=maybe_prefix(prefix, "language_model"),
157 + )
158 logit_scale = getattr(config, "logit_scale", 1.0)
159 self.language_model.logits_processor.scale *= logit_scale
160
vllm/model_executor/models/phi3v.py
+14 -25
@@ -29,24 +29,22 @@ from vllm.config import ModelConfig, VllmConfig
29 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30 InputContext, token_inputs)
31 from vllm.logger import init_logger
32 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
32 from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding)
36 from vllm.model_executor.models.clip import CLIPVisionModel
38 -from vllm.model_executor.models.llama import LlamaForCausalLM
39 -from vllm.model_executor.pooling_metadata import PoolingMetadata
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
39 from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
40 from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
44 -from vllm.sequence import IntermediateTensors, PoolerOutput
41 +from vllm.sequence import IntermediateTensors
42 from vllm.utils import is_list_of
43
44 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
45 from .interfaces import SupportsMultiModal, SupportsPP
49 -from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,
46 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
47 + init_vllm_registered_model, maybe_prefix,
48 merge_multimodal_embeddings)
49
50 logger = init_logger(__name__)
@@ -536,7 +534,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
534 super().__init__()
535 config = vllm_config.model_config.hf_config
536 quant_config = vllm_config.quant_config
539 - pooler_config = vllm_config.model_config.pooler_config
537 multimodal_config = vllm_config.model_config.multimodal_config
538 self.config = config
539 self.multimodal_config = multimodal_config
@@ -556,18 +553,17 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
553 quant_config,
554 prefix=maybe_prefix(prefix, "model.vision_embed_tokens"))
555
559 - # The prefix is empty intentionally because default prefix of
560 - # LlamaForCausalLM is "model"
561 - self.language_model = LlamaForCausalLM(vllm_config=vllm_config,
562 - prefix="")
563 -
564 - # The same model class supports both language generation and embedding
565 - # because the architecture name is the same
566 - self._pooler = Pooler.from_config_with_defaults(
567 - pooler_config,
568 - pooling_type=PoolingType.LAST,
569 - normalize=True,
570 - softmax=False)
556 + self.language_model = init_vllm_registered_model(
557 + vllm_config=vllm_config,
558 + # The prefix is empty intentionally because default prefix of
559 + # LlamaForCausalLM is "model"
560 + prefix="",
561 + # We don't directly initialize vLLM's LlamaForCausalLM so we
562 + # can automatically apply embedding wrapper if this model is
563 + # initialized as an embedding model
564 + architectures=["LlamaForCausalLM"],
565 + )
566 +
567 self.make_empty_intermediate_tensors = (
568 self.language_model.make_empty_intermediate_tensors)
569
@@ -739,13 +735,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
735 ) -> Optional[SamplerOutput]:
736 return self.language_model.sample(logits, sampling_metadata)
737
742 - def pooler(
743 - self,
744 - hidden_states: torch.Tensor,
745 - pooling_metadata: PoolingMetadata,
746 - ) -> Optional[PoolerOutput]:
747 - return self._pooler(hidden_states, pooling_metadata)
748 -
738 def load_weights(self, weights: Iterable[Tuple[str,
739 torch.Tensor]]) -> Set[str]:
740 hf_to_vllm_mapper = WeightsMapper(
vllm/model_executor/models/pixtral.py
+3 -2
@@ -172,9 +172,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
172
173 # init MistralForCausalLM
174 self.language_model = init_vllm_registered_model(
175 - config.text_config,
175 vllm_config=vllm_config,
177 - prefix=maybe_prefix(prefix, "language_model"))
176 + hf_config=config.text_config,
177 + prefix=maybe_prefix(prefix, "language_model"),
178 + )
179
180 self.vision_encoder = VisionTransformer(self.vision_args)
181 self.vision_language_adapter = VisionLanguageAdapter(
vllm/model_executor/models/qwen2.py
+12 -16
@@ -31,6 +31,7 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
31 from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 +from vllm.logger import init_logger
35 from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -55,6 +56,8 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
56 make_empty_intermediate_tensors_factory, make_layers,
57 maybe_prefix)
58
59 +logger = init_logger(__name__)
60 +
61
62 class Qwen2MLP(nn.Module):
63
@@ -433,7 +436,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
436 config = vllm_config.model_config.hf_config
437 quant_config = vllm_config.quant_config
438 lora_config = vllm_config.lora_config
436 - pooler_config = vllm_config.model_config.pooler_config
439
440 self.config = config
441 self.lora_config = lora_config
@@ -454,14 +456,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
456 self.logits_processor = LogitsProcessor(config.vocab_size)
457 self.sampler = get_sampler()
458
457 - # The same model class supports both language generation and embedding
458 - # because the architecture name is the same
459 - self._pooler = Pooler.from_config_with_defaults(
460 - pooler_config,
461 - pooling_type=PoolingType.LAST,
462 - normalize=True,
463 - softmax=False)
464 -
459 self.make_empty_intermediate_tensors = (
460 self.model.make_empty_intermediate_tensors)
461
@@ -499,13 +493,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
493 next_tokens = self.sampler(logits, sampling_metadata)
494 return next_tokens
495
502 - def pooler(
503 - self,
504 - hidden_states: torch.Tensor,
505 - pooling_metadata: PoolingMetadata,
506 - ) -> Optional[PoolerOutput]:
507 - return self._pooler(hidden_states, pooling_metadata)
508 -
496 def load_weights(self, weights: Iterable[Tuple[str,
497 torch.Tensor]]) -> Set[str]:
498 loader = AutoWeightsLoader(
@@ -553,6 +540,15 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
540 self.model = Qwen2Model(vllm_config=vllm_config,
541 prefix=maybe_prefix(prefix, "model"))
542
543 + # TODO: Replace this model class with for_embedding(Qwen2ForCausalLM),
544 + # after changing the default pooling method
545 + if pooler_config.pooling_type is None:
546 + logger.warning(
547 + "This embedding model will default to last-token pooling in "
548 + "an upcoming version. To avoid breaking changes, you should "
549 + "pass `--override-pooler-config '{\"pooling_type\": \"MEAN\"}'`"
550 + " explicitly.")
551 +
552 self._pooler = Pooler.from_config_with_defaults(
553 pooler_config,
554 pooling_type=PoolingType.MEAN,
vllm/model_executor/models/qwen2_vl.py
+2 -16
@@ -50,7 +50,6 @@ from vllm.model_executor.layers.activation import QuickGELU
50 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51 RowParallelLinear)
52 from vllm.model_executor.layers.logits_processor import LogitsProcessor
53 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
53 from vllm.model_executor.layers.quantization import QuantizationConfig
54 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
55 from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -59,14 +58,13 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
58 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
59 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
60 from vllm.model_executor.models.qwen2 import Qwen2Model
62 -from vllm.model_executor.pooling_metadata import PoolingMetadata
61 from vllm.multimodal import MULTIMODAL_REGISTRY
62 from vllm.multimodal.image import cached_get_image_processor
63 from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
64 MultiModalKwargs, NestedTensors)
65 from vllm.multimodal.utils import cached_get_tokenizer
66 from vllm.platforms import _Backend
69 -from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
67 +from vllm.sequence import IntermediateTensors, SequenceData
68 from vllm.transformers_utils.config import uses_mrope
69 from vllm.transformers_utils.processor import cached_get_processor
70
@@ -1070,7 +1068,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1068 config = vllm_config.model_config.hf_config
1069 cache_config = vllm_config.cache_config
1070 quant_config = vllm_config.quant_config
1073 - pooler_config = vllm_config.model_config.pooler_config
1071 multimodal_config = vllm_config.model_config.multimodal_config
1072 assert not cache_config.enable_prefix_caching, \
1073 "Qwen2-VL currently does not support prefix caching"
@@ -1102,11 +1099,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1099
1100 self.logits_processor = LogitsProcessor(config.vocab_size)
1101 self.sampler = get_sampler()
1105 - self._pooler = Pooler.from_config_with_defaults(
1106 - pooler_config,
1107 - pooling_type=PoolingType.LAST,
1108 - normalize=True,
1109 - softmax=False)
1102 +
1103 self.make_empty_intermediate_tensors = (
1104 make_empty_intermediate_tensors_factory(
1105 ["hidden_states", "residual"], config.hidden_size))
@@ -1361,13 +1354,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1354 next_tokens = self.sampler(logits, sampling_metadata)
1355 return next_tokens
1356
1364 - def pooler(
1365 - self,
1366 - hidden_states: torch.Tensor,
1367 - pooling_metadata: PoolingMetadata,
1368 - ) -> Optional[PoolerOutput]:
1369 - return self._pooler(hidden_states, pooling_metadata)
1370 -
1357 def load_weights(self, weights: Iterable[Tuple[str,
1358 torch.Tensor]]) -> Set[str]:
1359 stacked_params_mapping = [
vllm/model_executor/models/registry.py
+41 -18
@@ -20,6 +20,7 @@ import torch.nn as nn
20 from vllm.logger import init_logger
21 from vllm.platforms import current_platform
22
23 +from .adapters import as_embedding_model
24 from .interfaces import (has_inner_state, is_attention_free,
25 supports_cross_encoding, supports_multimodal,
26 supports_pp)
@@ -107,15 +108,15 @@ _EMBEDDING_MODELS = {
108 "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
109 "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
110 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
110 - "Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
111 + "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
112 "GlmForCausalLM": ("glm", "GlmForCausalLM"),
112 - "LlamaModel": ("llama", "LlamaEmbeddingModel"),
113 + "LlamaModel": ("llama", "LlamaForCausalLM"),
114 **{
115 # Multiple models share the same architecture, so we include them all
116 k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
117 if arch == "LlamaForCausalLM"
118 },
118 - "MistralModel": ("llama", "LlamaEmbeddingModel"),
119 + "MistralModel": ("llama", "LlamaForCausalLM"),
120 "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
121 "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
122 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
@@ -125,7 +126,7 @@ _EMBEDDING_MODELS = {
126 # [Multimodal]
127 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
128 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
128 - "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration") # noqa: E501,
129 + "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
130 }
131
132 _CROSS_ENCODER_MODELS = {
@@ -208,6 +209,7 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
209
210 @dataclass(frozen=True)
211 class _ModelInfo:
212 + architecture: str
213 is_text_generation_model: bool
214 is_embedding_model: bool
215 supports_cross_encoding: bool
@@ -218,9 +220,19 @@ class _ModelInfo:
220
221 @staticmethod
222 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
223 + is_embedding_model_ = is_embedding_model(model)
224 + if not is_embedding_model_:
225 + try:
226 + as_embedding_model(model)
227 + except Exception:
228 + pass
229 + else:
230 + is_embedding_model_ = True
231 +
232 return _ModelInfo(
233 + architecture=model.__name__,
234 is_text_generation_model=is_text_generation_model(model),
223 - is_embedding_model=is_embedding_model(model),
235 + is_embedding_model=is_embedding_model_,
236 supports_cross_encoding=supports_cross_encoding(model),
237 supports_multimodal=supports_multimodal(model),
238 supports_pp=supports_pp(model),
@@ -399,13 +411,13 @@ class _ModelRegistry:
411 def inspect_model_cls(
412 self,
413 architectures: Union[str, List[str]],
402 - ) -> _ModelInfo:
414 + ) -> Tuple[_ModelInfo, str]:
415 architectures = self._normalize_archs(architectures)
416
417 for arch in architectures:
418 model_info = self._try_inspect_model_cls(arch)
419 if model_info is not None:
408 - return model_info
420 + return (model_info, arch)
421
422 return self._raise_for_unsupported(architectures)
423
@@ -426,39 +438,50 @@ class _ModelRegistry:
438 self,
439 architectures: Union[str, List[str]],
440 ) -> bool:
429 - return self.inspect_model_cls(architectures).is_text_generation_model
441 + model_cls, _ = self.inspect_model_cls(architectures)
442 + return model_cls.is_text_generation_model
443
444 def is_embedding_model(
445 self,
446 architectures: Union[str, List[str]],
447 ) -> bool:
435 - return self.inspect_model_cls(architectures).is_embedding_model
448 + model_cls, _ = self.inspect_model_cls(architectures)
449 + return model_cls.is_embedding_model
450
451 def is_cross_encoder_model(
452 self,
453 architectures: Union[str, List[str]],
454 ) -> bool:
441 - return self.inspect_model_cls(architectures).supports_cross_encoding
455 + model_cls, _ = self.inspect_model_cls(architectures)
456 + return model_cls.supports_cross_encoding
457
458 def is_multimodal_model(
459 self,
460 architectures: Union[str, List[str]],
461 ) -> bool:
447 - return self.inspect_model_cls(architectures).supports_multimodal
462 + model_cls, _ = self.inspect_model_cls(architectures)
463 + return model_cls.supports_multimodal
464
465 def is_pp_supported_model(
466 self,
467 architectures: Union[str, List[str]],
468 ) -> bool:
453 - return self.inspect_model_cls(architectures).supports_pp
469 + model_cls, _ = self.inspect_model_cls(architectures)
470 + return model_cls.supports_pp
471
455 - def model_has_inner_state(self, architectures: Union[str,
456 - List[str]]) -> bool:
457 - return self.inspect_model_cls(architectures).has_inner_state
472 + def model_has_inner_state(
473 + self,
474 + architectures: Union[str, List[str]],
475 + ) -> bool:
476 + model_cls, _ = self.inspect_model_cls(architectures)
477 + return model_cls.has_inner_state
478
459 - def is_attention_free_model(self, architectures: Union[str,
460 - List[str]]) -> bool:
461 - return self.inspect_model_cls(architectures).is_attention_free
479 + def is_attention_free_model(
480 + self,
481 + architectures: Union[str, List[str]],
482 + ) -> bool:
483 + model_cls, _ = self.inspect_model_cls(architectures)
484 + return model_cls.is_attention_free
485
486
487 ModelRegistry = _ModelRegistry({
vllm/model_executor/models/ultravox.py
+3 -2
@@ -360,9 +360,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
360 ))
361 self.multi_modal_projector = UltravoxProjector(config)
362 self.language_model = init_vllm_registered_model(
363 - config.text_config,
363 vllm_config=vllm_config,
365 - prefix=maybe_prefix(prefix, "language_model"))
364 + hf_config=config.text_config,
365 + prefix=maybe_prefix(prefix, "language_model"),
366 + )
367 if config.text_model_id is not None:
368 # this prefix is not for initialization, but for loading weights
369 # note the trailing dot
vllm/model_executor/models/utils.py
+19 -5
@@ -173,8 +173,15 @@ class AutoWeightsLoader:
173 module_load_weights = getattr(module, "load_weights", None)
174 if callable(module_load_weights):
175 loaded_params = module_load_weights(weights)
176 - yield from map(lambda x: self._get_qualname(base_prefix, x),
177 - loaded_params)
176 + if loaded_params is None:
177 + logger.warning(
178 + "Unable to collect loaded parameters "
179 + "for module %s", module)
180 + else:
181 + yield from map(
182 + lambda x: self._get_qualname(base_prefix, x),
183 + loaded_params,
184 + )
185
186 child_modules = dict(module.named_children())
187 child_params = dict(module.named_parameters(recurse=False))
@@ -232,17 +239,24 @@ class AutoWeightsLoader:
239
240
241 def init_vllm_registered_model(
235 - hf_config: PretrainedConfig,
242 vllm_config: VllmConfig,
243 + *,
244 prefix: str = "",
245 + hf_config: Optional[PretrainedConfig] = None,
246 + architectures: Optional[list[str]] = None,
247 ) -> nn.Module:
248 """
249 Helper function to initialize an inner model registered to vLLM,
250 based on the arguments passed to the outer vLLM model.
251 """
252 from vllm.model_executor.model_loader.loader import _initialize_model
244 - vllm_config = vllm_config.with_hf_config(hf_config)
245 - return _initialize_model(vllm_config, prefix)
253 +
254 + if hf_config is not None:
255 + vllm_config = vllm_config.with_hf_config(hf_config)
256 +
257 + return _initialize_model(vllm_config=vllm_config,
258 + prefix=prefix,
259 + architectures=architectures)
260
261
262 @overload
vllm/multimodal/base.py
+3 -3
@@ -7,7 +7,7 @@ from torch import nn
7
8 from vllm.inputs import InputContext
9 from vllm.logger import init_logger
10 -from vllm.utils import (get_allowed_kwarg_only_overrides,
10 +from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
11 resolve_mm_processor_kwargs)
12
13 if TYPE_CHECKING:
@@ -54,8 +54,8 @@ class MultiModalPlugin(ABC):
54 """
55
56 def __init__(self) -> None:
57 - self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {}
58 - self._max_mm_tokens: Dict[Type[nn.Module], MultiModalTokensCalc] = {}
57 + self._input_mappers = ClassRegistry[nn.Module, MultiModalInputMapper]()
58 + self._max_mm_tokens = ClassRegistry[nn.Module, MultiModalTokensCalc]()
59
60 @abstractmethod
61 def get_data_key(self) -> str:
vllm/multimodal/registry.py
+3 -2
@@ -9,6 +9,7 @@ from typing_extensions import TypeAlias
9 from vllm.inputs import InputProcessingContext
10 from vllm.logger import init_logger
11 from vllm.transformers_utils.tokenizer import AnyTokenizer
12 +from vllm.utils import ClassRegistry
13
14 from .audio import AudioPlugin
15 from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
@@ -62,8 +63,8 @@ class MultiModalRegistry:
63 plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
64 self._plugins = {p.get_data_key(): p for p in plugins}
65
65 - self._processor_factories: Dict[Type[nn.Module],
66 - MultiModalProcessorFactory] = {}
66 + self._processor_factories = ClassRegistry[nn.Module,
67 + MultiModalProcessorFactory]()
68
69 # This is used for non-multimodal models
70 self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
vllm/utils.py
+19 -3
@@ -20,7 +20,7 @@ import uuid
20 import warnings
21 import weakref
22 from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
23 -from collections import defaultdict
23 +from collections import UserDict, defaultdict
24 from collections.abc import Iterable, Mapping
25 from functools import lru_cache, partial, wraps
26 from platform import uname
@@ -1517,13 +1517,13 @@ class AtomicCounter:
1517
1518
1519 # Adapted from: https://stackoverflow.com/a/47212782/5082708
1520 -class LazyDict(Mapping, Generic[T]):
1520 +class LazyDict(Mapping[str, T], Generic[T]):
1521
1522 def __init__(self, factory: Dict[str, Callable[[], T]]):
1523 self._factory = factory
1524 self._dict: Dict[str, T] = {}
1525
1526 - def __getitem__(self, key) -> T:
1526 + def __getitem__(self, key: str) -> T:
1527 if key not in self._dict:
1528 if key not in self._factory:
1529 raise KeyError(key)
@@ -1540,6 +1540,22 @@ class LazyDict(Mapping, Generic[T]):
1540 return len(self._factory)
1541
1542
1543 +class ClassRegistry(UserDict[type[T], _V]):
1544 +
1545 + def __getitem__(self, key: type[T]) -> _V:
1546 + for cls in key.mro():
1547 + if cls in self.data:
1548 + return self.data[cls]
1549 +
1550 + raise KeyError(key)
1551 +
1552 + def __contains__(self, key: object) -> bool:
1553 + if not isinstance(key, type):
1554 + return False
1555 +
1556 + return any(cls in self.data for cls in key.mro())
1557 +
1558 +
1559 def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1560 """
1561 Create a weak reference to a tensor.