A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Replace embedding models with pooling adapter (#10769)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 1, 2024 at 08:02 UTC
133707123e730a3544875d432a9435bdfe5e34cf
32 files changed
+387
-323
.buildkite/test-pipeline.yaml
+2
-2
@@ -334,7 +334,6 @@ steps:
334
commands:
335
- pytest -v -s models/decoder_only/language -m 'core_model or quant_model'
336
- pytest -v -s models/embedding/language -m core_model
337
- - pytest -v -s models/embedding/vision_language -m core_model
337
338
- label: Language Models Test (Extended) # 50min
339
optional: true
@@ -346,7 +345,6 @@ steps:
345
commands:
346
- pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
347
- pytest -v -s models/embedding/language -m 'not core_model'
349
- - pytest -v -s models/embedding/vision_language -m 'not core_model'
348
349
- label: Multi-Modal Models Test (Standard) # 26min
350
#mirror_hardwares: [amd]
@@ -359,6 +357,7 @@ steps:
357
commands:
358
- pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
359
- pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
360
+ - pytest -v -s models/embedding/vision_language -m core_model
361
- pytest -v -s models/encoder_decoder/language -m core_model
362
- pytest -v -s models/encoder_decoder/vision_language -m core_model
363
@@ -376,6 +375,7 @@ steps:
375
# https://github.com/huggingface/transformers/issues/34307
376
- pytest -v -s models/decoder_only/vision_language/test_phi3v.py
377
- pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
378
+ - pytest -v -s models/embedding/vision_language -m 'not core_model'
379
- pytest -v -s models/encoder_decoder/language -m 'not core_model'
380
- pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
381
docs/source/models/supported_models.rst
+14
-1
@@ -357,7 +357,7 @@ Text Embedding
357
- ✅︎
358
* - :code:`Qwen2Model`, :code:`Qwen2ForCausalLM`
359
- Qwen2-based
360
- - :code:`ssmits/Qwen2-7B-Instruct-embed-base`, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc.
360
+ - :code:`ssmits/Qwen2-7B-Instruct-embed-base` (see note), :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc.
361
- ✅︎
362
- ✅︎
363
* - :code:`RobertaModel`, :code:`RobertaForMaskedLM`
@@ -378,6 +378,10 @@ Text Embedding
378
.. tip::
379
You can override the model's pooling method by passing :code:`--override-pooler-config`.
380
381
+.. note::
382
+ :code:`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
383
+ You should manually set mean pooling by passing :code:`--override-pooler-config '{"pooling_type": "MEAN"}'`.
384
+
385
.. note::
386
Unlike base Qwen2, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` uses bi-directional attention.
387
You can set :code:`--hf-overrides '{"is_causal": false}'` to change the attention mask accordingly.
@@ -397,12 +401,21 @@ Reward Modeling
401
- Example HF Models
402
- :ref:`LoRA <lora>`
403
- :ref:`PP <distributed_serving>`
404
+ * - :code:`LlamaForCausalLM`
405
+ - Llama-based
406
+ - :code:`peiyi9979/math-shepherd-mistral-7b-prm`, etc.
407
+ - ✅︎
408
+ - ✅︎
409
* - :code:`Qwen2ForRewardModel`
410
- Qwen2-based
411
- :code:`Qwen/Qwen2.5-Math-RM-72B`, etc.
412
- ✅︎
413
- ✅︎
414
415
+.. important::
416
+ For process-supervised reward models such as :code:`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
417
+ e.g.: :code:`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
418
+
419
.. note::
420
As an interim measure, these models are supported in both offline and online inference via Embeddings API.
421
tests/conftest.py
-1
@@ -263,7 +263,6 @@ class HfRunner:
263
dtype: str = "half",
264
*,
265
model_kwargs: Optional[Dict[str, Any]] = None,
266
- is_embedding_model: bool = False,
266
is_sentence_transformer: bool = False,
267
is_cross_encoder: bool = False,
268
skip_tokenizer_init: bool = False,
tests/models/embedding/language/test_embedding.py
+5
@@ -4,6 +4,8 @@ Run `pytest tests/models/embedding/language/test_embedding.py`.
4
"""
5
import pytest
6
7
+from vllm.config import PoolerConfig
8
+
9
from ..utils import check_embeddings_close
10
11
@@ -33,6 +35,9 @@ def test_models(
35
dtype: str,
36
) -> None:
37
vllm_extra_kwargs = {}
38
+ if model == "ssmits/Qwen2-7B-Instruct-embed-base":
39
+ vllm_extra_kwargs["override_pooler_config"] = \
40
+ PoolerConfig(pooling_type="MEAN")
41
if model == "Alibaba-NLP/gte-Qwen2-7B-instruct":
42
vllm_extra_kwargs["hf_overrides"] = {"is_causal": False}
43
tests/models/test_registry.py
+14
-17
@@ -6,11 +6,8 @@ import torch.cuda
6
from vllm.model_executor.models import (is_embedding_model,
7
is_text_generation_model,
8
supports_multimodal)
9
-# yapf conflicts with isort for this block
10
-# yapf: disable
11
-from vllm.model_executor.models.registry import (_CROSS_ENCODER_MODELS,
12
- _EMBEDDING_MODELS,
13
- _MULTIMODAL_MODELS,
9
+from vllm.model_executor.models.adapters import as_embedding_model
10
+from vllm.model_executor.models.registry import (_MULTIMODAL_MODELS,
11
_SPECULATIVE_DECODING_MODELS,
12
_TEXT_GENERATION_MODELS,
13
ModelRegistry)
@@ -26,18 +23,18 @@ def test_registry_imports(model_arch):
23
model_cls, _ = ModelRegistry.resolve_model_cls(model_arch)
24
25
if model_arch in _SPECULATIVE_DECODING_MODELS:
29
- pass # Ignore these models which do not have a unified format
30
- else:
31
- assert is_text_generation_model(model_cls) is (
32
- model_arch in _TEXT_GENERATION_MODELS
33
- or model_arch in _MULTIMODAL_MODELS)
34
-
35
- embedding_models = {**_EMBEDDING_MODELS, **_CROSS_ENCODER_MODELS}
36
- assert is_embedding_model(model_cls) is (model_arch
37
- in embedding_models)
38
-
39
- assert supports_multimodal(model_cls) is (model_arch
40
- in _MULTIMODAL_MODELS)
26
+ return # Ignore these models which do not have a unified format
27
+
28
+ if (model_arch in _TEXT_GENERATION_MODELS
29
+ or model_arch in _MULTIMODAL_MODELS):
30
+ assert is_text_generation_model(model_cls)
31
+
32
+ # All vLLM models should be convertible to an embedding model
33
+ embed_model = as_embedding_model(model_cls)
34
+ assert is_embedding_model(embed_model)
35
+
36
+ if model_arch in _MULTIMODAL_MODELS:
37
+ assert supports_multimodal(model_cls)
38
39
40
@fork_new_process_for_each_test
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py
+40
-5
@@ -1,13 +1,34 @@
1
-from typing import List, Optional, Union
1
+from typing import Iterable, List, Optional, Tuple, Union
2
3
import torch
4
+import torch.nn as nn
5
6
from vllm.attention import AttentionMetadata
6
-from vllm.model_executor.models.gemma2 import Gemma2EmbeddingModel
7
-from vllm.sequence import IntermediateTensors
7
+from vllm.config import VllmConfig
8
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
9
+from vllm.model_executor.models.gemma2 import Gemma2Model
10
+from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
11
+from vllm.model_executor.pooling_metadata import PoolingMetadata
12
+from vllm.sequence import IntermediateTensors, PoolerOutput
13
14
10
-class MyGemma2Embedding(Gemma2EmbeddingModel):
15
+class MyGemma2Embedding(nn.Module):
16
+
17
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
18
+ super().__init__()
19
+
20
+ self.model = Gemma2Model(vllm_config=vllm_config,
21
+ prefix=maybe_prefix(prefix, "model"))
22
+
23
+ self._pooler = Pooler.from_config_with_defaults(
24
+ vllm_config.model_config.pooler_config,
25
+ pooling_type=PoolingType.LAST,
26
+ normalize=True,
27
+ softmax=False,
28
+ )
29
+
30
+ self.make_empty_intermediate_tensors = (
31
+ self.model.make_empty_intermediate_tensors)
32
33
def forward(
34
self,
@@ -18,7 +39,7 @@ class MyGemma2Embedding(Gemma2EmbeddingModel):
39
intermediate_tensors: Optional[IntermediateTensors] = None,
40
inputs_embeds: Optional[torch.Tensor] = None,
41
) -> Union[torch.Tensor, IntermediateTensors]:
21
- hidden_states = super().forward(
42
+ hidden_states = self.model(
43
input_ids,
44
positions,
45
kv_caches,
@@ -32,3 +53,17 @@ class MyGemma2Embedding(Gemma2EmbeddingModel):
53
54
# Return all-zero embeddings
55
return torch.zeros_like(hidden_states)
56
+
57
+ def pooler(
58
+ self,
59
+ hidden_states: torch.Tensor,
60
+ pooling_metadata: PoolingMetadata,
61
+ ) -> Optional[PoolerOutput]:
62
+ return self._pooler(hidden_states, pooling_metadata)
63
+
64
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
65
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
66
+ weights = hf_to_vllm_mapper.apply(weights)
67
+ weights = ((name, data) for name, data in weights
68
+ if not name.startswith("lm_head."))
69
+ return self.model.load_weights(weights)
tests/test_config.py
+1
-2
@@ -26,8 +26,7 @@ def test_auto_task(model_id, expected_task):
26
27
28
@pytest.mark.parametrize(("model_id", "bad_task"), [
29
- ("facebook/opt-125m", "embedding"),
30
- ("intfloat/e5-mistral-7b-instruct", "generate"),
29
+ ("Qwen/Qwen2.5-Math-RM-72B", "generate"),
30
])
31
def test_incorrect_task(model_id, bad_task):
32
with pytest.raises(ValueError, match=r"does not support the .* task"):
vllm/config.py
+25
@@ -370,6 +370,31 @@ class ModelConfig:
370
selected_task = next(iter(supported_tasks_lst))
371
372
if len(supported_tasks) > 1:
373
+ suffix_to_preferred_task: List[Tuple[str, _Task]] = [
374
+ # Hardcode the models that are exceptions
375
+ ("AquilaModel", "generate"),
376
+ ("ChatGLMModel", "generate"),
377
+ # Other models follow this pattern
378
+ ("ForCausalLM", "generate"),
379
+ ("ForConditionalGeneration", "generate"),
380
+ ("ChatModel", "generate"),
381
+ ("LMHeadModel", "generate"),
382
+ ("EmbeddingModel", "embedding"),
383
+ ("RewardModel", "embedding"),
384
+ ("ForSequenceClassification", "embedding"),
385
+ ]
386
+ info, arch = ModelRegistry.inspect_model_cls(architectures)
387
+
388
+ for suffix, pref_task in suffix_to_preferred_task:
389
+ if arch.endswith(suffix) and pref_task in supported_tasks:
390
+ selected_task = pref_task
391
+ break
392
+ else:
393
+ if (arch.endswith("Model")
394
+ and info.architecture.endswith("ForCausalLM")
395
+ and "embedding" in supported_tasks):
396
+ selected_task = "embedding"
397
+
398
logger.info(
399
"This model supports multiple tasks: %s. "
400
"Defaulting to '%s'.", supported_tasks, selected_task)
vllm/inputs/registry.py
+8
-8
@@ -11,8 +11,8 @@ from typing_extensions import TypeVar, assert_never
11
from vllm.logger import init_logger
12
from vllm.transformers_utils.processor import cached_get_processor
13
from vllm.transformers_utils.tokenizer import AnyTokenizer
14
-from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
15
- resolve_mm_processor_kwargs)
14
+from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
15
+ print_warning_once, resolve_mm_processor_kwargs)
16
17
from .data import ProcessorInputs, SingletonInputs
18
from .parse import is_encoder_decoder_inputs
@@ -136,12 +136,12 @@ class InputRegistry:
136
"""
137
138
def __init__(self) -> None:
139
- self._dummy_factories_by_model_type: Dict[Type[nn.Module],
140
- DummyDataFactory] = {}
141
- self._dummy_encoder_factories_by_model_type: Dict[
142
- Type[nn.Module], DummyDataFactory] = {}
143
- self._input_processors_by_model_type: Dict[Type[nn.Module],
144
- InputProcessor] = {}
139
+ self._dummy_factories_by_model_type = \
140
+ ClassRegistry[nn.Module, DummyDataFactory]()
141
+ self._dummy_encoder_factories_by_model_type = \
142
+ ClassRegistry[nn.Module, DummyDataFactory]()
143
+ self._input_processors_by_model_type = \
144
+ ClassRegistry[nn.Module, InputProcessor]()
145
146
def _default_dummy_data_factory(
147
self,
vllm/model_executor/layers/pooler.py
+1
-3
@@ -60,9 +60,7 @@ class Pooler(nn.Module):
60
softmax: bool,
61
step_tag_id: Optional[int] = None,
62
returned_token_ids: Optional[List[int]] = None,
63
- ) -> Optional["Pooler"]:
64
- if pooler_config is None:
65
- return None
63
+ ) -> "Pooler":
64
return cls(
65
pooling_type=PoolingType[pooler_config.pooling_type]
66
if pooler_config.pooling_type is not None else pooling_type,
vllm/model_executor/model_loader/loader.py
+14
-4
@@ -9,6 +9,7 @@ import itertools
9
import json
10
import math
11
import os
12
+import warnings
13
from abc import ABC, abstractmethod
14
from contextlib import contextmanager
15
from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, cast
@@ -97,22 +98,31 @@ def device_loading_context(module: torch.nn.Module,
98
logger = init_logger(__name__)
99
100
100
-def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
101
+def _initialize_model(
102
+ vllm_config: VllmConfig,
103
+ *,
104
+ prefix: str = "",
105
+ architectures: Optional[list[str]] = None,
106
+) -> nn.Module:
107
"""Initialize a model with the given configurations."""
108
model_config = vllm_config.model_config
103
- model_class, _ = get_model_architecture(model_config)
109
+ model_class, _ = get_model_architecture(model_config,
110
+ architectures=architectures)
111
+
112
signatures = inspect.signature(model_class.__init__)
113
all_params = [param.name for param in signatures.parameters.values()]
114
if "vllm_config" in all_params and "prefix" in all_params:
115
# new-style model class
116
with set_current_vllm_config(vllm_config):
117
return model_class(vllm_config=vllm_config, prefix=prefix)
118
+
119
msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
120
"input arguments. Possibly you have an old-style model class"
121
" registered from out of tree and it is used for new vLLM version. "
122
"Check https://docs.vllm.ai/en/latest/design/arch_overview.html "
123
"for the design and update the model class accordingly.")
115
- logger.warning(msg)
124
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
125
+
126
logger.warning(
127
"Trying to guess the arguments for old-style model class %s",
128
model_class,
@@ -356,7 +366,7 @@ class DefaultModelLoader(BaseModelLoader):
366
weights_to_load = {name for name, _ in model.named_parameters()}
367
loaded_weights = model.load_weights(
368
self._get_all_weights(model_config, model))
359
- # We only enable strict check for non-quantiized models
369
+ # We only enable strict check for non-quantized models
370
# that have loaded weights tracking currently.
371
if model_config.quantization is None and loaded_weights is not None:
372
weights_not_loaded = weights_to_load - loaded_weights
vllm/model_executor/model_loader/utils.py
+14
-4
@@ -1,12 +1,13 @@
1
"""Utilities for selecting and loading models."""
2
import contextlib
3
-from typing import Tuple, Type
3
+from typing import Optional, Tuple, Type
4
5
import torch
6
from torch import nn
7
8
from vllm.config import ModelConfig
9
from vllm.model_executor.models import ModelRegistry
10
+from vllm.model_executor.models.adapters import as_embedding_model
11
12
13
@contextlib.contextmanager
@@ -19,8 +20,13 @@ def set_default_torch_dtype(dtype: torch.dtype):
20
21
22
def get_model_architecture(
22
- model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
23
- architectures = getattr(model_config.hf_config, "architectures", [])
23
+ model_config: ModelConfig,
24
+ *,
25
+ architectures: Optional[list[str]] = None,
26
+) -> Tuple[Type[nn.Module], str]:
27
+ if architectures is None:
28
+ architectures = getattr(model_config.hf_config, "architectures", [])
29
+
30
# Special handling for quantized Mixtral.
31
# FIXME(woosuk): This is a temporary hack.
32
mixtral_supported = [
@@ -32,7 +38,11 @@ def get_model_architecture(
38
and "MixtralForCausalLM" in architectures):
39
architectures = ["QuantMixtralForCausalLM"]
40
35
- return ModelRegistry.resolve_model_cls(architectures)
41
+ model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
42
+ if model_config.task == "embedding":
43
+ model_cls = as_embedding_model(model_cls)
44
+
45
+ return model_cls, arch
46
47
48
def get_architecture_class_name(model_config: ModelConfig) -> str:
vllm/model_executor/models/adapters.py
new
+98
@@ -0,0 +1,98 @@
1
+from collections.abc import Iterable
2
+from typing import Any, TypeVar
3
+
4
+import torch
5
+import torch.nn as nn
6
+
7
+from .interfaces_base import VllmModelForEmbedding, is_embedding_model
8
+
9
+_T = TypeVar("_T", bound=type[nn.Module])
10
+
11
+
12
+def as_embedding_model(cls: _T) -> _T:
13
+ """Subclass an existing vLLM model to support embeddings."""
14
+ # Avoid modifying existing embedding models
15
+ if is_embedding_model(cls):
16
+ return cls
17
+
18
+ # Lazy import
19
+ from vllm.config import VllmConfig
20
+ from vllm.model_executor.layers.pooler import (Pooler, PoolerOutput,
21
+ PoolingType)
22
+ from vllm.model_executor.pooling_metadata import PoolingMetadata
23
+
24
+ from .utils import AutoWeightsLoader, WeightsMapper
25
+
26
+ class ModelForEmbedding(cls, VllmModelForEmbedding):
27
+
28
+ def __init__(
29
+ self,
30
+ *,
31
+ vllm_config: "VllmConfig",
32
+ prefix: str = "",
33
+ **kwargs: Any,
34
+ ) -> None:
35
+ super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
36
+
37
+ # These are not used in embedding models
38
+ for attr in ("lm_head", "logits_processor"):
39
+ if hasattr(self, attr):
40
+ delattr(self, attr)
41
+
42
+ pooler_config = vllm_config.model_config.pooler_config
43
+ assert pooler_config is not None
44
+
45
+ # If the model already defines a pooler instance, don't overwrite it
46
+ if not getattr(self, "_pooler", None):
47
+ self._pooler = Pooler.from_config_with_defaults(
48
+ pooler_config,
49
+ pooling_type=PoolingType.LAST,
50
+ normalize=True,
51
+ softmax=False,
52
+ )
53
+
54
+ def pooler(
55
+ self,
56
+ hidden_states: torch.Tensor,
57
+ pooling_metadata: PoolingMetadata,
58
+ ) -> PoolerOutput:
59
+ return self._pooler(hidden_states, pooling_metadata)
60
+
61
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
62
+ # TODO: Support uninitialized params tracking
63
+
64
+ # We have deleted this attribute, so don't load it
65
+ weights = ((name, data) for name, data in weights
66
+ if not name.startswith("lm_head."))
67
+
68
+ # If `*ForCausalLM` defines `load_weights` on the inner model
69
+ # and there are no other inner modules with parameters,
70
+ # we support loading from both `*Model` and `*ForCausalLM`
71
+ if hasattr(self, "model") and hasattr(self.model, "load_weights"):
72
+ # Whether only `self.model` contains parameters
73
+ model_is_only_param = all(
74
+ name == "model" or next(child.parameters(), None) is None
75
+ for name, child in self.named_children())
76
+
77
+ if model_is_only_param:
78
+ mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
79
+ weights = mapper.apply(weights)
80
+
81
+ self.model.load_weights(weights)
82
+ return
83
+
84
+ # For most other models
85
+ if hasattr(cls, "load_weights"):
86
+ cls.load_weights(self, weights) # type: ignore
87
+ # Fallback
88
+ else:
89
+ loader = AutoWeightsLoader(self)
90
+ loader.load_weights(weights)
91
+
92
+ ModelForEmbedding.__name__ = cls.__name__ \
93
+ .removesuffix("ForCausalLM") \
94
+ .removesuffix("ForConditionalGeneration") \
95
+ .removesuffix("ChatModel") \
96
+ .removesuffix("LMHeadModel") + "ForEmbedding"
97
+
98
+ return ModelForEmbedding # type: ignore
vllm/model_executor/models/blip2.py
+3
-2
@@ -512,9 +512,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
512
)
513
514
self.language_model = init_vllm_registered_model(
515
- config.text_config,
515
vllm_config=vllm_config,
517
- prefix=maybe_prefix(prefix, "language_model"))
516
+ hf_config=config.text_config,
517
+ prefix=maybe_prefix(prefix, "language_model"),
518
+ )
519
520
self.make_empty_intermediate_tensors = (
521
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/gemma2.py
+2
-56
@@ -30,19 +30,17 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
30
QKVParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
-from vllm.model_executor.pooling_metadata import PoolingMetadata
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, PoolerOutput
40
+from vllm.sequence import IntermediateTensors
41
42
from .interfaces import SupportsLoRA, SupportsPP
45
-from .utils import (AutoWeightsLoader, WeightsMapper, extract_layer_index,
43
+from .utils import (AutoWeightsLoader, extract_layer_index,
44
is_pp_missing_parameter,
45
make_empty_intermediate_tensors_factory, make_layers,
46
maybe_prefix)
@@ -455,55 +453,3 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
453
if self.config.tie_word_embeddings else None),
454
)
455
return loader.load_weights(weights)
458
-
459
-
460
-class Gemma2EmbeddingModel(nn.Module, SupportsPP):
461
- """
462
- A model that uses Gemma2 with additional embedding functionalities.
463
-
464
- This class encapsulates the Gemma2Model and provides an interface for
465
- embedding operations and customized pooling functions.
466
-
467
- Attributes:
468
- model: An instance of Gemma2Model used for forward operations.
469
- _pooler: An instance of Pooler used for pooling operations.
470
- """
471
-
472
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
473
- super().__init__()
474
-
475
- self.model = Gemma2Model(vllm_config=vllm_config,
476
- prefix=maybe_prefix(prefix, "model"))
477
- self._pooler = Pooler.from_config_with_defaults(
478
- vllm_config.model_config.pooler_config,
479
- pooling_type=PoolingType.LAST,
480
- normalize=True,
481
- softmax=False)
482
- self.make_empty_intermediate_tensors = (
483
- self.model.make_empty_intermediate_tensors)
484
-
485
- def forward(
486
- self,
487
- input_ids: Optional[torch.Tensor],
488
- positions: torch.Tensor,
489
- kv_caches: List[torch.Tensor],
490
- attn_metadata: AttentionMetadata,
491
- intermediate_tensors: Optional[IntermediateTensors] = None,
492
- inputs_embeds: Optional[torch.Tensor] = None,
493
- ) -> Union[torch.Tensor, IntermediateTensors]:
494
- return self.model(input_ids, positions, kv_caches, attn_metadata,
495
- intermediate_tensors, inputs_embeds)
496
-
497
- def pooler(
498
- self,
499
- hidden_states: torch.Tensor,
500
- pooling_metadata: PoolingMetadata,
501
- ) -> Optional[PoolerOutput]:
502
- return self._pooler(hidden_states, pooling_metadata)
503
-
504
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
505
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
506
- weights = hf_to_vllm_mapper.apply(weights)
507
- weights = ((name, data) for name, data in weights
508
- if not name.startswith("lm_head."))
509
- self.model.load_weights(weights)
vllm/model_executor/models/internvl.py
+3
-2
@@ -474,9 +474,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
474
)
475
476
self.language_model = init_vllm_registered_model(
477
- config.text_config,
477
vllm_config=vllm_config,
479
- prefix=maybe_prefix(prefix, "language_model"))
478
+ hf_config=config.text_config,
479
+ prefix=maybe_prefix(prefix, "language_model"),
480
+ )
481
482
self.mlp1 = self._init_mlp1(config)
483
vllm/model_executor/models/llama.py
+7
-95
@@ -37,7 +37,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
QKVParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
40
from vllm.model_executor.layers.quantization import QuantizationConfig
41
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
42
get_compressed_tensors_cache_scale)
@@ -47,14 +46,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
46
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
47
from vllm.model_executor.model_loader.weight_utils import (
48
default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
50
-from vllm.model_executor.pooling_metadata import PoolingMetadata
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.platforms import current_platform
53
-from vllm.sequence import IntermediateTensors, PoolerOutput
51
+from vllm.sequence import IntermediateTensors
52
53
from .interfaces import SupportsLoRA, SupportsPP
56
-from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
57
- extract_layer_index, is_pp_missing_parameter,
54
+from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,
55
+ is_pp_missing_parameter,
56
make_empty_intermediate_tensors_factory, make_layers,
57
maybe_prefix)
58
@@ -511,11 +509,12 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
509
config = vllm_config.model_config.hf_config
510
quant_config = vllm_config.quant_config
511
lora_config = vllm_config.lora_config
514
- pooler_config = vllm_config.model_config.pooler_config
512
self.config = config
513
self.lora_config = lora_config
514
518
- self.model = self._init_model(vllm_config=vllm_config, prefix=prefix)
515
+ self.model = self._init_model(vllm_config=vllm_config,
516
+ prefix=maybe_prefix(prefix, "model"))
517
+
518
if get_pp_group().is_last_rank:
519
self.unpadded_vocab_size = config.vocab_size
520
if lora_config:
@@ -544,13 +543,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
543
self.sampler = get_sampler()
544
else:
545
self.lm_head = PPMissingLayer()
546
+
547
self.make_empty_intermediate_tensors = (
548
self.model.make_empty_intermediate_tensors)
549
- self._pooler = Pooler.from_config_with_defaults(
550
- pooler_config,
551
- pooling_type=PoolingType.STEP,
552
- normalize=False,
553
- softmax=False)
549
550
def _init_model(self, vllm_config: VllmConfig, prefix: str = ""):
551
return LlamaModel(vllm_config=vllm_config, prefix=prefix)
@@ -581,14 +576,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
576
sampling_metadata)
577
return logits
578
584
- def pooler(
585
- self,
586
- hidden_states: torch.Tensor,
587
- pooling_metadata: PoolingMetadata,
588
- ) -> Optional[PoolerOutput]:
589
- logits = self.compute_logits(hidden_states, None)
590
- return self._pooler(logits, pooling_metadata)
591
-
579
def sample(self, logits: torch.Tensor,
580
sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:
581
next_tokens = self.sampler(logits, sampling_metadata)
@@ -639,78 +626,3 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
626
name = name.replace(item, mapping[item])
627
628
return name, loaded_weight
642
-
643
-
644
-class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
645
- """
646
- A model that uses Llama with additional embedding functionalities.
647
-
648
- This class encapsulates the LlamaModel and provides an interface for
649
- embedding operations and customized pooling functions.
650
-
651
- Attributes:
652
- model: An instance of LlamaModel used for forward operations.
653
- _pooler: An instance of Pooler used for pooling operations.
654
- """
655
- packed_modules_mapping = {
656
- "qkv_proj": ["q_proj", "k_proj", "v_proj"],
657
- "gate_up_proj": ["gate_proj", "up_proj"]
658
- }
659
-
660
- # LoRA specific attributes
661
- supported_lora_modules = [
662
- "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
663
- ]
664
- embedding_modules = {
665
- "embed_tokens": "input_embeddings",
666
- }
667
- embedding_padding_modules = []
668
-
669
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
670
- super().__init__()
671
-
672
- pooler_config = vllm_config.model_config.pooler_config
673
-
674
- self.model = LlamaModel(vllm_config=vllm_config,
675
- prefix=maybe_prefix(prefix, "model"))
676
- self._pooler = Pooler.from_config_with_defaults(
677
- pooler_config,
678
- pooling_type=PoolingType.LAST,
679
- normalize=True,
680
- softmax=False)
681
- self.make_empty_intermediate_tensors = (
682
- self.model.make_empty_intermediate_tensors)
683
-
684
- def forward(
685
- self,
686
- input_ids: Optional[torch.Tensor],
687
- positions: torch.Tensor,
688
- kv_caches: List[torch.Tensor],
689
- attn_metadata: AttentionMetadata,
690
- intermediate_tensors: Optional[IntermediateTensors] = None,
691
- inputs_embeds: Optional[torch.Tensor] = None,
692
- ) -> Union[torch.Tensor, IntermediateTensors]:
693
- return self.model(input_ids, positions, kv_caches, attn_metadata,
694
- intermediate_tensors, inputs_embeds)
695
-
696
- def pooler(
697
- self,
698
- hidden_states: torch.Tensor,
699
- pooling_metadata: PoolingMetadata,
700
- ) -> Optional[PoolerOutput]:
701
- return self._pooler(hidden_states, pooling_metadata)
702
-
703
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
704
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
705
- weights = hf_to_vllm_mapper.apply(weights)
706
- weights = ((name, data) for name, data in weights
707
- if not name.startswith("lm_head."))
708
- self.model.load_weights(weights)
709
-
710
- def load_kv_cache_scales(self, quantization_param_path: str) -> None:
711
- self.model.load_kv_cache_scales(quantization_param_path)
712
-
713
- # LRUCacheWorkerLoRAManager instantiation requires model config.
714
- @property
715
- def config(self):
716
- return self.model.config
vllm/model_executor/models/llava.py
+3
-2
@@ -319,9 +319,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
319
projector_hidden_act=config.projector_hidden_act)
320
321
self.language_model = init_vllm_registered_model(
322
- config.text_config,
322
vllm_config=vllm_config,
324
- prefix=maybe_prefix(prefix, "language_model"))
323
+ hf_config=config.text_config,
324
+ prefix=maybe_prefix(prefix, "language_model"),
325
+ )
326
327
self.make_empty_intermediate_tensors = (
328
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_next.py
+5
-21
@@ -14,13 +14,11 @@ from vllm.attention import AttentionMetadata
14
from vllm.config import VllmConfig
15
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16
InputContext)
17
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
17
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19
-from vllm.model_executor.pooling_metadata import PoolingMetadata
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.multimodal.inputs import NestedTensors
23
-from vllm.sequence import IntermediateTensors, PoolerOutput
21
+from vllm.sequence import IntermediateTensors
22
from vllm.utils import is_list_of
23
24
from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -286,7 +284,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
284
super().__init__()
285
config = vllm_config.model_config.hf_config
286
quant_config = vllm_config.quant_config
289
- pooler_config = vllm_config.model_config.pooler_config
287
multimodal_config = vllm_config.model_config.multimodal_config
288
289
vision_feature_layer = config.vision_feature_layer
@@ -321,17 +318,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
318
projector_hidden_act=config.projector_hidden_act)
319
320
self.language_model = init_vllm_registered_model(
324
- config.text_config,
321
vllm_config=vllm_config,
326
- prefix=maybe_prefix(prefix, "language_model"))
327
-
328
- # The same model class supports both language generation and embedding
329
- # because the architecture name is the same
330
- self._pooler = Pooler.from_config_with_defaults(
331
- pooler_config,
332
- pooling_type=PoolingType.LAST,
333
- normalize=True,
334
- softmax=False)
322
+ hf_config=config.text_config,
323
+ prefix=maybe_prefix(prefix, "language_model"),
324
+ )
325
+
326
self.make_empty_intermediate_tensors = (
327
self.language_model.make_empty_intermediate_tensors)
328
@@ -678,13 +669,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
669
) -> Optional[SamplerOutput]:
670
return self.language_model.sample(logits, sampling_metadata)
671
681
- def pooler(
682
- self,
683
- hidden_states: torch.Tensor,
684
- pooling_metadata: PoolingMetadata,
685
- ) -> Optional[PoolerOutput]:
686
- return self._pooler(hidden_states, pooling_metadata)
687
-
672
def load_weights(self, weights: Iterable[Tuple[str,
673
torch.Tensor]]) -> Set[str]:
674
loader = AutoWeightsLoader(self)
vllm/model_executor/models/llava_next_video.py
+3
-2
@@ -275,9 +275,10 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
275
text_hidden_size=config.text_config.hidden_size,
276
projector_hidden_act=config.projector_hidden_act)
277
self.language_model = init_vllm_registered_model(
278
- config.text_config,
278
vllm_config=vllm_config,
280
- prefix=maybe_prefix(prefix, "language_model"))
279
+ hf_config=config.text_config,
280
+ prefix=maybe_prefix(prefix, "language_model"),
281
+ )
282
283
self.make_empty_intermediate_tensors = (
284
self.language_model.model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_onevision.py
+3
-2
@@ -422,9 +422,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
422
prefix=maybe_prefix(prefix, "vision_tower"))
423
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
424
self.language_model = init_vllm_registered_model(
425
- config.text_config,
425
vllm_config=vllm_config,
427
- prefix=maybe_prefix(prefix, "language_model"))
426
+ hf_config=config.text_config,
427
+ prefix=maybe_prefix(prefix, "language_model"),
428
+ )
429
self.image_newline = nn.Parameter(
430
torch.empty(config.text_config.hidden_size))
431
vllm/model_executor/models/paligemma.py
+3
-2
@@ -151,9 +151,10 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
151
self.quant_config = quant_config
152
config.text_config.architectures = ["GemmaForCausalLM"]
153
self.language_model = init_vllm_registered_model(
154
- config.text_config,
154
vllm_config=vllm_config,
156
- prefix=maybe_prefix(prefix, "language_model"))
155
+ hf_config=config.text_config,
156
+ prefix=maybe_prefix(prefix, "language_model"),
157
+ )
158
logit_scale = getattr(config, "logit_scale", 1.0)
159
self.language_model.logits_processor.scale *= logit_scale
160
vllm/model_executor/models/phi3v.py
+14
-25
@@ -29,24 +29,22 @@ from vllm.config import ModelConfig, VllmConfig
29
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30
InputContext, token_inputs)
31
from vllm.logger import init_logger
32
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
32
from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding)
36
from vllm.model_executor.models.clip import CLIPVisionModel
38
-from vllm.model_executor.models.llama import LlamaForCausalLM
39
-from vllm.model_executor.pooling_metadata import PoolingMetadata
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
40
from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
44
-from vllm.sequence import IntermediateTensors, PoolerOutput
41
+from vllm.sequence import IntermediateTensors
42
from vllm.utils import is_list_of
43
44
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
45
from .interfaces import SupportsMultiModal, SupportsPP
49
-from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,
46
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
47
+ init_vllm_registered_model, maybe_prefix,
48
merge_multimodal_embeddings)
49
50
logger = init_logger(__name__)
@@ -536,7 +534,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
534
super().__init__()
535
config = vllm_config.model_config.hf_config
536
quant_config = vllm_config.quant_config
539
- pooler_config = vllm_config.model_config.pooler_config
537
multimodal_config = vllm_config.model_config.multimodal_config
538
self.config = config
539
self.multimodal_config = multimodal_config
@@ -556,18 +553,17 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
553
quant_config,
554
prefix=maybe_prefix(prefix, "model.vision_embed_tokens"))
555
559
- # The prefix is empty intentionally because default prefix of
560
- # LlamaForCausalLM is "model"
561
- self.language_model = LlamaForCausalLM(vllm_config=vllm_config,
562
- prefix="")
563
-
564
- # The same model class supports both language generation and embedding
565
- # because the architecture name is the same
566
- self._pooler = Pooler.from_config_with_defaults(
567
- pooler_config,
568
- pooling_type=PoolingType.LAST,
569
- normalize=True,
570
- softmax=False)
556
+ self.language_model = init_vllm_registered_model(
557
+ vllm_config=vllm_config,
558
+ # The prefix is empty intentionally because default prefix of
559
+ # LlamaForCausalLM is "model"
560
+ prefix="",
561
+ # We don't directly initialize vLLM's LlamaForCausalLM so we
562
+ # can automatically apply embedding wrapper if this model is
563
+ # initialized as an embedding model
564
+ architectures=["LlamaForCausalLM"],
565
+ )
566
+
567
self.make_empty_intermediate_tensors = (
568
self.language_model.make_empty_intermediate_tensors)
569
@@ -739,13 +735,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
735
) -> Optional[SamplerOutput]:
736
return self.language_model.sample(logits, sampling_metadata)
737
742
- def pooler(
743
- self,
744
- hidden_states: torch.Tensor,
745
- pooling_metadata: PoolingMetadata,
746
- ) -> Optional[PoolerOutput]:
747
- return self._pooler(hidden_states, pooling_metadata)
748
-
738
def load_weights(self, weights: Iterable[Tuple[str,
739
torch.Tensor]]) -> Set[str]:
740
hf_to_vllm_mapper = WeightsMapper(
vllm/model_executor/models/pixtral.py
+3
-2
@@ -172,9 +172,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
172
173
# init MistralForCausalLM
174
self.language_model = init_vllm_registered_model(
175
- config.text_config,
175
vllm_config=vllm_config,
177
- prefix=maybe_prefix(prefix, "language_model"))
176
+ hf_config=config.text_config,
177
+ prefix=maybe_prefix(prefix, "language_model"),
178
+ )
179
180
self.vision_encoder = VisionTransformer(self.vision_args)
181
self.vision_language_adapter = VisionLanguageAdapter(
vllm/model_executor/models/qwen2.py
+12
-16
@@ -31,6 +31,7 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
31
from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, VllmConfig
33
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
+from vllm.logger import init_logger
35
from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -55,6 +56,8 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
56
make_empty_intermediate_tensors_factory, make_layers,
57
maybe_prefix)
58
59
+logger = init_logger(__name__)
60
+
61
62
class Qwen2MLP(nn.Module):
63
@@ -433,7 +436,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
436
config = vllm_config.model_config.hf_config
437
quant_config = vllm_config.quant_config
438
lora_config = vllm_config.lora_config
436
- pooler_config = vllm_config.model_config.pooler_config
439
440
self.config = config
441
self.lora_config = lora_config
@@ -454,14 +456,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
456
self.logits_processor = LogitsProcessor(config.vocab_size)
457
self.sampler = get_sampler()
458
457
- # The same model class supports both language generation and embedding
458
- # because the architecture name is the same
459
- self._pooler = Pooler.from_config_with_defaults(
460
- pooler_config,
461
- pooling_type=PoolingType.LAST,
462
- normalize=True,
463
- softmax=False)
464
-
459
self.make_empty_intermediate_tensors = (
460
self.model.make_empty_intermediate_tensors)
461
@@ -499,13 +493,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
493
next_tokens = self.sampler(logits, sampling_metadata)
494
return next_tokens
495
502
- def pooler(
503
- self,
504
- hidden_states: torch.Tensor,
505
- pooling_metadata: PoolingMetadata,
506
- ) -> Optional[PoolerOutput]:
507
- return self._pooler(hidden_states, pooling_metadata)
508
-
496
def load_weights(self, weights: Iterable[Tuple[str,
497
torch.Tensor]]) -> Set[str]:
498
loader = AutoWeightsLoader(
@@ -553,6 +540,15 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
540
self.model = Qwen2Model(vllm_config=vllm_config,
541
prefix=maybe_prefix(prefix, "model"))
542
543
+ # TODO: Replace this model class with for_embedding(Qwen2ForCausalLM),
544
+ # after changing the default pooling method
545
+ if pooler_config.pooling_type is None:
546
+ logger.warning(
547
+ "This embedding model will default to last-token pooling in "
548
+ "an upcoming version. To avoid breaking changes, you should "
549
+ "pass `--override-pooler-config '{\"pooling_type\": \"MEAN\"}'`"
550
+ " explicitly.")
551
+
552
self._pooler = Pooler.from_config_with_defaults(
553
pooler_config,
554
pooling_type=PoolingType.MEAN,
vllm/model_executor/models/qwen2_vl.py
+2
-16
@@ -50,7 +50,6 @@ from vllm.model_executor.layers.activation import QuickGELU
50
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51
RowParallelLinear)
52
from vllm.model_executor.layers.logits_processor import LogitsProcessor
53
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
53
from vllm.model_executor.layers.quantization import QuantizationConfig
54
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
55
from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -59,14 +58,13 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
58
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
59
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
60
from vllm.model_executor.models.qwen2 import Qwen2Model
62
-from vllm.model_executor.pooling_metadata import PoolingMetadata
61
from vllm.multimodal import MULTIMODAL_REGISTRY
62
from vllm.multimodal.image import cached_get_image_processor
63
from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
64
MultiModalKwargs, NestedTensors)
65
from vllm.multimodal.utils import cached_get_tokenizer
66
from vllm.platforms import _Backend
69
-from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
67
+from vllm.sequence import IntermediateTensors, SequenceData
68
from vllm.transformers_utils.config import uses_mrope
69
from vllm.transformers_utils.processor import cached_get_processor
70
@@ -1070,7 +1068,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1068
config = vllm_config.model_config.hf_config
1069
cache_config = vllm_config.cache_config
1070
quant_config = vllm_config.quant_config
1073
- pooler_config = vllm_config.model_config.pooler_config
1071
multimodal_config = vllm_config.model_config.multimodal_config
1072
assert not cache_config.enable_prefix_caching, \
1073
"Qwen2-VL currently does not support prefix caching"
@@ -1102,11 +1099,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1099
1100
self.logits_processor = LogitsProcessor(config.vocab_size)
1101
self.sampler = get_sampler()
1105
- self._pooler = Pooler.from_config_with_defaults(
1106
- pooler_config,
1107
- pooling_type=PoolingType.LAST,
1108
- normalize=True,
1109
- softmax=False)
1102
+
1103
self.make_empty_intermediate_tensors = (
1104
make_empty_intermediate_tensors_factory(
1105
["hidden_states", "residual"], config.hidden_size))
@@ -1361,13 +1354,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1354
next_tokens = self.sampler(logits, sampling_metadata)
1355
return next_tokens
1356
1364
- def pooler(
1365
- self,
1366
- hidden_states: torch.Tensor,
1367
- pooling_metadata: PoolingMetadata,
1368
- ) -> Optional[PoolerOutput]:
1369
- return self._pooler(hidden_states, pooling_metadata)
1370
-
1357
def load_weights(self, weights: Iterable[Tuple[str,
1358
torch.Tensor]]) -> Set[str]:
1359
stacked_params_mapping = [
vllm/model_executor/models/registry.py
+41
-18
@@ -20,6 +20,7 @@ import torch.nn as nn
20
from vllm.logger import init_logger
21
from vllm.platforms import current_platform
22
23
+from .adapters import as_embedding_model
24
from .interfaces import (has_inner_state, is_attention_free,
25
supports_cross_encoding, supports_multimodal,
26
supports_pp)
@@ -107,15 +108,15 @@ _EMBEDDING_MODELS = {
108
"RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
109
"XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
110
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
110
- "Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
111
+ "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
112
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
112
- "LlamaModel": ("llama", "LlamaEmbeddingModel"),
113
+ "LlamaModel": ("llama", "LlamaForCausalLM"),
114
**{
115
# Multiple models share the same architecture, so we include them all
116
k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
117
if arch == "LlamaForCausalLM"
118
},
118
- "MistralModel": ("llama", "LlamaEmbeddingModel"),
119
+ "MistralModel": ("llama", "LlamaForCausalLM"),
120
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
121
"Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
122
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
@@ -125,7 +126,7 @@ _EMBEDDING_MODELS = {
126
# [Multimodal]
127
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
128
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
128
- "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration") # noqa: E501,
129
+ "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
130
}
131
132
_CROSS_ENCODER_MODELS = {
@@ -208,6 +209,7 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
209
210
@dataclass(frozen=True)
211
class _ModelInfo:
212
+ architecture: str
213
is_text_generation_model: bool
214
is_embedding_model: bool
215
supports_cross_encoding: bool
@@ -218,9 +220,19 @@ class _ModelInfo:
220
221
@staticmethod
222
def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
223
+ is_embedding_model_ = is_embedding_model(model)
224
+ if not is_embedding_model_:
225
+ try:
226
+ as_embedding_model(model)
227
+ except Exception:
228
+ pass
229
+ else:
230
+ is_embedding_model_ = True
231
+
232
return _ModelInfo(
233
+ architecture=model.__name__,
234
is_text_generation_model=is_text_generation_model(model),
223
- is_embedding_model=is_embedding_model(model),
235
+ is_embedding_model=is_embedding_model_,
236
supports_cross_encoding=supports_cross_encoding(model),
237
supports_multimodal=supports_multimodal(model),
238
supports_pp=supports_pp(model),
@@ -399,13 +411,13 @@ class _ModelRegistry:
411
def inspect_model_cls(
412
self,
413
architectures: Union[str, List[str]],
402
- ) -> _ModelInfo:
414
+ ) -> Tuple[_ModelInfo, str]:
415
architectures = self._normalize_archs(architectures)
416
417
for arch in architectures:
418
model_info = self._try_inspect_model_cls(arch)
419
if model_info is not None:
408
- return model_info
420
+ return (model_info, arch)
421
422
return self._raise_for_unsupported(architectures)
423
@@ -426,39 +438,50 @@ class _ModelRegistry:
438
self,
439
architectures: Union[str, List[str]],
440
) -> bool:
429
- return self.inspect_model_cls(architectures).is_text_generation_model
441
+ model_cls, _ = self.inspect_model_cls(architectures)
442
+ return model_cls.is_text_generation_model
443
444
def is_embedding_model(
445
self,
446
architectures: Union[str, List[str]],
447
) -> bool:
435
- return self.inspect_model_cls(architectures).is_embedding_model
448
+ model_cls, _ = self.inspect_model_cls(architectures)
449
+ return model_cls.is_embedding_model
450
451
def is_cross_encoder_model(
452
self,
453
architectures: Union[str, List[str]],
454
) -> bool:
441
- return self.inspect_model_cls(architectures).supports_cross_encoding
455
+ model_cls, _ = self.inspect_model_cls(architectures)
456
+ return model_cls.supports_cross_encoding
457
458
def is_multimodal_model(
459
self,
460
architectures: Union[str, List[str]],
461
) -> bool:
447
- return self.inspect_model_cls(architectures).supports_multimodal
462
+ model_cls, _ = self.inspect_model_cls(architectures)
463
+ return model_cls.supports_multimodal
464
465
def is_pp_supported_model(
466
self,
467
architectures: Union[str, List[str]],
468
) -> bool:
453
- return self.inspect_model_cls(architectures).supports_pp
469
+ model_cls, _ = self.inspect_model_cls(architectures)
470
+ return model_cls.supports_pp
471
455
- def model_has_inner_state(self, architectures: Union[str,
456
- List[str]]) -> bool:
457
- return self.inspect_model_cls(architectures).has_inner_state
472
+ def model_has_inner_state(
473
+ self,
474
+ architectures: Union[str, List[str]],
475
+ ) -> bool:
476
+ model_cls, _ = self.inspect_model_cls(architectures)
477
+ return model_cls.has_inner_state
478
459
- def is_attention_free_model(self, architectures: Union[str,
460
- List[str]]) -> bool:
461
- return self.inspect_model_cls(architectures).is_attention_free
479
+ def is_attention_free_model(
480
+ self,
481
+ architectures: Union[str, List[str]],
482
+ ) -> bool:
483
+ model_cls, _ = self.inspect_model_cls(architectures)
484
+ return model_cls.is_attention_free
485
486
487
ModelRegistry = _ModelRegistry({
vllm/model_executor/models/ultravox.py
+3
-2
@@ -360,9 +360,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
360
))
361
self.multi_modal_projector = UltravoxProjector(config)
362
self.language_model = init_vllm_registered_model(
363
- config.text_config,
363
vllm_config=vllm_config,
365
- prefix=maybe_prefix(prefix, "language_model"))
364
+ hf_config=config.text_config,
365
+ prefix=maybe_prefix(prefix, "language_model"),
366
+ )
367
if config.text_model_id is not None:
368
# this prefix is not for initialization, but for loading weights
369
# note the trailing dot
vllm/model_executor/models/utils.py
+19
-5
@@ -173,8 +173,15 @@ class AutoWeightsLoader:
173
module_load_weights = getattr(module, "load_weights", None)
174
if callable(module_load_weights):
175
loaded_params = module_load_weights(weights)
176
- yield from map(lambda x: self._get_qualname(base_prefix, x),
177
- loaded_params)
176
+ if loaded_params is None:
177
+ logger.warning(
178
+ "Unable to collect loaded parameters "
179
+ "for module %s", module)
180
+ else:
181
+ yield from map(
182
+ lambda x: self._get_qualname(base_prefix, x),
183
+ loaded_params,
184
+ )
185
186
child_modules = dict(module.named_children())
187
child_params = dict(module.named_parameters(recurse=False))
@@ -232,17 +239,24 @@ class AutoWeightsLoader:
239
240
241
def init_vllm_registered_model(
235
- hf_config: PretrainedConfig,
242
vllm_config: VllmConfig,
243
+ *,
244
prefix: str = "",
245
+ hf_config: Optional[PretrainedConfig] = None,
246
+ architectures: Optional[list[str]] = None,
247
) -> nn.Module:
248
"""
249
Helper function to initialize an inner model registered to vLLM,
250
based on the arguments passed to the outer vLLM model.
251
"""
252
from vllm.model_executor.model_loader.loader import _initialize_model
244
- vllm_config = vllm_config.with_hf_config(hf_config)
245
- return _initialize_model(vllm_config, prefix)
253
+
254
+ if hf_config is not None:
255
+ vllm_config = vllm_config.with_hf_config(hf_config)
256
+
257
+ return _initialize_model(vllm_config=vllm_config,
258
+ prefix=prefix,
259
+ architectures=architectures)
260
261
262
@overload
vllm/multimodal/base.py
+3
-3
@@ -7,7 +7,7 @@ from torch import nn
7
8
from vllm.inputs import InputContext
9
from vllm.logger import init_logger
10
-from vllm.utils import (get_allowed_kwarg_only_overrides,
10
+from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
11
resolve_mm_processor_kwargs)
12
13
if TYPE_CHECKING:
@@ -54,8 +54,8 @@ class MultiModalPlugin(ABC):
54
"""
55
56
def __init__(self) -> None:
57
- self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {}
58
- self._max_mm_tokens: Dict[Type[nn.Module], MultiModalTokensCalc] = {}
57
+ self._input_mappers = ClassRegistry[nn.Module, MultiModalInputMapper]()
58
+ self._max_mm_tokens = ClassRegistry[nn.Module, MultiModalTokensCalc]()
59
60
@abstractmethod
61
def get_data_key(self) -> str:
vllm/multimodal/registry.py
+3
-2
@@ -9,6 +9,7 @@ from typing_extensions import TypeAlias
9
from vllm.inputs import InputProcessingContext
10
from vllm.logger import init_logger
11
from vllm.transformers_utils.tokenizer import AnyTokenizer
12
+from vllm.utils import ClassRegistry
13
14
from .audio import AudioPlugin
15
from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
@@ -62,8 +63,8 @@ class MultiModalRegistry:
63
plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
64
self._plugins = {p.get_data_key(): p for p in plugins}
65
65
- self._processor_factories: Dict[Type[nn.Module],
66
- MultiModalProcessorFactory] = {}
66
+ self._processor_factories = ClassRegistry[nn.Module,
67
+ MultiModalProcessorFactory]()
68
69
# This is used for non-multimodal models
70
self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
vllm/utils.py
+19
-3
@@ -20,7 +20,7 @@ import uuid
20
import warnings
21
import weakref
22
from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
23
-from collections import defaultdict
23
+from collections import UserDict, defaultdict
24
from collections.abc import Iterable, Mapping
25
from functools import lru_cache, partial, wraps
26
from platform import uname
@@ -1517,13 +1517,13 @@ class AtomicCounter:
1517
1518
1519
# Adapted from: https://stackoverflow.com/a/47212782/5082708
1520
-class LazyDict(Mapping, Generic[T]):
1520
+class LazyDict(Mapping[str, T], Generic[T]):
1521
1522
def __init__(self, factory: Dict[str, Callable[[], T]]):
1523
self._factory = factory
1524
self._dict: Dict[str, T] = {}
1525
1526
- def __getitem__(self, key) -> T:
1526
+ def __getitem__(self, key: str) -> T:
1527
if key not in self._dict:
1528
if key not in self._factory:
1529
raise KeyError(key)
@@ -1540,6 +1540,22 @@ class LazyDict(Mapping, Generic[T]):
1540
return len(self._factory)
1541
1542
1543
+class ClassRegistry(UserDict[type[T], _V]):
1544
+
1545
+ def __getitem__(self, key: type[T]) -> _V:
1546
+ for cls in key.mro():
1547
+ if cls in self.data:
1548
+ return self.data[cls]
1549
+
1550
+ raise KeyError(key)
1551
+
1552
+ def __contains__(self, key: object) -> bool:
1553
+ if not isinstance(key, type):
1554
+ return False
1555
+
1556
+ return any(cls in self.data for cls in key.mro())
1557
+
1558
+
1559
def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1560
"""
1561
Create a weak reference to a tensor.