A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Refactor Transformers backend to use mixins (#26906)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Oct 16, 2025 at 22:50 UTC
fb5e10d3fbb79323d5d9444543a143a864fe29cc
17 files changed
+1510
-1248
.github/CODEOWNERS
+1
-1
@@ -57,7 +57,7 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
57
/tests/v1/offloading @ApostaC
58
59
# Transformers backend
60
-/vllm/model_executor/models/transformers.py @hmellor
60
+/vllm/model_executor/models/transformers @hmellor
61
/tests/models/test_transformers.py @hmellor
62
63
# Docs
tests/models/registry.py
+6
-2
@@ -912,11 +912,11 @@ _TRANSFORMERS_BACKEND_MODELS = {
912
"TransformersForCausalLM": _HfExamplesInfo(
913
"hmellor/Ilama-3.2-1B", trust_remote_code=True
914
),
915
- "TransformersForMultimodalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
915
+ "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
916
"TransformersMoEForCausalLM": _HfExamplesInfo(
917
"allenai/OLMoE-1B-7B-0924", min_transformers_version="4.57.0.dev0"
918
),
919
- "TransformersMoEForMultimodalLM": _HfExamplesInfo(
919
+ "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
920
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="4.57.0.dev0"
921
),
922
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
@@ -925,6 +925,10 @@ _TRANSFORMERS_BACKEND_MODELS = {
925
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
926
"Qwen/Qwen3-30B-A3B", min_transformers_version="4.57.0.dev0"
927
),
928
+ "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
929
+ "TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
930
+ "google/gemma-3-4b-it"
931
+ ),
932
}
933
934
_EXAMPLE_MODELS = {
tests/models/test_initialization.py
+1
-1
@@ -37,7 +37,7 @@ MINIMAL_MODEL_ARCH_LIST = [
37
"JinaVLForRanking",
38
"InternVLChatModel",
39
"InternLM2ForRewardModel",
40
- "TransformersForMultimodalLM",
40
+ "TransformersMultiModalForCausalLM",
41
"PrithviGeoSpatialMAE",
42
"UltravoxModel",
43
"DeepSeekMTPModel",
tests/models/test_transformers.py
+1
-5
@@ -211,11 +211,7 @@ def test_embed_loading(vllm_runner, model):
211
def test_pooling(hf_runner, vllm_runner, example_prompts, arch):
212
model = get_model(arch)
213
214
- vllm_kwargs = dict(
215
- max_model_len=None,
216
- model_impl="transformers",
217
- compilation_config=dict(cudagraph_capture_sizes=[8]),
218
- )
214
+ vllm_kwargs = dict(max_model_len=None, model_impl="transformers")
215
216
hf_kwargs = dict()
217
if arch == "TransformersEmbeddingModel":
vllm/config/model.py
+14
-11
@@ -147,6 +147,10 @@ class ModelConfig:
147
seed: int | None = None
148
"""Random seed for reproducibility. Initialized to None in V0, but
149
initialized to 0 in V1."""
150
+ hf_config: PretrainedConfig = field(init=False)
151
+ """The Hugging Face config of the model."""
152
+ hf_text_config: PretrainedConfig = field(init=False)
153
+ """The Hugging Face config of the text model (same as hf_config for text models)."""
154
hf_config_path: str | None = None
155
"""Name or path of the Hugging Face config to use. If unspecified, model
156
name or path will be used."""
@@ -771,8 +775,10 @@ class ModelConfig:
775
def _get_transformers_backend_cls(self) -> str:
776
"""Determine which Transformers backend class will be used if
777
`model_impl` is set to `transformers` or `auto`."""
774
- prefix = "Transformers"
775
- prefix += "MoE" if self.get_num_experts() > 1 else ""
778
+ cls = "Transformers"
779
+ # If 'hf_config != hf_text_config' it's a nested config, i.e. multimodal
780
+ cls += "MultiModal" if self.hf_config != self.hf_text_config else ""
781
+ cls += "MoE" if self.get_num_experts() > 1 else ""
782
# Check if the architecture we're wrapping has defaults
783
runner = None
784
convert = None
@@ -788,18 +794,15 @@ class ModelConfig:
794
runner = "generate"
795
if convert in {None, "none"}:
796
convert = "embed"
791
- # Resolve Transformers backend pooling classes
797
+ # Resolve Transformers backend task
798
if runner == "pooling":
799
if convert == "embed":
794
- return prefix + "EmbeddingModel"
800
+ return cls + "EmbeddingModel"
801
if convert == "classify":
796
- return prefix + "ForSequenceClassification"
797
- # Resolve Transformers backend generate classes
798
- if self.hf_config != self.hf_text_config:
799
- # If 'hf_text_config' is the same as 'hf_config'. If not, it is
800
- # probably a composite config, i.e. multimodal
801
- return prefix + "ForMultimodalLM"
802
- return prefix + "ForCausalLM"
802
+ return cls + "ForSequenceClassification"
803
+ else:
804
+ cls += "ForCausalLM"
805
+ return cls
806
807
def using_transformers_backend(self) -> bool:
808
"""Check if the model is using the Transformers backend class."""
vllm/model_executor/models/deepseek_vl2.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.config.multimodal import BaseDummyOptions
19
from vllm.distributed import get_tensor_model_parallel_world_size
20
from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
22
-from vllm.model_executor.models.transformers import replace_linear_class
22
+from vllm.model_executor.models.transformers.utils import replace_linear_class
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import (
25
MultiModalDataDict,
vllm/model_executor/models/registry.py
+26
-14
@@ -401,32 +401,44 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
401
# Text generation models
402
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
403
# Multimodal models
404
- "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
404
+ "Emu3ForConditionalGeneration": (
405
+ "transformers",
406
+ "TransformersMultiModalForCausalLM",
407
+ ),
408
}
409
410
_TRANSFORMERS_BACKEND_MODELS = {
411
+ # Text generation models
412
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
409
- "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
410
- "TransformersMoEForCausalLM": ("transformers_moe", "TransformersMoEForCausalLM"), # noqa: E501
411
- "TransformersMoEForMultimodalLM": (
412
- "transformers_moe",
413
- "TransformersMoEForMultimodalLM",
413
+ "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
414
+ # Multimodal models
415
+ "TransformersMultiModalForCausalLM": (
416
+ "transformers",
417
+ "TransformersMultiModalForCausalLM",
418
+ ),
419
+ "TransformersMultiModalMoEForCausalLM": (
420
+ "transformers",
421
+ "TransformersMultiModalMoEForCausalLM",
422
),
415
- "TransformersEmbeddingModel": (
416
- "transformers_pooling",
417
- "TransformersEmbeddingModel",
423
+ # Embedding models
424
+ "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
425
+ "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
426
+ "TransformersMultiModalEmbeddingModel": (
427
+ "transformers",
428
+ "TransformersMultiModalEmbeddingModel",
429
),
430
+ # Sequence classification models
431
"TransformersForSequenceClassification": (
420
- "transformers_pooling",
432
+ "transformers",
433
"TransformersForSequenceClassification",
434
),
435
"TransformersMoEForSequenceClassification": (
424
- "transformers_pooling",
436
+ "transformers",
437
"TransformersMoEForSequenceClassification",
438
),
427
- "TransformersMoEEmbeddingModel": (
428
- "transformers_pooling",
429
- "TransformersMoEEmbeddingModel",
439
+ "TransformersMultiModalForSequenceClassification": (
440
+ "transformers",
441
+ "TransformersMultiModalForSequenceClassification",
442
),
443
}
444
vllm/model_executor/models/transformers.py
deleted
-961
@@ -1,961 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Copyright 2024 The vLLM team.
5
-#
6
-# Licensed under the Apache License, Version 2.0 (the "License");
7
-# you may not use this file except in compliance with the License.
8
-# You may obtain a copy of the License at
9
-#
10
-# http://www.apache.org/licenses/LICENSE-2.0
11
-#
12
-# Unless required by applicable law or agreed to in writing, software
13
-# distributed under the License is distributed on an "AS IS" BASIS,
14
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
-# See the License for the specific language governing permissions and
16
-# limitations under the License.
17
-"""Wrapper around `transformers` models"""
18
-
19
-from collections.abc import Iterable, Mapping
20
-from contextlib import contextmanager
21
-from pathlib import Path
22
-from typing import Literal
23
-
24
-import regex as re
25
-import torch
26
-import transformers
27
-from packaging.version import Version
28
-from torch import nn
29
-from transformers import AutoModel, BatchFeature, PretrainedConfig, PreTrainedModel
30
-from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
31
-
32
-from vllm.attention import Attention, AttentionType
33
-from vllm.compilation.decorators import support_torch_compile
34
-from vllm.config import (
35
- CacheConfig,
36
- DeviceConfig,
37
- ModelConfig,
38
- ParallelConfig,
39
- VllmConfig,
40
-)
41
-from vllm.config.multimodal import BaseDummyOptions
42
-from vllm.config.utils import getattr_iter
43
-from vllm.distributed import get_pp_group, get_tp_group
44
-from vllm.distributed.utils import get_pp_indices
45
-from vllm.logger import init_logger
46
-from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
47
-from vllm.model_executor.layers.linear import (
48
- ColumnParallelLinear,
49
- ReplicatedLinear,
50
- RowParallelLinear,
51
-)
52
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
53
-from vllm.model_executor.layers.quantization import QuantizationConfig
54
-from vllm.model_executor.layers.vocab_parallel_embedding import (
55
- ParallelLMHead,
56
- VocabParallelEmbedding,
57
-)
58
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
59
-from vllm.multimodal.inputs import (
60
- MultiModalDataDict,
61
- MultiModalFieldConfig,
62
- MultiModalInputs,
63
- MultiModalUUIDDict,
64
- PlaceholderRange,
65
-)
66
-from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
67
-from vllm.multimodal.processing import BaseMultiModalProcessor, BaseProcessingInfo
68
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
69
-from vllm.sequence import IntermediateTensors
70
-
71
-from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsQuant
72
-from .utils import (
73
- AutoWeightsLoader,
74
- PPMissingLayer,
75
- WeightsMapper,
76
- make_empty_intermediate_tensors_factory,
77
- maybe_prefix,
78
-)
79
-
80
-logger = init_logger(__name__)
81
-
82
-
83
-def get_feature_request_tip(
84
- model: str,
85
- trust_remote_code: bool,
86
-) -> str:
87
- hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new"
88
- gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose"
89
- url = hf_url if trust_remote_code else gh_url
90
- prefix = f"Please open {url} to request support for this feature. "
91
- if Path(model).exists():
92
- prefix = ""
93
- doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models"
94
- tip = f"See {doc_url} for instructions on how to add support yourself."
95
- return f"{prefix}{tip}"
96
-
97
-
98
-def vllm_flash_attention_forward(
99
- # Transformers args
100
- module: torch.nn.Module,
101
- query: torch.Tensor,
102
- key: torch.Tensor,
103
- value: torch.Tensor,
104
- attention_mask: torch.Tensor,
105
- # Transformers kwargs
106
- scaling: float | None = None,
107
- # vLLM kwargs
108
- attention_instances: dict[Attention] | None = None,
109
- **kwargs,
110
-):
111
- self_attn = attention_instances[module.layer_idx]
112
- if scaling is not None:
113
- self_attn.impl.scale = float(scaling)
114
- hidden = query.shape[-2]
115
- query, key, value = (x.transpose(1, 2) for x in (query, key, value))
116
- query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
117
- return self_attn.forward(query, key, value), None
118
-
119
-
120
-ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
121
-
122
-
123
-def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
124
- logger.debug("%s: %s -> %s", name, old_module, new_module)
125
-
126
-
127
-def can_enable_torch_compile(vllm_config: VllmConfig) -> bool:
128
- """
129
- Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
130
-
131
- Defaults to `True` but is disabled in the following situations:
132
-
133
- - The model uses dynamic rope scaling.
134
- """
135
- enable = True
136
- text_config = vllm_config.model_config.hf_config.get_text_config()
137
- # Dynamic rope scaling is not compatible with torch.compile
138
- rope_scaling: dict = getattr(text_config, "rope_scaling", None) or {}
139
- if rope_scaling.get("rope_type") == "dynamic":
140
- enable = False
141
- return enable
142
-
143
-
144
-Style = Literal["colwise", "colwise_rep", "rowwise", "rowwise_rep", "replicate"]
145
-
146
-
147
-def replace_linear_class(
148
- linear: nn.Linear,
149
- style: Style = "replicate",
150
- quant_config: QuantizationConfig | None = None,
151
- *,
152
- prefix: str = "",
153
-) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
154
- """
155
- Replace nn.Linear with one of vLLM's tensor parallel linear classes.
156
-
157
- Args:
158
- linear: `nn.Linear` to be replaced.
159
- style: Tensor parallel style of the new linear, e.g. "colwise".
160
- quant_config: Quantization config for the new linear.
161
- Returns:
162
- The new linear.
163
- """
164
-
165
- if not isinstance(style, str):
166
- raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
167
-
168
- vllm_linear_cls, vllm_linear_kwargs = {
169
- "colwise": (ColumnParallelLinear, {}),
170
- "colwise_rep": (ColumnParallelLinear, {"gather_output": True}),
171
- "rowwise": (RowParallelLinear, {}),
172
- "rowwise_rep": (RowParallelLinear, {"input_is_parallel": False}),
173
- "replicate": (ReplicatedLinear, {}),
174
- }.get(style, (ReplicatedLinear, {}))
175
-
176
- return vllm_linear_cls(
177
- input_size=linear.in_features,
178
- output_size=linear.out_features,
179
- bias=linear.bias is not None,
180
- quant_config=quant_config,
181
- prefix=prefix,
182
- return_bias=False,
183
- **vllm_linear_kwargs,
184
- )
185
-
186
-
187
-def replace_rms_norm_class(rms_norm: nn.Module, hidden_size: int) -> RMSNorm:
188
- """Replace a Transformers RMSNorm with vLLM's RMSNorm.
189
-
190
- This method assumes:
191
- - Weight is stored as `weight`.
192
- - Epsilon is stored as `eps` or `variance_epsilon`.
193
- - `with_scale` indicates whether the layer has a weight (Gemma3n only).
194
- - `var_hidden_size` is only ever used for Intern vision encoder in vLLM
195
- and Transformers doesn't appear to have the same concept.
196
- """
197
- eps = getattr_iter(rms_norm, ("eps", "variance_epsilon"), 1e-6)
198
- kwargs = {"hidden_size": hidden_size, "eps": eps}
199
- # Update hidden size if weight is available
200
- weight_meta = getattr(rms_norm, "weight", None)
201
- if weight_meta is not None:
202
- kwargs["hidden_size"] = weight_meta.size(0)
203
- # Check if weight is all zeros, which indicates GemmaRMSNorm
204
- # We must create a new instance because rms_norm is on meta
205
- try:
206
- with torch.device("cpu"):
207
- weight_test = getattr(rms_norm.__class__(1), "weight", None)
208
- except Exception:
209
- logger.warning(
210
- "Failed to determine if RMSNorm weight is centered on zero or one. "
211
- "Defaulting to one."
212
- )
213
- weight_test = None
214
- if weight_test is not None and torch.all(weight_test == 0):
215
- return GemmaRMSNorm(**kwargs)
216
- # Otherwise assume it's a regular RMSNorm
217
- kwargs["has_weight"] = getattr(rms_norm, "with_scale", True)
218
- if weight_meta is not None:
219
- kwargs["dtype"] = weight_meta.dtype
220
- else:
221
- # No weight, fall back to weightless RMSNorm
222
- kwargs["has_weight"] = False
223
- return RMSNorm(**kwargs)
224
-
225
-
226
-# Copied from `accelerate`
227
-@contextmanager
228
-def init_on_device_without_buffers(device: torch.device):
229
- """
230
- A context manager under which models are initialized with all
231
- parameters on the specified device. However buffers are not
232
- initialized on specified device.
233
-
234
- Args:
235
- device (`torch.device`):
236
- Device to initialize all parameters on.
237
- """
238
-
239
- old_register_parameter = nn.Module.register_parameter
240
-
241
- def register_empty_parameter(module, name, param):
242
- old_register_parameter(module, name, param)
243
- if param is not None:
244
- param_cls = type(module._parameters[name])
245
- kwargs = module._parameters[name].__dict__
246
- kwargs["requires_grad"] = param.requires_grad
247
- module._parameters[name] = param_cls(
248
- module._parameters[name].to(device), **kwargs
249
- )
250
-
251
- tensor_constructors_to_patch = {}
252
-
253
- def patch_tensor_constructor(fn):
254
- def wrapper(*args, **kwargs):
255
- kwargs["device"] = device
256
- return fn(*args, **kwargs)
257
-
258
- return wrapper
259
-
260
- try:
261
- nn.Module.register_parameter = register_empty_parameter
262
- for torch_function_name in tensor_constructors_to_patch:
263
- setattr(
264
- torch,
265
- torch_function_name,
266
- patch_tensor_constructor(getattr(torch, torch_function_name)),
267
- )
268
- yield
269
- finally:
270
- nn.Module.register_parameter = old_register_parameter
271
- for (
272
- torch_function_name,
273
- old_torch_function,
274
- ) in tensor_constructors_to_patch.items():
275
- setattr(torch, torch_function_name, old_torch_function)
276
-
277
-
278
-class MultiModalProcessingInfo(BaseProcessingInfo):
279
- def get_supported_mm_limits(self):
280
- return {"image": None}
281
-
282
- def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
283
- return {"image": self.get_max_image_tokens()}
284
-
285
- def get_max_image_tokens(self) -> int:
286
- width, height = self.get_max_image_size()
287
- processor = self.get_hf_processor()
288
- multimodal_config = self.ctx.model_config.multimodal_config
289
- mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
290
- mm_tokens = processor._get_num_multimodal_tokens(
291
- image_sizes=([height, width],), **mm_processor_kwargs
292
- )
293
- image_tokens = mm_tokens["num_image_tokens"][0]
294
- return image_tokens
295
-
296
- def get_max_image_size(self):
297
- return 10_000, 10_000 # hardcode for arbitrary very large size
298
-
299
-
300
-class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
301
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
302
- num_images = mm_counts.get("image", 0)
303
-
304
- processor = self.info.get_hf_processor()
305
- if "gemma3" in processor.__class__.__name__.lower():
306
- image_token = processor.boi_token
307
- else:
308
- image_token = getattr(processor, "image_token", "")
309
- return image_token * num_images
310
-
311
- def get_dummy_mm_data(
312
- self,
313
- seq_len: int,
314
- mm_counts: Mapping[str, int],
315
- mm_options: Mapping[str, BaseDummyOptions] | None = None,
316
- ) -> MultiModalDataDict:
317
- num_images = mm_counts.get("image", 0)
318
-
319
- target_width, target_height = self.info.get_max_image_size()
320
-
321
- image_overrides = mm_options.get("image") if mm_options else None
322
-
323
- return {
324
- "image": self._get_dummy_images(
325
- width=target_width,
326
- height=target_height,
327
- num_images=num_images,
328
- overrides=image_overrides,
329
- ),
330
- }
331
-
332
-
333
-class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
334
- def _get_prompt_updates(
335
- self,
336
- mm_items: MultiModalDataItems,
337
- hf_processor_mm_kwargs: Mapping[str, object],
338
- out_mm_kwargs: MultiModalKwargsItems,
339
- ):
340
- """
341
- Given the original multi-modal items for this modality
342
- and HF-processed data, output the updates to perform.
343
-
344
- The information returned by this method is used to update token inputs
345
- which bypass the HF processor. It is also used to update the output of
346
- HF processor if the HF process does not apply prompt updates to text
347
- inputs.
348
-
349
- Moreover, this information is critical to determine the token positions
350
- in order to construct :class:`~vllm-multimodal.input.PlaceholderRange`
351
- for each multi-modal item.
352
- """
353
- return None
354
-
355
- def _get_mm_fields_config(
356
- self,
357
- hf_inputs: BatchFeature,
358
- hf_processor_mm_kwargs: Mapping[str, object],
359
- ) -> Mapping[str, MultiModalFieldConfig]:
360
- # HF Processors always return a mask but vLLM doesn't need it
361
- hf_inputs.pop("attention_mask", None)
362
- num_image_patches = hf_inputs.get("num_image_patches")
363
- mm_fields = {
364
- key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
365
- for key in hf_inputs
366
- }
367
- mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
368
- "image", num_image_patches
369
- )
370
-
371
- # Keep these as batched, as they always have batch size as first dim
372
- mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
373
- mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
374
- mm_fields["num_image_patches"] = MultiModalFieldConfig.batched("image")
375
- return mm_fields
376
-
377
- def _get_hf_mm_data(
378
- self,
379
- mm_items: MultiModalDataItems,
380
- ) -> tuple[Mapping[str, object], Mapping[str, object]]:
381
- """
382
- In contrast to the base class, this method always adds
383
- `return_mm_token_type_ids` to the processor data
384
- """
385
- processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
386
- processor_data["return_mm_token_type_ids"] = True
387
- return processor_data, passthrough_data
388
-
389
- def apply(
390
- self,
391
- prompt: str | list[int],
392
- mm_data: MultiModalDataDict,
393
- hf_processor_mm_kwargs: Mapping[str, object],
394
- tokenization_kwargs: Mapping[str, object] | None = None,
395
- mm_uuids: MultiModalUUIDDict | None = None,
396
- ) -> MultiModalInputs:
397
- """
398
- Process multi-modal inputs to be used in vLLM.
399
-
400
- Apply HF Processor on prompt text and multi-modal data together,
401
- outputting token IDs and processed tensors.
402
- """
403
- if tokenization_kwargs is None:
404
- tokenization_kwargs = {}
405
-
406
- mm_items = self._to_mm_items(mm_data)
407
- hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
408
- if not isinstance(prompt, str):
409
- # the prompt is the tokenized ids which is not supported
410
- # by the hf_processor, which is why we would need to decode the ids
411
- # into string
412
- prompt = hf_processor.decode(prompt)
413
-
414
- # Bypass cached processor and always apply to the full set of mm inputs
415
- # NOTE: we can't just set caching=False because base class method
416
- # transforms outputs to `MultiModalKwargs` which is not going to
417
- # work for Transformers. We have a lot of logic tied to
418
- # `mm_tokens_per_modality` below
419
- prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
420
- prompt_text=prompt,
421
- mm_items=mm_items,
422
- hf_processor_mm_kwargs=hf_processor_mm_kwargs,
423
- tokenization_kwargs=tokenization_kwargs,
424
- )
425
-
426
- # For gemma3 we check `token_type_ids` as the key
427
- token_type_key = (
428
- "mm_token_type_ids"
429
- if "mm_token_type_ids" in processed_data
430
- else "token_type_ids"
431
- )
432
- mm_token_type_ids = processed_data.pop(token_type_key)
433
-
434
- # We can infer vLLM style placeholder from token type ids, if we split
435
- # it for each input `mm_data`.
436
- mm_positions = torch.where(mm_token_type_ids == 1)[1]
437
- images = mm_items.get_items("image", ImageProcessorItems)
438
- multimodal_config = self.info.ctx.model_config.multimodal_config
439
- mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
440
- image_sizes = []
441
- for item_idx in range(len(images)):
442
- image_size = images.get_image_size(item_idx)
443
- image_sizes.append((image_size.height, image_size.width))
444
-
445
- mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
446
- image_sizes=image_sizes, **mm_processor_kwargs
447
- )
448
-
449
- mm_placeholders = {}
450
- split_sizes = mm_tokens_per_modality["num_image_tokens"]
451
- if split_sizes:
452
- chunked_mm_positions = torch.split(mm_positions, split_sizes)
453
- mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
454
- chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
455
- ranges = [
456
- PlaceholderRange(
457
- offset=positions[0].item(),
458
- length=positions.shape[0],
459
- is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
460
- )
461
- for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
462
- ]
463
- mm_placeholders = {"image": ranges}
464
-
465
- processed_data["num_image_patches"] = torch.tensor(
466
- mm_tokens_per_modality["num_image_patches"]
467
- )
468
- mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
469
- processed_data,
470
- self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
471
- )
472
-
473
- # Use overrides if provided; fallback to data-dependent hashing.
474
- mm_hashes = self._hash_mm_items(
475
- mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
476
- )
477
-
478
- return MultiModalInputs(
479
- type="multimodal",
480
- prompt_token_ids=prompt_ids,
481
- mm_kwargs=mm_kwargs,
482
- mm_hashes=mm_hashes,
483
- mm_placeholders=mm_placeholders,
484
- )
485
-
486
-
487
-class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
488
- embedding_padding_modules = ["lm_head"]
489
- embedding_modules = ["embed_tokens"] # TODO transformers will have a util to get it
490
-
491
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
492
- super().__init__()
493
- logger.info("Using Transformers backend.")
494
-
495
- self.config: PretrainedConfig = vllm_config.model_config.hf_config
496
- self.text_config: PretrainedConfig = self.config.get_text_config()
497
- self.cache_config: CacheConfig = vllm_config.cache_config
498
- self.device_config: DeviceConfig = vllm_config.device_config
499
- self.model_config: ModelConfig = vllm_config.model_config
500
- self.parallel_config: ParallelConfig = vllm_config.parallel_config
501
- self.quant_config: QuantizationConfig | None = vllm_config.quant_config
502
-
503
- self.pp_group = get_pp_group()
504
- self.tp_group = get_tp_group()
505
-
506
- # Weights to skip in `self.load_weights`
507
- self.skip_prefixes: list[str] = []
508
- """Skip loading weights whose qualname starts with these prefixes."""
509
- self.skip_substrs: list[str] = []
510
- """Skip loading weights whose qualname contains these substrings."""
511
- self.ignore_unexpected_prefixes: list[str] = []
512
- """Ignore unexpected weights whose qualname starts with these prefixes.
513
- """
514
- self.ignore_unexpected_suffixes: list[str] = []
515
- """Ignore unexpected weights whose qualname ends with these suffixes."""
516
-
517
- if self.quant_config:
518
- quant_method_name = self.quant_config.get_name()
519
- # Check for unsupported quantization methods.
520
- if quant_method_name == "mxfp4":
521
- raise NotImplementedError(
522
- "Transformers backend does not support MXFP4 quantization yet."
523
- )
524
- # Skip loading extra bias for GPTQ models.
525
- if "gptq" in quant_method_name:
526
- self.ignore_unexpected_suffixes.append(".bias")
527
-
528
- # Set correct attn and init on "meta" to delay allocating GPU tensors
529
- self.text_config._attn_implementation = "vllm"
530
- with init_on_device_without_buffers("meta"):
531
- self.model: PreTrainedModel = AutoModel.from_config(
532
- self.config,
533
- dtype=self.model_config.dtype,
534
- trust_remote_code=self.model_config.trust_remote_code,
535
- )
536
-
537
- # Remove layers not on this pipeline parallel rank
538
- self.pipeline_parallel()
539
- # Substitute remaining layers with vLLM's layers as needed
540
- self.recursive_replace()
541
- # Create attention instances for KV cache allocation
542
- self.attention_instances = self.create_attention_instances()
543
-
544
- # Input embeddings
545
- input_embeddings = self.model.get_input_embeddings()
546
- if not isinstance(input_embeddings, PPMissingLayer):
547
- # Some models use embedding scales
548
- self.embed_scale = getattr(input_embeddings, "embed_scale", None)
549
- names = ("embedding_size", "hidden_size")
550
- embedding_dim = getattr_iter(self.text_config, names, None)
551
- assert embedding_dim is not None
552
- self.model.set_input_embeddings(
553
- VocabParallelEmbedding(
554
- self.text_config.vocab_size,
555
- embedding_dim=embedding_dim,
556
- org_num_embeddings=self.text_config.vocab_size,
557
- quant_config=self.quant_config,
558
- )
559
- )
560
-
561
- # Initialize any parameters that have not had their modules replaced
562
- self.init_parameters(self.model)
563
-
564
- # Pipeline parallel intermediate tensors
565
- self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
566
- ["hidden_states"], self.text_config.hidden_size
567
- )
568
-
569
- def pipeline_parallel(self):
570
- """
571
- Apply the model's pipeline parallelization plan.
572
- """
573
- if self.pp_group.world_size <= 1:
574
- return
575
-
576
- if not self.model.supports_pp_plan:
577
- tip = get_feature_request_tip(
578
- self.model_config.model, self.model_config.trust_remote_code
579
- )
580
- raise ValueError(
581
- f"{type(self.model)} does not support pipeline parallel. {tip}"
582
- )
583
-
584
- module_lists = []
585
- module_list_idx = None
586
- pp_plan = list(self.model._pp_plan.keys())
587
- for i, name in enumerate(pp_plan):
588
- if isinstance(getattr(self.model, name), nn.ModuleList):
589
- module_lists.append(name)
590
- module_list_idx = i
591
-
592
- if len(module_lists) > 1:
593
- raise ValueError(
594
- "Pipeline parallel of models with multiple `ModuleList`s "
595
- "in the base model are not supported yet!"
596
- )
597
- if module_list_idx is None:
598
- raise ValueError(f"Could not find `ModuleList` in {type(self.model)}")
599
-
600
- # Layers before module list
601
- for name in pp_plan[:module_list_idx]:
602
- if self.pp_group.is_first_rank or (
603
- self.text_config.tie_word_embeddings and self.pp_group.is_last_rank
604
- ):
605
- continue
606
- setattr(self.model, name, PPMissingLayer())
607
-
608
- # Module list
609
- start_layer, end_layer = get_pp_indices(
610
- self.text_config.num_hidden_layers,
611
- self.pp_group.rank_in_group,
612
- self.pp_group.world_size,
613
- )
614
- layers_name = pp_plan[module_list_idx]
615
- layers = getattr(self.model, layers_name)
616
- for i in range(len(layers)):
617
- if start_layer <= i and i < end_layer:
618
- continue
619
- layers[i] = PPMissingLayer()
620
-
621
- # Layers after module list
622
- for name in pp_plan[module_list_idx + 1 :]:
623
- # Modules that should be on last rank
624
- if not self.pp_group.is_last_rank:
625
- setattr(self.model, name, PPMissingLayer())
626
-
627
- def recursive_replace(self):
628
- """Recursively replace modules in the model as needed.
629
-
630
- Currently, this replaces:
631
-
632
- - `nn.Linear` with vLLM's tensor parallel linear classes
633
- - `*RMSNorm` with vLLM's `RMSNorm`
634
- """
635
- tp_plan = self.model.tp_plan
636
-
637
- if not tp_plan and self.tp_group.world_size > 1:
638
- tip = get_feature_request_tip(
639
- self.model_config.model, self.model_config.trust_remote_code
640
- )
641
- raise ValueError(
642
- f"{type(self.model)} does not support tensor parallel. {tip}"
643
- )
644
-
645
- # Prefix the patterns because we always start from `self.model`
646
- tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
647
-
648
- def _recursive_replace(module: nn.Module, prefix: str):
649
- for child_name, child_module in module.named_children():
650
- new_module = child_module
651
- qual_name = maybe_prefix(prefix, child_name)
652
- if isinstance(child_module, nn.Linear):
653
- generator = (p for p in tp_plan if re.match(p, qual_name))
654
- pattern = next(generator, None)
655
- # Some weight loaders expect all linear layers to inherit
656
- # LinearBase, so we set a default style which causes any
657
- # unspecified layers to be replaced with ReplicatedLinear
658
- style = tp_plan.get(pattern, "replicate")
659
- new_module = replace_linear_class(
660
- child_module, style, self.quant_config, prefix=qual_name
661
- )
662
- elif child_module.__class__.__name__.endswith("RMSNorm"):
663
- new_module = replace_rms_norm_class(
664
- child_module, self.text_config.hidden_size
665
- )
666
- else:
667
- _recursive_replace(child_module, prefix=qual_name)
668
-
669
- if new_module is not child_module:
670
- setattr(module, child_name, new_module)
671
- log_replacement(qual_name, child_module, new_module)
672
-
673
- _recursive_replace(self.model, prefix="model")
674
-
675
- def create_attention_instances(
676
- self, attn_type: AttentionType = AttentionType.DECODER
677
- ) -> dict[int, Attention]:
678
- """
679
- Create `Attention` instances to inform KV cache allocation.
680
- """
681
- num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
682
- head_size = self.model_config.get_head_size()
683
- num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
684
- logits_soft_cap = getattr(self.text_config, "attn_logit_softcapping", None)
685
- start, end = get_pp_indices(
686
- self.text_config.num_hidden_layers,
687
- self.pp_group.rank_in_group,
688
- self.pp_group.world_size,
689
- )
690
-
691
- attention_instances = {}
692
- for i in range(start, end):
693
- # Handle interleaved sliding window attention
694
- per_layer_sliding_window = None
695
- if (
696
- hasattr(self.config, "layer_types")
697
- and self.config.layer_types[i] == "sliding_attention"
698
- ):
699
- per_layer_sliding_window = self.config.sliding_window
700
-
701
- attention_instances[i] = Attention(
702
- num_heads=num_heads,
703
- head_size=head_size,
704
- # NOTE: We use Llama scale as default, if it's set by
705
- # Transformers, it's updated in vllm_flash_attention_forward
706
- scale=head_size**-0.5,
707
- num_kv_heads=num_kv_heads,
708
- cache_config=self.cache_config,
709
- quant_config=self.quant_config,
710
- logits_soft_cap=logits_soft_cap,
711
- per_layer_sliding_window=per_layer_sliding_window,
712
- prefix=f"{i}.attn",
713
- attn_type=attn_type,
714
- )
715
- return attention_instances
716
-
717
- def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
718
- """
719
- If a `parameter` is on the `meta` device, then its parent
720
- `module` is the original module created by:
721
-
722
- ```python
723
- with torch.device("meta"):
724
- self.model: PreTrainedModel = AutoModel.from_config(...)
725
- ```
726
- """
727
-
728
- def _init_parameters(module: nn.Module, dtype: torch.dtype | None):
729
- for name, param in module.named_parameters(recurse=False):
730
- if param.device == torch.device("meta"):
731
- new_param = nn.Parameter(
732
- torch.empty_like(
733
- param.data,
734
- dtype=dtype or self.model_config.dtype,
735
- device=self.device_config.device,
736
- )
737
- )
738
- setattr(module, name, new_param)
739
- for child in module.children():
740
- _init_parameters(child, dtype)
741
-
742
- _init_parameters(module, dtype)
743
-
744
- def forward(
745
- self,
746
- input_ids: torch.Tensor | None,
747
- positions: torch.Tensor,
748
- intermediate_tensors: IntermediateTensors | None = None,
749
- inputs_embeds: torch.Tensor | None = None,
750
- **kwargs,
751
- ) -> torch.Tensor | IntermediateTensors:
752
- if not self.pp_group.is_first_rank:
753
- assert intermediate_tensors is not None
754
- input_ids = None
755
- inputs_embeds = intermediate_tensors["hidden_states"]
756
-
757
- if input_ids is not None:
758
- input_ids = input_ids[None, ...]
759
- if inputs_embeds is not None:
760
- inputs_embeds = inputs_embeds[None, ...]
761
-
762
- if self.model_config.uses_mrope:
763
- position_ids = positions[:, None]
764
- else:
765
- position_ids = positions[None, ...]
766
-
767
- hidden_states = self.model(
768
- input_ids=input_ids,
769
- inputs_embeds=inputs_embeds,
770
- use_cache=False,
771
- position_ids=position_ids,
772
- attention_instances=self.attention_instances,
773
- return_dict=False,
774
- **kwargs,
775
- )[0][0, ...] # we remove batch dimension for now
776
-
777
- if not self.pp_group.is_last_rank:
778
- return IntermediateTensors({"hidden_states": hidden_states})
779
-
780
- return hidden_states
781
-
782
- def load_weights(
783
- self,
784
- weights: Iterable[tuple[str, torch.Tensor]],
785
- ) -> set[str]:
786
- loader = AutoWeightsLoader(
787
- self,
788
- skip_prefixes=self.skip_prefixes,
789
- skip_substrs=self.skip_substrs,
790
- ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
791
- ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
792
- )
793
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
794
-
795
- def check_version(self, min_version: str, feature: str):
796
- installed = Version(transformers.__version__)
797
- required = Version(min_version)
798
- if installed < required:
799
- raise ImportError(
800
- f"Transformers backend requires transformers>={required} "
801
- f"for {feature}, but got {installed}"
802
- )
803
-
804
-
805
-@support_torch_compile(enable_if=can_enable_torch_compile)
806
-class TransformersForCausalLM(TransformersBase):
807
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
808
- super().__init__(vllm_config=vllm_config, prefix=prefix)
809
-
810
- # Tell `TransformersBase.load_weights` to skip
811
- # `lm_head` if the model has tied word embeddings
812
- if self.text_config.tie_word_embeddings:
813
- self.skip_prefixes.append("lm_head.")
814
-
815
- if self.pp_group.is_last_rank:
816
- self.unpadded_vocab_size = self.text_config.vocab_size
817
- self.lm_head = ParallelLMHead(
818
- self.text_config.vocab_size,
819
- self.text_config.hidden_size,
820
- quant_config=self.quant_config,
821
- prefix=maybe_prefix(prefix, "lm_head"),
822
- )
823
- if self.text_config.tie_word_embeddings:
824
- self.lm_head = self.lm_head.tie_weights(
825
- self.model.get_input_embeddings()
826
- )
827
-
828
- logit_scale = getattr(self.text_config, "logit_scale", 1.0)
829
- self.logits_processor = LogitsProcessor(
830
- self.unpadded_vocab_size, self.text_config.vocab_size, logit_scale
831
- )
832
- else:
833
- self.lm_head = PPMissingLayer()
834
-
835
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
836
- inputs_embeds = self.model.get_input_embeddings()(input_ids)
837
- if self.embed_scale is not None:
838
- inputs_embeds *= self.embed_scale
839
- return inputs_embeds
840
-
841
- def compute_logits(
842
- self,
843
- hidden_states: torch.Tensor,
844
- ) -> torch.Tensor | None:
845
- logits = self.logits_processor(self.lm_head, hidden_states)
846
- return logits
847
-
848
-
849
-@MULTIMODAL_REGISTRY.register_processor(
850
- MultiModalProcessor,
851
- info=MultiModalProcessingInfo,
852
- dummy_inputs=MultiModalDummyInputsBuilder,
853
-)
854
-@support_torch_compile(
855
- # set `positions` to last dim to support Qwen-mrope
856
- dynamic_arg_dims={
857
- "input_ids": 0,
858
- "positions": -1,
859
- "intermediate_tensors": 0,
860
- "inputs_embeds": 0,
861
- },
862
- enable_if=can_enable_torch_compile,
863
-)
864
-class TransformersForMultimodalLM(TransformersForCausalLM, SupportsMultiModal):
865
- supports_multimodal_raw_input_only = True
866
- merge_by_field_config = True
867
- # Backwards compatibility for prev released models. State dicts back then
868
- # had different formats and cannot be loaded with `AutoModel` mapping as is
869
- hf_to_vllm_mapper = WeightsMapper(
870
- orig_to_new_prefix={
871
- "language_model.model": "model.language_model",
872
- "text_model.model": "model.text_model",
873
- "vision_tower": "model.vision_tower",
874
- "vqmodel": "model.vqmodel",
875
- "visual": "model.visual",
876
- "vision_model": "model.vision_model",
877
- "vision_embed_tokens": "model.vision_embed_tokens",
878
- "image_newline": "model.image_newline",
879
- "multi_modal_projector": "model.multi_modal_projector",
880
- "text_model.lm_head": "lm_head",
881
- "language_model.lm_head": "lm_head",
882
- # Qwen models used "model" as the name for the language model.
883
- # Therefore, we must map each of submodule explicitly to avoid
884
- # conflicts with newer models that use "model.language_model".
885
- "model.embed_tokens": "model.language_model.embed_tokens",
886
- "model.layers": "model.language_model.layers",
887
- "model.norm": "model.language_model.norm",
888
- }
889
- )
890
-
891
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
892
- super().__init__(vllm_config=vllm_config, prefix=prefix)
893
-
894
- self.dtype = vllm_config.model_config.dtype
895
-
896
- def forward(
897
- self,
898
- input_ids: torch.Tensor | None,
899
- positions: torch.Tensor,
900
- intermediate_tensors: IntermediateTensors | None = None,
901
- inputs_embeds: torch.Tensor | None = None,
902
- **kwargs: object,
903
- ) -> torch.Tensor | IntermediateTensors:
904
- # Gemma3 and PaliGemma needs `token_type_ids` to work correctly
905
- # Other models will not have `token_type_ids` in kwargs
906
- kwargs = {k: v for k, v in kwargs.items() if k == "token_type_ids"}
907
- model_output = super().forward(
908
- input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
909
- )
910
- return model_output
911
-
912
- def get_language_model(self) -> torch.nn.Module:
913
- """`TransformersForMultimodalLM` does not contain a vLLM language model class.
914
- Therefore, in order to return a language model vLLM class, we use a wrapper to
915
- give `self` the same interface as `TransformersForCausalLM`."""
916
-
917
- class LanguageModelWrapper(TransformersForCausalLM):
918
- def __init__(self, multimodal_model):
919
- # Don't call super().__init__() to avoid re-initialization
920
- self.__dict__.update(multimodal_model.__dict__)
921
-
922
- model = getattr_iter(self.model, ("language_model", "text_model"), None)
923
-
924
- return LanguageModelWrapper(self)
925
-
926
- def get_multimodal_embeddings(self, **kwargs):
927
- pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
928
- image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
929
- # Model might use `image_patches` instead of `pixel_values`
930
- if pixel_values is None:
931
- pixel_values = kwargs.pop("image_patches", None)
932
-
933
- if image_embeds is not None:
934
- return image_embeds
935
-
936
- if pixel_values is None:
937
- return None
938
-
939
- num_image_patches = kwargs.pop("num_image_patches")
940
- kwargs.pop("token_type_ids", None) # used only in `forward`
941
- if pixel_values is not None:
942
- vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
943
-
944
- if isinstance(vision_embeddings, torch.Tensor):
945
- if vision_embeddings.ndim == 2:
946
- vision_embeddings = vision_embeddings.unsqueeze(0)
947
-
948
- # Embeddings have to be 2D tensors of length `num_images`
949
- # but transformers returns concat tensors if each patch
950
- # is of different size. We split it back to make vLLM happy
951
- vision_embeddings = torch.split(
952
- vision_embeddings, num_image_patches.flatten().tolist()
953
- )
954
- vision_embeddings = [
955
- embed.flatten(start_dim=0, end_dim=-2)
956
- for embed in vision_embeddings
957
- ]
958
-
959
- return vision_embeddings
960
-
961
- get_input_embeddings = SupportsMultiModal.get_input_embeddings
vllm/model_executor/models/transformers/__init__.py
new
+127
@@ -0,0 +1,127 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Wrapper around `transformers` models"""
18
+
19
+from vllm.compilation.decorators import support_torch_compile
20
+from vllm.model_executor.models.transformers.base import Base
21
+from vllm.model_executor.models.transformers.causal import CausalMixin
22
+from vllm.model_executor.models.transformers.legacy import LegacyMixin
23
+from vllm.model_executor.models.transformers.moe import MoEMixin
24
+from vllm.model_executor.models.transformers.multimodal import (
25
+ DYNAMIC_ARG_DIMS,
26
+ MultiModalDummyInputsBuilder,
27
+ MultiModalMixin,
28
+ MultiModalProcessingInfo,
29
+ MultiModalProcessor,
30
+)
31
+from vllm.model_executor.models.transformers.pooling import (
32
+ EmbeddingMixin,
33
+ SequenceClassificationMixin,
34
+)
35
+from vllm.model_executor.models.transformers.utils import can_enable_torch_compile
36
+from vllm.multimodal import MULTIMODAL_REGISTRY
37
+
38
+
39
+# Text only models
40
+@support_torch_compile(enable_if=can_enable_torch_compile)
41
+class TransformersForCausalLM(CausalMixin, Base): ...
42
+
43
+
44
+@support_torch_compile(enable_if=can_enable_torch_compile)
45
+class TransformersMoEForCausalLM(MoEMixin, CausalMixin, Base): ...
46
+
47
+
48
+# Multimodal models
49
+@MULTIMODAL_REGISTRY.register_processor(
50
+ MultiModalProcessor,
51
+ info=MultiModalProcessingInfo,
52
+ dummy_inputs=MultiModalDummyInputsBuilder,
53
+)
54
+@support_torch_compile(
55
+ dynamic_arg_dims=DYNAMIC_ARG_DIMS, enable_if=can_enable_torch_compile
56
+)
57
+class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin, Base): ...
58
+
59
+
60
+@MULTIMODAL_REGISTRY.register_processor(
61
+ MultiModalProcessor,
62
+ info=MultiModalProcessingInfo,
63
+ dummy_inputs=MultiModalDummyInputsBuilder,
64
+)
65
+@support_torch_compile(
66
+ dynamic_arg_dims=DYNAMIC_ARG_DIMS, enable_if=can_enable_torch_compile
67
+)
68
+class TransformersMultiModalMoEForCausalLM(
69
+ MoEMixin, MultiModalMixin, CausalMixin, Base
70
+): ...
71
+
72
+
73
+# Embedding models
74
+@support_torch_compile(enable_if=can_enable_torch_compile)
75
+class TransformersEmbeddingModel(EmbeddingMixin, LegacyMixin, Base): ...
76
+
77
+
78
+@support_torch_compile(enable_if=can_enable_torch_compile)
79
+class TransformersMoEEmbeddingModel(EmbeddingMixin, MoEMixin, Base): ...
80
+
81
+
82
+@MULTIMODAL_REGISTRY.register_processor(
83
+ MultiModalProcessor,
84
+ info=MultiModalProcessingInfo,
85
+ dummy_inputs=MultiModalDummyInputsBuilder,
86
+)
87
+@support_torch_compile(
88
+ dynamic_arg_dims=DYNAMIC_ARG_DIMS, enable_if=can_enable_torch_compile
89
+)
90
+class TransformersMultiModalEmbeddingModel(EmbeddingMixin, MultiModalMixin, Base): ...
91
+
92
+
93
+# Sequence classification models
94
+@support_torch_compile(enable_if=can_enable_torch_compile)
95
+class TransformersForSequenceClassification(
96
+ SequenceClassificationMixin, LegacyMixin, Base
97
+): ...
98
+
99
+
100
+@support_torch_compile(enable_if=can_enable_torch_compile)
101
+class TransformersMoEForSequenceClassification(
102
+ SequenceClassificationMixin, MoEMixin, Base
103
+): ...
104
+
105
+
106
+@MULTIMODAL_REGISTRY.register_processor(
107
+ MultiModalProcessor,
108
+ info=MultiModalProcessingInfo,
109
+ dummy_inputs=MultiModalDummyInputsBuilder,
110
+)
111
+@support_torch_compile(
112
+ dynamic_arg_dims=DYNAMIC_ARG_DIMS, enable_if=can_enable_torch_compile
113
+)
114
+class TransformersMultiModalForSequenceClassification(
115
+ SequenceClassificationMixin, MultiModalMixin, Base
116
+): ...
117
+
118
+
119
+def __getattr__(name: str):
120
+ """Handle imports of non-existent classes with a helpful error message."""
121
+ if name not in globals():
122
+ raise AttributeError(
123
+ "The Transformers backend does not currently have a class to handle "
124
+ f"the requested model type: {name}. Please open an issue at "
125
+ "https://github.com/vllm-project/vllm/issues/new"
126
+ )
127
+ return globals()[name]
vllm/model_executor/models/transformers/base.py
new
+435
@@ -0,0 +1,435 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Transformers backend base class."""
18
+
19
+from collections.abc import Iterable
20
+from typing import TYPE_CHECKING
21
+
22
+import regex as re
23
+import torch
24
+import transformers
25
+from packaging.version import Version
26
+from torch import nn
27
+from transformers import AutoModel
28
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
29
+
30
+from vllm.attention import Attention, AttentionType
31
+from vllm.config.utils import getattr_iter
32
+from vllm.distributed import get_pp_group, get_tp_group
33
+from vllm.distributed.utils import get_pp_indices
34
+from vllm.logger import init_logger
35
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
36
+from vllm.model_executor.models.interfaces import (
37
+ SupportsLoRA,
38
+ SupportsPP,
39
+ SupportsQuant,
40
+)
41
+from vllm.model_executor.models.interfaces_base import VllmModel
42
+from vllm.model_executor.models.transformers.utils import (
43
+ get_feature_request_tip,
44
+ init_on_device_without_buffers,
45
+ log_replacement,
46
+ replace_linear_class,
47
+ replace_rms_norm_class,
48
+)
49
+from vllm.model_executor.models.utils import (
50
+ AutoWeightsLoader,
51
+ PPMissingLayer,
52
+ make_empty_intermediate_tensors_factory,
53
+ maybe_prefix,
54
+)
55
+from vllm.sequence import IntermediateTensors
56
+
57
+if TYPE_CHECKING:
58
+ from transformers import PreTrainedModel
59
+
60
+ from vllm.config import VllmConfig
61
+else:
62
+ PreTrainedModel = object
63
+
64
+logger = init_logger(__name__)
65
+
66
+
67
+def vllm_flash_attention_forward(
68
+ # Transformers args
69
+ module: torch.nn.Module,
70
+ query: torch.Tensor,
71
+ key: torch.Tensor,
72
+ value: torch.Tensor,
73
+ attention_mask: torch.Tensor,
74
+ # Transformers kwargs
75
+ scaling: float | None = None,
76
+ # vLLM kwargs
77
+ attention_instances: dict[int, Attention] | None = None,
78
+ **kwargs,
79
+):
80
+ self_attn = attention_instances[module.layer_idx]
81
+ if scaling is not None:
82
+ self_attn.impl.scale = float(scaling)
83
+ hidden = query.shape[-2]
84
+ query, key, value = (x.transpose(1, 2) for x in (query, key, value))
85
+ query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
86
+ return self_attn.forward(query, key, value), None
87
+
88
+
89
+ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
90
+
91
+
92
+class Base(nn.Module, VllmModel, SupportsQuant, SupportsLoRA, SupportsPP):
93
+ embedding_padding_modules = ["lm_head"]
94
+ embedding_modules = ["embed_tokens"] # TODO transformers will have a util to get it
95
+
96
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
97
+ super().__init__()
98
+ logger.info("Using Transformers backend.")
99
+
100
+ self.config = vllm_config.model_config.hf_config
101
+ self.text_config = self.config.get_text_config()
102
+ self.cache_config = vllm_config.cache_config
103
+ self.device_config = vllm_config.device_config
104
+ self.model_config = vllm_config.model_config
105
+ self.parallel_config = vllm_config.parallel_config
106
+ self.quant_config = vllm_config.quant_config
107
+
108
+ self.pp_group = get_pp_group()
109
+ self.tp_group = get_tp_group()
110
+
111
+ # Weights to skip in `self.load_weights`
112
+ self.skip_prefixes: list[str] = []
113
+ """Skip loading weights whose qualname starts with these prefixes."""
114
+ self.skip_substrs: list[str] = []
115
+ """Skip loading weights whose qualname contains these substrings."""
116
+ self.ignore_unexpected_prefixes: list[str] = []
117
+ """Ignore unexpected weights whose qualname starts with these prefixes.
118
+ """
119
+ self.ignore_unexpected_suffixes: list[str] = []
120
+ """Ignore unexpected weights whose qualname ends with these suffixes."""
121
+
122
+ if self.quant_config:
123
+ quant_method_name = self.quant_config.get_name()
124
+ # Check for unsupported quantization methods.
125
+ if quant_method_name == "mxfp4":
126
+ raise NotImplementedError(
127
+ "Transformers backend does not support MXFP4 quantization yet."
128
+ )
129
+ # Skip loading extra bias for GPTQ models.
130
+ if "gptq" in quant_method_name:
131
+ self.ignore_unexpected_suffixes.append(".bias")
132
+
133
+ # Set correct attn and init on "meta" to delay allocating GPU tensors
134
+ self.text_config._attn_implementation = "vllm"
135
+ with init_on_device_without_buffers("meta"):
136
+ self.model: PreTrainedModel = AutoModel.from_config(
137
+ self.config,
138
+ dtype=self.model_config.dtype,
139
+ trust_remote_code=self.model_config.trust_remote_code,
140
+ )
141
+
142
+ # Remove layers not on this pipeline parallel rank
143
+ self.pipeline_parallel()
144
+ # Substitute remaining layers with vLLM's layers as needed
145
+ self.recursive_replace()
146
+ # Create attention instances for KV cache allocation
147
+ self.attention_instances = self.create_attention_instances()
148
+
149
+ # Input embeddings
150
+ input_embeddings = self.model.get_input_embeddings()
151
+ if not isinstance(input_embeddings, PPMissingLayer):
152
+ # Some models scale embeddings inside the input embedding layer
153
+ self.embed_scale = getattr(input_embeddings, "embed_scale", None)
154
+ names = ("embedding_size", "hidden_size")
155
+ embedding_dim = getattr_iter(self.text_config, names, None)
156
+ assert embedding_dim is not None
157
+ self.model.set_input_embeddings(
158
+ VocabParallelEmbedding(
159
+ self.text_config.vocab_size,
160
+ embedding_dim=embedding_dim,
161
+ org_num_embeddings=self.text_config.vocab_size,
162
+ quant_config=self.quant_config,
163
+ )
164
+ )
165
+
166
+ # Initialize any parameters that have not had their modules replaced
167
+ self.init_parameters(self.model)
168
+
169
+ # Pipeline parallel intermediate tensors
170
+ self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
171
+ ["hidden_states"], self.text_config.hidden_size
172
+ )
173
+
174
+ def pipeline_parallel(self):
175
+ """
176
+ Apply the model's pipeline parallelization plan.
177
+ """
178
+ if self.pp_group.world_size <= 1:
179
+ return
180
+
181
+ if not self.model.supports_pp_plan:
182
+ tip = get_feature_request_tip(
183
+ self.model_config.model, self.model_config.trust_remote_code
184
+ )
185
+ raise ValueError(
186
+ f"{type(self.model)} does not support pipeline parallel. {tip}"
187
+ )
188
+
189
+ module_lists = []
190
+ module_list_idx = None
191
+ pp_plan = list(self.model._pp_plan.keys())
192
+ for i, name in enumerate(pp_plan):
193
+ if isinstance(getattr(self.model, name), nn.ModuleList):
194
+ module_lists.append(name)
195
+ module_list_idx = i
196
+
197
+ if len(module_lists) > 1:
198
+ raise ValueError(
199
+ "Pipeline parallel of models with multiple `ModuleList`s "
200
+ "in the base model are not supported yet!"
201
+ )
202
+ if module_list_idx is None:
203
+ raise ValueError(f"Could not find `ModuleList` in {type(self.model)}")
204
+
205
+ # Layers before module list
206
+ for name in pp_plan[:module_list_idx]:
207
+ if self.pp_group.is_first_rank or (
208
+ self.text_config.tie_word_embeddings and self.pp_group.is_last_rank
209
+ ):
210
+ continue
211
+ setattr(self.model, name, PPMissingLayer())
212
+
213
+ # Module list
214
+ start_layer, end_layer = get_pp_indices(
215
+ self.text_config.num_hidden_layers,
216
+ self.pp_group.rank_in_group,
217
+ self.pp_group.world_size,
218
+ )
219
+ layers_name = pp_plan[module_list_idx]
220
+ layers = getattr(self.model, layers_name)
221
+ for i in range(len(layers)):
222
+ if start_layer <= i and i < end_layer:
223
+ continue
224
+ layers[i] = PPMissingLayer()
225
+
226
+ # Layers after module list
227
+ for name in pp_plan[module_list_idx + 1 :]:
228
+ # Modules that should be on last rank
229
+ if not self.pp_group.is_last_rank:
230
+ setattr(self.model, name, PPMissingLayer())
231
+
232
+ def recursive_replace(self):
233
+ """Recursively replace modules in the model as needed.
234
+
235
+ Currently, this replaces:
236
+
237
+ - `nn.Linear` with vLLM's tensor parallel linear classes
238
+ - `*RMSNorm` with vLLM's `RMSNorm`
239
+ """
240
+ tp_plan = self.model.tp_plan
241
+
242
+ if not tp_plan and self.tp_group.world_size > 1:
243
+ tip = get_feature_request_tip(
244
+ self.model_config.model, self.model_config.trust_remote_code
245
+ )
246
+ raise ValueError(
247
+ f"{type(self.model)} does not support tensor parallel. {tip}"
248
+ )
249
+
250
+ # Prefix the patterns because we always start from `self.model`
251
+ tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
252
+
253
+ def _recursive_replace(module: nn.Module, prefix: str):
254
+ for child_name, child_module in module.named_children():
255
+ new_module = child_module
256
+ qual_name = maybe_prefix(prefix, child_name)
257
+ if isinstance(child_module, nn.Linear):
258
+ generator = (p for p in tp_plan if re.match(p, qual_name))
259
+ pattern = next(generator, None)
260
+ # Some weight loaders expect all linear layers to inherit
261
+ # LinearBase, so we set a default style which causes any
262
+ # unspecified layers to be replaced with ReplicatedLinear
263
+ style = tp_plan.get(pattern, "replicate")
264
+ new_module = replace_linear_class(
265
+ child_module, style, self.quant_config, prefix=qual_name
266
+ )
267
+ elif child_module.__class__.__name__.endswith("RMSNorm"):
268
+ new_module = replace_rms_norm_class(
269
+ child_module, self.text_config.hidden_size
270
+ )
271
+ else:
272
+ _recursive_replace(child_module, prefix=qual_name)
273
+
274
+ if new_module is not child_module:
275
+ setattr(module, child_name, new_module)
276
+ log_replacement(qual_name, child_module, new_module)
277
+
278
+ _recursive_replace(self.model, prefix="model")
279
+
280
+ def create_attention_instances(self) -> dict[int, Attention]:
281
+ """
282
+ Create `Attention` instances to inform KV cache allocation.
283
+ """
284
+ text_config = self.text_config
285
+
286
+ num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
287
+ head_size = self.model_config.get_head_size()
288
+ num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
289
+ logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
290
+
291
+ # In encoder models, the attention layers will have `is_causal=False`
292
+ is_encoder = lambda module: not getattr(module, "is_causal", True)
293
+ has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
294
+ is_multimodal = lambda config: config != config.get_text_config()
295
+ # vLLM does not support encoder-decoder models, so if any encoder layer is
296
+ # found in a text only model, we assume the whole model is an encoder model
297
+ if has_encoder(self.model) and not is_multimodal(self.config):
298
+ self.check_version("4.57.0.dev0", "encoder models support")
299
+ attn_type = AttentionType.ENCODER_ONLY
300
+ else:
301
+ attn_type = AttentionType.DECODER
302
+
303
+ pp_rank = self.pp_group.rank_in_group
304
+ pp_size = self.pp_group.world_size
305
+ start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
306
+
307
+ attention_instances = {}
308
+ for i in range(start, end):
309
+ # Handle interleaved sliding window attention
310
+ per_layer_sliding_window = None
311
+ if (
312
+ hasattr(self.config, "layer_types")
313
+ and self.config.layer_types[i] == "sliding_attention"
314
+ ):
315
+ per_layer_sliding_window = self.config.sliding_window
316
+
317
+ attention_instances[i] = Attention(
318
+ num_heads=num_heads,
319
+ head_size=head_size,
320
+ # NOTE: We use Llama scale as default, if it's set by
321
+ # Transformers, it's updated in vllm_flash_attention_forward
322
+ scale=head_size**-0.5,
323
+ num_kv_heads=num_kv_heads,
324
+ cache_config=self.cache_config,
325
+ quant_config=self.quant_config,
326
+ logits_soft_cap=logits_soft_cap,
327
+ per_layer_sliding_window=per_layer_sliding_window,
328
+ prefix=f"{i}.attn",
329
+ attn_type=attn_type,
330
+ )
331
+ return attention_instances
332
+
333
+ def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
334
+ """
335
+ If a `parameter` is on the `meta` device, then its parent
336
+ `module` is the original module created by:
337
+
338
+ ```python
339
+ with torch.device("meta"):
340
+ self.model: "PreTrainedModel" = AutoModel.from_config(...)
341
+ ```
342
+ """
343
+
344
+ def _init_parameters(module: nn.Module, dtype: torch.dtype | None):
345
+ for name, param in module.named_parameters(recurse=False):
346
+ if param.device == torch.device("meta"):
347
+ new_param = nn.Parameter(
348
+ torch.empty_like(
349
+ param.data,
350
+ dtype=dtype or self.model_config.dtype,
351
+ device=self.device_config.device,
352
+ )
353
+ )
354
+ setattr(module, name, new_param)
355
+ for child in module.children():
356
+ _init_parameters(child, dtype)
357
+
358
+ _init_parameters(module, dtype)
359
+
360
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
361
+ inputs_embeds = self.model.get_input_embeddings()(input_ids)
362
+ if self.embed_scale is not None:
363
+ inputs_embeds *= self.embed_scale
364
+ return inputs_embeds
365
+
366
+ def forward(
367
+ self,
368
+ input_ids: torch.Tensor | None,
369
+ positions: torch.Tensor,
370
+ intermediate_tensors: IntermediateTensors | None = None,
371
+ inputs_embeds: torch.Tensor | None = None,
372
+ **kwargs,
373
+ ) -> torch.Tensor | IntermediateTensors:
374
+ if not self.pp_group.is_first_rank:
375
+ assert intermediate_tensors is not None
376
+ input_ids = None
377
+ inputs_embeds = intermediate_tensors["hidden_states"]
378
+
379
+ if input_ids is not None:
380
+ input_ids = input_ids[None, ...]
381
+ if inputs_embeds is not None:
382
+ inputs_embeds = inputs_embeds[None, ...]
383
+
384
+ # If the model scales embeddings inside the input embedding layer we must
385
+ # ensure they are scaled here since VocabParallelEmbedding will not do it
386
+ if (
387
+ self.embed_scale is not None
388
+ and input_ids is not None
389
+ and inputs_embeds is None
390
+ ):
391
+ inputs_embeds = self.get_input_embeddings(input_ids)
392
+ input_ids = None
393
+
394
+ if self.model_config.uses_mrope:
395
+ position_ids = positions[:, None]
396
+ else:
397
+ position_ids = positions[None, ...]
398
+
399
+ hidden_states = self.model(
400
+ input_ids=input_ids,
401
+ inputs_embeds=inputs_embeds,
402
+ use_cache=False,
403
+ position_ids=position_ids,
404
+ attention_instances=self.attention_instances,
405
+ return_dict=False,
406
+ **kwargs,
407
+ )[0][0, ...] # we remove batch dimension for now
408
+
409
+ if not self.pp_group.is_last_rank:
410
+ return IntermediateTensors({"hidden_states": hidden_states})
411
+
412
+ return hidden_states
413
+
414
+ def load_weights(
415
+ self,
416
+ weights: Iterable[tuple[str, torch.Tensor]],
417
+ ) -> set[str]:
418
+ loader = AutoWeightsLoader(
419
+ self,
420
+ skip_prefixes=self.skip_prefixes,
421
+ skip_substrs=self.skip_substrs,
422
+ ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
423
+ ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
424
+ )
425
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
426
+
427
+ @staticmethod
428
+ def check_version(min_version: str, feature: str):
429
+ installed = Version(transformers.__version__)
430
+ required = Version(min_version)
431
+ if installed < required:
432
+ raise ImportError(
433
+ f"Transformers backend requires transformers>={required} "
434
+ f"for {feature}, but got {installed}"
435
+ )
vllm/model_executor/models/transformers/causal.py
new
+66
@@ -0,0 +1,66 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Transformers backend mixin for causal language models."""
18
+
19
+from typing import TYPE_CHECKING
20
+
21
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
22
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23
+from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
24
+from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
25
+
26
+if TYPE_CHECKING:
27
+ import torch
28
+
29
+ from vllm.config import VllmConfig
30
+
31
+
32
+class CausalMixin(VllmModelForTextGeneration):
33
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
34
+ # Skip VllmModelForTextGeneration.__init__ and call the next class in MRO
35
+ super(VllmModelForTextGeneration, self).__init__(
36
+ vllm_config=vllm_config, prefix=prefix
37
+ )
38
+
39
+ # Tell `Base.load_weights` to skip
40
+ # `lm_head` if the model has tied word embeddings
41
+ if self.text_config.tie_word_embeddings:
42
+ self.skip_prefixes.append("lm_head.")
43
+
44
+ if self.pp_group.is_last_rank:
45
+ self.unpadded_vocab_size = self.text_config.vocab_size
46
+ self.lm_head = ParallelLMHead(
47
+ self.text_config.vocab_size,
48
+ self.text_config.hidden_size,
49
+ quant_config=self.quant_config,
50
+ prefix=maybe_prefix(prefix, "lm_head"),
51
+ )
52
+ if self.text_config.tie_word_embeddings:
53
+ self.lm_head = self.lm_head.tie_weights(
54
+ self.model.get_input_embeddings()
55
+ )
56
+
57
+ logit_scale = getattr(self.text_config, "logit_scale", 1.0)
58
+ self.logits_processor = LogitsProcessor(
59
+ self.unpadded_vocab_size, self.text_config.vocab_size, logit_scale
60
+ )
61
+ else:
62
+ self.lm_head = PPMissingLayer()
63
+
64
+ def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
65
+ logits = self.logits_processor(self.lm_head, hidden_states)
66
+ return logits
vllm/model_executor/models/transformers/legacy.py
new
+97
@@ -0,0 +1,97 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Transformers backend mixin for legacy models."""
18
+
19
+from typing import TYPE_CHECKING
20
+
21
+import torch
22
+
23
+from vllm.model_executor.models.utils import WeightsMapper
24
+from vllm.sequence import IntermediateTensors
25
+
26
+if TYPE_CHECKING:
27
+ from vllm.config import VllmConfig
28
+
29
+
30
+class LegacyMixin:
31
+ hf_to_vllm_mapper = WeightsMapper(
32
+ # These are applied in order, so the order matters!
33
+ orig_to_new_prefix={
34
+ # Handle BERT-like models
35
+ "roberta": "model",
36
+ "bert": "model",
37
+ # Add `model.` prefix for base model checkpoints
38
+ "": "model.",
39
+ # Remove `model.` prefix if it was already there
40
+ "model.model.": "model.",
41
+ # Classifier/scoring heads will be adjacent to `model`
42
+ "model.score": "classifier",
43
+ "model.classifier": "classifier",
44
+ },
45
+ orig_to_new_suffix={
46
+ # Replace legacy suffixes used for norms
47
+ ".gamma": ".weight",
48
+ ".beta": ".bias",
49
+ },
50
+ )
51
+
52
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
53
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
54
+
55
+ # Skip unsupported/unwanted output embeddings layers
56
+ self.skip_prefixes.extend(
57
+ [
58
+ "model.lm_head.",
59
+ "model.predictions.",
60
+ "model.qa_outputs.",
61
+ "model.embeddings_project.",
62
+ "model.discriminator_predictions.",
63
+ ]
64
+ )
65
+
66
+ # Some encoder models have the position_ids buffer in the checkpoint.
67
+ # vLLM will always pass position_ids as an argument, so we skip loading
68
+ # the buffer if it exists
69
+ self.skip_substrs.append("position_ids")
70
+
71
+ # Some encoder models have the bias of the final classifier layer
72
+ # in the checkpoint. vLLM does not use this bias, so we skip loading
73
+ # it if it exists
74
+ self.skip_substrs.append("score.bias")
75
+
76
+ # roberta-like models an extra padding in positions.
77
+ # FIXME(Isotr0py): This is quite hacky for roberta edge case,
78
+ # we should find a better way to handle this.
79
+ self.is_roberta = "roberta" in self.text_config.model_type
80
+ self.padding_idx = self.text_config.pad_token_id
81
+
82
+ def forward(
83
+ self,
84
+ input_ids: torch.Tensor | None,
85
+ positions: torch.Tensor,
86
+ intermediate_tensors: IntermediateTensors | None = None,
87
+ inputs_embeds: torch.Tensor | None = None,
88
+ ) -> torch.Tensor | IntermediateTensors:
89
+ if self.is_roberta:
90
+ # RoBERTa-specific positions padding
91
+ positions += self.padding_idx + 1
92
+ return super().forward(
93
+ input_ids=input_ids,
94
+ positions=positions,
95
+ intermediate_tensors=intermediate_tensors,
96
+ inputs_embeds=inputs_embeds,
97
+ )
vllm/model_executor/models/transformers/moe.py
renamed
+14
-37
@@ -14,31 +14,27 @@
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
-"""Wrapper around `transformers` MoE models."""
17
+"""Transformers backend mixin for Mixture of Experts (MoE) models."""
18
19
-from typing import Any
19
+from typing import TYPE_CHECKING, Any
20
21
import torch
22
import torch.nn as nn
23
24
-from vllm.compilation.decorators import support_torch_compile
24
from vllm.config.utils import getattr_iter
25
from vllm.distributed import get_dp_group, get_ep_group
26
from vllm.forward_context import ForwardContext, get_forward_context
27
from vllm.model_executor.custom_op import CustomOp
28
from vllm.model_executor.layers.fused_moe import FusedMoE
29
+from vllm.model_executor.models.interfaces import MixtureOfExperts
30
+from vllm.model_executor.models.utils import maybe_prefix
31
from vllm.platforms import current_platform
32
from vllm.utils import direct_register_custom_op
33
33
-from .interfaces import MixtureOfExperts, SupportsMultiModal
34
-from .transformers import (
35
- TransformersBase,
36
- TransformersForCausalLM,
37
- TransformersForMultimodalLM,
38
- can_enable_torch_compile,
39
- log_replacement,
40
-)
41
-from .utils import maybe_prefix
34
+from .utils import log_replacement
35
+
36
+if TYPE_CHECKING:
37
+ from vllm.config import VllmConfig
38
39
40
@CustomOp.register("transformers_fused_moe")
@@ -117,11 +113,11 @@ direct_register_custom_op(
113
)
114
115
120
-class TransformersMoEBase(TransformersBase, MixtureOfExperts):
121
- def __init__(self, *, vllm_config, prefix=""):
116
+class MoEMixin(MixtureOfExperts):
117
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
118
self.check_version("4.57.0.dev0", "MoE models support")
123
- self.ep_group = get_ep_group()
124
- super().__init__(vllm_config=vllm_config, prefix=prefix)
119
+ # Skip MixtureOfExperts.__init__ and call the next class in MRO
120
+ super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
121
122
def set_eplb_state(
123
self,
@@ -242,7 +238,7 @@ class TransformersMoEBase(TransformersBase, MixtureOfExperts):
238
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
239
240
# MixtureOfExperts mixin settings
245
- ep_size = self.ep_group.world_size
241
+ ep_size = get_ep_group().world_size
242
243
self.mlp_layers = [] # Used for MixtureOfExperts methods
244
self.expert_weights = []
@@ -316,24 +312,5 @@ class TransformersMoEBase(TransformersBase, MixtureOfExperts):
312
_recursive_replace(child_module, prefix=qual_name)
313
314
_recursive_replace(self.model, prefix="model")
319
- # Continue with the replacement of layers in TransformersBase
315
+ # Continue with the replacement of layers in Base
316
super().recursive_replace()
321
-
322
-
323
-@support_torch_compile(enable_if=can_enable_torch_compile)
324
-class TransformersMoEForCausalLM(TransformersMoEBase, TransformersForCausalLM):
325
- pass
326
-
327
-
328
-@support_torch_compile(
329
- # set `positions` to last dim to support Qwen-mrope
330
- dynamic_arg_dims={
331
- "input_ids": 0,
332
- "positions": -1,
333
- "intermediate_tensors": 0,
334
- "inputs_embeds": 0,
335
- },
336
- enable_if=can_enable_torch_compile,
337
-)
338
-class TransformersMoEForMultimodalLM(TransformersMoEBase, TransformersForMultimodalLM):
339
- get_input_embeddings = SupportsMultiModal.get_input_embeddings
vllm/model_executor/models/transformers/multimodal.py
new
+396
@@ -0,0 +1,396 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Transformers backend mixin for multi-modal models."""
18
+
19
+from collections.abc import Mapping
20
+from typing import TYPE_CHECKING
21
+
22
+import torch
23
+
24
+from vllm.config.utils import getattr_iter
25
+from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal
26
+from vllm.model_executor.models.utils import WeightsMapper
27
+from vllm.multimodal import MultiModalKwargsItems
28
+from vllm.multimodal.inputs import (
29
+ MultiModalDataDict,
30
+ MultiModalFieldConfig,
31
+ MultiModalInputs,
32
+ MultiModalUUIDDict,
33
+ PlaceholderRange,
34
+)
35
+from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
36
+from vllm.multimodal.processing import BaseMultiModalProcessor, BaseProcessingInfo
37
+from vllm.multimodal.profiling import BaseDummyInputsBuilder
38
+from vllm.sequence import IntermediateTensors
39
+
40
+if TYPE_CHECKING:
41
+ from transformers import BatchFeature, PretrainedConfig
42
+
43
+ from vllm.config import VllmConfig
44
+ from vllm.config.multimodal import BaseDummyOptions
45
+
46
+DYNAMIC_ARG_DIMS = {
47
+ "input_ids": 0,
48
+ # set `positions` to last dim to support Qwen-mrope
49
+ "positions": -1,
50
+ "intermediate_tensors": 0,
51
+ "inputs_embeds": 0,
52
+}
53
+
54
+
55
+class MultiModalProcessingInfo(BaseProcessingInfo):
56
+ def get_supported_mm_limits(self):
57
+ return {"image": None}
58
+
59
+ def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
60
+ return {"image": self.get_max_image_tokens()}
61
+
62
+ def get_max_image_tokens(self) -> int:
63
+ width, height = self.get_max_image_size()
64
+ processor = self.get_hf_processor()
65
+ multimodal_config = self.ctx.model_config.multimodal_config
66
+ mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
67
+ mm_tokens = processor._get_num_multimodal_tokens(
68
+ image_sizes=([height, width],), **mm_processor_kwargs
69
+ )
70
+ image_tokens = mm_tokens["num_image_tokens"][0]
71
+ return image_tokens
72
+
73
+ def get_max_image_size(self):
74
+ return 10_000, 10_000 # hardcode for arbitrary very large size
75
+
76
+
77
+class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
78
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
79
+ num_images = mm_counts.get("image", 0)
80
+
81
+ processor = self.info.get_hf_processor()
82
+ if "gemma3" in processor.__class__.__name__.lower():
83
+ image_token = processor.boi_token
84
+ else:
85
+ image_token = getattr(processor, "image_token", "")
86
+ return image_token * num_images
87
+
88
+ def get_dummy_mm_data(
89
+ self,
90
+ seq_len: int,
91
+ mm_counts: Mapping[str, int],
92
+ mm_options: Mapping[str, "BaseDummyOptions"] | None = None,
93
+ ) -> MultiModalDataDict:
94
+ num_images = mm_counts.get("image", 0)
95
+
96
+ target_width, target_height = self.info.get_max_image_size()
97
+
98
+ image_overrides = mm_options.get("image") if mm_options else None
99
+
100
+ return {
101
+ "image": self._get_dummy_images(
102
+ width=target_width,
103
+ height=target_height,
104
+ num_images=num_images,
105
+ overrides=image_overrides,
106
+ ),
107
+ }
108
+
109
+
110
+class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
111
+ def _get_prompt_updates(
112
+ self,
113
+ mm_items: MultiModalDataItems,
114
+ hf_processor_mm_kwargs: Mapping[str, object],
115
+ out_mm_kwargs: MultiModalKwargsItems,
116
+ ):
117
+ """
118
+ Given the original multi-modal items for this modality
119
+ and HF-processed data, output the updates to perform.
120
+
121
+ The information returned by this method is used to update token inputs
122
+ which bypass the HF processor. It is also used to update the output of
123
+ HF processor if the HF process does not apply prompt updates to text
124
+ inputs.
125
+
126
+ Moreover, this information is critical to determine the token positions
127
+ in order to construct :class:`~vllm-multimodal.input.PlaceholderRange`
128
+ for each multi-modal item.
129
+ """
130
+ return None
131
+
132
+ def _get_mm_fields_config(
133
+ self,
134
+ hf_inputs: "BatchFeature",
135
+ hf_processor_mm_kwargs: Mapping[str, object],
136
+ ) -> Mapping[str, MultiModalFieldConfig]:
137
+ # HF Processors always return a mask but vLLM doesn't need it
138
+ hf_inputs.pop("attention_mask", None)
139
+ num_image_patches = hf_inputs.get("num_image_patches")
140
+ mm_fields = {
141
+ key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
142
+ for key in hf_inputs
143
+ }
144
+ mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
145
+ "image", num_image_patches
146
+ )
147
+
148
+ # Keep these as batched, as they always have batch size as first dim
149
+ mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
150
+ mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
151
+ mm_fields["num_image_patches"] = MultiModalFieldConfig.batched("image")
152
+ return mm_fields
153
+
154
+ def _get_hf_mm_data(
155
+ self,
156
+ mm_items: MultiModalDataItems,
157
+ ) -> tuple[Mapping[str, object], Mapping[str, object]]:
158
+ """
159
+ In contrast to the base class, this method always adds
160
+ `return_mm_token_type_ids` to the processor data
161
+ """
162
+ processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
163
+ processor_data["return_mm_token_type_ids"] = True
164
+ return processor_data, passthrough_data
165
+
166
+ def apply(
167
+ self,
168
+ prompt: str | list[int],
169
+ mm_data: MultiModalDataDict,
170
+ hf_processor_mm_kwargs: Mapping[str, object],
171
+ tokenization_kwargs: Mapping[str, object] | None = None,
172
+ mm_uuids: MultiModalUUIDDict | None = None,
173
+ ) -> MultiModalInputs:
174
+ """
175
+ Process multi-modal inputs to be used in vLLM.
176
+
177
+ Apply HF Processor on prompt text and multi-modal data together,
178
+ outputting token IDs and processed tensors.
179
+ """
180
+ if tokenization_kwargs is None:
181
+ tokenization_kwargs = {}
182
+
183
+ mm_items = self._to_mm_items(mm_data)
184
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
185
+ if not isinstance(prompt, str):
186
+ # the prompt is the tokenized ids which is not supported
187
+ # by the hf_processor, which is why we would need to decode the ids
188
+ # into string
189
+ prompt = hf_processor.decode(prompt)
190
+
191
+ # Bypass cached processor and always apply to the full set of mm inputs
192
+ # NOTE: we can't just set caching=False because base class method
193
+ # transforms outputs to `MultiModalKwargs` which is not going to
194
+ # work for Transformers. We have a lot of logic tied to
195
+ # `mm_tokens_per_modality` below
196
+ prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
197
+ prompt_text=prompt,
198
+ mm_items=mm_items,
199
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
200
+ tokenization_kwargs=tokenization_kwargs,
201
+ )
202
+
203
+ # For gemma3 we check `token_type_ids` as the key
204
+ token_type_key = (
205
+ "mm_token_type_ids"
206
+ if "mm_token_type_ids" in processed_data
207
+ else "token_type_ids"
208
+ )
209
+ mm_token_type_ids = processed_data.pop(token_type_key)
210
+
211
+ # We can infer vLLM style placeholder from token type ids, if we split
212
+ # it for each input `mm_data`.
213
+ mm_positions = torch.where(mm_token_type_ids == 1)[1]
214
+ images = mm_items.get_items("image", ImageProcessorItems)
215
+ multimodal_config = self.info.ctx.model_config.multimodal_config
216
+ mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
217
+ image_sizes = []
218
+ for item_idx in range(len(images)):
219
+ image_size = images.get_image_size(item_idx)
220
+ image_sizes.append((image_size.height, image_size.width))
221
+
222
+ mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
223
+ image_sizes=image_sizes, **mm_processor_kwargs
224
+ )
225
+
226
+ mm_placeholders = {}
227
+ split_sizes = mm_tokens_per_modality["num_image_tokens"]
228
+ if split_sizes:
229
+ chunked_mm_positions = torch.split(mm_positions, split_sizes)
230
+ mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
231
+ chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
232
+ ranges = [
233
+ PlaceholderRange(
234
+ offset=positions[0].item(),
235
+ length=positions.shape[0],
236
+ is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
237
+ )
238
+ for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
239
+ ]
240
+ mm_placeholders = {"image": ranges}
241
+
242
+ processed_data["num_image_patches"] = torch.tensor(
243
+ mm_tokens_per_modality["num_image_patches"]
244
+ )
245
+ mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
246
+ processed_data,
247
+ self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
248
+ )
249
+
250
+ # Use overrides if provided; fallback to data-dependent hashing.
251
+ mm_hashes = self._hash_mm_items(
252
+ mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
253
+ )
254
+
255
+ return MultiModalInputs(
256
+ type="multimodal",
257
+ prompt_token_ids=prompt_ids,
258
+ mm_kwargs=mm_kwargs,
259
+ mm_hashes=mm_hashes,
260
+ mm_placeholders=mm_placeholders,
261
+ )
262
+
263
+
264
+class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
265
+ supports_multimodal_raw_input_only = True
266
+ merge_by_field_config = True
267
+ # Backwards compatibility for prev released models. State dicts back then
268
+ # had different formats and cannot be loaded with `AutoModel` mapping as is
269
+ hf_to_vllm_mapper = WeightsMapper(
270
+ orig_to_new_prefix={
271
+ "language_model.model": "model.language_model",
272
+ "text_model.model": "model.text_model",
273
+ "vision_tower": "model.vision_tower",
274
+ "vqmodel": "model.vqmodel",
275
+ "visual": "model.visual",
276
+ "vision_model": "model.vision_model",
277
+ "vision_embed_tokens": "model.vision_embed_tokens",
278
+ "image_newline": "model.image_newline",
279
+ "multi_modal_projector": "model.multi_modal_projector",
280
+ "text_model.lm_head": "lm_head",
281
+ "language_model.lm_head": "lm_head",
282
+ # Qwen models used "model" as the name for the language model.
283
+ # Therefore, we must map each of submodule explicitly to avoid
284
+ # conflicts with newer models that use "model.language_model".
285
+ "model.embed_tokens": "model.language_model.embed_tokens",
286
+ "model.layers": "model.language_model.layers",
287
+ "model.norm": "model.language_model.norm",
288
+ }
289
+ )
290
+
291
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
292
+ # Skip SupportsMRoPE.__init__ and call the next class in MRO
293
+ super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix)
294
+
295
+ def forward(
296
+ self,
297
+ input_ids: torch.Tensor | None,
298
+ positions: torch.Tensor,
299
+ intermediate_tensors: IntermediateTensors | None = None,
300
+ inputs_embeds: torch.Tensor | None = None,
301
+ **kwargs: object,
302
+ ) -> torch.Tensor | IntermediateTensors:
303
+ # Gemma3 and PaliGemma needs `token_type_ids` to work correctly
304
+ # Other models will not have `token_type_ids` in kwargs
305
+ kwargs = {k: v for k, v in kwargs.items() if k == "token_type_ids"}
306
+ model_output = super().forward(
307
+ input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
308
+ )
309
+ return model_output
310
+
311
+ def get_language_model(self) -> torch.nn.Module:
312
+ """Transformers backend multimodal classes do not contain a separate vLLM
313
+ language model class. Therefore, in order to return a language model vLLM class,
314
+ we use a wrapper to give `self` the same interface as a text model."""
315
+
316
+ # Exclude self and object
317
+ bases = self.__class__.mro()[1:-1]
318
+ # Keep only classes defined in `vllm.model_executor.models.transformers`
319
+ bases = [b for b in bases if ".transformers." in b.__module__]
320
+ # Exclude MultiModalMixin itself
321
+ bases = [b for b in bases if b is not MultiModalMixin]
322
+
323
+ class LanguageModel(*bases):
324
+ def __init__(self, multimodal_model):
325
+ # Don't call super().__init__() to avoid re-initialization
326
+ self.__dict__.update(multimodal_model.__dict__)
327
+
328
+ model = getattr_iter(self.model, ("language_model", "text_model"), None)
329
+
330
+ return LanguageModel(self)
331
+
332
+ def get_multimodal_embeddings(self, **kwargs):
333
+ pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
334
+ image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
335
+ # Model might use `image_patches` instead of `pixel_values`
336
+ if pixel_values is None:
337
+ pixel_values = kwargs.pop("image_patches", None)
338
+
339
+ if image_embeds is not None:
340
+ return image_embeds
341
+
342
+ if pixel_values is None:
343
+ return None
344
+
345
+ num_image_patches = kwargs.pop("num_image_patches")
346
+ kwargs.pop("token_type_ids", None) # used only in `forward`
347
+ if pixel_values is not None:
348
+ vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
349
+
350
+ if isinstance(vision_embeddings, torch.Tensor):
351
+ if vision_embeddings.ndim == 2:
352
+ vision_embeddings = vision_embeddings.unsqueeze(0)
353
+
354
+ # Embeddings have to be 2D tensors of length `num_images`
355
+ # but transformers returns concat tensors if each patch
356
+ # is of different size. We split it back to make vLLM happy
357
+ vision_embeddings = torch.split(
358
+ vision_embeddings, num_image_patches.flatten().tolist()
359
+ )
360
+ vision_embeddings = [
361
+ embed.flatten(start_dim=0, end_dim=-2)
362
+ for embed in vision_embeddings
363
+ ]
364
+
365
+ return vision_embeddings
366
+
367
+ def get_mrope_input_positions(
368
+ self,
369
+ input_tokens: list[int],
370
+ hf_config: "PretrainedConfig",
371
+ image_grid_thw: list[list[int]] | torch.Tensor | None,
372
+ video_grid_thw: list[list[int]] | torch.Tensor | None,
373
+ second_per_grid_ts: list[float] | None = None,
374
+ context_len: int = 0,
375
+ seq_len: int | None = None,
376
+ audio_feature_lengths: torch.Tensor | None = None,
377
+ use_audio_in_video: bool = False,
378
+ ) -> tuple[torch.Tensor, int]:
379
+ if any((second_per_grid_ts, audio_feature_lengths, use_audio_in_video)):
380
+ raise NotImplementedError("Transformers backend only supports images.")
381
+
382
+ if isinstance(image_grid_thw, list):
383
+ image_grid_thw = torch.tensor(image_grid_thw)
384
+ if isinstance(video_grid_thw, list):
385
+ video_grid_thw = torch.tensor(video_grid_thw)
386
+
387
+ mrope_positions, mrope_position_delta = self.model.get_rope_index(
388
+ input_ids=torch.tensor(input_tokens).unsqueeze(0),
389
+ image_grid_thw=image_grid_thw,
390
+ video_grid_thw=video_grid_thw,
391
+ )
392
+
393
+ mrope_positions = mrope_positions[:, 0, context_len:seq_len]
394
+ mrope_position_delta = mrope_position_delta[0].item()
395
+
396
+ return mrope_positions, mrope_position_delta
vllm/model_executor/models/transformers/pooling.py
new
+118
@@ -0,0 +1,118 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Transformers backend mixins for pooling models."""
18
+
19
+from typing import TYPE_CHECKING
20
+
21
+import torch
22
+from transformers import AutoModelForSequenceClassification
23
+
24
+from vllm.model_executor.layers.pooler import (
25
+ ClassifierPooler,
26
+ CLSPool,
27
+ DispatchPooler,
28
+ Pooler,
29
+)
30
+from vllm.model_executor.models.interfaces import SupportsCrossEncoding
31
+from vllm.model_executor.models.interfaces_base import VllmModelForPooling
32
+
33
+if TYPE_CHECKING:
34
+ from vllm.config import VllmConfig
35
+
36
+
37
+class EmbeddingMixin(VllmModelForPooling):
38
+ default_pooling_type = "CLS"
39
+
40
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
41
+ # Skip VllmModelForPooling.__init__ and call the next class in MRO
42
+ super(VllmModelForPooling, self).__init__(
43
+ vllm_config=vllm_config, prefix=prefix
44
+ )
45
+
46
+ pooler_config = vllm_config.model_config.pooler_config
47
+ assert pooler_config is not None
48
+
49
+ self.pooler = DispatchPooler(
50
+ {
51
+ "token_embed": Pooler.for_token_embed(pooler_config),
52
+ "embed": Pooler.for_embed(pooler_config),
53
+ }
54
+ )
55
+
56
+
57
+class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling):
58
+ default_pooling_type = "CLS"
59
+
60
+ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
61
+ # Skip VllmModelForPooling.__init__ and call the next class in MRO
62
+ super(VllmModelForPooling, self).__init__(
63
+ vllm_config=vllm_config, prefix=prefix
64
+ )
65
+
66
+ pooler_config = vllm_config.model_config.pooler_config
67
+ assert pooler_config is not None
68
+
69
+ # Certain information about the the model and classifier can only be
70
+ # inferred from the `ForSequenceClassification` class. Therefore, we
71
+ # instantiate it on the "meta" device to avoid allocating GPU memory.
72
+ with torch.device("meta"):
73
+ seq_cls_model = AutoModelForSequenceClassification.from_config(
74
+ self.config,
75
+ dtype=self.model_config.dtype,
76
+ trust_remote_code=self.model_config.trust_remote_code,
77
+ )
78
+
79
+ # When used for sequence classification, some models have their
80
+ # pooling layers removed. Make sure this is reflected in vLLM.
81
+ for module in seq_cls_model.modules():
82
+ if hasattr(module, "pooler") and module.pooler is None:
83
+ self.model.pooler = None
84
+ break
85
+ if self.model.pooler is not None:
86
+ raise ValueError(
87
+ "Sequence classification models with pooling layers are not "
88
+ "supported yet in the Transformers backend."
89
+ )
90
+
91
+ # Unlike `lm_head`, `classifier` is not always `nn.Linear`.
92
+ self.classifier = seq_cls_model.classifier
93
+ self.init_parameters(self.classifier, dtype=self.model_config.head_dtype)
94
+
95
+ class ClassifierWithReshape(self.classifier.__class__):
96
+ """CLSPool has already been applied in `pooling`.
97
+ Add dim to match expected input shape of `classifier.forward`."""
98
+
99
+ def forward(self, *args, **kwargs):
100
+ if len(args) > 0:
101
+ args = (args[0].unsqueeze(1), *args[1:])
102
+ return super().forward(*args, **kwargs)
103
+
104
+ self.classifier.__class__ = ClassifierWithReshape
105
+
106
+ self.pooler = DispatchPooler(
107
+ {
108
+ "token_classify": Pooler.for_token_classify(
109
+ pooler_config, classifier=self.classifier
110
+ ),
111
+ "classify": ClassifierPooler(
112
+ pooling=CLSPool(), classifier=self.classifier, act_fn="classify"
113
+ ),
114
+ "score": ClassifierPooler(
115
+ pooling=CLSPool(), classifier=self.classifier, act_fn="score"
116
+ ),
117
+ }
118
+ )
vllm/model_executor/models/transformers/utils.py
new
+207
@@ -0,0 +1,207 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2024 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Transformers backend utilities."""
18
+
19
+from contextlib import contextmanager
20
+from pathlib import Path
21
+from typing import TYPE_CHECKING, Literal
22
+
23
+import torch
24
+from torch import nn
25
+
26
+from vllm.config.utils import getattr_iter
27
+from vllm.logger import init_logger
28
+from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
29
+from vllm.model_executor.layers.linear import (
30
+ ColumnParallelLinear,
31
+ ReplicatedLinear,
32
+ RowParallelLinear,
33
+)
34
+
35
+if TYPE_CHECKING:
36
+ from vllm.config import VllmConfig
37
+ from vllm.model_executor.layers.quantization import QuantizationConfig
38
+
39
+
40
+logger = init_logger(__name__)
41
+
42
+
43
+# Copied from `accelerate`
44
+@contextmanager
45
+def init_on_device_without_buffers(device: torch.device):
46
+ """
47
+ A context manager under which models are initialized with all
48
+ parameters on the specified device. However buffers are not
49
+ initialized on specified device.
50
+
51
+ Args:
52
+ device (`torch.device`):
53
+ Device to initialize all parameters on.
54
+ """
55
+
56
+ old_register_parameter = nn.Module.register_parameter
57
+
58
+ def register_empty_parameter(module, name, param):
59
+ old_register_parameter(module, name, param)
60
+ if param is not None:
61
+ param_cls = type(module._parameters[name])
62
+ kwargs = module._parameters[name].__dict__
63
+ kwargs["requires_grad"] = param.requires_grad
64
+ module._parameters[name] = param_cls(
65
+ module._parameters[name].to(device), **kwargs
66
+ )
67
+
68
+ tensor_constructors_to_patch = {}
69
+
70
+ def patch_tensor_constructor(fn):
71
+ def wrapper(*args, **kwargs):
72
+ kwargs["device"] = device
73
+ return fn(*args, **kwargs)
74
+
75
+ return wrapper
76
+
77
+ try:
78
+ nn.Module.register_parameter = register_empty_parameter
79
+ for torch_function_name in tensor_constructors_to_patch:
80
+ setattr(
81
+ torch,
82
+ torch_function_name,
83
+ patch_tensor_constructor(getattr(torch, torch_function_name)),
84
+ )
85
+ yield
86
+ finally:
87
+ nn.Module.register_parameter = old_register_parameter
88
+ for (
89
+ torch_function_name,
90
+ old_torch_function,
91
+ ) in tensor_constructors_to_patch.items():
92
+ setattr(torch, torch_function_name, old_torch_function)
93
+
94
+
95
+Style = Literal["colwise", "colwise_rep", "rowwise", "rowwise_rep", "replicate"]
96
+
97
+
98
+def replace_linear_class(
99
+ linear: nn.Linear,
100
+ style: Style = "replicate",
101
+ quant_config: "QuantizationConfig | None" = None,
102
+ *,
103
+ prefix: str = "",
104
+) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
105
+ """
106
+ Replace nn.Linear with one of vLLM's tensor parallel linear classes.
107
+
108
+ Args:
109
+ linear: `nn.Linear` to be replaced.
110
+ style: Tensor parallel style of the new linear, e.g. "colwise".
111
+ quant_config: Quantization config for the new linear.
112
+ Returns:
113
+ The new linear.
114
+ """
115
+
116
+ if not isinstance(style, str):
117
+ raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
118
+
119
+ vllm_linear_cls, vllm_linear_kwargs = {
120
+ "colwise": (ColumnParallelLinear, {}),
121
+ "colwise_rep": (ColumnParallelLinear, {"gather_output": True}),
122
+ "rowwise": (RowParallelLinear, {}),
123
+ "rowwise_rep": (RowParallelLinear, {"input_is_parallel": False}),
124
+ "replicate": (ReplicatedLinear, {}),
125
+ }.get(style, (ReplicatedLinear, {}))
126
+
127
+ return vllm_linear_cls(
128
+ input_size=linear.in_features,
129
+ output_size=linear.out_features,
130
+ bias=linear.bias is not None,
131
+ quant_config=quant_config,
132
+ prefix=prefix,
133
+ return_bias=False,
134
+ **vllm_linear_kwargs,
135
+ )
136
+
137
+
138
+def replace_rms_norm_class(rms_norm: nn.Module, hidden_size: int) -> RMSNorm:
139
+ """Replace a Transformers RMSNorm with vLLM's RMSNorm.
140
+
141
+ This method assumes:
142
+ - Weight is stored as `weight`.
143
+ - Epsilon is stored as `eps` or `variance_epsilon`.
144
+ - `with_scale` indicates whether the layer has a weight (Gemma3n only).
145
+ - `var_hidden_size` is only ever used for Intern vision encoder in vLLM
146
+ and Transformers doesn't appear to have the same concept.
147
+ """
148
+ eps = getattr_iter(rms_norm, ("eps", "variance_epsilon"), 1e-6)
149
+ kwargs = {"hidden_size": hidden_size, "eps": eps}
150
+ # Update hidden size if weight is available
151
+ weight_meta = getattr(rms_norm, "weight", None)
152
+ if weight_meta is not None:
153
+ kwargs["hidden_size"] = weight_meta.size(0)
154
+ # Check if weight is all zeros, which indicates GemmaRMSNorm
155
+ # We must create a new instance because rms_norm is on meta
156
+ try:
157
+ with torch.device("cpu"):
158
+ weight_test = getattr(rms_norm.__class__(1), "weight", None)
159
+ except Exception:
160
+ logger.warning(
161
+ "Failed to determine if RMSNorm weight is centered on zero or one. "
162
+ "Defaulting to one."
163
+ )
164
+ weight_test = None
165
+ if weight_test is not None and torch.all(weight_test == 0):
166
+ return GemmaRMSNorm(**kwargs)
167
+ # Otherwise assume it's a regular RMSNorm
168
+ kwargs["has_weight"] = getattr(rms_norm, "with_scale", True)
169
+ if weight_meta is not None:
170
+ kwargs["dtype"] = weight_meta.dtype
171
+ else:
172
+ # No weight, fall back to weightless RMSNorm
173
+ kwargs["has_weight"] = False
174
+ return RMSNorm(**kwargs)
175
+
176
+
177
+def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
178
+ logger.debug("%s: %s -> %s", name, old_module, new_module)
179
+
180
+
181
+def get_feature_request_tip(
182
+ model: str,
183
+ trust_remote_code: bool,
184
+) -> str:
185
+ hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new"
186
+ gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose"
187
+ url = hf_url if trust_remote_code else gh_url
188
+ prefix = f"Please open {url} to request support for this feature. "
189
+ if Path(model).exists():
190
+ prefix = ""
191
+ doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models"
192
+ tip = f"See {doc_url} for instructions on how to add support yourself."
193
+ return f"{prefix}{tip}"
194
+
195
+
196
+def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
197
+ """
198
+ Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
199
+
200
+ Defaults to `True` but is disabled in the following situations:
201
+
202
+ - The model uses dynamic rope scaling.
203
+ """
204
+ text_config = vllm_config.model_config.hf_config.get_text_config()
205
+ # Dynamic rope scaling is not compatible with torch.compile
206
+ rope_scaling: dict = getattr(text_config, "rope_scaling", None) or {}
207
+ return rope_scaling.get("rope_type") != "dynamic"
vllm/model_executor/models/transformers_pooling.py
deleted
-215
@@ -1,215 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Copyright 2024 The vLLM team.
5
-#
6
-# Licensed under the Apache License, Version 2.0 (the "License");
7
-# you may not use this file except in compliance with the License.
8
-# You may obtain a copy of the License at
9
-#
10
-# http://www.apache.org/licenses/LICENSE-2.0
11
-#
12
-# Unless required by applicable law or agreed to in writing, software
13
-# distributed under the License is distributed on an "AS IS" BASIS,
14
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
-# See the License for the specific language governing permissions and
16
-# limitations under the License.
17
-"""Wrapper around `transformers` models for pooling tasks."""
18
-
19
-import torch
20
-from transformers import AutoModelForSequenceClassification
21
-
22
-from vllm.attention import Attention, AttentionType
23
-from vllm.compilation.decorators import support_torch_compile
24
-from vllm.config import VllmConfig
25
-from vllm.model_executor.layers.pooler import (
26
- ClassifierPooler,
27
- CLSPool,
28
- DispatchPooler,
29
- Pooler,
30
-)
31
-from vllm.sequence import IntermediateTensors
32
-
33
-from .interfaces_base import VllmModelForPooling
34
-from .transformers import TransformersBase, can_enable_torch_compile
35
-from .transformers_moe import TransformersMoEBase
36
-from .utils import WeightsMapper
37
-
38
-
39
-class TransformersPoolingBase(TransformersBase, VllmModelForPooling):
40
- hf_to_vllm_mapper = WeightsMapper(
41
- # These are applied in order, so the order matters!
42
- orig_to_new_prefix={
43
- # Handle BERT-like models
44
- "roberta": "model",
45
- "bert": "model",
46
- # Add `model.` prefix for base model checkpoints
47
- "": "model.",
48
- # Remove `model.` prefix if it was already there
49
- "model.model.": "model.",
50
- # Classifier/scoring heads will be adjacent to `model`
51
- "model.score": "classifier",
52
- "model.classifier": "classifier",
53
- },
54
- orig_to_new_suffix={
55
- # Replace legacy suffixes used for norms
56
- ".gamma": ".weight",
57
- ".beta": ".bias",
58
- },
59
- )
60
-
61
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
62
- super().__init__(vllm_config=vllm_config, prefix=prefix)
63
-
64
- # Skip unsupported/unwanted output embeddings layers
65
- self.skip_prefixes.extend(
66
- [
67
- "model.lm_head.",
68
- "model.predictions.",
69
- "model.qa_outputs.",
70
- "model.embeddings_project.",
71
- "model.discriminator_predictions.",
72
- ]
73
- )
74
-
75
- # Some encoder models have the position_ids buffer in the checkpoint.
76
- # vLLM will always pass position_ids as an argument, so we skip loading
77
- # the buffer if it exists
78
- self.skip_substrs.append("position_ids")
79
-
80
- # Some encoder models have the bias of the final classifier layer
81
- # in the checkpoint. vLLM does not use this bias, so we skip loading
82
- # it if it exists
83
- self.skip_substrs.append("score.bias")
84
-
85
- # roberta-like models an extra padding in positions.
86
- # FIXME(Isotr0py): This is quite hacky for roberta edge case,
87
- # we should find a better way to handle this.
88
- self.is_roberta = "roberta" in self.text_config.model_type
89
- self.padding_idx = self.text_config.pad_token_id
90
-
91
- def create_attention_instances(
92
- self, attn_type: AttentionType = AttentionType.DECODER
93
- ) -> dict[int, Attention]:
94
- # TODO(hmellor): Better way to detect encoder models
95
- # In encoder models, the attention layers will have `is_causal=False`
96
- is_encoder = lambda m: not getattr(m, "is_causal", True)
97
- # vLLM does not support encoder-decoder models, so if any encoder layer
98
- # is found, we assume the whole model is an encoder model
99
- if any(is_encoder(m) for m in self.model.modules()):
100
- attn_type = AttentionType.ENCODER_ONLY
101
-
102
- # Check minimum transformers version for encoder models support
103
- if attn_type == AttentionType.ENCODER_ONLY:
104
- self.check_version("4.57.0.dev0", "encoder models support")
105
-
106
- return super().create_attention_instances(attn_type)
107
-
108
- def forward(
109
- self,
110
- input_ids: torch.Tensor | None,
111
- positions: torch.Tensor,
112
- intermediate_tensors: IntermediateTensors | None = None,
113
- inputs_embeds: torch.Tensor | None = None,
114
- ) -> torch.Tensor | IntermediateTensors:
115
- if self.is_roberta:
116
- # RoBERTa-specific positions padding
117
- positions += self.padding_idx + 1
118
- return super().forward(
119
- input_ids=input_ids,
120
- positions=positions,
121
- intermediate_tensors=intermediate_tensors,
122
- inputs_embeds=inputs_embeds,
123
- )
124
-
125
-
126
-@support_torch_compile(enable_if=can_enable_torch_compile)
127
-class TransformersEmbeddingModel(TransformersPoolingBase):
128
- default_pooling_type = "CLS"
129
-
130
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
131
- super().__init__(vllm_config=vllm_config, prefix=prefix)
132
-
133
- pooler_config = vllm_config.model_config.pooler_config
134
- assert pooler_config is not None
135
-
136
- self.pooler = DispatchPooler(
137
- {
138
- "token_embed": Pooler.for_token_embed(pooler_config),
139
- "embed": Pooler.for_embed(pooler_config),
140
- }
141
- )
142
-
143
-
144
-@support_torch_compile(enable_if=can_enable_torch_compile)
145
-class TransformersForSequenceClassification(TransformersPoolingBase):
146
- default_pooling_type = "CLS"
147
-
148
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
149
- super().__init__(vllm_config=vllm_config, prefix=prefix)
150
-
151
- pooler_config = vllm_config.model_config.pooler_config
152
- assert pooler_config is not None
153
-
154
- # Certain information about the the model and classifier can only be
155
- # inferred from the `ForSequenceClassification` class. Therefore, we
156
- # instantiate it on the "meta" device to avoid allocating GPU memory.
157
- with torch.device("meta"):
158
- seq_cls_model = AutoModelForSequenceClassification.from_config(
159
- self.config,
160
- dtype=self.model_config.dtype,
161
- trust_remote_code=self.model_config.trust_remote_code,
162
- )
163
-
164
- # When used for sequence classification, some models have their
165
- # pooling layers removed. Make sure this is reflected in vLLM.
166
- for module in seq_cls_model.modules():
167
- if hasattr(module, "pooler") and module.pooler is None:
168
- self.model.pooler = None
169
- break
170
- if self.model.pooler is not None:
171
- raise ValueError(
172
- "Sequence classification models with pooling layers are not "
173
- "supported yet in the Transformers backend."
174
- )
175
-
176
- # Unlike `lm_head`, `classifier` is not always `nn.Linear`.
177
- self.classifier = seq_cls_model.classifier
178
- self.init_parameters(self.classifier, dtype=self.model_config.head_dtype)
179
-
180
- class ClassifierWithReshape(self.classifier.__class__):
181
- """CLSPool has already been applied in `pooling`.
182
- Add dim to match expected input shape of `classifier.forward`."""
183
-
184
- def forward(self, *args, **kwargs):
185
- if len(args) > 0:
186
- args = (args[0].unsqueeze(1), *args[1:])
187
- return super().forward(*args, **kwargs)
188
-
189
- self.classifier.__class__ = ClassifierWithReshape
190
-
191
- self.pooler = DispatchPooler(
192
- {
193
- "token_classify": Pooler.for_token_classify(
194
- pooler_config, classifier=self.classifier
195
- ),
196
- "classify": ClassifierPooler(
197
- pooling=CLSPool(), classifier=self.classifier, act_fn="classify"
198
- ),
199
- "score": ClassifierPooler(
200
- pooling=CLSPool(), classifier=self.classifier, act_fn="score"
201
- ),
202
- }
203
- )
204
-
205
-
206
-@support_torch_compile(enable_if=can_enable_torch_compile)
207
-class TransformersMoEEmbeddingModel(TransformersMoEBase, TransformersEmbeddingModel):
208
- pass
209
-
210
-
211
-@support_torch_compile(enable_if=can_enable_torch_compile)
212
-class TransformersMoEForSequenceClassification(
213
- TransformersMoEBase, TransformersForSequenceClassification
214
-):
215
- pass