A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 19:30 UTC
[Bugfix] Fix encoder-only model support for transformers backend (#28021)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Isotr0py committed
Nov 5, 2025 at 14:24 UTC
0ff05e3770ad5d819c813666b72f1954cf317b34
4 files changed
+16
-10
tests/models/registry.py
+6
-6
@@ -899,27 +899,27 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
899
900
_TRANSFORMERS_BACKEND_MODELS = {
901
"TransformersEmbeddingModel": _HfExamplesInfo(
902
- "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
902
+ "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0.dev"
903
),
904
"TransformersForSequenceClassification": _HfExamplesInfo(
905
"papluca/xlm-roberta-base-language-detection",
906
- min_transformers_version="5.0.0",
906
+ min_transformers_version="5.0.0.dev",
907
),
908
"TransformersForCausalLM": _HfExamplesInfo(
909
"hmellor/Ilama-3.2-1B", trust_remote_code=True
910
),
911
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
912
"TransformersMoEForCausalLM": _HfExamplesInfo(
913
- "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
913
+ "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0.dev"
914
),
915
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
916
- "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
916
+ "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0.dev"
917
),
918
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
919
- "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
919
+ "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0.dev"
920
),
921
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
922
- "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
922
+ "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0.dev"
923
),
924
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
925
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
tests/models/test_transformers.py
+1
-1
@@ -82,7 +82,7 @@ def test_models(
82
from packaging.version import Version
83
84
installed = Version(transformers.__version__)
85
- required = Version("5.0.0")
85
+ required = Version("5.0.0.dev")
86
if model == "allenai/OLMoE-1B-7B-0924" and installed < required:
87
pytest.skip(
88
"MoE models with the Transformers backend require "
vllm/model_executor/models/transformers/base.py
+8
-2
@@ -28,6 +28,7 @@ from transformers import AutoModel
28
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
29
30
from vllm.attention import Attention, AttentionType
31
+from vllm.attention.layers.encoder_only_attention import EncoderOnlyAttention
32
from vllm.config.utils import getattr_iter
33
from vllm.distributed import get_pp_group, get_tp_group
34
from vllm.distributed.utils import get_pp_indices
@@ -317,7 +318,7 @@ class Base(nn.Module, VllmModel, SupportsQuant, SupportsLoRA, SupportsPP):
318
# vLLM does not support encoder-decoder models, so if any encoder layer is
319
# found in a text only model, we assume the whole model is an encoder model
320
if has_encoder(self.model) and not is_multimodal(self.config):
320
- self.check_version("4.57.0.dev0", "encoder models support")
321
+ self.check_version("5.0.0.dev0", "encoder models support")
322
attn_type = AttentionType.ENCODER_ONLY
323
else:
324
attn_type = AttentionType.DECODER
@@ -336,7 +337,12 @@ class Base(nn.Module, VllmModel, SupportsQuant, SupportsLoRA, SupportsPP):
337
):
338
per_layer_sliding_window = self.config.sliding_window
339
339
- attention_instances[i] = Attention(
340
+ attn_cls = (
341
+ EncoderOnlyAttention
342
+ if attn_type == AttentionType.ENCODER_ONLY
343
+ else Attention
344
+ )
345
+ attention_instances[i] = attn_cls(
346
num_heads=num_heads,
347
head_size=head_size,
348
# NOTE: We use Llama scale as default, if it's set by
vllm/model_executor/models/transformers/moe.py
+1
-1
@@ -115,7 +115,7 @@ direct_register_custom_op(
115
116
class MoEMixin(MixtureOfExperts):
117
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
118
- self.check_version("4.57.0.dev0", "MoE models support")
118
+ self.check_version("5.0.0.dev0", "MoE models support")
119
# Skip MixtureOfExperts.__init__ and call the next class in MRO
120
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
121