A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
Fix some Transformers nightly tests (#29802)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Dec 2, 2025 at 07:05 UTC
f5b0846ba0aa6a4b6ab788ff257d0a00eb376e75
3 files changed
+29
-28
vllm/model_executor/models/jina_vl.py
+1
-1
@@ -29,7 +29,7 @@ logger = init_logger(__name__)
29
class JinaVLScorer(nn.Module):
30
def __init__(self, model_config: "ModelConfig"):
31
super().__init__()
32
- config = model_config.hf_config
32
+ config = model_config.hf_config.get_text_config()
33
head_dtype = model_config.head_dtype
34
self.dense = ColumnParallelLinear(
35
config.hidden_size, config.hidden_size, params_dtype=head_dtype, bias=True
vllm/model_executor/models/modernbert.py
+27
-26
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.pooler import (
20
PoolingParamsUpdate,
21
PoolingType,
22
)
23
-from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
23
+from vllm.model_executor.layers.rotary_embedding import get_rope
24
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
25
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26
from vllm.sequence import IntermediateTensors
@@ -62,19 +62,6 @@ class ModernBertEmbeddings(nn.Module):
62
return embeddings
63
64
65
-class ModernBertRotaryEmbedding(RotaryEmbedding):
66
- def __init__(self, config: ModernBertConfig, head_size: int, dim: int, base: float):
67
- super().__init__(
68
- head_size=head_size,
69
- rotary_dim=dim,
70
- max_position_embeddings=config.max_position_embeddings,
71
- base=base,
72
- is_neox_style=True,
73
- dtype=torch.float16,
74
- )
75
- self.config = config
76
-
77
-
65
class ModernBertAttention(nn.Module):
66
def __init__(self, config: ModernBertConfig, layer_id: int | None = None):
67
super().__init__()
@@ -95,19 +82,33 @@ class ModernBertAttention(nn.Module):
82
bias=config.attention_bias,
83
)
84
98
- sliding_window = None
99
- if layer_id % config.global_attn_every_n_layers != 0:
100
- sliding_window = config.local_attention // 2
101
- rope_theta = (
102
- config.local_rope_theta
103
- if config.local_rope_theta is not None
104
- else config.global_rope_theta
105
- )
85
+ if layer_types := getattr(config, "layer_types", None):
86
+ # Transformers v5
87
+ layer_type = layer_types[layer_id]
88
+ rope_parameters = config.rope_parameters[layer_type]
89
+ sliding_window: int | None = None
90
+ if layer_type == "sliding_attention":
91
+ sliding_window = config.local_attention // 2
92
else:
107
- rope_theta = config.global_rope_theta
108
-
109
- self.rotary_emb = ModernBertRotaryEmbedding(
110
- config=config, head_size=self.head_dim, dim=self.head_dim, base=rope_theta
93
+ # Transformers v4
94
+ sliding_window = None
95
+ if layer_id % config.global_attn_every_n_layers != 0:
96
+ sliding_window = config.local_attention // 2
97
+ rope_theta = (
98
+ config.local_rope_theta
99
+ if config.local_rope_theta is not None
100
+ else config.global_rope_theta
101
+ )
102
+ else:
103
+ rope_theta = config.global_rope_theta
104
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
105
+
106
+ self.rotary_emb = get_rope(
107
+ head_size=self.head_dim,
108
+ rotary_dim=self.head_dim,
109
+ max_position=config.max_position_embeddings,
110
+ rope_parameters=rope_parameters,
111
+ dtype=torch.float16,
112
)
113
self.attn = EncoderOnlyAttention(
114
self.num_heads,
vllm/model_executor/models/qwen2.py
+1
-1
@@ -503,7 +503,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
503
504
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
505
super().__init__()
506
- config = vllm_config.model_config.hf_config
506
+ config = vllm_config.model_config.hf_config.get_text_config()
507
quant_config = vllm_config.quant_config
508
509
self.config = config