@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf3eacfe58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 15:00 UTC

Standardise `get_rope` to use `rope_parameters["partial_rotary_factor"]`, not `rotary_dim` (#30389)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Dec 11, 2025 at 20:45 UTC cf3eacfe58fa9e745c2854782ada884a9f992cf7
83 files changed +238 -292
benchmarks/kernels/benchmark_mrope.py
-1
@@ -99,7 +99,6 @@ def benchmark_mrope(
99 # the parameters to compute the q k v size based on tp_size
100 mrope_helper_class = get_rope(
101 head_size=head_dim,
102 - rotary_dim=head_dim,
102 max_position=max_position,
103 is_neox_style=is_neox_style,
104 rope_parameters=rope_parameters,
benchmarks/kernels/benchmark_rope.py
+2 -2
@@ -32,8 +32,8 @@ def get_benchmark(head_size, rotary_dim, is_neox_style, device):
32 def benchmark(batch_size, seq_len, num_heads, provider):
33 dtype = torch.bfloat16
34 max_position = 8192
35 - base = 10000
36 - rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
35 + rope_parameters = {"partial_rotary_factor": rotary_dim / head_size}
36 + rope = get_rope(head_size, max_position, is_neox_style, rope_parameters)
37 rope = rope.to(dtype=dtype, device=device)
38 cos_sin_cache = rope.cos_sin_cache.to(dtype=torch.float, device=device)
39
tests/compile/test_functionalization.py
+1 -4
@@ -128,14 +128,12 @@ class TestFusedAddRMSNorm(torch.nn.Module):
128
129
130 class TestRotaryEmbedding(torch.nn.Module):
131 - def __init__(self, head_dim=64, rotary_dim=None, max_position=2048, base=10000):
131 + def __init__(self, head_dim=64, max_position=2048, base=10000):
132 super().__init__()
133 self.head_dim = head_dim
134 - self.rotary_dim = rotary_dim or head_dim
134
135 self.rotary_emb = get_rope(
136 self.head_dim,
138 - rotary_dim=self.rotary_dim,
137 max_position=max_position,
138 rope_parameters={"rope_type": "default", "rope_theta": base},
139 )
@@ -170,7 +168,6 @@ class TestRotaryEmbeddingSliceScatter(torch.nn.Module):
168
169 self.rotary_emb = get_rope(
170 self.head_dim,
173 - rotary_dim=self.head_dim,
171 max_position=max_position,
172 rope_parameters={"rope_type": "default", "rope_theta": base},
173 )
tests/kernels/core/test_mrope.py
-2
@@ -116,7 +116,6 @@ def test_mrope(
116
117 mrope_helper_class = get_rope(
118 head_size=head_dim,
119 - rotary_dim=head_dim,
119 max_position=max_position,
120 is_neox_style=is_neox_style,
121 rope_parameters=config.rope_parameters,
@@ -185,7 +184,6 @@ def test_mrope_torch_compile_tracing(
184
185 mrope_helper_class = get_rope(
186 head_size=head_dim,
188 - rotary_dim=head_dim,
187 max_position=max_position,
188 is_neox_style=is_neox_style,
189 rope_parameters=config.rope_parameters,
tests/kernels/core/test_pos_encoding.py
+8 -4
@@ -83,8 +83,12 @@ def test_rotary_embedding(
83 torch.set_default_device(device)
84 if rotary_dim is None:
85 rotary_dim = head_size
86 - rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
87 - rope = get_rope(head_size, rotary_dim, max_position, is_neox_style, rope_parameters)
86 + rope_parameters = {
87 + "rope_type": "default",
88 + "rope_theta": rope_theta,
89 + "partial_rotary_factor": rotary_dim / head_size,
90 + }
91 + rope = get_rope(head_size, max_position, is_neox_style, rope_parameters)
92 rope = rope.to(dtype=dtype, device=torch.get_default_device())
93
94 positions = torch.randint(0, max_position, (batch_size, seq_len))
@@ -150,9 +154,9 @@ def test_rope_module_cache():
154 if rotary_dim is None:
155 rotary_dim = head_size
156 rope_parameters["rope_theta"] = rope_theta
157 + rope_parameters["partial_rotary_factor"] = rotary_dim / head_size
158 rope = get_rope(
159 head_size,
155 - rotary_dim,
160 max_position,
161 is_neox_style,
162 rope_parameters,
@@ -177,9 +181,9 @@ def test_rope_module_cache():
181 if rotary_dim is None:
182 rotary_dim = head_size
183 rope_parameters["rope_theta"] = rope_theta
184 + rope_parameters["partial_rotary_factor"] = rotary_dim / head_size
185 rope = get_rope(
186 head_size,
182 - rotary_dim,
187 max_position,
188 is_neox_style,
189 rope_parameters,
vllm/config/utils.py
+16 -2
@@ -73,14 +73,28 @@ def get_field(cls: ConfigType, name: str) -> Field:
73 )
74
75
76 -def getattr_iter(object: object, names: Iterable[str], default: Any) -> Any:
76 +def getattr_iter(
77 + object: object, names: Iterable[str], default: Any, warn: bool = False
78 +) -> Any:
79 """
80 A helper function that retrieves an attribute from an object which may
81 have multiple possible names. This is useful when fetching attributes from
82 arbitrary `transformers.PretrainedConfig` instances.
83 +
84 + In the case where the first name in `names` is the preferred name, and
85 + any other names are deprecated aliases, setting `warn=True` will log a
86 + warning when a deprecated name is used.
87 """
82 - for name in names:
88 + for i, name in enumerate(names):
89 if hasattr(object, name):
90 + if warn and i > 0:
91 + logger.warning_once(
92 + "%s contains a deprecated attribute name '%s'. "
93 + "Please use the preferred attribute name '%s' instead.",
94 + type(object).__name__,
95 + name,
96 + names[0],
97 + )
98 return getattr(object, name)
99 return default
100
vllm/model_executor/layers/rotary_embedding/__init__.py
+160 -166
@@ -25,7 +25,6 @@ _ROPE_DICT: dict[tuple, RotaryEmbedding] = {}
25
26 def get_rope(
27 head_size: int,
28 - rotary_dim: int,
28 max_position: int,
29 is_neox_style: bool = True,
30 rope_parameters: dict[str, Any] | None = None,
@@ -54,12 +53,15 @@ def get_rope(
53 else:
54 dual_chunk_attention_args = None
55
57 - partial_rotary_factor = 1.0
58 - if rope_parameters is not None:
59 - partial_rotary_factor = rope_parameters.get("partial_rotary_factor", 1.0)
56 + rope_parameters = rope_parameters or {}
57 + base = rope_parameters.get("rope_theta", 10000)
58 + scaling_type = rope_parameters.get("rope_type", "default")
59 + partial_rotary_factor = rope_parameters.get("partial_rotary_factor", 1.0)
60 +
61 + if partial_rotary_factor <= 0.0 or partial_rotary_factor > 1.0:
62 + raise ValueError(f"{partial_rotary_factor=} must be between 0.0 and 1.0")
63 + rotary_dim = int(head_size * partial_rotary_factor)
64
61 - if partial_rotary_factor < 1.0:
62 - rotary_dim = int(rotary_dim * partial_rotary_factor)
65 key = (
66 head_size,
67 rotary_dim,
@@ -72,7 +74,6 @@ def get_rope(
74 if key in _ROPE_DICT:
75 return _ROPE_DICT[key]
76
75 - base = rope_parameters["rope_theta"] if rope_parameters else 10000
77 if dual_chunk_attention_config is not None:
78 extra_kwargs = {
79 k: v
@@ -88,208 +89,201 @@ def get_rope(
89 dtype,
90 **extra_kwargs,
91 )
91 - elif not rope_parameters:
92 - rotary_emb = RotaryEmbedding(
93 - head_size, rotary_dim, max_position, base, is_neox_style, dtype
94 - )
95 - else:
96 - scaling_type = rope_parameters["rope_type"]
97 -
98 - if scaling_type == "llama3":
99 - scaling_factor = rope_parameters["factor"]
100 - low_freq_factor = rope_parameters["low_freq_factor"]
101 - high_freq_factor = rope_parameters["high_freq_factor"]
102 - original_max_position = rope_parameters["original_max_position_embeddings"]
103 - rotary_emb = Llama3RotaryEmbedding(
92 + elif scaling_type == "default":
93 + if "mrope_section" in rope_parameters:
94 + rotary_emb = MRotaryEmbedding(
95 head_size,
96 rotary_dim,
97 max_position,
98 base,
99 is_neox_style,
100 dtype,
110 - scaling_factor,
111 - low_freq_factor,
112 - high_freq_factor,
113 - original_max_position,
101 + mrope_section=rope_parameters["mrope_section"],
102 + mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
103 )
115 - elif scaling_type == "mllama4":
116 - rotary_emb = Llama4VisionRotaryEmbedding(
117 - head_size, rotary_dim, max_position, base, is_neox_style, dtype
118 - )
119 - elif scaling_type == "default":
120 - if "mrope_section" in rope_parameters:
121 - rotary_emb = MRotaryEmbedding(
122 - head_size,
123 - rotary_dim,
124 - max_position,
125 - base,
126 - is_neox_style,
127 - dtype,
128 - mrope_section=rope_parameters["mrope_section"],
129 - mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
130 - )
131 - else:
132 - rotary_emb = RotaryEmbedding(
133 - head_size,
134 - rotary_dim,
135 - max_position,
136 - base,
137 - is_neox_style,
138 - dtype,
139 - )
140 - elif scaling_type == "linear":
141 - scaling_factor = rope_parameters["factor"]
142 - rotary_emb = LinearScalingRotaryEmbedding(
104 + else:
105 + rotary_emb = RotaryEmbedding(
106 head_size,
107 rotary_dim,
108 max_position,
109 base,
110 is_neox_style,
148 - scaling_factor,
111 dtype,
112 )
151 - elif scaling_type == "ntk":
152 - scaling_factor = rope_parameters["factor"]
153 - mixed_b = rope_parameters.get("mixed_b")
154 - rotary_emb = NTKScalingRotaryEmbedding(
113 + elif scaling_type == "llama3":
114 + scaling_factor = rope_parameters["factor"]
115 + low_freq_factor = rope_parameters["low_freq_factor"]
116 + high_freq_factor = rope_parameters["high_freq_factor"]
117 + original_max_position = rope_parameters["original_max_position_embeddings"]
118 + rotary_emb = Llama3RotaryEmbedding(
119 + head_size,
120 + rotary_dim,
121 + max_position,
122 + base,
123 + is_neox_style,
124 + dtype,
125 + scaling_factor,
126 + low_freq_factor,
127 + high_freq_factor,
128 + original_max_position,
129 + )
130 + elif scaling_type == "mllama4":
131 + rotary_emb = Llama4VisionRotaryEmbedding(
132 + head_size, rotary_dim, max_position, base, is_neox_style, dtype
133 + )
134 + elif scaling_type == "linear":
135 + scaling_factor = rope_parameters["factor"]
136 + rotary_emb = LinearScalingRotaryEmbedding(
137 + head_size,
138 + rotary_dim,
139 + max_position,
140 + base,
141 + is_neox_style,
142 + scaling_factor,
143 + dtype,
144 + )
145 + elif scaling_type == "ntk":
146 + scaling_factor = rope_parameters["factor"]
147 + mixed_b = rope_parameters.get("mixed_b")
148 + rotary_emb = NTKScalingRotaryEmbedding(
149 + head_size,
150 + rotary_dim,
151 + max_position,
152 + base,
153 + is_neox_style,
154 + scaling_factor,
155 + dtype,
156 + mixed_b,
157 + )
158 + elif scaling_type == "dynamic":
159 + if "alpha" in rope_parameters:
160 + scaling_alpha = rope_parameters["alpha"]
161 + rotary_emb = DynamicNTKAlphaRotaryEmbedding(
162 head_size,
163 rotary_dim,
164 max_position,
165 base,
166 is_neox_style,
160 - scaling_factor,
167 + scaling_alpha,
168 dtype,
162 - mixed_b,
169 )
164 - elif scaling_type == "dynamic":
165 - if "alpha" in rope_parameters:
166 - scaling_alpha = rope_parameters["alpha"]
167 - rotary_emb = DynamicNTKAlphaRotaryEmbedding(
168 - head_size,
169 - rotary_dim,
170 - max_position,
171 - base,
172 - is_neox_style,
173 - scaling_alpha,
174 - dtype,
175 - )
176 - elif "factor" in rope_parameters:
177 - scaling_factor = rope_parameters["factor"]
178 - rotary_emb = DynamicNTKScalingRotaryEmbedding(
179 - head_size,
180 - rotary_dim,
181 - max_position,
182 - base,
183 - is_neox_style,
184 - scaling_factor,
185 - dtype,
186 - )
187 - else:
188 - raise ValueError(
189 - "Dynamic rope scaling must contain either 'alpha' or 'factor' field"
190 - )
191 - elif scaling_type == "xdrope":
192 - scaling_alpha = rope_parameters["alpha"]
193 - rotary_emb = XDRotaryEmbedding(
170 + elif "factor" in rope_parameters:
171 + scaling_factor = rope_parameters["factor"]
172 + rotary_emb = DynamicNTKScalingRotaryEmbedding(
173 head_size,
174 rotary_dim,
175 max_position,
176 base,
177 is_neox_style,
199 - scaling_alpha,
178 + scaling_factor,
179 dtype,
201 - xdrope_section=rope_parameters["xdrope_section"],
180 )
203 - elif scaling_type == "yarn":
204 - scaling_factor = rope_parameters["factor"]
205 - original_max_position = rope_parameters["original_max_position_embeddings"]
206 - extra_kwargs = {
207 - k: v
208 - for k, v in rope_parameters.items()
209 - if k
210 - in (
211 - "extrapolation_factor",
212 - "attn_factor",
213 - "beta_fast",
214 - "beta_slow",
215 - "apply_yarn_scaling",
216 - "truncate",
217 - )
218 - }
219 - if "mrope_section" in rope_parameters:
220 - extra_kwargs.pop("apply_yarn_scaling", None)
221 - rotary_emb = MRotaryEmbedding(
222 - head_size,
223 - rotary_dim,
224 - original_max_position,
225 - base,
226 - is_neox_style,
227 - dtype,
228 - mrope_section=rope_parameters["mrope_section"],
229 - mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
230 - scaling_factor=scaling_factor,
231 - **extra_kwargs,
232 - )
233 - else:
234 - rotary_emb = YaRNScalingRotaryEmbedding(
235 - head_size,
236 - rotary_dim,
237 - original_max_position,
238 - base,
239 - is_neox_style,
240 - scaling_factor,
241 - dtype,
242 - **extra_kwargs,
243 - )
244 - elif scaling_type in ["deepseek_yarn", "deepseek_llama_scaling"]:
245 - scaling_factor = rope_parameters["factor"]
246 - original_max_position = rope_parameters["original_max_position_embeddings"]
247 - # assert max_position == original_max_position * scaling_factor
248 - extra_kwargs = {
249 - k: v
250 - for k, v in rope_parameters.items()
251 - if k
252 - in (
253 - "extrapolation_factor",
254 - "attn_factor",
255 - "beta_fast",
256 - "beta_slow",
257 - "mscale",
258 - "mscale_all_dim",
259 - )
260 - }
261 - rotary_emb = DeepseekScalingRotaryEmbedding(
181 + else:
182 + raise ValueError(
183 + "Dynamic rope scaling must contain either 'alpha' or 'factor' field"
184 + )
185 + elif scaling_type == "xdrope":
186 + scaling_alpha = rope_parameters["alpha"]
187 + rotary_emb = XDRotaryEmbedding(
188 + head_size,
189 + rotary_dim,
190 + max_position,
191 + base,
192 + is_neox_style,
193 + scaling_alpha,
194 + dtype,
195 + xdrope_section=rope_parameters["xdrope_section"],
196 + )
197 + elif scaling_type == "yarn":
198 + scaling_factor = rope_parameters["factor"]
199 + original_max_position = rope_parameters["original_max_position_embeddings"]
200 + extra_kwargs = {
201 + k: v
202 + for k, v in rope_parameters.items()
203 + if k
204 + in (
205 + "extrapolation_factor",
206 + "attn_factor",
207 + "beta_fast",
208 + "beta_slow",
209 + "apply_yarn_scaling",
210 + "truncate",
211 + )
212 + }
213 + if "mrope_section" in rope_parameters:
214 + extra_kwargs.pop("apply_yarn_scaling", None)
215 + rotary_emb = MRotaryEmbedding(
216 head_size,
217 rotary_dim,
218 original_max_position,
219 base,
220 is_neox_style,
267 - scaling_factor,
221 dtype,
222 + mrope_section=rope_parameters["mrope_section"],
223 + mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
224 + scaling_factor=scaling_factor,
225 **extra_kwargs,
226 )
271 - elif scaling_type == "longrope":
272 - short_factor = rope_parameters["short_factor"]
273 - long_factor = rope_parameters["long_factor"]
274 - original_max_position = rope_parameters["original_max_position_embeddings"]
275 - extra_kwargs = {
276 - k: v
277 - for k, v in rope_parameters.items()
278 - if k in ("short_mscale", "long_mscale")
279 - }
280 - rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(
227 + else:
228 + rotary_emb = YaRNScalingRotaryEmbedding(
229 head_size,
230 rotary_dim,
283 - max_position,
231 original_max_position,
232 base,
233 is_neox_style,
234 + scaling_factor,
235 dtype,
288 - short_factor,
289 - long_factor,
236 **extra_kwargs,
237 )
292 - else:
293 - raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
238 + elif scaling_type in ["deepseek_yarn", "deepseek_llama_scaling"]:
239 + scaling_factor = rope_parameters["factor"]
240 + original_max_position = rope_parameters["original_max_position_embeddings"]
241 + # assert max_position == original_max_position * scaling_factor
242 + extra_kwargs = {
243 + k: v
244 + for k, v in rope_parameters.items()
245 + if k
246 + in (
247 + "extrapolation_factor",
248 + "attn_factor",
249 + "beta_fast",
250 + "beta_slow",
251 + "mscale",
252 + "mscale_all_dim",
253 + )
254 + }
255 + rotary_emb = DeepseekScalingRotaryEmbedding(
256 + head_size,
257 + rotary_dim,
258 + original_max_position,
259 + base,
260 + is_neox_style,
261 + scaling_factor,
262 + dtype,
263 + **extra_kwargs,
264 + )
265 + elif scaling_type == "longrope":
266 + short_factor = rope_parameters["short_factor"]
267 + long_factor = rope_parameters["long_factor"]
268 + original_max_position = rope_parameters["original_max_position_embeddings"]
269 + extra_kwargs = {
270 + k: v
271 + for k, v in rope_parameters.items()
272 + if k in ("short_mscale", "long_mscale")
273 + }
274 + rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(
275 + head_size,
276 + rotary_dim,
277 + max_position,
278 + original_max_position,
279 + base,
280 + is_neox_style,
281 + dtype,
282 + short_factor,
283 + long_factor,
284 + **extra_kwargs,
285 + )
286 + else:
287 + raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
288 _ROPE_DICT[key] = rotary_emb
289 return rotary_emb
vllm/model_executor/models/afmoe.py
-1
@@ -241,7 +241,6 @@ class AfmoeAttention(nn.Module):
241 if self.is_local_attention:
242 self.rotary_emb = get_rope(
243 self.head_dim,
244 - rotary_dim=self.head_dim,
244 max_position=max_position_embeddings,
245 rope_parameters=config["rope_parameters"],
246 is_neox_style=True,
vllm/model_executor/models/apertus.py
-1
@@ -226,7 +226,6 @@ class ApertusAttention(nn.Module):
226
227 self.rotary_emb = get_rope(
228 self.head_dim,
229 - rotary_dim=self.head_dim,
229 max_position=self.max_position_embeddings,
230 rope_parameters=config.rope_parameters,
231 is_neox_style=is_neox_style,
vllm/model_executor/models/arctic.py
-1
@@ -314,7 +314,6 @@ class ArcticAttention(nn.Module):
314
315 self.rotary_emb = get_rope(
316 self.head_dim,
317 - rotary_dim=self.head_dim,
317 max_position=self.max_position_embeddings,
318 rope_parameters=config.rope_parameters,
319 is_neox_style=True,
vllm/model_executor/models/baichuan.py
-1
@@ -189,7 +189,6 @@ class BaiChuanAttention(nn.Module):
189 else:
190 self.rotary_emb = get_rope(
191 self.head_dim,
192 - rotary_dim=self.head_dim,
192 max_position=self.max_position_embeddings,
193 rope_parameters=rope_parameters,
194 )
vllm/model_executor/models/bailing_moe.py
+2 -2
@@ -127,11 +127,11 @@ class BailingAttention(nn.Module):
127 prefix=f"{prefix}.dense",
128 )
129
130 - self.rotary_dim = getattr(config, "rotary_dim", self.head_dim)
130 + rotary_dim = getattr(config, "rotary_dim", self.head_dim)
131 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / self.head_dim
132
133 self.rotary_emb = get_rope(
134 self.head_dim,
134 - rotary_dim=self.rotary_dim,
135 max_position=config.max_position_embeddings,
136 rope_parameters=config.rope_parameters,
137 is_neox_style=True,
vllm/model_executor/models/bamba.py
+2 -5
@@ -178,14 +178,11 @@ class BambaAttentionDecoderLayer(nn.Module):
178 self.scaling = self.head_dim**-0.5
179 self.max_position_embeddings = max_position_embeddings
180
181 - if hasattr(config, "attn_rotary_emb"):
182 - rotary_dim = config.attn_rotary_emb # for backward compatibility
183 - else:
184 - rotary_dim = self.head_dim # default
181 + rotary_dim = getattr(config, "attn_rotary_emb", self.head_dim)
182 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / self.head_dim
183
184 self.rotary_emb = get_rope(
185 head_size=self.head_dim,
188 - rotary_dim=rotary_dim,
186 max_position=max_position_embeddings,
187 rope_parameters=config.rope_parameters,
188 is_neox_style=True,
vllm/model_executor/models/chameleon.py
-1
@@ -314,7 +314,6 @@ class ChameleonAttention(nn.Module):
314 self.k_norm = ChameleonLayerNorm((self.num_kv_heads, self.head_dim))
315 self.rotary_emb = get_rope(
316 self.head_dim,
317 - rotary_dim=self.head_dim,
317 max_position=max_position_embeddings,
318 rope_parameters=rope_parameters,
319 )
vllm/model_executor/models/chatglm.py
+5 -2
@@ -99,13 +99,16 @@ class GLMAttention(nn.Module):
99 # https://huggingface.co/zai-org/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
100 rope_ratio = getattr(config, "rope_ratio", 1.0)
101 max_positions = getattr(config, "seq_length", 8192)
102 - rope_parameters = {"rope_type": "default", "rope_theta": 10000 * rope_ratio}
102 + rope_parameters = {
103 + "rope_type": "default",
104 + "rope_theta": 10000 * rope_ratio,
105 + "partial_rotary_factor": 0.5,
106 + }
107 # NOTE: zai-org/cogagent-9b-20241220 uses original_rope=False,
108 # which is equivalent to is_neox_style=True
109 is_neox_style = not config.original_rope
110 self.rotary_emb = get_rope(
111 self.head_dim,
108 - rotary_dim=self.head_dim // 2,
112 max_position=max_positions,
113 rope_parameters=rope_parameters,
114 is_neox_style=is_neox_style,
vllm/model_executor/models/commandr.py
-1
@@ -175,7 +175,6 @@ class CohereAttention(nn.Module):
175 )
176 self.rotary_emb = get_rope(
177 self.head_dim,
178 - rotary_dim=self.head_dim,
178 max_position=self.max_position_embeddings,
179 rope_parameters=config.rope_parameters,
180 is_neox_style=False,
vllm/model_executor/models/config.py
+7 -5
@@ -42,9 +42,10 @@ class GteNewModelConfig(VerifyAndUpdateConfig):
42 config.hidden_act = "geglu"
43
44 head_dim = config.hidden_size // config.num_attention_heads
45 + rotary_dim = getattr(config, "rotary_emb_dim", head_dim)
46 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
47 config.rotary_kwargs = {
48 "head_size": head_dim,
47 - "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
49 "max_position": config.max_position_embeddings,
50 "rope_parameters": config.rope_parameters,
51 }
@@ -77,9 +78,11 @@ class JinaRobertaModelConfig(VerifyAndUpdateConfig):
78 if not model_config.enforce_eager:
79 max_position = round_up(max_position, 8)
80
81 + rotary_dim = getattr(config, "rotary_emb_dim", head_dim)
82 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
83 +
84 config.rotary_kwargs = {
85 "head_size": head_dim,
82 - "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
86 "max_position": max_position,
87 "rope_parameters": config.rope_parameters,
88 }
@@ -113,12 +116,10 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
116 config.num_hidden_layers = config.n_layer
117
118 head_dim = config.hidden_size // config.num_attention_heads
116 - rotary_emb_dim = int(head_dim * config.rotary_emb_fraction)
119 max_trained_positions = getattr(config, "max_trained_positions", 2048)
120
121 config.rotary_kwargs = {
122 "head_size": head_dim,
121 - "rotary_dim": rotary_emb_dim,
123 "max_position": max_trained_positions,
124 "rope_parameters": config.rope_parameters,
125 }
@@ -240,9 +241,10 @@ class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
241 config.hidden_act = "geglu"
242
243 head_dim = config.hidden_size // config.num_attention_heads
244 + rotary_dim = getattr(config, "rotary_emb_dim", head_dim)
245 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
246 config.rotary_kwargs = {
247 "head_size": head_dim,
245 - "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
248 "max_position": config.max_position_embeddings,
249 "rope_parameters": config.rope_parameters,
250 }
vllm/model_executor/models/dbrx.py
-1
@@ -222,7 +222,6 @@ class DbrxAttention(nn.Module):
222 )
223 self.rotary_emb = get_rope(
224 self.head_dim,
225 - rotary_dim=self.head_dim,
225 max_position=self.max_position,
226 rope_parameters=rope_parameters,
227 is_neox_style=True,
vllm/model_executor/models/deepseek_v2.py
-4
@@ -156,7 +156,6 @@ class DeepseekAttention(nn.Module):
156
157 self.rotary_emb = get_rope(
158 self.head_dim,
159 - rotary_dim=self.head_dim,
159 max_position=max_position_embeddings,
160 rope_parameters=config.rope_parameters,
161 )
@@ -499,7 +498,6 @@ class DeepseekV2Attention(nn.Module):
498
499 self.rotary_emb = get_rope(
500 qk_rope_head_dim,
502 - rotary_dim=qk_rope_head_dim,
501 max_position=max_position_embeddings,
502 rope_parameters=config.rope_parameters,
503 is_neox_style=False,
@@ -1018,7 +1016,6 @@ class DeepseekV2MLAAttention(nn.Module):
1016
1017 self.rotary_emb = get_rope(
1018 qk_rope_head_dim,
1021 - rotary_dim=qk_rope_head_dim,
1019 max_position=max_position_embeddings,
1020 rope_parameters=config.rope_parameters,
1021 is_neox_style=False,
@@ -1038,7 +1035,6 @@ class DeepseekV2MLAAttention(nn.Module):
1035 if self.is_v32:
1036 self.indexer_rope_emb = get_rope(
1037 qk_rope_head_dim,
1041 - rotary_dim=qk_rope_head_dim,
1038 max_position=max_position_embeddings,
1039 rope_parameters=config.rope_parameters,
1040 is_neox_style=True,
vllm/model_executor/models/dots1.py
-1
@@ -250,7 +250,6 @@ class Dots1Attention(nn.Module):
250
251 self.rotary_emb = get_rope(
252 self.head_dim,
253 - rotary_dim=self.head_dim,
253 max_position=max_position_embeddings,
254 rope_parameters=config.rope_parameters,
255 )
vllm/model_executor/models/ernie45_moe.py
-1
@@ -288,7 +288,6 @@ class Ernie4_5_MoeAttention(nn.Module):
288
289 self.rotary_emb = get_rope(
290 self.head_dim,
291 - rotary_dim=self.head_dim,
291 max_position=max_position_embeddings,
292 rope_parameters=rope_parameters,
293 is_neox_style=False,
vllm/model_executor/models/exaone.py
-1
@@ -167,7 +167,6 @@ class ExaoneAttention(nn.Module):
167
168 self.rotary_emb = get_rope(
169 self.head_dim,
170 - rotary_dim=self.head_dim,
170 max_position=max_position_embeddings,
171 rope_parameters=config.rope_parameters,
172 is_neox_style=is_neox_style,
vllm/model_executor/models/exaone4.py
-1
@@ -176,7 +176,6 @@ class Exaone4Attention(nn.Module):
176 set_default_rope_theta(config, default_theta=1000000)
177 self.rotary_emb = get_rope(
178 self.head_dim,
179 - rotary_dim=self.head_dim,
179 max_position=max_position_embeddings,
180 rope_parameters=config.rope_parameters,
181 is_neox_style=is_neox_style,
vllm/model_executor/models/falcon.py
-1
@@ -167,7 +167,6 @@ class FalconAttention(nn.Module):
167 max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
168 self.rotary_emb = get_rope(
169 self.head_dim,
170 - rotary_dim=self.head_dim,
170 max_position=max_position_embeddings,
171 rope_parameters=config.rope_parameters,
172 )
vllm/model_executor/models/falcon_h1.py
+2 -5
@@ -242,14 +242,11 @@ class FalconH1AttentionDecoderLayer(nn.Module):
242 self.scaling = self.head_dim**-0.5
243 self.max_position_embeddings = max_position_embeddings
244
245 - if hasattr(config, "attn_rotary_emb"):
246 - rotary_dim = config.attn_rotary_emb # for backward compatibility
247 - else:
248 - rotary_dim = self.head_dim # default
245 + rotary_dim = getattr(config, "attn_rotary_emb", self.head_dim)
246 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / self.head_dim
247
248 self.rotary_emb = get_rope(
249 head_size=self.head_dim,
252 - rotary_dim=rotary_dim,
250 max_position=max_position_embeddings,
251 rope_parameters=config.rope_parameters,
252 is_neox_style=True,
vllm/model_executor/models/gemma.py
-1
@@ -174,7 +174,6 @@ class GemmaAttention(nn.Module):
174
175 self.rotary_emb = get_rope(
176 self.head_dim,
177 - rotary_dim=self.head_dim,
177 max_position=max_position_embeddings,
178 rope_parameters=rope_parameters,
179 is_neox_style=True,
vllm/model_executor/models/gemma2.py
-1
@@ -152,7 +152,6 @@ class Gemma2Attention(nn.Module):
152 )
153 self.rotary_emb = get_rope(
154 self.head_dim,
155 - rotary_dim=self.head_dim,
155 max_position=max_position_embeddings,
156 rope_parameters=config.rope_parameters,
157 is_neox_style=True,
vllm/model_executor/models/gemma3.py
-1
@@ -176,7 +176,6 @@ class Gemma3Attention(nn.Module):
176
177 self.rotary_emb = get_rope(
178 self.head_dim,
179 - rotary_dim=self.head_dim,
179 max_position=max_position_embeddings,
180 rope_parameters=rope_parameters,
181 is_neox_style=True,
vllm/model_executor/models/gemma3n.py
-1
@@ -384,7 +384,6 @@ class Gemma3nAttention(nn.Module):
384
385 self.rotary_emb = get_rope(
386 self.head_dim,
387 - rotary_dim=self.head_dim,
387 max_position=max_position_embeddings,
388 rope_parameters=rope_parameters,
389 is_neox_style=True,
vllm/model_executor/models/glm4.py
-2
@@ -81,7 +81,6 @@ class Glm4Attention(nn.Module):
81 config.rope_parameters.setdefault("partial_rotary_factor", 0.5)
82 self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
83 self.head_dim = head_dim or hidden_size // self.total_num_heads
84 - self.rotary_dim = self.head_dim
84 self.q_size = self.num_heads * self.head_dim
85 self.kv_size = self.num_kv_heads * self.head_dim
86 self.scaling = self.head_dim**-0.5
@@ -103,7 +102,6 @@ class Glm4Attention(nn.Module):
102 )
103 self.rotary_emb = get_rope(
104 self.head_dim,
106 - rotary_dim=self.rotary_dim,
105 max_position=max_position,
106 rope_parameters=config.rope_parameters,
107 is_neox_style=False,
vllm/model_executor/models/glm4_1v.py
+1 -1
@@ -678,9 +678,9 @@ class Glm4vVisionTransformer(nn.Module):
678 head_dim = self.hidden_size // self.num_heads
679 self.rotary_pos_emb = get_rope(
680 head_size=head_dim,
681 - rotary_dim=head_dim // 2,
681 max_position=8192,
682 is_neox_style=True,
683 + rope_parameters={"partial_rotary_factor": 0.5},
684 )
685 self.blocks = nn.ModuleList(
686 [
vllm/model_executor/models/glm4_moe.py
-1
@@ -285,7 +285,6 @@ class Glm4MoeAttention(nn.Module):
285 config.rope_parameters.setdefault("partial_rotary_factor", 0.5)
286 self.rotary_emb = get_rope(
287 self.head_dim,
288 - rotary_dim=self.head_dim,
288 max_position=max_position_embeddings,
289 rope_parameters=config.rope_parameters,
290 )
vllm/model_executor/models/gpt_j.py
+3 -2
@@ -95,12 +95,13 @@ class GPTJAttention(nn.Module):
95 scaling = self.head_size**-0.5
96 assert getattr(config, "rotary", True)
97 assert config.rotary_dim % 2 == 0
98 + rope_parameters = getattr(config, "rope_parameters", {})
99 + rope_parameters["partial_rotary_factor"] = config.rotary_dim / self.head_size
100 max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
101 self.rotary_emb = get_rope(
102 self.head_size,
101 - rotary_dim=config.rotary_dim,
103 max_position=max_position_embeddings,
103 - rope_parameters=getattr(config, "rope_parameters", None),
104 + rope_parameters=rope_parameters,
105 is_neox_style=False,
106 )
107 self.attn = Attention(
vllm/model_executor/models/gpt_neox.py
-1
@@ -92,7 +92,6 @@ class GPTNeoXAttention(nn.Module):
92 max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
93 self.rotary_emb = get_rope(
94 self.head_size,
95 - rotary_dim=self.head_size,
95 max_position=max_position_embeddings,
96 rope_parameters=config.rope_parameters,
97 )
vllm/model_executor/models/gpt_oss.py
-1
@@ -67,7 +67,6 @@ class OAIAttention(nn.Module):
67
68 self.rotary_emb = get_rope(
69 self.head_dim,
70 - rotary_dim=self.head_dim,
70 max_position=config.max_position_embeddings,
71 dtype=torch.float32,
72 rope_parameters={
vllm/model_executor/models/granite.py
-1
@@ -160,7 +160,6 @@ class GraniteAttention(nn.Module):
160
161 self.rotary_emb = get_rope(
162 self.head_dim,
163 - rotary_dim=self.head_dim,
163 max_position=max_position_embeddings,
164 rope_parameters=config.rope_parameters,
165 )
vllm/model_executor/models/granitemoe.py
-1
@@ -190,7 +190,6 @@ class GraniteMoeAttention(nn.Module):
190 )
191 self.rotary_emb = get_rope(
192 self.head_dim,
193 - rotary_dim=self.head_dim,
193 max_position=max_position,
194 rope_parameters=rope_parameters,
195 is_neox_style=True,
vllm/model_executor/models/granitemoehybrid.py
-1
@@ -271,7 +271,6 @@ class GraniteMoeHybridAttention(nn.Module):
271 if config.position_embedding_type == "rope":
272 self.rotary_emb = get_rope(
273 self.head_dim,
274 - rotary_dim=self.head_dim,
274 max_position=config.max_position_embeddings,
275 rope_parameters=config.rope_parameters,
276 is_neox_style=True,
vllm/model_executor/models/grok1.py
-1
@@ -181,7 +181,6 @@ class Grok1Attention(nn.Module):
181 )
182 self.rotary_emb = get_rope(
183 self.head_dim,
184 - rotary_dim=self.head_dim,
184 max_position=max_position,
185 rope_parameters=rope_parameters,
186 is_neox_style=True,
vllm/model_executor/models/hunyuan_v1.py
-2
@@ -199,7 +199,6 @@ class HunYuanAttention(nn.Module):
199
200 self.rotary_emb = get_rope(
201 self.head_dim,
202 - rotary_dim=self.head_dim,
202 max_position=max_position_embeddings,
203 rope_parameters=config.rope_parameters,
204 is_neox_style=True,
@@ -305,7 +304,6 @@ class HunYuanCrossAttention(nn.Module):
304
305 self.rotary_emb = get_rope(
306 self.head_dim,
308 - rotary_dim=self.head_dim,
307 max_position=max_position_embeddings,
308 rope_parameters=config.rope_parameters,
309 is_neox_style=True,
vllm/model_executor/models/internlm2.py
-1
@@ -140,7 +140,6 @@ class InternLM2Attention(nn.Module):
140
141 self.rotary_emb = get_rope(
142 self.head_dim,
143 - rotary_dim=self.head_dim,
143 max_position=max_position_embeddings,
144 rope_parameters=rope_parameters,
145 )
vllm/model_executor/models/lfm2.py
-1
@@ -143,7 +143,6 @@ class Lfm2Attention(nn.Module):
143 )
144 self.rotary_emb = get_rope(
145 self.head_dim,
146 - rotary_dim=self.head_dim,
146 max_position=self.max_position_embeddings,
147 rope_parameters=config.rope_parameters,
148 is_neox_style=True,
vllm/model_executor/models/lfm2_moe.py
-1
@@ -236,7 +236,6 @@ class Lfm2MoeAttention(nn.Module):
236 )
237 self.rotary_emb = get_rope(
238 self.head_dim,
239 - rotary_dim=self.head_dim,
239 max_position=self.max_position_embeddings,
240 rope_parameters=config.rope_parameters,
241 is_neox_style=True,
vllm/model_executor/models/llama.py
-1
@@ -259,7 +259,6 @@ class LlamaAttention(nn.Module):
259
260 self.rotary_emb = get_rope(
261 self.head_dim,
262 - rotary_dim=self.head_dim,
262 max_position=self.max_position_embeddings,
263 rope_parameters=getattr(config, "rope_parameters", None),
264 is_neox_style=is_neox_style,
vllm/model_executor/models/llama4.py
-1
@@ -243,7 +243,6 @@ class Llama4Attention(nn.Module):
243 self.rotary_emb = (
244 get_rope(
245 self.head_dim,
246 - rotary_dim=self.head_dim,
246 max_position=max_position_embeddings,
247 rope_parameters=config.rope_parameters,
248 is_neox_style=is_neox_style,
vllm/model_executor/models/minicpm.py
-1
@@ -277,7 +277,6 @@ class MiniCPMAttention(nn.Module):
277
278 self.rotary_emb = get_rope(
279 self.head_dim,
280 - rotary_dim=self.head_dim,
280 max_position=max_position_embeddings,
281 rope_parameters=rope_parameters,
282 )
vllm/model_executor/models/minicpm3.py
-1
@@ -120,7 +120,6 @@ class MiniCPM3Attention(nn.Module):
120
121 self.rotary_emb = get_rope(
122 self.qk_rope_head_dim,
123 - rotary_dim=self.qk_rope_head_dim,
123 max_position=max_position_embeddings,
124 rope_parameters=config.rope_parameters,
125 )
vllm/model_executor/models/minimax_m2.py
+5 -1
@@ -199,9 +199,13 @@ class MiniMaxM2Attention(nn.Module):
199 prefix=f"{prefix}.o_proj",
200 )
201
202 + if (
203 + rope_parameters is not None
204 + and "partial_rotary_factor" not in rope_parameters
205 + ):
206 + rope_parameters["partial_rotary_factor"] = rotary_dim / self.head_dim
207 self.rotary_emb = get_rope(
208 self.head_dim,
204 - rotary_dim=self.head_dim,
209 max_position=max_position_embeddings,
210 rope_parameters=rope_parameters,
211 )
vllm/model_executor/models/minimax_text_01.py
+2 -5
@@ -187,7 +187,6 @@ class MiniMaxText01Attention(nn.Module):
187 num_heads: int,
188 head_dim: int,
189 num_kv_heads: int,
190 - rotary_dim: int,
190 max_position: int = 4096 * 32,
191 rope_parameters: dict | None = None,
192 sliding_window: int | None = None,
@@ -245,7 +244,6 @@ class MiniMaxText01Attention(nn.Module):
244 )
245 self.rotary_emb = get_rope(
246 head_size=self.head_dim,
248 - rotary_dim=rotary_dim,
247 max_position=max_position,
248 rope_parameters=rope_parameters,
249 is_neox_style=True,
@@ -290,6 +288,8 @@ class MiniMaxText01DecoderLayer(nn.Module):
288 head_dim = getattr(config, "head_dim", None)
289 if head_dim is None:
290 head_dim = config.hidden_size // config.num_attention_heads
291 + rotary_dim = getattr(config, "rotary_dim", head_dim)
292 + config.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
293 if hasattr(config, "max_model_len") and isinstance(config.max_model_len, int):
294 max_position_embeddings = min(
295 config.max_position_embeddings, config.max_model_len
@@ -321,9 +321,6 @@ class MiniMaxText01DecoderLayer(nn.Module):
321 hidden_size=self.hidden_size,
322 num_heads=config.num_attention_heads,
323 head_dim=head_dim,
324 - rotary_dim=config.rotary_dim
325 - if hasattr(config, "rotary_dim")
326 - else head_dim,
324 num_kv_heads=config.num_key_value_heads,
325 max_position=max_position_embeddings,
326 rope_parameters=config.rope_parameters,
vllm/model_executor/models/mixtral.py
-1
@@ -206,7 +206,6 @@ class MixtralAttention(nn.Module):
206 )
207 self.rotary_emb = get_rope(
208 self.head_dim,
209 - rotary_dim=self.head_dim,
209 max_position=max_position,
210 rope_parameters=config.rope_parameters,
211 is_neox_style=True,
vllm/model_executor/models/mllama4.py
+1 -1
@@ -295,11 +295,11 @@ class Llama4VisionAttention(nn.Module):
295 rope_parameters = {
296 "rope_type": "mllama4",
297 "rope_theta": config.rope_parameters["rope_theta"],
298 + "partial_rotary_factor": 0.5,
299 }
300
301 self.rotary_emb = get_rope(
302 head_size=self.head_dim,
302 - rotary_dim=config.hidden_size // config.num_attention_heads // 2,
303 # number of image patches
304 max_position=(config.image_size // config.patch_size) ** 2,
305 rope_parameters=rope_parameters,
vllm/model_executor/models/modernbert.py
-1
@@ -105,7 +105,6 @@ class ModernBertAttention(nn.Module):
105
106 self.rotary_emb = get_rope(
107 head_size=self.head_dim,
108 - rotary_dim=self.head_dim,
108 max_position=config.max_position_embeddings,
109 rope_parameters=rope_parameters,
110 dtype=torch.float16,
vllm/model_executor/models/molmo.py
-1
@@ -433,7 +433,6 @@ class MolmoAttention(nn.Module):
433 # Rotary embeddings.
434 self.rotary_emb = get_rope(
435 self.head_dim,
436 - rotary_dim=self.head_dim,
436 max_position=self.max_position_embeddings,
437 rope_parameters=config.rope_parameters,
438 )
vllm/model_executor/models/nemotron.py
-1
@@ -199,7 +199,6 @@ class NemotronAttention(nn.Module):
199
200 self.rotary_emb = get_rope(
201 self.head_dim,
202 - rotary_dim=self.head_dim,
202 max_position=max_position_embeddings,
203 rope_parameters=config.rope_parameters,
204 )
vllm/model_executor/models/nemotron_nas.py
-1
@@ -118,7 +118,6 @@ class DeciLMAttention(LlamaAttention):
118
119 self.rotary_emb = get_rope(
120 self.head_dim,
121 - rotary_dim=self.head_dim,
121 max_position=self.max_position_embeddings,
122 rope_parameters=config.rope_parameters,
123 is_neox_style=is_neox_style,
vllm/model_executor/models/olmo.py
-1
@@ -102,7 +102,6 @@ class OlmoAttention(nn.Module):
102 # Rotary embeddings.
103 self.rotary_emb = get_rope(
104 self.head_dim,
105 - rotary_dim=self.head_dim,
105 max_position=self.max_position_embeddings,
106 rope_parameters=config.rope_parameters,
107 )
vllm/model_executor/models/olmo2.py
-1
@@ -146,7 +146,6 @@ class Olmo2Attention(nn.Module):
146 rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
147 self.rotary_emb = get_rope(
148 self.head_dim,
149 - rotary_dim=self.head_dim,
149 max_position=self.max_position_embeddings,
150 rope_parameters=rope_parameters,
151 )
vllm/model_executor/models/olmoe.py
-1
@@ -171,7 +171,6 @@ class OlmoeAttention(nn.Module):
171
172 self.rotary_emb = get_rope(
173 self.head_dim,
174 - rotary_dim=self.head_dim,
174 max_position=max_position_embeddings,
175 rope_parameters=config.rope_parameters,
176 is_neox_style=True,
vllm/model_executor/models/openpangu.py
-2
@@ -352,7 +352,6 @@ class OpenPanguMLAAttention(nn.Module):
352 }
353 self.rotary_emb = get_rope(
354 qk_rope_head_dim,
355 - rotary_dim=qk_rope_head_dim,
355 max_position=max_position_embeddings,
356 rope_parameters=rope_parameters,
357 is_neox_style=False,
@@ -525,7 +524,6 @@ class OpenPanguEmbeddedAttention(nn.Module):
524
525 self.rotary_emb = get_rope(
526 self.head_dim,
528 - rotary_dim=self.head_dim,
527 max_position=self.max_position_embeddings,
528 rope_parameters=config.rope_parameters,
529 is_neox_style=is_neox_style,
vllm/model_executor/models/orion.py
-1
@@ -135,7 +135,6 @@ class OrionAttention(nn.Module):
135
136 self.rotary_emb = get_rope(
137 self.head_dim,
138 - rotary_dim=self.head_dim,
138 max_position=max_position_embeddings,
139 rope_parameters=rope_parameters,
140 )
vllm/model_executor/models/ouro.py
-1
@@ -166,7 +166,6 @@ class OuroAttention(nn.Module):
166
167 self.rotary_emb = get_rope(
168 self.head_dim,
169 - rotary_dim=self.head_dim,
169 max_position=max_position,
170 rope_parameters=config.rope_parameters,
171 dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/persimmon.py
-1
@@ -134,7 +134,6 @@ class PersimmonAttention(nn.Module):
134
135 self.rotary_emb = get_rope(
136 self.head_dim,
137 - rotary_dim=self.head_dim,
137 max_position=self.max_position_embeddings,
138 rope_parameters=config.rope_parameters,
139 )
vllm/model_executor/models/phi.py
+4 -8
@@ -84,19 +84,18 @@ class PhiAttention(nn.Module):
84 prefix: str = "",
85 ):
86 super().__init__()
87 - self.total_num_heads = config.num_attention_heads
87 self.hidden_size = config.hidden_size
89 - self.head_size = self.hidden_size // self.total_num_heads
88 + self.head_size = self.hidden_size // config.num_attention_heads
89
90 tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
92 - assert self.total_num_heads % tensor_model_parallel_world_size == 0
93 - self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
91 + assert config.num_attention_heads % tensor_model_parallel_world_size == 0
92 + self.num_heads = config.num_attention_heads // tensor_model_parallel_world_size
93
94 # pylint: disable=C0103
95 self.qkv_proj = QKVParallelLinear(
96 self.hidden_size,
97 self.head_size,
99 - self.total_num_heads,
98 + config.num_attention_heads,
99 bias=True,
100 quant_config=quant_config,
101 prefix=f"{prefix}.qkv_proj",
@@ -109,13 +108,10 @@ class PhiAttention(nn.Module):
108 )
109
110 scaling = self.head_size**-0.5
112 - rotary_dim = config.hidden_size // config.num_attention_heads
113 - assert rotary_dim % 2 == 0
111
112 max_position_embeddings = getattr(config, "max_position_embeddings", 2048)
113 self.rotary_emb = get_rope(
114 self.head_size,
118 - rotary_dim=rotary_dim,
115 max_position=max_position_embeddings,
116 rope_parameters=config.rope_parameters,
117 )
vllm/model_executor/models/phimoe.py
-1
@@ -352,7 +352,6 @@ class PhiMoEAttention(nn.Module):
352 )
353 self.rotary_emb = get_rope(
354 self.head_dim,
355 - rotary_dim=self.head_dim,
355 max_position=max_position,
356 rope_parameters=rope_parameters,
357 is_neox_style=True,
vllm/model_executor/models/plamo2.py
-1
@@ -574,7 +574,6 @@ class Plamo2AttentionMixer(nn.Module):
574
575 self.rotary_emb = get_rope(
576 self.head_dim,
577 - rotary_dim=self.head_dim,
577 max_position=max_position,
578 rope_parameters=config.rope_parameters,
579 )
vllm/model_executor/models/plamo3.py
-1
@@ -179,7 +179,6 @@ class Plamo3AttentionMixer(nn.Module):
179
180 self.rotary_emb = get_rope(
181 self.head_dim,
182 - rotary_dim=self.head_dim,
182 max_position=max_position,
183 rope_parameters=rope_parameters,
184 )
vllm/model_executor/models/qwen.py
-1
@@ -114,7 +114,6 @@ class QWenAttention(nn.Module):
114
115 self.rotary_emb = get_rope(
116 self.head_dim,
117 - rotary_dim=self.head_dim,
117 max_position=max_position_embeddings,
118 rope_parameters=rope_parameters,
119 )
vllm/model_executor/models/qwen2.py
-1
@@ -164,7 +164,6 @@ class Qwen2Attention(nn.Module):
164
165 self.rotary_emb = get_rope(
166 self.head_dim,
167 - rotary_dim=self.head_dim,
167 max_position=max_position,
168 rope_parameters=rope_parameters,
169 dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/qwen2_5_vl.py
+1 -1
@@ -624,9 +624,9 @@ class Qwen2_5_VisionTransformer(nn.Module):
624 head_dim = self.hidden_size // self.num_heads
625 self.rotary_pos_emb = get_rope(
626 head_size=head_dim,
627 - rotary_dim=head_dim // 2,
627 max_position=8192,
628 is_neox_style=True,
629 + rope_parameters={"partial_rotary_factor": 0.5},
630 )
631
632 self.attn_backend = get_vit_attn_backend(
vllm/model_executor/models/qwen2_moe.py
-1
@@ -244,7 +244,6 @@ class Qwen2MoeAttention(nn.Module):
244
245 self.rotary_emb = get_rope(
246 self.head_dim,
247 - rotary_dim=self.head_dim,
247 max_position=max_position_embeddings,
248 rope_parameters=rope_parameters,
249 dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/qwen2_vl.py
+1 -1
@@ -621,9 +621,9 @@ class Qwen2VisionTransformer(nn.Module):
621 head_dim = embed_dim // num_heads
622 self.rotary_pos_emb = get_rope(
623 head_size=head_dim,
624 - rotary_dim=head_dim // 2,
624 max_position=8192,
625 is_neox_style=True,
626 + rope_parameters={"partial_rotary_factor": 0.5},
627 )
628
629 self.blocks = nn.ModuleList(
vllm/model_executor/models/qwen3.py
-1
@@ -111,7 +111,6 @@ class Qwen3Attention(nn.Module):
111
112 self.rotary_emb = get_rope(
113 self.head_dim,
114 - rotary_dim=self.head_dim,
114 max_position=max_position,
115 rope_parameters=rope_parameters,
116 dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/qwen3_moe.py
-1
@@ -269,7 +269,6 @@ class Qwen3MoeAttention(nn.Module):
269
270 self.rotary_emb = get_rope(
271 self.head_dim,
272 - rotary_dim=self.head_dim,
272 max_position=max_position_embeddings,
273 rope_parameters=rope_parameters,
274 dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/qwen3_next.py
-1
@@ -747,7 +747,6 @@ class Qwen3NextAttention(nn.Module):
747
748 self.rotary_emb = get_rope(
749 head_size=self.head_dim,
750 - rotary_dim=self.head_dim,
750 max_position=config.max_position_embeddings,
751 rope_parameters=config.rope_parameters,
752 dual_chunk_attention_config=self.dual_chunk_attention_config,
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+1 -1
@@ -333,9 +333,9 @@ class Qwen3Omni_VisionTransformer(nn.Module):
333 head_dim = self.hidden_size // self.num_heads
334 self.rotary_pos_emb = get_rope(
335 head_size=head_dim,
336 - rotary_dim=head_dim // 2,
336 max_position=8192,
337 is_neox_style=True,
338 + rope_parameters={"partial_rotary_factor": 0.5},
339 )
340
341 self.blocks = nn.ModuleList(
vllm/model_executor/models/qwen3_vl.py
+1 -1
@@ -340,9 +340,9 @@ class Qwen3_VisionTransformer(nn.Module):
340 head_dim = self.hidden_size // self.num_heads
341 self.rotary_pos_emb = get_rope(
342 head_size=head_dim,
343 - rotary_dim=head_dim // 2,
343 max_position=8192,
344 is_neox_style=True,
345 + rope_parameters={"partial_rotary_factor": 0.5},
346 )
347
348 self.merger = Qwen3_VisionPatchMerger(
vllm/model_executor/models/seed_oss.py
-1
@@ -161,7 +161,6 @@ class SeedOssAttention(nn.Module):
161
162 self.rotary_emb = get_rope(
163 self.head_dim,
164 - rotary_dim=self.head_dim,
164 max_position=max_position,
165 rope_parameters=rope_parameters,
166 )
vllm/model_executor/models/solar.py
-1
@@ -160,7 +160,6 @@ class SolarAttention(nn.Module):
160
161 self.rotary_emb = get_rope(
162 self.head_dim,
163 - rotary_dim=self.head_dim,
163 max_position=max_position_embeddings,
164 rope_parameters=config.rope_parameters,
165 )
vllm/model_executor/models/stablelm.py
-1
@@ -148,7 +148,6 @@ class StablelmAttention(nn.Module):
148 )
149 self.rotary_emb = get_rope(
150 self.head_dim,
151 - rotary_dim=self.head_dim,
151 max_position=self.config.max_position_embeddings,
152 rope_parameters=self.config.rope_parameters,
153 )
vllm/model_executor/models/starcoder2.py
-1
@@ -112,7 +112,6 @@ class Starcoder2Attention(nn.Module):
112 )
113 self.rotary_emb = get_rope(
114 self.head_dim,
115 - rotary_dim=self.head_dim,
115 max_position=self.max_position_embeddings,
116 rope_parameters=config.rope_parameters,
117 is_neox_style=True,
vllm/model_executor/models/step3_text.py
-1
@@ -196,7 +196,6 @@ class Step3TextAttention(nn.Module):
196 )
197 self.rotary_emb = get_rope(
198 self.head_dim,
199 - rotary_dim=self.head_dim,
199 max_position=max_position_embedding,
200 rope_parameters=rope_parameters,
201 )
vllm/model_executor/models/zamba2.py
-1
@@ -230,7 +230,6 @@ class Zamba2Attention(nn.Module):
230 if config.use_mem_rope:
231 self.rotary_emb = get_rope(
232 head_size=self.attention_head_dim,
233 - rotary_dim=self.attention_head_dim,
233 max_position=config.max_position_embeddings,
234 rope_parameters=config.rope_parameters,
235 is_neox_style=True,
vllm/transformers_utils/config.py
+13 -4
@@ -306,8 +306,13 @@ def patch_rope_parameters(config: PretrainedConfig) -> None:
306 """Provide backwards compatibility for RoPE."""
307 from vllm.config.utils import getattr_iter
308
309 - rope_theta_names = ("rope_theta", "rotary_emb_base")
310 - rope_theta = getattr_iter(config, rope_theta_names, None)
309 + # Older custom models may use non-standard field names
310 + # which need patching for both Transformers v4 and v5.
311 + names = ["rope_theta", "rotary_emb_base"]
312 + rope_theta = getattr_iter(config, names, None, warn=True)
313 + names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"]
314 + partial_rotary_factor = getattr_iter(config, names, None, warn=True)
315 +
316 if Version(version("transformers")) < Version("5.0.0.dev0"):
317 # Transformers v4 installed, legacy config fields may be present
318 if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
@@ -316,14 +321,18 @@ def patch_rope_parameters(config: PretrainedConfig) -> None:
321 if not hasattr(config, "rope_parameters"):
322 config.rope_parameters = {"rope_type": "default"}
323 config.rope_parameters["rope_theta"] = rope_theta
319 - partial_rotary_factor_names = ("partial_rotary_factor", "rotary_pct")
320 - partial_rotary_factor = getattr_iter(config, partial_rotary_factor_names, None)
324 if partial_rotary_factor is not None:
325 if not hasattr(config, "rope_parameters"):
326 config.rope_parameters = {"rope_type": "default"}
327 config.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
328 elif rope_theta is not None or hasattr(config, "rope_parameters"):
329 # Transformers v5 installed
330 + # Patch these fields in case they used non-standard names
331 + if rope_theta is not None:
332 + config.rope_theta = rope_theta
333 + if partial_rotary_factor is not None:
334 + config.partial_rotary_factor = partial_rotary_factor
335 + # Standardize and validate RoPE parameters
336 config.standardize_rope_params()
337 config.validate_rope()
338