A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 22:15 UTC
Update `rope_scaling` to `rope_parameters` in preparation for Transformers v5 (#28542)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Nov 19, 2025 at 18:06 UTC
a8b70304d68497ac1c432a2ff343e9bfb516c227
104 files changed
+544
-912
.buildkite/test-pipeline.yaml
+3
-3
@@ -872,12 +872,12 @@ steps:
872
optional: true
873
commands:
874
- pip install --upgrade git+https://github.com/huggingface/transformers
875
- - pytest -v -s tests/models/test_initialization.py -k 'not (Gemma3 or ModernBert or Qwen2_5_VL or Qwen2_5vl or Qwen2VL or TransformersMultiModalEmbeddingModel or TransformersMultiModalForSequenceClassification or Ultravox or Phi4Multimodal or LlavaNextVideo or MiniCPMO or Lfm2Moe or PaliGemma or RobertaForSequenceClassification or Ovis2_5 or Fuyu or DeepseekOCR or KimiVL)'
875
+ - pytest -v -s tests/models/test_initialization.py -k 'not (Ultravox or Phi4Multimodal or MiniCPMO or Lfm2Moe or RobertaForSequenceClassification or Ovis2_5 or DeepseekOCR or KimiVL)'
876
- pytest -v -s tests/models/test_transformers.py
877
# - pytest -v -s tests/models/multimodal/processing/
878
- - pytest -v -s tests/models/multimodal/test_mapping.py -k 'not (Gemma3 or Qwen2VL or Qwen2_5_VL)'
878
+ - pytest -v -s tests/models/multimodal/test_mapping.py
879
- python3 examples/offline_inference/basic/chat.py
880
- # - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
880
+ - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
881
# Whisper needs spawn method to avoid deadlock
882
- VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
883
benchmarks/kernels/benchmark_mrope.py
+7
-12
@@ -6,7 +6,7 @@
6
#
7
# The CSV file (named with current date/time) contains these columns:
8
# model_name, tp_size, num_tokens, num_heads, num_kv_heads, head_dim, max_position,
9
-# rope_theta, is_neox_style, rope_scaling, dtype, torch_mean, torch_median, torch_p99,
9
+# is_neox_style, rope_parameters, dtype, torch_mean, torch_median, torch_p99,
10
# torch_min, torch_max, triton_mean, triton_median, triton_p99, triton_min, triton_max,
11
# speedup
12
#
@@ -86,9 +86,8 @@ def benchmark_mrope(
86
num_heads: int,
87
num_kv_heads: int,
88
max_position: int = 8192,
89
- rope_theta: float = 10000,
89
is_neox_style: bool = True,
91
- rope_scaling: dict[str, Any] = None,
90
+ rope_parameters: dict[str, Any] | None = None,
91
dtype: torch.dtype = torch.bfloat16,
92
seed: int = 0,
93
warmup_iter: int = 10,
@@ -102,9 +101,8 @@ def benchmark_mrope(
101
head_size=head_dim,
102
rotary_dim=head_dim,
103
max_position=max_position,
105
- base=rope_theta,
104
is_neox_style=is_neox_style,
107
- rope_scaling=rope_scaling,
105
+ rope_parameters=rope_parameters,
106
dtype=dtype,
107
).to(device=device)
108
@@ -203,9 +201,8 @@ def benchmark_mrope(
201
num_kv_heads,
202
head_dim,
203
max_position,
206
- rope_theta,
204
is_neox_style,
208
- str(rope_scaling),
205
+ str(rope_parameters),
206
str(dtype).split(".")[-1],
207
torch_stats["mean"],
208
torch_stats["median"],
@@ -255,9 +252,8 @@ if __name__ == "__main__":
252
"num_kv_heads",
253
"head_dim",
254
"max_position",
258
- "rope_theta",
255
"is_neox_style",
260
- "rope_scaling",
256
+ "rope_parameters",
257
"dtype",
258
"torch_mean",
259
"torch_median",
@@ -303,7 +299,7 @@ if __name__ == "__main__":
299
q_size = num_heads * head_dim
300
kv_size = num_kv_heads * head_dim
301
is_neox_style = True
306
- rope_theta = config.rope_theta
302
+ rope_parameters = config.rope_parameters
303
max_position = config.max_position_embeddings
304
305
for num_tokens in num_tokens_list:
@@ -315,9 +311,8 @@ if __name__ == "__main__":
311
num_heads=num_heads,
312
num_kv_heads=num_kv_heads,
313
max_position=max_position,
318
- rope_theta=rope_theta,
314
is_neox_style=is_neox_style,
320
- rope_scaling=config.rope_scaling,
315
+ rope_parameters=rope_parameters,
316
dtype=getattr(torch, args.dtype),
317
seed=args.seed,
318
warmup_iter=args.warmup_iter,
examples/offline_inference/context_extension.py
+3
-3
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""
4
This script demonstrates how to extend the context length
5
-of a Qwen model using the YARN method (rope_scaling)
5
+of a Qwen model using the YARN method (rope_parameters)
6
and run a simple chat example.
7
8
Usage:
@@ -19,8 +19,8 @@ def create_llm():
19
20
# Use yarn to extend context
21
hf_overrides = {
22
- "rope_theta": rope_theta,
23
- "rope_scaling": {
22
+ "rope_parameters": {
23
+ "rope_theta": rope_theta,
24
"rope_type": "yarn",
25
"factor": factor,
26
"original_max_position_embeddings": original_max_position_embeddings,
tests/compile/test_functionalization.py
+2
-2
@@ -137,7 +137,7 @@ class TestRotaryEmbedding(torch.nn.Module):
137
self.head_dim,
138
rotary_dim=self.rotary_dim,
139
max_position=max_position,
140
- base=base,
140
+ rope_parameters={"rope_type": "default", "rope_theta": base},
141
)
142
143
def forward(self, positions, q, k):
@@ -172,7 +172,7 @@ class TestRotaryEmbeddingSliceScatter(torch.nn.Module):
172
self.head_dim,
173
rotary_dim=self.head_dim,
174
max_position=max_position,
175
- base=base,
175
+ rope_parameters={"rope_type": "default", "rope_theta": base},
176
)
177
178
def forward(self, positions, hidden_states):
tests/kernels/core/test_mrope.py
+5
-11
@@ -5,11 +5,11 @@ from typing import NamedTuple
5
import pytest
6
import torch
7
from packaging.version import Version
8
-from transformers import AutoConfig
8
from transformers import __version__ as TRANSFORMERS_VERSION
9
10
from vllm.model_executor.layers.rotary_embedding import get_rope
11
from vllm.platforms import current_platform
12
+from vllm.transformers_utils.config import get_config
13
14
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
15
@@ -98,8 +98,7 @@ def test_mrope(
98
atol = model_info.atol
99
rtol = model_info.rtol
100
101
- config = AutoConfig.from_pretrained(model_name)
102
- config = config.get_text_config()
101
+ config = get_config(model_name, False).get_text_config()
102
103
# get the model config
104
total_num_kv_heads = config.num_key_value_heads
@@ -113,7 +112,6 @@ def test_mrope(
112
)
113
is_neox_style = True
114
116
- rope_theta = config.rope_theta
115
max_position = config.max_position_embeddings
116
partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
117
rotary_dim = int(head_dim * partial_rotary_factor)
@@ -122,9 +120,8 @@ def test_mrope(
120
head_size=head_dim,
121
rotary_dim=rotary_dim,
122
max_position=max_position,
125
- base=rope_theta,
123
is_neox_style=is_neox_style,
127
- rope_scaling=config.rope_scaling,
124
+ rope_parameters=config.rope_parameters,
125
dtype=dtype,
126
).to(device=device)
127
@@ -173,8 +170,7 @@ def test_mrope_torch_compile_tracing(
170
atol = model_info.atol
171
rtol = model_info.rtol
172
176
- config = AutoConfig.from_pretrained(model_name)
177
- config = config.get_text_config()
173
+ config = get_config(model_name, False).get_text_config()
174
175
# get the model config
176
total_num_kv_heads = config.num_key_value_heads
@@ -187,7 +183,6 @@ def test_mrope_torch_compile_tracing(
183
else config.hidden_size // total_num_heads
184
)
185
is_neox_style = True
190
- rope_theta = config.rope_theta
186
max_position = config.max_position_embeddings
187
partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
188
rotary_dim = int(head_dim * partial_rotary_factor)
@@ -196,9 +191,8 @@ def test_mrope_torch_compile_tracing(
191
head_size=head_dim,
192
rotary_dim=rotary_dim,
193
max_position=max_position,
199
- base=rope_theta,
194
is_neox_style=is_neox_style,
201
- rope_scaling=config.rope_scaling,
195
+ rope_parameters=config.rope_parameters,
196
dtype=dtype,
197
).to(device=device)
198
tests/kernels/core/test_pos_encoding.py
+20
-19
@@ -74,7 +74,7 @@ def test_rotary_embedding(
74
device: str,
75
use_key: bool,
76
max_position: int = 8192,
77
- base: float = 10000,
77
+ rope_theta: float = 10000,
78
) -> None:
79
if rotary_dim is None:
80
rotary_dim = head_size
@@ -83,7 +83,8 @@ def test_rotary_embedding(
83
torch.set_default_device(device)
84
if rotary_dim is None:
85
rotary_dim = head_size
86
- rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
86
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
87
+ rope = get_rope(head_size, rotary_dim, max_position, is_neox_style, rope_parameters)
88
rope = rope.to(dtype=dtype, device=torch.get_default_device())
89
90
positions = torch.randint(0, max_position, (batch_size, seq_len))
@@ -120,9 +121,9 @@ def test_rotary_embedding(
121
@torch.inference_mode()
122
def test_rope_module_cache():
123
MAX_POSITIONS = [123, 1234]
123
- BASES = [10000, 1000000]
124
- ROPE_SCALINGS = (
125
- None,
124
+ ROPE_THETAS = [10000, 1000000]
125
+ ROPE_PARAMETERS = (
126
+ {"rope_type": "default"},
127
{"rope_type": "linear", "factor": (1,)},
128
{"rope_type": "dynamic", "factor": 1},
129
)
@@ -130,9 +131,9 @@ def test_rope_module_cache():
131
HEAD_SIZES,
132
ROTARY_DIMS,
133
MAX_POSITIONS,
133
- BASES,
134
+ ROPE_THETAS,
135
IS_NEOX_STYLE,
135
- ROPE_SCALINGS,
136
+ ROPE_PARAMETERS,
137
DTYPES,
138
)
139
rope_setting_id_map: dict[str, int] = {}
@@ -141,20 +142,20 @@ def test_rope_module_cache():
142
head_size,
143
rotary_dim,
144
max_position,
144
- base,
145
- is_neox_stype,
146
- rope_scaling,
145
+ rope_theta,
146
+ is_neox_style,
147
+ rope_parameters,
148
dtype,
149
) = setting
150
if rotary_dim is None:
151
rotary_dim = head_size
152
+ rope_parameters["rope_theta"] = rope_theta
153
rope = get_rope(
154
head_size,
155
rotary_dim,
156
max_position,
155
- base,
156
- is_neox_stype,
157
- rope_scaling,
157
+ is_neox_style,
158
+ rope_parameters,
159
dtype,
160
)
161
# different settings cannot share the same rope module
@@ -168,20 +169,20 @@ def test_rope_module_cache():
169
head_size,
170
rotary_dim,
171
max_position,
171
- base,
172
- is_neox_stype,
173
- rope_scaling,
172
+ rope_theta,
173
+ is_neox_style,
174
+ rope_parameters,
175
dtype,
176
) = setting
177
if rotary_dim is None:
178
rotary_dim = head_size
179
+ rope_parameters["rope_theta"] = rope_theta
180
rope = get_rope(
181
head_size,
182
rotary_dim,
183
max_position,
182
- base,
183
- is_neox_stype,
184
- rope_scaling,
184
+ is_neox_style,
185
+ rope_parameters,
186
dtype,
187
)
188
# check if cache take effect
tests/kernels/moe/test_gpt_oss_triton_kernels.py
+1
-1
@@ -201,7 +201,7 @@ class ModelConfig:
201
sliding_window: int = 128
202
initial_context_length: int = 4096
203
rope_theta: float = 150000.0
204
- rope_scaling_factor: float = 32.0
204
+ rope_parameters_factor: float = 32.0
205
rope_ntk_alpha: float = 1.0
206
rope_ntk_beta: float = 32.0
207
tests/models/language/pooling/test_nomic_max_model_len.py
+9
-7
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
# ruff: noqa: SIM117
4
+from typing import Any
5
+
6
import pytest
7
8
from ...utils import EmbedModelInfo
@@ -79,8 +81,8 @@ def test_set_max_model_len_illegal(model_info, vllm_runner):
81
@pytest.mark.parametrize("model_info", MODELS)
82
def test_use_rope_scaling_legal(model_info, vllm_runner):
83
hf_overrides = {
82
- "rope_theta": rope_theta,
83
- "rope_scaling": {
84
+ "rope_parameters": {
85
+ "rope_theta": rope_theta,
86
"rope_type": "yarn",
87
"factor": factor,
88
"original_max_position_embeddings": original_max_position_embeddings,
@@ -96,9 +98,9 @@ def test_use_rope_scaling_legal(model_info, vllm_runner):
98
99
@pytest.mark.parametrize("model_info", MODELS)
100
def test_use_rope_scaling_illegal(model_info, vllm_runner):
99
- hf_overrides = {
100
- "rope_theta": rope_theta,
101
- "rope_scaling": {
101
+ hf_overrides: dict[str, Any] = {
102
+ "rope_parameters": {
103
+ "rope_theta": rope_theta,
104
"rope_type": "yarn",
105
"factor": factor,
106
"original_max_position_embeddings": original_max_position_embeddings,
@@ -115,8 +117,8 @@ def test_use_rope_scaling_illegal(model_info, vllm_runner):
117
pass
118
119
hf_overrides = {
118
- "rope_theta": rope_theta,
119
- "rope_scaling": {
120
+ "rope_parameters": {
121
+ "rope_theta": rope_theta,
122
"rope_type": "yarn",
123
"factor": factor,
124
"original_max_position_embeddings": original_max_position_embeddings,
tests/test_config.py
+20
-17
@@ -249,45 +249,48 @@ def test_get_bert_tokenization_sentence_transformer_config():
249
250
251
def test_rope_customization():
252
- TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
253
- TEST_ROPE_THETA = 16_000_000.0
254
- LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
252
+ TEST_ROPE_PARAMETERS = {
253
+ "rope_theta": 16_000_000.0,
254
+ "rope_type": "dynamic",
255
+ "factor": 2.0,
256
+ }
257
+ LLAMA_ROPE_PARAMETERS = {"rope_theta": 500000.0, "rope_type": "default"}
258
+ LONGCHAT_ROPE_PARAMETERS = {"rope_type": "linear", "factor": 8.0}
259
260
llama_model_config = ModelConfig("meta-llama/Meta-Llama-3-8B-Instruct")
257
- assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
258
- assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
261
+ assert (
262
+ getattr(llama_model_config.hf_config, "rope_parameters", None)
263
+ == LLAMA_ROPE_PARAMETERS
264
+ )
265
assert llama_model_config.max_model_len == 8192
266
267
llama_model_config = ModelConfig(
268
"meta-llama/Meta-Llama-3-8B-Instruct",
263
- hf_overrides={
264
- "rope_scaling": TEST_ROPE_SCALING,
265
- "rope_theta": TEST_ROPE_THETA,
266
- },
269
+ hf_overrides={"rope_parameters": TEST_ROPE_PARAMETERS},
270
)
271
assert (
269
- getattr(llama_model_config.hf_config, "rope_scaling", None) == TEST_ROPE_SCALING
272
+ getattr(llama_model_config.hf_config, "rope_parameters", None)
273
+ == TEST_ROPE_PARAMETERS
274
)
271
- assert getattr(llama_model_config.hf_config, "rope_theta", None) == TEST_ROPE_THETA
275
assert llama_model_config.max_model_len == 16384
276
277
longchat_model_config = ModelConfig("lmsys/longchat-13b-16k")
275
- # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
278
+ # Check if LONGCHAT_ROPE_PARAMETERS entries are in longchat_model_config
279
assert all(
277
- longchat_model_config.hf_config.rope_scaling.get(key) == value
278
- for key, value in LONGCHAT_ROPE_SCALING.items()
280
+ longchat_model_config.hf_config.rope_parameters.get(key) == value
281
+ for key, value in LONGCHAT_ROPE_PARAMETERS.items()
282
)
283
assert longchat_model_config.max_model_len == 16384
284
285
longchat_model_config = ModelConfig(
286
"lmsys/longchat-13b-16k",
287
hf_overrides={
285
- "rope_scaling": TEST_ROPE_SCALING,
288
+ "rope_parameters": TEST_ROPE_PARAMETERS,
289
},
290
)
291
assert (
289
- getattr(longchat_model_config.hf_config, "rope_scaling", None)
290
- == TEST_ROPE_SCALING
292
+ getattr(longchat_model_config.hf_config, "rope_parameters", None)
293
+ == TEST_ROPE_PARAMETERS
294
)
295
assert longchat_model_config.max_model_len == 4096
296
vllm/config/model.py
+29
-34
@@ -11,6 +11,7 @@ import torch
11
from pydantic import ConfigDict, SkipValidation, field_validator, model_validator
12
from pydantic.dataclasses import dataclass
13
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
14
+from transformers.configuration_utils import ALLOWED_LAYER_TYPES
15
16
import vllm.envs as envs
17
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MultiModalConfig
@@ -2100,31 +2101,32 @@ def _get_and_verify_max_len(
2101
)
2102
derived_max_model_len = default_max_len
2103
2103
- rope_scaling = getattr(hf_config, "rope_scaling", None)
2104
+ # In Transformers v5 rope_parameters could be TypedDict or dict[str, TypedDict].
2105
+ # To simplify the verification, we convert it to dict[str, TypedDict].
2106
+ rope_parameters = getattr(hf_config, "rope_parameters", None)
2107
+ if rope_parameters and not set(rope_parameters.keys()).issubset(
2108
+ ALLOWED_LAYER_TYPES
2109
+ ):
2110
+ rope_parameters = {"": rope_parameters}
2111
+
2112
# NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE
2113
# scaling, so we skip applying the scaling factor again.
2106
- if rope_scaling is not None and "gemma3" not in hf_config.model_type:
2107
- # No need to consider "type" key because of patch_rope_scaling when
2108
- # loading HF config
2109
- rope_type = rope_scaling["rope_type"]
2110
-
2111
- if rope_type not in ("su", "longrope", "llama3"):
2112
- if disable_sliding_window:
2113
- # TODO(robertgshaw): Find a model that supports rope_scaling
2114
- # with sliding window to see if this case should be allowed.
2115
- raise NotImplementedError(
2116
- "Disabling sliding window is not supported for models "
2117
- "with rope_scaling. Please raise an issue so we can "
2118
- "investigate."
2119
- )
2120
-
2121
- # NOTE: rope_type == "default" does not define factor
2122
- # https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
2123
- scaling_factor = rope_scaling.get("factor", 1.0)
2124
-
2125
- if rope_type == "yarn":
2126
- derived_max_model_len = rope_scaling["original_max_position_embeddings"]
2127
- derived_max_model_len *= scaling_factor
2114
+ if rope_parameters is not None and "gemma3" not in hf_config.model_type:
2115
+ scaling_factor = 1.0
2116
+ for rp in rope_parameters.values():
2117
+ # No need to consider "type" key because of patch_rope_parameters when
2118
+ # loading HF config
2119
+ rope_type = rp["rope_type"]
2120
+
2121
+ if rope_type not in ("su", "longrope", "llama3"):
2122
+ # NOTE: rope_type == "default" does not define factor https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
2123
+ # NOTE: This assumes all layer types have the same scaling factor.
2124
+ scaling_factor = rp.get("factor", scaling_factor)
2125
+
2126
+ if rope_type == "yarn":
2127
+ derived_max_model_len = rp["original_max_position_embeddings"]
2128
+ # Do this outside loop since all layer types should have the same scaling
2129
+ derived_max_model_len *= scaling_factor
2130
2131
if encoder_config and "max_seq_length" in encoder_config:
2132
derived_max_model_len = encoder_config["max_seq_length"]
@@ -2134,7 +2136,9 @@ def _get_and_verify_max_len(
2136
if max_model_len is None:
2137
# For LongRoPE, default to original_max_position_embeddings to avoid
2138
# performance degradation for shorter sequences
2137
- if rope_scaling is not None and rope_scaling["rope_type"] == "longrope":
2139
+ if rope_parameters is not None and any(
2140
+ rp["rope_type"] == "longrope" for rp in rope_parameters.values()
2141
+ ):
2142
max_model_len = int(
2143
getattr(
2144
hf_config, "original_max_position_embeddings", derived_max_model_len
@@ -2151,16 +2155,7 @@ def _get_and_verify_max_len(
2155
# that will be bigger than derived_max_model_len. We compare user input
2156
# with model_max_length and allow this override when it's smaller.
2157
model_max_length = getattr(hf_config, "model_max_length", None)
2154
- if model_max_length is not None and max_model_len <= model_max_length:
2155
- if disable_sliding_window:
2156
- # TODO(robertgshaw): Find a model that has model_max_length
2157
- # with sliding window to see if this case should be allowed.
2158
- raise NotImplementedError(
2159
- "Disabling sliding window is not supported for models "
2160
- "model_max_length in the config. Please raise an issue "
2161
- "so we can investigate."
2162
- )
2163
- else:
2158
+ if model_max_length is None or max_model_len > model_max_length:
2159
msg = (
2160
f"User-specified max_model_len ({max_model_len}) is greater "
2161
f"than the derived max_model_len ({max_len_key}="
vllm/model_executor/layers/rotary_embedding/__init__.py
+38
-38
@@ -26,23 +26,23 @@ def get_rope(
26
head_size: int,
27
rotary_dim: int,
28
max_position: int,
29
- base: float,
29
is_neox_style: bool = True,
31
- rope_scaling: dict[str, Any] | None = None,
30
+ rope_parameters: dict[str, Any] | None = None,
31
dtype: torch.dtype | None = None,
32
partial_rotary_factor: float = 1.0,
33
dual_chunk_attention_config: dict[str, Any] | None = None,
34
) -> RotaryEmbedding:
35
if dtype is None:
36
dtype = torch.get_default_dtype()
38
- if rope_scaling is not None:
37
+ if rope_parameters is not None:
38
# Transforms every value that is a list into a tuple for caching calls
40
- rope_scaling_tuple = {
41
- k: tuple(v) if isinstance(v, list) else v for k, v in rope_scaling.items()
39
+ rope_parameters_tuple = {
40
+ k: tuple(v) if isinstance(v, list) else v
41
+ for k, v in rope_parameters.items()
42
}
43
- rope_scaling_args = tuple(rope_scaling_tuple.items())
43
+ rope_parameters_args = tuple(rope_parameters_tuple.items())
44
else:
45
- rope_scaling_args = None
45
+ rope_parameters_args = None
46
47
if dual_chunk_attention_config is not None:
48
dual_chunk_attention_tuple = {
@@ -60,15 +60,15 @@ def get_rope(
60
head_size,
61
rotary_dim,
62
max_position,
63
- base,
63
is_neox_style,
65
- rope_scaling_args,
64
+ rope_parameters_args,
65
dual_chunk_attention_args,
66
dtype,
67
)
68
if key in _ROPE_DICT:
69
return _ROPE_DICT[key]
70
71
+ base = rope_parameters["rope_theta"] if rope_parameters else 10000
72
if dual_chunk_attention_config is not None:
73
extra_kwargs = {
74
k: v
@@ -84,18 +84,18 @@ def get_rope(
84
dtype,
85
**extra_kwargs,
86
)
87
- elif not rope_scaling:
87
+ elif not rope_parameters:
88
rotary_emb = RotaryEmbedding(
89
head_size, rotary_dim, max_position, base, is_neox_style, dtype
90
)
91
else:
92
- scaling_type = rope_scaling["rope_type"]
92
+ scaling_type = rope_parameters["rope_type"]
93
94
if scaling_type == "llama3":
95
- scaling_factor = rope_scaling["factor"]
96
- low_freq_factor = rope_scaling["low_freq_factor"]
97
- high_freq_factor = rope_scaling["high_freq_factor"]
98
- original_max_position = rope_scaling["original_max_position_embeddings"]
95
+ scaling_factor = rope_parameters["factor"]
96
+ low_freq_factor = rope_parameters["low_freq_factor"]
97
+ high_freq_factor = rope_parameters["high_freq_factor"]
98
+ original_max_position = rope_parameters["original_max_position_embeddings"]
99
rotary_emb = Llama3RotaryEmbedding(
100
head_size,
101
rotary_dim,
@@ -113,7 +113,7 @@ def get_rope(
113
head_size, rotary_dim, max_position, base, is_neox_style, dtype
114
)
115
elif scaling_type == "default":
116
- if "mrope_section" in rope_scaling:
116
+ if "mrope_section" in rope_parameters:
117
rotary_emb = MRotaryEmbedding(
118
head_size,
119
rotary_dim,
@@ -121,8 +121,8 @@ def get_rope(
121
base,
122
is_neox_style,
123
dtype,
124
- mrope_section=rope_scaling["mrope_section"],
125
- mrope_interleaved=rope_scaling.get("mrope_interleaved", False),
124
+ mrope_section=rope_parameters["mrope_section"],
125
+ mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
126
)
127
else:
128
rotary_emb = RotaryEmbedding(
@@ -134,7 +134,7 @@ def get_rope(
134
dtype,
135
)
136
elif scaling_type == "linear":
137
- scaling_factor = rope_scaling["factor"]
137
+ scaling_factor = rope_parameters["factor"]
138
rotary_emb = LinearScalingRotaryEmbedding(
139
head_size,
140
rotary_dim,
@@ -145,8 +145,8 @@ def get_rope(
145
dtype,
146
)
147
elif scaling_type == "ntk":
148
- scaling_factor = rope_scaling["factor"]
149
- mixed_b = rope_scaling.get("mixed_b", None)
148
+ scaling_factor = rope_parameters["factor"]
149
+ mixed_b = rope_parameters.get("mixed_b")
150
rotary_emb = NTKScalingRotaryEmbedding(
151
head_size,
152
rotary_dim,
@@ -158,8 +158,8 @@ def get_rope(
158
mixed_b,
159
)
160
elif scaling_type == "dynamic":
161
- if "alpha" in rope_scaling:
162
- scaling_alpha = rope_scaling["alpha"]
161
+ if "alpha" in rope_parameters:
162
+ scaling_alpha = rope_parameters["alpha"]
163
rotary_emb = DynamicNTKAlphaRotaryEmbedding(
164
head_size,
165
rotary_dim,
@@ -169,8 +169,8 @@ def get_rope(
169
scaling_alpha,
170
dtype,
171
)
172
- elif "factor" in rope_scaling:
173
- scaling_factor = rope_scaling["factor"]
172
+ elif "factor" in rope_parameters:
173
+ scaling_factor = rope_parameters["factor"]
174
rotary_emb = DynamicNTKScalingRotaryEmbedding(
175
head_size,
176
rotary_dim,
@@ -185,11 +185,11 @@ def get_rope(
185
"Dynamic rope scaling must contain either 'alpha' or 'factor' field"
186
)
187
elif scaling_type == "yarn":
188
- scaling_factor = rope_scaling["factor"]
189
- original_max_position = rope_scaling["original_max_position_embeddings"]
188
+ scaling_factor = rope_parameters["factor"]
189
+ original_max_position = rope_parameters["original_max_position_embeddings"]
190
extra_kwargs = {
191
k: v
192
- for k, v in rope_scaling.items()
192
+ for k, v in rope_parameters.items()
193
if k
194
in (
195
"extrapolation_factor",
@@ -199,7 +199,7 @@ def get_rope(
199
"apply_yarn_scaling",
200
)
201
}
202
- if "mrope_section" in rope_scaling:
202
+ if "mrope_section" in rope_parameters:
203
extra_kwargs.pop("apply_yarn_scaling", None)
204
rotary_emb = MRotaryEmbedding(
205
head_size,
@@ -208,8 +208,8 @@ def get_rope(
208
base,
209
is_neox_style,
210
dtype,
211
- mrope_section=rope_scaling["mrope_section"],
212
- mrope_interleaved=rope_scaling.get("mrope_interleaved", False),
211
+ mrope_section=rope_parameters["mrope_section"],
212
+ mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
213
scaling_factor=scaling_factor,
214
**extra_kwargs,
215
)
@@ -225,12 +225,12 @@ def get_rope(
225
**extra_kwargs,
226
)
227
elif scaling_type == "deepseek_yarn":
228
- scaling_factor = rope_scaling["factor"]
229
- original_max_position = rope_scaling["original_max_position_embeddings"]
228
+ scaling_factor = rope_parameters["factor"]
229
+ original_max_position = rope_parameters["original_max_position_embeddings"]
230
# assert max_position == original_max_position * scaling_factor
231
extra_kwargs = {
232
k: v
233
- for k, v in rope_scaling.items()
233
+ for k, v in rope_parameters.items()
234
if k
235
in (
236
"extrapolation_factor",
@@ -252,12 +252,12 @@ def get_rope(
252
**extra_kwargs,
253
)
254
elif scaling_type == "longrope":
255
- short_factor = rope_scaling["short_factor"]
256
- long_factor = rope_scaling["long_factor"]
257
- original_max_position = rope_scaling["original_max_position_embeddings"]
255
+ short_factor = rope_parameters["short_factor"]
256
+ long_factor = rope_parameters["long_factor"]
257
+ original_max_position = rope_parameters["original_max_position_embeddings"]
258
extra_kwargs = {
259
k: v
260
- for k, v in rope_scaling.items()
260
+ for k, v in rope_parameters.items()
261
if k in ("short_mscale", "long_mscale")
262
}
263
rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(
vllm/model_executor/models/afmoe.py
+1
-16
@@ -5,7 +5,6 @@
5
import typing
6
from collections.abc import Callable, Iterable
7
from itertools import islice
8
-from typing import Any
8
9
import torch
10
from torch import nn
@@ -171,8 +170,6 @@ class AfmoeAttention(nn.Module):
170
hidden_size: int,
171
num_heads: int,
172
num_kv_heads: int,
174
- rope_theta: float = 10000,
175
- rope_scaling: dict[str, Any] | None = None,
173
max_position_embeddings: int = 131072,
174
head_dim: int | None = None,
175
rms_norm_eps: float = 1e-05,
@@ -202,7 +199,6 @@ class AfmoeAttention(nn.Module):
199
self.q_size = self.num_heads * self.head_dim
200
self.kv_size = self.num_kv_heads * self.head_dim
201
self.scaling = self.head_dim**-0.5
205
- self.rope_theta = rope_theta
202
self.max_position_embeddings = max_position_embeddings
203
204
# Check if this is a local attention layer
@@ -246,8 +242,7 @@ class AfmoeAttention(nn.Module):
242
self.head_dim,
243
rotary_dim=self.head_dim,
244
max_position=max_position_embeddings,
249
- base=rope_theta,
250
- rope_scaling=rope_scaling,
245
+ rope_parameters=config["rope_parameters"],
246
is_neox_style=True,
247
)
248
else:
@@ -303,14 +298,6 @@ class AfmoeDecoderLayer(nn.Module):
298
) -> None:
299
super().__init__()
300
self.hidden_size = config.hidden_size
306
- rope_theta = getattr(config, "rope_theta", 10000)
307
- rope_scaling = getattr(config, "rope_scaling", None)
308
- if rope_scaling is not None and getattr(
309
- config, "original_max_position_embeddings", None
310
- ):
311
- rope_scaling["original_max_position_embeddings"] = (
312
- config.original_max_position_embeddings
313
- )
301
max_position_embeddings = getattr(config, "max_position_embeddings", 131072)
302
303
# DecoderLayers are created with `make_layers` which passes the prefix
@@ -323,8 +310,6 @@ class AfmoeDecoderLayer(nn.Module):
310
hidden_size=self.hidden_size,
311
num_heads=config.num_attention_heads,
312
num_kv_heads=config.num_key_value_heads,
326
- rope_theta=rope_theta,
327
- rope_scaling=rope_scaling,
313
max_position_embeddings=max_position_embeddings,
314
head_dim=config.head_dim,
315
rms_norm_eps=config.rms_norm_eps,
vllm/model_executor/models/apertus.py
+2
-20
@@ -27,7 +27,6 @@
27
28
from collections.abc import Iterable
29
from itertools import islice
30
-from typing import Any
30
31
import torch
32
from torch import nn
@@ -118,8 +117,6 @@ class ApertusAttention(nn.Module):
117
hidden_size: int,
118
num_heads: int,
119
num_kv_heads: int,
121
- rope_theta: float = 10000,
122
- rope_scaling: dict[str, Any] | None = None,
120
max_position_embeddings: int = 8192,
121
quant_config: QuantizationConfig | None = None,
122
bias: bool = False,
@@ -155,7 +152,6 @@ class ApertusAttention(nn.Module):
152
self.q_size = self.num_heads * self.head_dim
153
self.kv_size = self.num_kv_heads * self.head_dim
154
self.scaling = self.head_dim**-0.5
158
- self.rope_theta = rope_theta
155
self.max_position_embeddings = max_position_embeddings
156
157
self.qkv_proj = QKVParallelLinear(
@@ -176,9 +172,7 @@ class ApertusAttention(nn.Module):
172
prefix=f"{prefix}.o_proj",
173
)
174
179
- self._init_rotary_emb(
180
- config, rope_scaling=rope_scaling, quant_config=quant_config
181
- )
175
+ self._init_rotary_emb(config, quant_config=quant_config)
176
177
sliding_window = None
178
if layer_types := getattr(config, "layer_types", None):
@@ -224,7 +218,6 @@ class ApertusAttention(nn.Module):
218
def _init_rotary_emb(
219
self,
220
config: ApertusConfig,
227
- rope_scaling: dict[str, Any] | None,
221
quant_config: QuantizationConfig | None,
222
) -> None:
223
is_neox_style = True
@@ -236,8 +229,7 @@ class ApertusAttention(nn.Module):
229
self.head_dim,
230
rotary_dim=int(self.partial_rotary_factor * self.head_dim),
231
max_position=self.max_position_embeddings,
239
- base=self.rope_theta,
240
- rope_scaling=rope_scaling,
232
+ rope_parameters=config.rope_parameters,
233
is_neox_style=is_neox_style,
234
partial_rotary_factor=self.partial_rotary_factor,
235
)
@@ -253,14 +245,6 @@ class ApertusDecoderLayer(nn.Module):
245
) -> None:
246
super().__init__()
247
self.hidden_size = config.hidden_size
256
- rope_theta = getattr(config, "rope_theta", 10000)
257
- rope_scaling = getattr(config, "rope_scaling", None)
258
- if rope_scaling is not None and getattr(
259
- config, "original_max_position_embeddings", None
260
- ):
261
- rope_scaling["original_max_position_embeddings"] = (
262
- config.original_max_position_embeddings
263
- )
248
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
249
# Support abacusai/Smaug-72B-v0.1 with attention_bias
250
# Support internlm/internlm-7b with bias
@@ -288,8 +272,6 @@ class ApertusDecoderLayer(nn.Module):
272
num_kv_heads=getattr(
273
config, "num_key_value_heads", config.num_attention_heads
274
),
291
- rope_theta=rope_theta,
292
- rope_scaling=rope_scaling,
275
max_position_embeddings=max_position_embeddings,
276
quant_config=quant_config,
277
bias=attention_bias,
vllm/model_executor/models/arcee.py
-11
@@ -103,15 +103,6 @@ class ArceeDecoderLayer(nn.Module):
103
) -> None:
104
super().__init__()
105
self.hidden_size = config.hidden_size
106
- # Rotary embedding parameters (reuse LLaMA defaults)
107
- rope_theta = getattr(config, "rope_theta", 10000)
108
- rope_scaling = getattr(config, "rope_scaling", None)
109
- if rope_scaling is not None and getattr(
110
- config, "original_max_position_embeddings", None
111
- ):
112
- rope_scaling["original_max_position_embeddings"] = (
113
- config.original_max_position_embeddings
114
- )
106
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
107
# Determine if attention bias is needed (some variants use bias terms)
108
attention_bias = getattr(config, "attention_bias", False) or getattr(
@@ -133,8 +124,6 @@ class ArceeDecoderLayer(nn.Module):
124
num_kv_heads=getattr(
125
config, "num_key_value_heads", config.num_attention_heads
126
),
136
- rope_theta=rope_theta,
137
- rope_scaling=rope_scaling,
127
max_position_embeddings=max_position_embeddings,
128
quant_config=quant_config,
129
bias=attention_bias,
vllm/model_executor/models/arctic.py
+1
-2
@@ -292,7 +292,6 @@ class ArcticAttention(nn.Module):
292
self.kv_size = self.num_kv_heads * self.head_dim
293
294
self.max_position_embeddings = config.max_position_embeddings
295
- self.rope_theta = config.rope_theta
295
self.scaling = self.head_dim**-0.5
296
297
self.qkv_proj = QKVParallelLinear(
@@ -317,7 +316,7 @@ class ArcticAttention(nn.Module):
316
self.head_dim,
317
rotary_dim=self.head_dim,
318
max_position=self.max_position_embeddings,
320
- base=int(self.rope_theta),
319
+ rope_parameters=config.rope_parameters,
320
is_neox_style=True,
321
)
322
vllm/model_executor/models/baichuan.py
+3
-5
@@ -136,7 +136,7 @@ class BaiChuanAttention(nn.Module):
136
hidden_size: int,
137
num_heads: int,
138
position_embedding: str,
139
- rope_theta: float = 10000,
139
+ rope_parameters: dict,
140
max_position_embeddings: int = 8192,
141
cache_config: CacheConfig | None = None,
142
quant_config: QuantizationConfig | None = None,
@@ -150,7 +150,6 @@ class BaiChuanAttention(nn.Module):
150
self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
151
self.head_dim = hidden_size // self.total_num_heads
152
self.position_embedding = position_embedding
153
- self.rope_theta = rope_theta
153
self.max_position_embeddings = max_position_embeddings
154
155
# pylint: disable=invalid-name
@@ -192,7 +191,7 @@ class BaiChuanAttention(nn.Module):
191
self.head_dim,
192
rotary_dim=self.head_dim,
193
max_position=self.max_position_embeddings,
195
- base=self.rope_theta,
194
+ rope_parameters=rope_parameters,
195
)
196
self.scaling = self.head_dim**-0.5
197
self.attn = Attention(
@@ -229,13 +228,12 @@ class BaiChuanDecoderLayer(nn.Module):
228
):
229
super().__init__()
230
self.hidden_size = config.hidden_size
232
- rope_theta = getattr(config, "rope_theta", 10000)
231
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
232
self.self_attn = BaiChuanAttention(
233
hidden_size=self.hidden_size,
234
num_heads=config.num_attention_heads,
235
position_embedding=position_embedding,
238
- rope_theta=rope_theta,
236
+ rope_parameters=config.rope_parameters,
237
max_position_embeddings=max_position_embeddings,
238
cache_config=cache_config,
239
quant_config=quant_config,
vllm/model_executor/models/bailing_moe.py
+1
-2
@@ -135,9 +135,8 @@ class BailingAttention(nn.Module):
135
self.head_dim,
136
rotary_dim=self.rotary_dim,
137
max_position=config.max_position_embeddings,
138
- base=config.rope_theta,
138
+ rope_parameters=config.rope_parameters,
139
is_neox_style=True,
140
- rope_scaling=config.rope_scaling,
140
partial_rotary_factor=self.partial_rotary_factor,
141
)
142
vllm/model_executor/models/bamba.py
+1
-5
@@ -156,8 +156,6 @@ class BambaAttentionDecoderLayer(nn.Module):
156
prefix: str = "",
157
) -> None:
158
super().__init__()
159
- rope_theta = getattr(config, "rope_theta", 10000)
160
- rope_scaling = getattr(config, "rope_scaling", None)
159
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
160
self.hidden_size = config.hidden_size
161
tp_size = get_tensor_model_parallel_world_size()
@@ -178,7 +176,6 @@ class BambaAttentionDecoderLayer(nn.Module):
176
self.q_size = self.num_heads * self.head_dim
177
self.kv_size = self.num_kv_heads * self.head_dim
178
self.scaling = self.head_dim**-0.5
181
- self.rope_theta = rope_theta
179
self.max_position_embeddings = max_position_embeddings
180
181
if hasattr(config, "partial_rotary_factor"):
@@ -192,8 +189,7 @@ class BambaAttentionDecoderLayer(nn.Module):
189
head_size=self.head_dim,
190
rotary_dim=rotary_dim,
191
max_position=max_position_embeddings,
195
- rope_scaling=rope_scaling,
196
- base=rope_theta,
192
+ rope_parameters=config.rope_parameters,
193
is_neox_style=True,
194
dtype=torch.get_default_dtype(), # see impl of get_rope
195
)
vllm/model_executor/models/chameleon.py
+4
-25
@@ -265,8 +265,7 @@ class ChameleonAttention(nn.Module):
265
hidden_size: int,
266
num_heads: int,
267
num_kv_heads: int,
268
- rope_theta: float = 10000,
269
- rope_scaling: dict[str, Any] | None = None,
268
+ rope_parameters: dict[str, Any],
269
max_position_embeddings: int = 4096,
270
quant_config: QuantizationConfig | None = None,
271
bias: bool = False,
@@ -293,7 +292,6 @@ class ChameleonAttention(nn.Module):
292
self.q_size = self.num_heads * self.head_dim
293
self.kv_size = self.num_kv_heads * self.head_dim
294
self.scaling = self.head_dim**-0.5
296
- self.rope_theta = rope_theta
295
self.max_position_embeddings = max_position_embeddings
296
297
self.qkv_proj = QKVParallelLinear(
@@ -318,8 +316,7 @@ class ChameleonAttention(nn.Module):
316
self.head_dim,
317
rotary_dim=self.head_dim,
318
max_position=max_position_embeddings,
321
- base=rope_theta,
322
- rope_scaling=rope_scaling,
319
+ rope_parameters=rope_parameters,
320
)
321
322
self.attn = Attention(
@@ -369,14 +366,6 @@ class ChameleonDecoderLayer(nn.Module):
366
) -> None:
367
super().__init__()
368
self.hidden_size = config.hidden_size
372
- rope_theta = getattr(config, "rope_theta", 10000)
373
- rope_scaling = getattr(config, "rope_scaling", None)
374
- if rope_scaling is not None and getattr(
375
- config, "original_max_position_embeddings", None
376
- ):
377
- rope_scaling["original_max_position_embeddings"] = (
378
- config.original_max_position_embeddings
379
- )
369
max_position_embeddings = getattr(config, "max_position_embeddings", 4096)
370
371
self.self_attn = ChameleonAttention(
@@ -385,8 +374,7 @@ class ChameleonDecoderLayer(nn.Module):
374
num_kv_heads=getattr(
375
config, "num_key_value_heads", config.num_attention_heads
376
),
388
- rope_theta=rope_theta,
389
- rope_scaling=rope_scaling,
377
+ rope_parameters=config.rope_parameters,
378
max_position_embeddings=max_position_embeddings,
379
quant_config=quant_config,
380
bias=False,
@@ -439,14 +427,6 @@ class ChameleonSwinDecoderLayer(nn.Module):
427
) -> None:
428
super().__init__()
429
self.hidden_size = config.hidden_size
442
- rope_theta = getattr(config, "rope_theta", 10000)
443
- rope_scaling = getattr(config, "rope_scaling", None)
444
- if rope_scaling is not None and getattr(
445
- config, "original_max_position_embeddings", None
446
- ):
447
- rope_scaling["original_max_position_embeddings"] = (
448
- config.original_max_position_embeddings
449
- )
430
max_position_embeddings = getattr(config, "max_position_embeddings", 4096)
431
432
self.self_attn = ChameleonAttention(
@@ -455,8 +435,7 @@ class ChameleonSwinDecoderLayer(nn.Module):
435
num_kv_heads=getattr(
436
config, "num_key_value_heads", config.num_attention_heads
437
),
458
- rope_theta=rope_theta,
459
- rope_scaling=rope_scaling,
438
+ rope_parameters=config.rope_parameters,
439
max_position_embeddings=max_position_embeddings,
440
quant_config=quant_config,
441
bias=False,
vllm/model_executor/models/chatglm.py
+2
-1
@@ -99,6 +99,7 @@ class GLMAttention(nn.Module):
99
# https://huggingface.co/zai-org/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
100
rope_ratio = getattr(config, "rope_ratio", 1.0)
101
max_positions = getattr(config, "seq_length", 8192)
102
+ rope_parameters = {"rope_type": "default", "rope_theta": 10000 * rope_ratio}
103
# NOTE: zai-org/cogagent-9b-20241220 uses original_rope=False,
104
# which is equivalent to is_neox_style=True
105
is_neox_style = not config.original_rope
@@ -106,7 +107,7 @@ class GLMAttention(nn.Module):
107
self.head_dim,
108
rotary_dim=self.head_dim // 2,
109
max_position=max_positions,
109
- base=10000 * rope_ratio,
110
+ rope_parameters=rope_parameters,
111
is_neox_style=is_neox_style,
112
)
113
self.attn = Attention(
vllm/model_executor/models/commandr.py
+1
-4
@@ -156,8 +156,6 @@ class CohereAttention(nn.Module):
156
self.max_position_embeddings = getattr(
157
config, "model_max_length", None
158
) or getattr(config, "max_position_embeddings", 8192)
159
- self.rope_theta = config.rope_theta
160
- self.rope_scaling = getattr(config, "rope_scaling", None)
159
self.use_qk_norm = getattr(config, "use_qk_norm", False)
160
self.qkv_proj = QKVParallelLinear(
161
self.hidden_size,
@@ -179,8 +177,7 @@ class CohereAttention(nn.Module):
177
self.head_dim,
178
rotary_dim=self.head_dim,
179
max_position=self.max_position_embeddings,
182
- base=self.rope_theta,
183
- rope_scaling=self.rope_scaling,
180
+ rope_parameters=config.rope_parameters,
181
is_neox_style=False,
182
)
183
vllm/model_executor/models/config.py
+12
-10
@@ -8,6 +8,7 @@ import vllm.envs as envs
8
from vllm.logger import init_logger
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.platforms import current_platform
11
+from vllm.transformers_utils.config import set_default_rope_theta
12
from vllm.utils.math_utils import cdiv, round_up
13
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
14
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec, MLAAttentionSpec
@@ -46,8 +47,7 @@ class GteNewModelConfig(VerifyAndUpdateConfig):
47
"head_size": head_dim,
48
"rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
49
"max_position": config.max_position_embeddings,
49
- "base": config.rope_theta,
50
- "rope_scaling": getattr(config, "rope_scaling", None),
50
+ "rope_parameters": config.rope_parameters,
51
}
52
53
@@ -78,12 +78,13 @@ class JinaRobertaModelConfig(VerifyAndUpdateConfig):
78
if not model_config.enforce_eager:
79
max_position = round_up(max_position, 8)
80
81
+ set_default_rope_theta(config, default_theta=config.rotary_emb_base)
82
+
83
config.rotary_kwargs = {
84
"head_size": head_dim,
85
"rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
86
"max_position": max_position,
85
- "base": getattr(config, "rope_theta", config.rotary_emb_base),
86
- "rope_scaling": getattr(config, "rope_scaling", None),
87
+ "rope_parameters": config.rope_parameters,
88
}
89
90
@@ -117,18 +118,20 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
118
head_dim = config.hidden_size // config.num_attention_heads
119
rotary_emb_dim = int(head_dim * config.rotary_emb_fraction)
120
max_trained_positions = getattr(config, "max_trained_positions", 2048)
121
+
122
+ set_default_rope_theta(config, default_theta=config.rotary_emb_base)
123
+
124
config.rotary_kwargs = {
125
"head_size": head_dim,
126
"rotary_dim": rotary_emb_dim,
127
"max_position": max_trained_positions,
124
- "base": getattr(config, "rope_theta", config.rotary_emb_base),
125
- "rope_scaling": getattr(config, "rope_scaling", None),
128
+ "rope_parameters": config.rope_parameters,
129
}
130
131
# we ignore config.rotary_scaling_factor so that for datasets shorter
132
# than max_trained_positions 2048, the results are consistent
133
# with SentenceTransformer.
131
- # The context extension uses vllm style rope_theta and rope_scaling.
134
+ # The context extension uses vllm style rope_theta and rope_parameters.
135
# See #17785 #18755
136
if (
137
not vllm_config.model_config.hf_overrides
@@ -172,7 +175,7 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
175
if hasattr(hf_text_config, "max_model_len"):
176
delattr(hf_text_config, "max_model_len")
177
hf_text_config.max_position_embeddings = max_trained_positions
175
- hf_text_config.rope_scaling = config.rotary_kwargs["rope_scaling"]
178
+ hf_text_config.rope_parameters = config.rotary_kwargs["rope_parameters"]
179
180
# The priority of sentence_bert_config.json is higher
181
# than max_position_embeddings
@@ -246,8 +249,7 @@ class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
249
"head_size": head_dim,
250
"rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
251
"max_position": config.max_position_embeddings,
249
- "base": config.rope_theta,
250
- "rope_scaling": getattr(config, "rope_scaling", None),
252
+ "rope_parameters": config.rope_parameters,
253
}
254
255
vllm/model_executor/models/dbrx.py
+5
-2
@@ -197,7 +197,10 @@ class DbrxAttention(nn.Module):
197
self.head_dim = self.d_model // self.total_num_heads
198
self.total_num_kv_heads = config.attn_config.kv_n_heads
199
self.clip_qkv = config.attn_config.clip_qkv
200
- self.rope_theta = config.attn_config.rope_theta
200
+ rope_parameters = {
201
+ "rope_type": "default",
202
+ "rope_theta": int(config.attn_config.rope_theta),
203
+ }
204
self.max_position = config.max_seq_len
205
206
# pylint: disable=invalid-name
@@ -221,7 +224,7 @@ class DbrxAttention(nn.Module):
224
self.head_dim,
225
rotary_dim=self.head_dim,
226
max_position=self.max_position,
224
- base=int(self.rope_theta),
227
+ rope_parameters=rope_parameters,
228
is_neox_style=True,
229
)
230
vllm/model_executor/models/deepseek_v2.py
+13
-30
@@ -27,7 +27,6 @@
27
import typing
28
from collections.abc import Callable, Iterable
29
from itertools import islice
30
-from typing import Any
30
31
import torch
32
from torch import nn
@@ -111,8 +110,6 @@ class DeepseekAttention(nn.Module):
110
config: DeepseekV2Config | DeepseekV3Config,
111
hidden_size: int,
112
num_heads: int,
114
- rope_theta: float = 10000,
115
- rope_scaling: dict[str, Any] | None = None,
113
max_position_embeddings: int = 8192,
114
cache_config: CacheConfig | None = None,
115
quant_config: QuantizationConfig | None = None,
@@ -139,7 +136,6 @@ class DeepseekAttention(nn.Module):
136
self.q_size = self.num_heads * self.head_dim
137
self.kv_size = self.num_kv_heads * self.head_dim
138
self.scaling = self.head_dim**-0.5
142
- self.rope_theta = rope_theta
139
self.max_position_embeddings = max_position_embeddings
140
141
self.qkv_proj = QKVParallelLinear(
@@ -162,8 +158,7 @@ class DeepseekAttention(nn.Module):
158
self.head_dim,
159
rotary_dim=self.head_dim,
160
max_position=max_position_embeddings,
165
- base=rope_theta,
166
- rope_scaling=rope_scaling,
161
+ rope_parameters=config.rope_parameters,
162
)
163
self.attn = Attention(
164
self.num_heads,
@@ -409,8 +404,6 @@ class DeepseekV2Attention(nn.Module):
404
v_head_dim: int,
405
q_lora_rank: int,
406
kv_lora_rank: int,
412
- rope_theta: float = 10000,
413
- rope_scaling: dict[str, Any] | None = None,
407
max_position_embeddings: int = 8192,
408
cache_config: CacheConfig | None = None,
409
quant_config: QuantizationConfig | None = None,
@@ -430,7 +423,6 @@ class DeepseekV2Attention(nn.Module):
423
assert num_heads % tp_size == 0
424
self.num_local_heads = num_heads // tp_size
425
self.scaling = self.qk_head_dim**-0.5
433
- self.rope_theta = rope_theta
426
self.max_position_embeddings = max_position_embeddings
427
assert topk_indices_buffer is None, (
428
"topk_indices_buffer is not \
@@ -485,21 +477,20 @@ class DeepseekV2Attention(nn.Module):
477
quant_config=quant_config,
478
prefix=f"{prefix}.o_proj",
479
)
488
- if rope_scaling:
489
- rope_scaling["rope_type"] = "deepseek_yarn"
480
+ if config.rope_parameters["rope_type"] != "default":
481
+ config.rope_parameters["rope_type"] = "deepseek_yarn"
482
483
self.rotary_emb = get_rope(
484
qk_rope_head_dim,
485
rotary_dim=qk_rope_head_dim,
486
max_position=max_position_embeddings,
495
- base=rope_theta,
496
- rope_scaling=rope_scaling,
487
+ rope_parameters=config.rope_parameters,
488
is_neox_style=False,
489
)
490
500
- if rope_scaling:
501
- mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
502
- scaling_factor = rope_scaling["factor"]
491
+ if config.rope_parameters["rope_type"] != "default":
492
+ mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
493
+ scaling_factor = config.rope_parameters["factor"]
494
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
495
self.scaling = self.scaling * mscale * mscale
496
@@ -903,8 +894,6 @@ class DeepseekV2MLAAttention(nn.Module):
894
v_head_dim: int,
895
q_lora_rank: int | None,
896
kv_lora_rank: int,
906
- rope_theta: float = 10000,
907
- rope_scaling: dict[str, Any] | None = None,
897
max_position_embeddings: int = 8192,
898
cache_config: CacheConfig | None = None,
899
quant_config: QuantizationConfig | None = None,
@@ -927,7 +916,6 @@ class DeepseekV2MLAAttention(nn.Module):
916
self.num_local_heads = num_heads // tp_size
917
918
self.scaling = self.qk_head_dim**-0.5
930
- self.rope_theta = rope_theta
919
self.max_position_embeddings = max_position_embeddings
920
921
if self.q_lora_rank is not None:
@@ -981,19 +969,18 @@ class DeepseekV2MLAAttention(nn.Module):
969
prefix=f"{prefix}.o_proj",
970
)
971
984
- if rope_scaling:
985
- rope_scaling["rope_type"] = "deepseek_yarn"
972
+ if config.rope_parameters["rope_type"] != "default":
973
+ config.rope_parameters["rope_type"] = "deepseek_yarn"
974
self.rotary_emb = get_rope(
975
qk_rope_head_dim,
976
rotary_dim=qk_rope_head_dim,
977
max_position=max_position_embeddings,
990
- base=rope_theta,
991
- rope_scaling=rope_scaling,
978
+ rope_parameters=config.rope_parameters,
979
is_neox_style=False,
980
)
994
- if rope_scaling:
995
- mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
996
- scaling_factor = rope_scaling["factor"]
981
+ if config.rope_parameters["rope_type"] != "default":
982
+ mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
983
+ scaling_factor = config.rope_parameters["factor"]
984
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
985
self.scaling = self.scaling * mscale * mscale
986
@@ -1073,8 +1060,6 @@ class DeepseekV2DecoderLayer(nn.Module):
1060
parallel_config = vllm_config.parallel_config
1061
1062
self.hidden_size = config.hidden_size
1076
- rope_theta = getattr(config, "rope_theta", 10000)
1077
- rope_scaling = getattr(config, "rope_scaling", None)
1063
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
1064
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
1065
# DecoderLayers are created with `make_layers` which passes the prefix
@@ -1107,8 +1092,6 @@ class DeepseekV2DecoderLayer(nn.Module):
1092
v_head_dim=v_head_dim,
1093
q_lora_rank=config.q_lora_rank if hasattr(config, "q_lora_rank") else None,
1094
kv_lora_rank=kv_lora_rank,
1110
- rope_theta=rope_theta,
1111
- rope_scaling=rope_scaling,
1095
max_position_embeddings=max_position_embeddings,
1096
cache_config=cache_config,
1097
quant_config=quant_config,
vllm/model_executor/models/dots1.py
+1
-10
@@ -27,7 +27,6 @@
27
28
from collections.abc import Iterable
29
from itertools import islice
30
-from typing import Any
30
31
import torch
32
from torch import nn
@@ -202,8 +201,6 @@ class Dots1Attention(nn.Module):
201
num_heads: int,
202
num_kv_heads: int,
203
config: Dots1Config,
205
- rope_theta: float = 10000,
206
- rope_scaling: dict[str, Any] | None = None,
204
max_position_embeddings: int = 8192,
205
cache_config: CacheConfig | None = None,
206
quant_config: QuantizationConfig | None = None,
@@ -229,7 +226,6 @@ class Dots1Attention(nn.Module):
226
self.q_size = self.num_heads * self.head_dim
227
self.kv_size = self.num_kv_heads * self.head_dim
228
self.scaling = self.head_dim**-0.5
232
- self.rope_theta = rope_theta
229
self.max_position_embeddings = max_position_embeddings
230
attention_bias = config.attention_bias
231
@@ -255,8 +251,7 @@ class Dots1Attention(nn.Module):
251
self.head_dim,
252
rotary_dim=self.head_dim,
253
max_position=max_position_embeddings,
258
- base=rope_theta,
259
- rope_scaling=rope_scaling,
254
+ rope_parameters=config.rope_parameters,
255
)
256
self.attn = Attention(
257
self.num_heads,
@@ -296,8 +291,6 @@ class Dots1DecoderLayer(nn.Module):
291
) -> None:
292
super().__init__()
293
self.hidden_size = config.hidden_size
299
- rope_theta = getattr(config, "rope_theta", 10000)
300
- rope_scaling = getattr(config, "rope_scaling", None)
294
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
295
layer_idx = int(prefix.split(sep=".")[-1])
296
self.layer_idx = layer_idx
@@ -307,8 +300,6 @@ class Dots1DecoderLayer(nn.Module):
300
num_heads=config.num_attention_heads,
301
num_kv_heads=config.num_key_value_heads,
302
config=config,
310
- rope_theta=rope_theta,
311
- rope_scaling=rope_scaling,
303
max_position_embeddings=max_position_embeddings,
304
cache_config=cache_config,
305
quant_config=quant_config,
vllm/model_executor/models/ernie45_moe.py
+5
-9
@@ -62,6 +62,7 @@ from vllm.model_executor.model_loader.weight_utils import (
62
maybe_remap_kv_scale_name,
63
)
64
from vllm.sequence import IntermediateTensors
65
+from vllm.transformers_utils.config import set_default_rope_theta
66
67
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
68
from .utils import (
@@ -232,9 +233,8 @@ class Ernie4_5_MoeAttention(nn.Module):
233
hidden_size: int,
234
num_heads: int,
235
num_kv_heads: int,
236
+ rope_parameters: dict[str, Any],
237
head_dim: int | None = None,
236
- rope_theta: float = 500000,
237
- rope_scaling: dict[str, Any] | None = None,
238
max_position_embeddings: int = 131072,
239
rms_norm_eps: float = 1e-05,
240
qkv_bias: bool = False,
@@ -266,7 +266,6 @@ class Ernie4_5_MoeAttention(nn.Module):
266
self.q_size = self.num_heads * self.head_dim
267
self.kv_size = self.num_kv_heads * self.head_dim
268
self.scaling = self.head_dim**-0.5
269
- self.rope_theta = rope_theta
269
self.max_position_embeddings = max_position_embeddings
270
271
self.qkv_proj = QKVParallelLinear(
@@ -291,9 +290,8 @@ class Ernie4_5_MoeAttention(nn.Module):
290
self.head_dim,
291
rotary_dim=self.head_dim,
292
max_position=max_position_embeddings,
294
- base=rope_theta,
293
+ rope_parameters=rope_parameters,
294
is_neox_style=False,
296
- rope_scaling=rope_scaling,
295
)
296
self.attn = Attention(
297
self.num_heads,
@@ -333,16 +331,14 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
331
) -> None:
332
super().__init__()
333
self.hidden_size = config.hidden_size
336
- rope_theta = getattr(config, "rope_theta", 500000)
337
- rope_scaling = getattr(config, "rope_scaling", None)
334
+ set_default_rope_theta(config, default_theta=500000)
335
max_position_embeddings = getattr(config, "max_position_embeddings", 131072)
336
self.self_attn = Ernie4_5_MoeAttention(
337
hidden_size=self.hidden_size,
338
num_heads=config.num_attention_heads,
339
num_kv_heads=config.num_key_value_heads,
340
head_dim=getattr(config, "head_dim", None),
344
- rope_theta=rope_theta,
345
- rope_scaling=rope_scaling,
341
+ rope_parameters=config.rope_parameters,
342
max_position_embeddings=max_position_embeddings,
343
rms_norm_eps=config.rms_norm_eps,
344
qkv_bias=getattr(config, "use_bias", False),
vllm/model_executor/models/ernie45_vl_moe.py
+5
-8
@@ -58,6 +58,7 @@ from vllm.model_executor.model_loader.weight_utils import (
58
maybe_remap_kv_scale_name,
59
)
60
from vllm.sequence import IntermediateTensors
61
+from vllm.transformers_utils.config import set_default_rope_theta
62
63
from .ernie45_moe import Ernie4_5_MoeMLP
64
from .interfaces import SupportsPP
@@ -91,9 +92,8 @@ class Ernie4_5_VLMoeAttention(nn.Module):
92
hidden_size: int,
93
num_heads: int,
94
num_kv_heads: int,
95
+ rope_parameters: dict[str, Any],
96
head_dim: int | None = None,
95
- rope_theta: float = 500000,
96
- rope_scaling: dict[str, Any] | None = None,
97
freq_allocation: int = 20,
98
max_position_embeddings: int = 131072,
99
rms_norm_eps: float = 1e-05,
@@ -126,7 +126,6 @@ class Ernie4_5_VLMoeAttention(nn.Module):
126
self.q_size = self.num_heads * self.head_dim
127
self.kv_size = self.num_kv_heads * self.head_dim
128
self.scaling = self.head_dim**-0.5
129
- self.rope_theta = rope_theta
129
self.max_position_embeddings = max_position_embeddings
130
131
self.qkv_proj = QKVParallelLinear(
@@ -155,7 +154,7 @@ class Ernie4_5_VLMoeAttention(nn.Module):
154
head_size=self.head_dim,
155
rotary_dim=self.head_dim,
156
max_position_embeddings=max_position_embeddings,
158
- base=rope_theta,
157
+ base=rope_parameters["rope_theta"],
158
is_neox_style=False,
159
dtype=torch.get_default_dtype(),
160
mrope_section=[h_rope, w_rope, t_rope],
@@ -413,8 +412,7 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
412
) -> None:
413
super().__init__()
414
self.hidden_size = config.hidden_size
416
- rope_theta = getattr(config, "rope_theta", 500000)
417
- rope_scaling = getattr(config, "rope_scaling", None)
415
+ set_default_rope_theta(config, default_theta=500000)
416
freq_allocation = getattr(config, "freq_allocation", 20)
417
max_position_embeddings = getattr(config, "max_position_embeddings", 131072)
418
@@ -423,8 +421,7 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
421
num_heads=config.num_attention_heads,
422
num_kv_heads=config.num_key_value_heads,
423
head_dim=getattr(config, "head_dim", None),
426
- rope_theta=rope_theta,
427
- rope_scaling=rope_scaling,
424
+ rope_parameters=config.rope_parameters,
425
freq_allocation=freq_allocation,
426
max_position_embeddings=max_position_embeddings,
427
rms_norm_eps=config.rms_norm_eps,
vllm/model_executor/models/exaone.py
+1
-20
@@ -27,7 +27,6 @@
27
28
from collections.abc import Iterable
29
from itertools import islice
30
-from typing import Any
30
31
import torch
32
from torch import nn
@@ -113,8 +112,6 @@ class ExaoneAttention(nn.Module):
112
hidden_size: int,
113
num_heads: int,
114
num_kv_heads: int,
116
- rope_theta: float = 10000,
117
- rope_scaling: dict[str, Any] | None = None,
115
max_position_embeddings: int = 8192,
116
quant_config: QuantizationConfig | None = None,
117
bias: bool = False,
@@ -144,7 +141,6 @@ class ExaoneAttention(nn.Module):
141
self.q_size = self.num_heads * self.head_dim
142
self.kv_size = self.num_kv_heads * self.head_dim
143
self.scaling = self.head_dim**-0.5
147
- self.rope_theta = rope_theta
144
self.max_position_embeddings = max_position_embeddings
145
146
self.qkv_proj = QKVParallelLinear(
@@ -173,8 +169,7 @@ class ExaoneAttention(nn.Module):
169
self.head_dim,
170
rotary_dim=self.head_dim,
171
max_position=max_position_embeddings,
176
- base=rope_theta,
177
- rope_scaling=rope_scaling,
172
+ rope_parameters=config.rope_parameters,
173
is_neox_style=is_neox_style,
174
)
175
self.attn = Attention(
@@ -207,8 +202,6 @@ class ExaoneBlockAttention(nn.Module):
202
hidden_size: int,
203
num_heads: int,
204
num_kv_heads: int,
210
- rope_theta: float = 10000,
211
- rope_scaling: dict[str, Any] | None = None,
205
max_position_embeddings: int = 8192,
206
quant_config: QuantizationConfig | None = None,
207
bias: bool = False,
@@ -221,8 +214,6 @@ class ExaoneBlockAttention(nn.Module):
214
hidden_size=hidden_size,
215
num_heads=num_heads,
216
num_kv_heads=num_kv_heads,
224
- rope_theta=rope_theta,
225
- rope_scaling=rope_scaling,
217
max_position_embeddings=max_position_embeddings,
218
quant_config=quant_config,
219
bias=bias,
@@ -251,14 +242,6 @@ class ExaoneDecoderLayer(nn.Module):
242
) -> None:
243
super().__init__()
244
self.hidden_size = config.hidden_size
254
- rope_theta = getattr(config, "rope_theta", 10000)
255
- rope_scaling = getattr(config, "rope_scaling", None)
256
- if rope_scaling is not None and getattr(
257
- config, "original_max_position_embeddings", None
258
- ):
259
- rope_scaling["original_max_position_embeddings"] = (
260
- config.original_max_position_embeddings
261
- )
245
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
246
# Support abacusai/Smaug-72B-v0.1 with attention_bias
247
# Support internlm/internlm-7b with bias
@@ -272,8 +255,6 @@ class ExaoneDecoderLayer(nn.Module):
255
num_kv_heads=getattr(
256
config, "num_key_value_heads", config.num_attention_heads
257
),
275
- rope_theta=rope_theta,
276
- rope_scaling=rope_scaling,
258
max_position_embeddings=max_position_embeddings,
259
quant_config=quant_config,
260
bias=attention_bias,
vllm/model_executor/models/exaone4.py
+3
-16
@@ -23,7 +23,6 @@
23
24
from collections.abc import Iterable
25
from itertools import islice
26
-from typing import Any
26
27
import torch
28
from torch import nn
@@ -52,6 +51,7 @@ from vllm.model_executor.model_loader.weight_utils import (
51
maybe_remap_kv_scale_name,
52
)
53
from vllm.sequence import IntermediateTensors
54
+from vllm.transformers_utils.config import set_default_rope_theta
55
56
from .interfaces import SupportsLoRA, SupportsPP
57
from .utils import (
@@ -110,8 +110,6 @@ class Exaone4Attention(nn.Module):
110
hidden_size: int,
111
num_heads: int,
112
num_kv_heads: int,
113
- rope_theta: float = 1000000,
114
- rope_scaling: dict[str, Any] | None = None,
113
max_position_embeddings: int = 8192,
114
quant_config: QuantizationConfig | None = None,
115
bias: bool = False,
@@ -141,7 +139,6 @@ class Exaone4Attention(nn.Module):
139
self.q_size = self.num_heads * self.head_dim
140
self.kv_size = self.num_kv_heads * self.head_dim
141
self.scaling = self.head_dim**-0.5
144
- self.rope_theta = rope_theta
142
self.max_position_embeddings = max_position_embeddings
143
144
self.qkv_proj = QKVParallelLinear(
@@ -176,12 +173,12 @@ class Exaone4Attention(nn.Module):
173
# apply rotary embeddings to every layer in full attention models
174
self.apply_rope_all_layers = "sliding_attention" not in config.layer_types
175
176
+ set_default_rope_theta(config, default_theta=1000000)
177
self.rotary_emb = get_rope(
178
self.head_dim,
179
rotary_dim=self.head_dim,
180
max_position=max_position_embeddings,
183
- base=rope_theta,
184
- rope_scaling=rope_scaling,
181
+ rope_parameters=config.rope_parameters,
182
is_neox_style=is_neox_style,
183
)
184
self.attn = Attention(
@@ -227,14 +224,6 @@ class Exaone4DecoderLayer(nn.Module):
224
) -> None:
225
super().__init__()
226
self.hidden_size = config.hidden_size
230
- rope_theta = getattr(config, "rope_theta", 1000000)
231
- rope_scaling = getattr(config, "rope_scaling", None)
232
- if rope_scaling is not None and getattr(
233
- config, "original_max_position_embeddings", None
234
- ):
235
- rope_scaling["original_max_position_embeddings"] = (
236
- config.original_max_position_embeddings
237
- )
227
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
228
# Support abacusai/Smaug-72B-v0.1 with attention_bias
229
# Support internlm/internlm-7b with bias
@@ -249,8 +238,6 @@ class Exaone4DecoderLayer(nn.Module):
238
num_kv_heads=getattr(
239
config, "num_key_value_heads", config.num_attention_heads
240
),
252
- rope_theta=rope_theta,
253
- rope_scaling=rope_scaling,
241
max_position_embeddings=max_position_embeddings,
242
quant_config=quant_config,
243
bias=attention_bias,
vllm/model_executor/models/falcon.py
+1
-2
@@ -164,13 +164,12 @@ class FalconAttention(nn.Module):
164
)
165
166
if self.use_rotary:
167
- rope_theta = getattr(config, "rope_theta", 10000)
167
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
168
self.rotary_emb = get_rope(
169
self.head_dim,
170
rotary_dim=self.head_dim,
171
max_position=max_position_embeddings,
173
- base=rope_theta,
172
+ rope_parameters=config.rope_parameters,
173
)
174
self.attn = Attention(
175
self.num_heads,
vllm/model_executor/models/falcon_h1.py
+3
-5
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35
)
36
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
37
from vllm.sequence import IntermediateTensors
38
+from vllm.transformers_utils.config import set_default_rope_theta
39
40
from .interfaces import (
41
HasInnerState,
@@ -214,8 +215,7 @@ class FalconH1AttentionDecoderLayer(nn.Module):
215
prefix: str = "",
216
) -> None:
217
super().__init__()
217
- rope_theta = getattr(config, "rope_theta", 1e11)
218
- rope_scaling = getattr(config, "rope_scaling", None)
218
+ set_default_rope_theta(config, default_theta=1e11)
219
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
220
self.hidden_size = config.hidden_size
221
tp_size = get_tensor_model_parallel_world_size()
@@ -240,7 +240,6 @@ class FalconH1AttentionDecoderLayer(nn.Module):
240
self.q_size = self.num_heads * self.head_dim
241
self.kv_size = self.num_kv_heads * self.head_dim
242
self.scaling = self.head_dim**-0.5
243
- self.rope_theta = rope_theta
243
self.max_position_embeddings = max_position_embeddings
244
245
if hasattr(config, "partial_rotary_factor"):
@@ -254,8 +253,7 @@ class FalconH1AttentionDecoderLayer(nn.Module):
253
head_size=self.head_dim,
254
rotary_dim=rotary_dim,
255
max_position=max_position_embeddings,
257
- rope_scaling=rope_scaling,
258
- base=rope_theta,
256
+ rope_parameters=config.rope_parameters,
257
is_neox_style=True,
258
dtype=None, # see impl of get_rope
259
)
vllm/model_executor/models/gemma.py
+4
-4
@@ -20,6 +20,7 @@
20
from collections.abc import Iterable
21
from functools import cache
22
from itertools import islice
23
+from typing import Any
24
25
import torch
26
from torch import nn
@@ -127,8 +128,8 @@ class GemmaAttention(nn.Module):
128
num_heads: int,
129
num_kv_heads: int,
130
head_dim: int,
131
+ rope_parameters: dict[str, Any],
132
max_position_embeddings: int = 8192,
131
- rope_theta: float = 10000,
133
cache_config: CacheConfig | None = None,
134
quant_config: QuantizationConfig | None = None,
135
prefix: str = "",
@@ -153,7 +154,6 @@ class GemmaAttention(nn.Module):
154
self.q_size = self.num_heads * self.head_dim
155
self.kv_size = self.num_kv_heads * self.head_dim
156
self.scaling = self.head_dim**-0.5
156
- self.rope_theta = rope_theta
157
158
self.qkv_proj = QKVParallelLinear(
159
hidden_size,
@@ -176,7 +176,7 @@ class GemmaAttention(nn.Module):
176
self.head_dim,
177
rotary_dim=self.head_dim,
178
max_position=max_position_embeddings,
179
- base=self.rope_theta,
179
+ rope_parameters=rope_parameters,
180
is_neox_style=True,
181
)
182
self.attn = Attention(
@@ -218,7 +218,7 @@ class GemmaDecoderLayer(nn.Module):
218
num_kv_heads=config.num_key_value_heads,
219
head_dim=config.head_dim,
220
max_position_embeddings=config.max_position_embeddings,
221
- rope_theta=config.rope_theta,
221
+ rope_parameters=config.rope_parameters,
222
cache_config=cache_config,
223
quant_config=quant_config,
224
prefix=f"{prefix}.self_attn",
vllm/model_executor/models/gemma2.py
+1
-4
@@ -107,7 +107,6 @@ class Gemma2Attention(nn.Module):
107
num_kv_heads: int,
108
head_dim: int,
109
max_position_embeddings: int,
110
- rope_theta: float,
110
cache_config: CacheConfig | None = None,
111
quant_config: QuantizationConfig | None = None,
112
attn_logits_soft_cap: float | None = None,
@@ -134,7 +133,6 @@ class Gemma2Attention(nn.Module):
133
self.q_size = self.num_heads * self.head_dim
134
self.kv_size = self.num_kv_heads * self.head_dim
135
self.scaling = config.query_pre_attn_scalar**-0.5
137
- self.rope_theta = rope_theta
136
137
self.qkv_proj = QKVParallelLinear(
138
hidden_size,
@@ -156,7 +154,7 @@ class Gemma2Attention(nn.Module):
154
self.head_dim,
155
rotary_dim=self.head_dim,
156
max_position=max_position_embeddings,
159
- base=self.rope_theta,
157
+ rope_parameters=config.rope_parameters,
158
is_neox_style=True,
159
)
160
@@ -206,7 +204,6 @@ class Gemma2DecoderLayer(nn.Module):
204
num_kv_heads=config.num_key_value_heads,
205
head_dim=config.head_dim,
206
max_position_embeddings=config.max_position_embeddings,
209
- rope_theta=config.rope_theta,
207
cache_config=cache_config,
208
quant_config=quant_config,
209
attn_logits_soft_cap=config.attn_logit_softcapping,
vllm/model_executor/models/gemma3.py
+12
-9
@@ -155,25 +155,28 @@ class Gemma3Attention(nn.Module):
155
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
156
157
layer_idx = extract_layer_index(prefix)
158
- self.is_sliding = config.layer_types[layer_idx] == "sliding_attention"
158
+ layer_type = config.layer_types[layer_idx]
159
+ self.is_sliding = layer_type == "sliding_attention"
160
sliding_window = config.sliding_window if self.is_sliding else None
161
162
# Initialize the rotary embedding.
162
- if self.is_sliding:
163
- # Local attention. Override the values in config.json.
164
- self.rope_theta = config.rope_local_base_freq
165
- self.rope_scaling = {"rope_type": "default"}
163
+ if layer_type in config.rope_parameters:
164
+ # Transformers v5 rope config.
165
+ rope_parameters = config.rope_parameters[layer_type]
166
else:
167
+ # Transformers v4 rope config.
168
# Global attention. Use the values in config.json.
168
- self.rope_theta = config.rope_theta
169
- self.rope_scaling = config.rope_scaling
169
+ rope_parameters = config.rope_parameters.copy()
170
+ # Local attention. Override the values in config.json.
171
+ if self.is_sliding:
172
+ rope_parameters["rope_theta"] = config.rope_local_base_freq
173
+
174
self.rotary_emb = get_rope(
175
self.head_dim,
176
rotary_dim=self.head_dim,
177
max_position=max_position_embeddings,
174
- base=self.rope_theta,
178
+ rope_parameters=rope_parameters,
179
is_neox_style=True,
176
- rope_scaling=self.rope_scaling,
180
)
181
182
if getattr(config, "is_causal", True):
vllm/model_executor/models/gemma3n.py
+11
-9
@@ -332,18 +332,21 @@ class Gemma3nAttention(nn.Module):
332
)
333
334
layer_idx = extract_layer_index(prefix)
335
- is_sliding = config.layer_types[layer_idx] == "sliding_attention"
335
+ layer_type = config.layer_types[layer_idx]
336
+ is_sliding = layer_type == "sliding_attention"
337
self.sliding_window = config.sliding_window if is_sliding else None
338
339
# Initialize the rotary embedding.
339
- if is_sliding:
340
- # Local attention. Override the values in config.json.
341
- rope_theta = config.rope_local_base_freq
342
- rope_scaling = {"rope_type": "default"}
340
+ if layer_type in config.rope_parameters:
341
+ # Transformers v5 rope config.
342
+ rope_parameters = config.rope_parameters[layer_type]
343
else:
344
+ # Transformers v4 rope config.
345
# Global attention. Use the values in config.json.
345
- rope_theta = config.rope_theta
346
- rope_scaling = config.rope_scaling
346
+ rope_parameters = config.rope_parameters.copy()
347
+ # Local attention. Override the values in config.json.
348
+ if is_sliding:
349
+ rope_parameters["rope_theta"] = config.rope_local_base_freq
350
351
first_kv_shared_layer_idx = (
352
config.num_hidden_layers - config.num_kv_shared_layers
@@ -383,9 +386,8 @@ class Gemma3nAttention(nn.Module):
386
self.head_dim,
387
rotary_dim=self.head_dim,
388
max_position=max_position_embeddings,
386
- base=rope_theta,
389
+ rope_parameters=rope_parameters,
390
is_neox_style=True,
388
- rope_scaling=rope_scaling,
391
)
392
393
self.attn = Attention(
vllm/model_executor/models/glm4.py
+1
-9
@@ -57,10 +57,8 @@ class Glm4Attention(nn.Module):
57
max_position: int = 4096 * 32,
58
head_dim: int | None = None,
59
qkv_bias: bool = False,
60
- rope_theta: float = 10000,
60
cache_config: CacheConfig | None = None,
61
quant_config: QuantizationConfig | None = None,
63
- rope_scaling: tuple | None = None,
62
prefix: str = "",
63
attn_type: str = AttentionType.DECODER,
64
) -> None:
@@ -86,7 +84,6 @@ class Glm4Attention(nn.Module):
84
self.q_size = self.num_heads * self.head_dim
85
self.kv_size = self.num_kv_heads * self.head_dim
86
self.scaling = self.head_dim**-0.5
89
- self.rope_theta = rope_theta
87
self.qkv_proj = QKVParallelLinear(
88
hidden_size,
89
self.head_dim,
@@ -107,8 +104,7 @@ class Glm4Attention(nn.Module):
104
self.head_dim,
105
rotary_dim=self.rotary_dim,
106
max_position=max_position,
110
- base=self.rope_theta,
111
- rope_scaling=rope_scaling,
107
+ rope_parameters=config.rope_parameters,
108
partial_rotary_factor=partial_rotary_factor,
109
is_neox_style=False,
110
)
@@ -150,8 +146,6 @@ class Glm4DecoderLayer(nn.Module):
146
quant_config = vllm_config.quant_config
147
148
self.hidden_size = config.hidden_size
153
- rope_theta = getattr(config, "rope_theta", 1000000)
154
- rope_scaling = getattr(config, "rope_scaling", None)
149
150
self.self_attn = Glm4Attention(
151
config=config,
@@ -159,12 +153,10 @@ class Glm4DecoderLayer(nn.Module):
153
num_heads=config.num_attention_heads,
154
max_position=config.max_position_embeddings,
155
num_kv_heads=config.num_key_value_heads,
162
- rope_theta=rope_theta,
156
qkv_bias=getattr(config, "attention_bias", False),
157
head_dim=getattr(config, "head_dim", None),
158
cache_config=cache_config,
159
quant_config=quant_config,
167
- rope_scaling=rope_scaling,
160
prefix=f"{prefix}.self_attn",
161
attn_type=AttentionType.DECODER,
162
)
vllm/model_executor/models/glm4_1v.py
-1
@@ -703,7 +703,6 @@ class Glm4vVisionTransformer(nn.Module):
703
head_size=head_dim,
704
rotary_dim=head_dim // 2,
705
max_position=8192,
706
- base=10000.0,
706
is_neox_style=True,
707
)
708
self.blocks = nn.ModuleList(
vllm/model_executor/models/glm4_moe.py
+1
-10
@@ -26,7 +26,6 @@
26
import typing
27
from collections.abc import Callable, Iterable
28
from itertools import islice
29
-from typing import Any
29
30
import torch
31
from torch import nn
@@ -233,8 +232,6 @@ class Glm4MoeAttention(nn.Module):
232
hidden_size: int,
233
num_heads: int,
234
num_kv_heads: int,
236
- rope_theta: float = 10000,
237
- rope_scaling: dict[str, Any] | None = None,
235
max_position_embeddings: int = 131072,
236
head_dim: int | None = None,
237
rms_norm_eps: float = 1e-05,
@@ -264,7 +261,6 @@ class Glm4MoeAttention(nn.Module):
261
self.q_size = self.num_heads * self.head_dim
262
self.kv_size = self.num_kv_heads * self.head_dim
263
self.scaling = self.head_dim**-0.5
267
- self.rope_theta = rope_theta
264
self.max_position_embeddings = max_position_embeddings
265
self.use_qk_norm = use_qk_norm
266
@@ -291,8 +287,7 @@ class Glm4MoeAttention(nn.Module):
287
self.head_dim,
288
rotary_dim=self.head_dim,
289
max_position=max_position_embeddings,
294
- base=rope_theta,
295
- rope_scaling=rope_scaling,
290
+ rope_parameters=config.rope_parameters,
291
partial_rotary_factor=partial_rotary_factor,
292
)
293
self.attn = Attention(
@@ -341,8 +336,6 @@ class Glm4MoeDecoderLayer(nn.Module):
336
) -> None:
337
super().__init__()
338
self.hidden_size = config.hidden_size
344
- rope_theta = getattr(config, "rope_theta", 10000)
345
- rope_scaling = getattr(config, "rope_scaling", None)
339
max_position_embeddings = getattr(config, "max_position_embeddings", 131072)
340
# DecoderLayers are created with `make_layers` which passes the prefix
341
# with the layer's index.
@@ -354,8 +347,6 @@ class Glm4MoeDecoderLayer(nn.Module):
347
hidden_size=self.hidden_size,
348
num_heads=config.num_attention_heads,
349
num_kv_heads=config.num_key_value_heads,
357
- rope_theta=rope_theta,
358
- rope_scaling=rope_scaling,
350
max_position_embeddings=max_position_embeddings,
351
head_dim=config.head_dim,
352
rms_norm_eps=config.rms_norm_eps,
vllm/model_executor/models/gpt_j.py
+1
-2
@@ -95,13 +95,12 @@ class GPTJAttention(nn.Module):
95
scaling = self.head_size**-0.5
96
assert getattr(config, "rotary", True)
97
assert config.rotary_dim % 2 == 0
98
- rope_theta = getattr(config, "rope_theta", 10000)
98
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
99
self.rotary_emb = get_rope(
100
self.head_size,
101
rotary_dim=config.rotary_dim,
102
max_position=max_position_embeddings,
104
- base=rope_theta,
103
+ rope_parameters=config.rope_parameters,
104
is_neox_style=False,
105
)
106
self.attn = Attention(
vllm/model_executor/models/gpt_neox.py
+1
-2
@@ -92,13 +92,12 @@ class GPTNeoXAttention(nn.Module):
92
scaling = self.head_size**-0.5
93
rotary_dim = int(self.head_size * config.rotary_pct)
94
assert rotary_dim % 2 == 0
95
- rope_theta = getattr(config, "rope_theta", 10000)
95
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
96
self.rotary_emb = get_rope(
97
self.head_size,
98
rotary_dim=rotary_dim,
99
max_position=max_position_embeddings,
101
- base=rope_theta,
100
+ rope_parameters=config.rope_parameters,
101
)
102
self.attn = Attention(
103
self.num_heads,
vllm/model_executor/models/gpt_oss.py
+6
-7
@@ -67,16 +67,16 @@ class OAIAttention(nn.Module):
67
self.head_dim,
68
rotary_dim=self.head_dim,
69
max_position=config.max_position_embeddings,
70
- base=config.rope_theta,
70
dtype=torch.float32,
72
- rope_scaling={
71
+ rope_parameters={
72
+ "rope_theta": config.rope_parameters["rope_theta"],
73
"rope_type": "yarn",
74
- "factor": config.rope_scaling["factor"],
75
- "original_max_position_embeddings": config.rope_scaling[
74
+ "factor": config.rope_parameters["factor"],
75
+ "original_max_position_embeddings": config.rope_parameters[
76
"original_max_position_embeddings"
77
],
78
- "beta_fast": config.rope_scaling["beta_fast"],
79
- "beta_slow": config.rope_scaling["beta_slow"],
78
+ "beta_fast": config.rope_parameters["beta_fast"],
79
+ "beta_slow": config.rope_parameters["beta_slow"],
80
},
81
is_neox_style=True,
82
)
@@ -90,7 +90,6 @@ class OAIAttention(nn.Module):
90
self.q_size = self.num_attention_heads * self.head_dim // tp_size
91
self.kv_size = self.num_key_value_heads * self.head_dim // tp_size
92
self.scaling = self.head_dim**-0.5
93
- self.rope_theta = config.rope_theta
93
94
self.qkv_proj = QKVParallelLinear(
95
hidden_size=self.hidden_size,
vllm/model_executor/models/granite.py
+1
-16
@@ -26,7 +26,6 @@
26
27
from collections.abc import Iterable
28
from itertools import islice
29
-from typing import Any
29
30
import torch
31
from torch import nn
@@ -112,8 +111,6 @@ class GraniteAttention(nn.Module):
111
hidden_size: int,
112
num_heads: int,
113
num_kv_heads: int,
115
- rope_theta: float = 10000,
116
- rope_scaling: dict[str, Any] | None = None,
114
max_position_embeddings: int = 8192,
115
quant_config: QuantizationConfig | None = None,
116
bias: bool = False,
@@ -143,7 +140,6 @@ class GraniteAttention(nn.Module):
140
self.q_size = self.num_heads * self.head_dim
141
self.kv_size = self.num_kv_heads * self.head_dim
142
self.scaling = config.attention_multiplier
146
- self.rope_theta = rope_theta
143
self.max_position_embeddings = max_position_embeddings
144
145
self.qkv_proj = QKVParallelLinear(
@@ -167,8 +163,7 @@ class GraniteAttention(nn.Module):
163
self.head_dim,
164
rotary_dim=self.head_dim,
165
max_position=max_position_embeddings,
170
- base=rope_theta,
171
- rope_scaling=rope_scaling,
166
+ rope_parameters=config.rope_parameters,
167
)
168
self.attn = Attention(
169
self.num_heads,
@@ -204,14 +199,6 @@ class GraniteDecoderLayer(nn.Module):
199
super().__init__()
200
self.hidden_size = config.hidden_size
201
self.residual_multiplier = config.residual_multiplier
207
- rope_theta = getattr(config, "rope_theta", 10000)
208
- rope_scaling = getattr(config, "rope_scaling", None)
209
- if rope_scaling is not None and getattr(
210
- config, "original_max_position_embeddings", None
211
- ):
212
- rope_scaling["original_max_position_embeddings"] = (
213
- config.original_max_position_embeddings
214
- )
202
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
203
# Support abacusai/Smaug-72B-v0.1 with attention_bias
204
# Support internlm/internlm-7b with bias
@@ -225,8 +212,6 @@ class GraniteDecoderLayer(nn.Module):
212
num_kv_heads=getattr(
213
config, "num_key_value_heads", config.num_attention_heads
214
),
228
- rope_theta=rope_theta,
229
- rope_scaling=rope_scaling,
215
max_position_embeddings=max_position_embeddings,
216
quant_config=quant_config,
217
bias=attention_bias,
vllm/model_executor/models/granitemoe.py
+3
-10
@@ -141,8 +141,7 @@ class GraniteMoeAttention(nn.Module):
141
num_heads: int,
142
num_kv_heads: int,
143
max_position: int = 4096 * 32,
144
- rope_theta: float = 10000,
145
- rope_scaling: dict[str, Any] | None = None,
144
+ rope_parameters: dict[str, Any] | None = None,
145
cache_config: CacheConfig | None = None,
146
quant_config: QuantizationConfig | None = None,
147
attention_multiplier: float | None = None,
@@ -172,7 +171,6 @@ class GraniteMoeAttention(nn.Module):
171
if attention_multiplier is not None
172
else self.head_dim**-1
173
)
175
- self.rope_theta = rope_theta
174
175
self.qkv_proj = QKVParallelLinear(
176
hidden_size,
@@ -194,9 +192,8 @@ class GraniteMoeAttention(nn.Module):
192
self.head_dim,
193
rotary_dim=self.head_dim,
194
max_position=max_position,
197
- base=int(self.rope_theta),
195
+ rope_parameters=rope_parameters,
196
is_neox_style=True,
199
- rope_scaling=rope_scaling,
197
)
198
self.attn = Attention(
199
self.num_heads,
@@ -235,16 +232,12 @@ class GraniteMoeDecoderLayer(nn.Module):
232
parallel_config = vllm_config.parallel_config
233
234
self.hidden_size = config.hidden_size
238
- # Requires transformers > 4.32.0
239
- rope_theta = getattr(config, "rope_theta", 10000)
240
- rope_scaling = getattr(config, "rope_scaling", None)
235
self.self_attn = GraniteMoeAttention(
236
hidden_size=self.hidden_size,
237
num_heads=config.num_attention_heads,
238
max_position=config.max_position_embeddings,
239
num_kv_heads=config.num_key_value_heads,
246
- rope_theta=rope_theta,
247
- rope_scaling=rope_scaling,
240
+ rope_parameters=config.rope_parameters,
241
cache_config=cache_config,
242
quant_config=quant_config,
243
prefix=f"{prefix}.self_attn",
vllm/model_executor/models/granitemoehybrid.py
+1
-4
@@ -273,10 +273,7 @@ class GraniteMoeHybridAttention(nn.Module):
273
self.head_dim,
274
rotary_dim=self.head_dim,
275
max_position=config.max_position_embeddings,
276
- base=int(config.rope_theta),
277
- rope_scaling=config.rope_scaling
278
- if hasattr(config, "rope_scaling") and config.rope_scaling is not None
279
- else None,
276
+ rope_parameters=config.rope_parameters,
277
is_neox_style=True,
278
)
279
else:
vllm/model_executor/models/granitemoeshared.py
+1
-5
@@ -84,16 +84,12 @@ class GraniteMoeSharedDecoderLayer(nn.Module):
84
) -> None:
85
super().__init__()
86
self.hidden_size = config.hidden_size
87
- # Requires transformers > 4.32.0
88
- rope_theta = getattr(config, "rope_theta", 10000)
89
- rope_scaling = getattr(config, "rope_scaling", None)
87
self.self_attn = GraniteMoeAttention(
88
hidden_size=self.hidden_size,
89
num_heads=config.num_attention_heads,
90
max_position=config.max_position_embeddings,
91
num_kv_heads=config.num_key_value_heads,
95
- rope_theta=rope_theta,
96
- rope_scaling=rope_scaling,
92
+ rope_parameters=config.rope_parameters,
93
cache_config=cache_config,
94
quant_config=quant_config,
95
prefix=f"{prefix}.self_attn",
vllm/model_executor/models/grok1.py
+4
-7
@@ -25,6 +25,7 @@
25
26
from collections.abc import Iterable
27
from itertools import islice
28
+from typing import Any
29
30
import torch
31
import torch.nn.functional as F
@@ -134,7 +135,7 @@ class Grok1Attention(nn.Module):
135
num_heads: int,
136
num_kv_heads: int,
137
max_position: int = 4096 * 32,
137
- rope_theta: float = 10000,
138
+ rope_parameters: dict[str, Any] | None = None,
139
cache_config: CacheConfig | None = None,
140
quant_config: QuantizationConfig | None = None,
141
prefix: str = "",
@@ -161,7 +162,6 @@ class Grok1Attention(nn.Module):
162
self.q_size = self.num_heads * self.head_dim
163
self.kv_size = self.num_kv_heads * self.head_dim
164
self.scaling = self.head_dim**-0.5
164
- self.rope_theta = rope_theta
165
166
self.qkv_proj = QKVParallelLinear(
167
hidden_size,
@@ -183,7 +183,7 @@ class Grok1Attention(nn.Module):
183
self.head_dim,
184
rotary_dim=self.head_dim,
185
max_position=max_position,
186
- base=int(self.rope_theta),
186
+ rope_parameters=rope_parameters,
187
is_neox_style=True,
188
)
189
@@ -234,15 +234,12 @@ class Grok1DecoderLayer(nn.Module):
234
if not self.use_fp8 and hasattr(quant_config, "is_fp8"):
235
self.use_fp8 = quant_config.is_fp8
236
237
- # Requires transformers > 4.32.0
238
- # Default rope_theta value if not in config
239
- rope_theta = 10000
237
self.attn = Grok1Attention(
238
hidden_size=self.hidden_size,
239
num_heads=config.num_attention_heads,
240
max_position=config.max_position_embeddings,
241
num_kv_heads=config.num_key_value_heads,
245
- rope_theta=rope_theta,
242
+ rope_parameters=config.rope_parameters,
243
cache_config=cache_config,
244
quant_config=quant_config,
245
prefix=f"{prefix}.attn",
vllm/model_executor/models/hunyuan_v1.py
+2
-23
@@ -27,7 +27,6 @@
27
import typing
28
from collections.abc import Callable, Iterable
29
from itertools import islice
30
-from typing import Any
30
31
import regex as re
32
import torch
@@ -142,8 +141,6 @@ class HunYuanAttention(nn.Module):
141
hidden_size: int,
142
num_heads: int,
143
num_kv_heads: int,
145
- rope_theta: float = 10000,
146
- rope_scaling: dict[str, Any] | None = None,
144
max_position_embeddings: int = 8192,
145
quant_config: QuantizationConfig | None = None,
146
bias: bool = False,
@@ -177,7 +174,6 @@ class HunYuanAttention(nn.Module):
174
self.q_size = self.num_heads * self.head_dim
175
self.kv_size = self.num_kv_heads * self.head_dim
176
self.scaling = self.head_dim**-0.5
180
- self.rope_theta = rope_theta
177
self.max_position_embeddings = max_position_embeddings
178
self.use_qk_norm = getattr(config, "use_qk_norm", False)
179
self.layer_id = layer_id
@@ -204,8 +200,7 @@ class HunYuanAttention(nn.Module):
200
self.head_dim,
201
rotary_dim=self.head_dim,
202
max_position=max_position_embeddings,
207
- base=rope_theta,
208
- rope_scaling=rope_scaling,
203
+ rope_parameters=config.rope_parameters,
204
is_neox_style=True,
205
)
206
self.attn = Attention(
@@ -254,8 +249,6 @@ class HunYuanCrossAttention(nn.Module):
249
hidden_size: int,
250
num_heads: int,
251
num_kv_heads: int,
257
- rope_theta: float = 10000,
258
- rope_scaling: dict[str, Any] | None = None,
252
max_position_embeddings: int = 8192,
253
quant_config: QuantizationConfig | None = None,
254
bias: bool = False,
@@ -289,7 +282,6 @@ class HunYuanCrossAttention(nn.Module):
282
self.q_size = self.num_heads * self.head_dim
283
self.kv_size = self.num_kv_heads * self.head_dim
284
self.scaling = self.head_dim**-0.5
292
- self.rope_theta = rope_theta
285
self.max_position_embeddings = max_position_embeddings
286
self.use_qk_norm = getattr(config, "use_qk_norm", False)
287
self.layer_id = layer_id
@@ -314,8 +306,7 @@ class HunYuanCrossAttention(nn.Module):
306
self.head_dim,
307
rotary_dim=self.head_dim,
308
max_position=max_position_embeddings,
317
- base=rope_theta,
318
- rope_scaling=rope_scaling,
309
+ rope_parameters=config.rope_parameters,
310
is_neox_style=True,
311
)
312
self.attn = Attention(
@@ -494,14 +485,6 @@ class HunYuanDecoderLayer(nn.Module):
485
if isinstance(config.intermediate_size, int)
486
else config.intermediate_size[layer_id]
487
)
497
- rope_theta = getattr(config, "rope_theta", 10000)
498
- rope_scaling = getattr(config, "rope_scaling", None)
499
- if rope_scaling is not None and getattr(
500
- config, "original_max_position_embeddings", None
501
- ):
502
- rope_scaling["original_max_position_embeddings"] = (
503
- config.original_max_position_embeddings
504
- )
488
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
489
attention_bias = getattr(config, "attention_bias", False) or getattr(
490
config, "bias", False
@@ -520,8 +503,6 @@ class HunYuanDecoderLayer(nn.Module):
503
num_kv_heads=getattr(
504
config, "num_key_value_heads", config.num_attention_heads
505
),
523
- rope_theta=rope_theta,
524
- rope_scaling=rope_scaling,
506
max_position_embeddings=max_position_embeddings,
507
quant_config=quant_config,
508
bias=attention_bias,
@@ -537,8 +518,6 @@ class HunYuanDecoderLayer(nn.Module):
518
num_kv_heads=getattr(
519
config, "num_key_value_heads", config.num_attention_heads
520
),
540
- rope_theta=rope_theta,
541
- rope_scaling=rope_scaling,
521
max_position_embeddings=max_position_embeddings,
522
quant_config=quant_config,
523
bias=attention_bias,
vllm/model_executor/models/internlm2.py
+3
-9
@@ -91,8 +91,7 @@ class InternLM2Attention(nn.Module):
91
hidden_size: int,
92
num_heads: int,
93
num_kv_heads: int,
94
- rope_theta: float = 10000,
95
- rope_scaling: dict[str, Any] | None = None,
94
+ rope_parameters: dict[str, Any] | None = None,
95
max_position_embeddings: int = 8192,
96
cache_config: CacheConfig | None = None,
97
quant_config: QuantizationConfig | None = None,
@@ -120,7 +119,6 @@ class InternLM2Attention(nn.Module):
119
self.kv_size = self.num_kv_heads * self.head_dim
120
self.key_value_groups = int(self.num_heads / self.num_kv_heads)
121
self.scaling = self.head_dim**-0.5
123
- self.rope_theta = rope_theta
122
self.max_position_embeddings = max_position_embeddings
123
124
self.wqkv = QKVParallelLinear(
@@ -144,8 +142,7 @@ class InternLM2Attention(nn.Module):
142
self.head_dim,
143
rotary_dim=self.head_dim,
144
max_position=max_position_embeddings,
147
- base=rope_theta,
148
- rope_scaling=rope_scaling,
145
+ rope_parameters=rope_parameters,
146
)
147
self.attn = Attention(
148
self.num_heads,
@@ -204,15 +201,12 @@ class InternLMDecoderLayer(nn.Module):
201
) -> None:
202
super().__init__()
203
self.hidden_size = config.hidden_size
207
- rope_theta = getattr(config, "rope_theta", 10000)
208
- rope_scaling = getattr(config, "rope_scaling", None)
204
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
205
self.attention = InternLM2Attention(
206
hidden_size=self.hidden_size,
207
num_heads=config.num_attention_heads,
208
num_kv_heads=config.num_key_value_heads,
214
- rope_theta=rope_theta,
215
- rope_scaling=rope_scaling,
209
+ rope_parameters=config.rope_parameters,
210
max_position_embeddings=max_position_embeddings,
211
cache_config=cache_config,
212
quant_config=quant_config,
vllm/model_executor/models/internlm2_ve.py
+1
-4
@@ -30,15 +30,12 @@ class InternLM2VEDecoderLayer(nn.Module):
30
) -> None:
31
super().__init__()
32
self.hidden_size = config.hidden_size
33
- rope_theta = getattr(config, "rope_theta", 10000)
34
- rope_scaling = getattr(config, "rope_scaling", None)
33
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
34
self.attention = InternLM2Attention(
35
hidden_size=self.hidden_size,
36
num_heads=config.num_attention_heads,
37
num_kv_heads=config.num_key_value_heads,
40
- rope_theta=rope_theta,
41
- rope_scaling=rope_scaling,
38
+ rope_parameters=config.rope_parameters,
39
max_position_embeddings=max_position_embeddings,
40
cache_config=cache_config,
41
quant_config=quant_config,
vllm/model_executor/models/kimi_linear.py
-5
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from collections.abc import Iterable
5
-from typing import Any
5
6
import torch
7
from torch import nn
@@ -190,9 +189,7 @@ class KimiMLAAttention(nn.Module):
189
v_head_dim: int,
190
q_lora_rank: int | None,
191
kv_lora_rank: int,
193
- rope_theta: float = 10000,
192
use_nope: bool = False,
195
- rope_scaling: dict[str, Any] | None = None,
193
cache_config: CacheConfig | None = None,
194
quant_config: QuantizationConfig | None = None,
195
prefix: str = "",
@@ -210,11 +207,9 @@ class KimiMLAAttention(nn.Module):
207
tp_size = get_tensor_model_parallel_world_size()
208
self.num_local_heads = num_heads // tp_size
209
self.scaling = self.qk_head_dim**-0.5
213
- self.rope_theta = rope_theta
210
self.use_nope = use_nope
211
assert self.use_nope is True
212
assert self.q_lora_rank is None
217
- assert rope_scaling is None
213
assert num_heads % tp_size == 0
214
self.kv_a_proj_with_mqa = ReplicatedLinear(
215
self.hidden_size,
vllm/model_executor/models/lfm2.py
+1
-16
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Iterable
4
from itertools import islice
5
-from typing import Any
5
6
import torch
7
import torch.nn as nn
@@ -96,8 +95,6 @@ class Lfm2Attention(nn.Module):
95
hidden_size: int,
96
num_heads: int,
97
num_kv_heads: int,
99
- rope_theta: float = 10000,
100
- rope_scaling: dict[str, Any] | None = None,
98
max_position_embeddings: int = 8192,
99
cache_config: CacheConfig | None = None,
100
quant_config: QuantizationConfig | None = None,
@@ -126,7 +123,6 @@ class Lfm2Attention(nn.Module):
123
self.q_size = self.num_heads * self.head_dim
124
self.kv_size = self.num_kv_heads * self.head_dim
125
self.scaling = self.head_dim**-0.5
129
- self.rope_theta = rope_theta
126
self.max_position_embeddings = max_position_embeddings
127
128
self.qkv_proj = QKVParallelLinear(
@@ -149,8 +145,7 @@ class Lfm2Attention(nn.Module):
145
self.head_dim,
146
rotary_dim=self.head_dim,
147
max_position=self.max_position_embeddings,
152
- base=self.rope_theta,
153
- rope_scaling=rope_scaling,
148
+ rope_parameters=config.rope_parameters,
149
is_neox_style=True,
150
)
151
self.attn = Attention(
@@ -199,14 +194,6 @@ class Lfm2AttentionDecoderLayer(nn.Module):
194
self.config = config
195
self.layer_idx = layer_idx
196
202
- rope_theta = getattr(config, "rope_theta", 10000)
203
- rope_scaling = getattr(config, "rope_scaling", None)
204
- if rope_scaling is not None and getattr(
205
- config, "original_max_position_embeddings", None
206
- ):
207
- rope_scaling["original_max_position_embeddings"] = (
208
- config.original_max_position_embeddings
209
- )
197
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
198
199
self.self_attn = Lfm2Attention(
@@ -215,8 +202,6 @@ class Lfm2AttentionDecoderLayer(nn.Module):
202
hidden_size=config.hidden_size,
203
num_heads=config.num_attention_heads,
204
num_kv_heads=config.num_key_value_heads,
218
- rope_theta=rope_theta,
219
- rope_scaling=rope_scaling,
205
max_position_embeddings=max_position_embeddings,
206
cache_config=cache_config,
207
quant_config=quant_config,
vllm/model_executor/models/lfm2_moe.py
+1
-16
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Iterable
4
from itertools import islice
5
-from typing import Any
5
6
import torch
7
import torch.nn as nn
@@ -189,8 +188,6 @@ class Lfm2MoeAttention(nn.Module):
188
hidden_size: int,
189
num_heads: int,
190
num_kv_heads: int,
192
- rope_theta: float = 10000,
193
- rope_scaling: dict[str, Any] | None = None,
191
max_position_embeddings: int = 8192,
192
cache_config: CacheConfig | None = None,
193
quant_config: QuantizationConfig | None = None,
@@ -219,7 +216,6 @@ class Lfm2MoeAttention(nn.Module):
216
self.q_size = self.num_heads * self.head_dim
217
self.kv_size = self.num_kv_heads * self.head_dim
218
self.scaling = self.head_dim**-0.5
222
- self.rope_theta = rope_theta
219
self.max_position_embeddings = max_position_embeddings
220
221
self.qkv_proj = QKVParallelLinear(
@@ -242,8 +238,7 @@ class Lfm2MoeAttention(nn.Module):
238
self.head_dim,
239
rotary_dim=self.head_dim,
240
max_position=self.max_position_embeddings,
245
- base=self.rope_theta,
246
- rope_scaling=rope_scaling,
241
+ rope_parameters=config.rope_parameters,
242
is_neox_style=True,
243
)
244
self.attn = Attention(
@@ -293,14 +288,6 @@ class Lfm2MoeAttentionDecoderLayer(nn.Module):
288
self.config = config
289
self.layer_idx = layer_idx
290
296
- rope_theta = getattr(config, "rope_theta", 10000)
297
- rope_scaling = getattr(config, "rope_scaling", None)
298
- if rope_scaling is not None and getattr(
299
- config, "original_max_position_embeddings", None
300
- ):
301
- rope_scaling["original_max_position_embeddings"] = (
302
- config.original_max_position_embeddings
303
- )
291
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
292
293
self.self_attn = Lfm2MoeAttention(
@@ -309,8 +296,6 @@ class Lfm2MoeAttentionDecoderLayer(nn.Module):
296
hidden_size=config.hidden_size,
297
num_heads=config.num_attention_heads,
298
num_kv_heads=config.num_key_value_heads,
312
- rope_theta=rope_theta,
313
- rope_scaling=rope_scaling,
299
max_position_embeddings=max_position_embeddings,
300
cache_config=cache_config,
301
quant_config=quant_config,
vllm/model_executor/models/llama.py
+2
-20
@@ -26,7 +26,6 @@
26
27
from collections.abc import Iterable
28
from itertools import islice
29
-from typing import Any
29
30
import torch
31
from torch import nn
@@ -120,8 +119,6 @@ class LlamaAttention(nn.Module):
119
hidden_size: int,
120
num_heads: int,
121
num_kv_heads: int,
123
- rope_theta: float = 10000,
124
- rope_scaling: dict[str, Any] | None = None,
122
max_position_embeddings: int = 8192,
123
quant_config: QuantizationConfig | None = None,
124
bias: bool = False,
@@ -157,7 +154,6 @@ class LlamaAttention(nn.Module):
154
self.q_size = self.num_heads * self.head_dim
155
self.kv_size = self.num_kv_heads * self.head_dim
156
self.scaling = self.head_dim**-0.5
160
- self.rope_theta = rope_theta
157
self.max_position_embeddings = max_position_embeddings
158
159
llama_4_scaling_config = getattr(config, "llama_4_scaling", None)
@@ -186,9 +182,7 @@ class LlamaAttention(nn.Module):
182
prefix=f"{prefix}.o_proj",
183
)
184
189
- self._init_rotary_emb(
190
- config, rope_scaling=rope_scaling, quant_config=quant_config
191
- )
185
+ self._init_rotary_emb(config, quant_config=quant_config)
186
187
sliding_window = None
188
if layer_types := getattr(config, "layer_types", None):
@@ -258,7 +252,6 @@ class LlamaAttention(nn.Module):
252
def _init_rotary_emb(
253
self,
254
config: LlamaConfig,
261
- rope_scaling: dict[str, Any] | None,
255
quant_config: QuantizationConfig | None,
256
) -> None:
257
is_neox_style = True
@@ -270,8 +263,7 @@ class LlamaAttention(nn.Module):
263
self.head_dim,
264
rotary_dim=self.head_dim,
265
max_position=self.max_position_embeddings,
273
- base=self.rope_theta,
274
- rope_scaling=rope_scaling,
266
+ rope_parameters=config.rope_parameters,
267
is_neox_style=is_neox_style,
268
partial_rotary_factor=self.partial_rotary_factor,
269
)
@@ -291,14 +283,6 @@ class LlamaDecoderLayer(nn.Module):
283
quant_config = self.get_quant_config(vllm_config)
284
285
self.hidden_size = config.hidden_size
294
- rope_theta = getattr(config, "rope_theta", 10000)
295
- rope_scaling = getattr(config, "rope_scaling", None)
296
- if rope_scaling is not None and getattr(
297
- config, "original_max_position_embeddings", None
298
- ):
299
- rope_scaling["original_max_position_embeddings"] = (
300
- config.original_max_position_embeddings
301
- )
286
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
287
# Support abacusai/Smaug-72B-v0.1 with attention_bias
288
# Support internlm/internlm-7b with bias
@@ -326,8 +310,6 @@ class LlamaDecoderLayer(nn.Module):
310
num_kv_heads=getattr(
311
config, "num_key_value_heads", config.num_attention_heads
312
),
329
- rope_theta=rope_theta,
330
- rope_scaling=rope_scaling,
313
max_position_embeddings=max_position_embeddings,
314
quant_config=quant_config,
315
bias=attention_bias,
vllm/model_executor/models/llama4.py
+1
-10
@@ -19,7 +19,6 @@
19
"""Inference-only LLaMA model compatible with HuggingFace weights."""
20
21
from collections.abc import Iterable
22
-from typing import Any
22
23
import torch
24
from torch import nn
@@ -171,8 +170,6 @@ class Llama4Attention(nn.Module):
170
hidden_size: int,
171
num_heads: int,
172
num_kv_heads: int,
174
- rope_theta: float = 10000,
175
- rope_scaling: dict[str, Any] | None = None,
173
max_position_embeddings: int = 8192,
174
quant_config: QuantizationConfig | None = None,
175
bias: bool = False,
@@ -208,7 +205,6 @@ class Llama4Attention(nn.Module):
205
206
self.floor_scale = getattr(config, "floor_scale", 8192.0)
207
self.attn_scale = getattr(config, "attn_scale", 0.1)
211
- self.rope_theta = rope_theta
208
self.max_position_embeddings = max_position_embeddings
209
self.n_rep = self.num_heads // self.num_kv_heads
210
self.qk_norm = (
@@ -248,8 +244,7 @@ class Llama4Attention(nn.Module):
244
self.head_dim,
245
rotary_dim=self.head_dim,
246
max_position=max_position_embeddings,
251
- base=int(rope_theta),
252
- rope_scaling=rope_scaling if rope_scaling != "default" else None,
247
+ rope_parameters=config.rope_parameters,
248
is_neox_style=is_neox_style,
249
)
250
if not self.nope
@@ -331,8 +326,6 @@ class Llama4DecoderLayer(nn.Module):
326
self.layer_idx = extract_layer_index(prefix)
327
self.global_layer = config.no_rope_layers[self.layer_idx] == 0
328
self.hidden_size = config.hidden_size
334
- rope_theta = config.rope_theta
335
- rope_scaling = config.rope_scaling
329
max_position_embeddings = config.max_position_embeddings
330
331
self.self_attn = Llama4Attention(
@@ -340,8 +333,6 @@ class Llama4DecoderLayer(nn.Module):
333
hidden_size=self.hidden_size,
334
num_heads=config.num_attention_heads,
335
num_kv_heads=config.num_key_value_heads,
343
- rope_theta=rope_theta,
344
- rope_scaling=rope_scaling,
336
max_position_embeddings=max_position_embeddings,
337
quant_config=quant_config,
338
bias=False,
vllm/model_executor/models/longcat_flash.py
+8
-14
@@ -108,8 +108,7 @@ class FlashConfig(PretrainedConfig):
108
eos_token_id=100001,
109
pretraining_tp=1,
110
tie_word_embeddings=False,
111
- rope_theta=1000000.0,
112
- rope_scaling=None,
111
+ rope_parameters=None,
112
attention_bias=False,
113
attention_dropout=0.0,
114
mla_scale_q_lora=False,
@@ -162,8 +161,13 @@ class FlashConfig(PretrainedConfig):
161
self.rms_norm_eps = rms_norm_eps
162
self.pretraining_tp = pretraining_tp
163
self.use_cache = use_cache
165
- self.rope_theta = rope_theta
166
- self.rope_scaling = rope_scaling
164
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
165
+ rope_scaling = kwargs.pop("rope_scaling", None)
166
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
167
+ rope_theta = kwargs.pop("rope_theta", 1000000.0)
168
+ if "rope_theta" not in rope_parameters:
169
+ rope_parameters["rope_theta"] = rope_theta
170
+ self.rope_parameters = rope_parameters
171
self.attention_bias = attention_bias
172
self.attention_dropout = attention_dropout
173
self.mla_scale_q_lora = mla_scale_q_lora
@@ -336,15 +340,7 @@ class FlashDecoderLayer(nn.Module):
340
super().__init__()
341
self.layer_idx = int(prefix.split(sep=".")[-1])
342
self.hidden_size = config.hidden_size
339
- rope_theta = getattr(config, "rope_theta", 10000)
340
- rope_scaling = getattr(config, "rope_scaling", None)
343
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
342
- if rope_scaling is not None and getattr(
343
- config, "original_max_position_embeddings", None
344
- ):
345
- rope_scaling["original_max_position_embeddings"] = (
346
- config.original_max_position_embeddings
347
- )
344
345
# Dual attention structure
346
self.self_attn = nn.ModuleList(
@@ -361,8 +357,6 @@ class FlashDecoderLayer(nn.Module):
357
config.q_lora_rank if hasattr(config, "q_lora_rank") else None
358
),
359
kv_lora_rank=config.kv_lora_rank,
364
- rope_theta=rope_theta,
365
- rope_scaling=rope_scaling,
360
max_position_embeddings=max_position_embeddings,
361
cache_config=cache_config,
362
quant_config=None
vllm/model_executor/models/minicpm.py
+3
-9
@@ -230,8 +230,7 @@ class MiniCPMAttention(nn.Module):
230
hidden_size: int,
231
num_heads: int,
232
num_kv_heads: int,
233
- rope_theta: float = 10000,
234
- rope_scaling: dict[str, Any] | None = None,
233
+ rope_parameters: dict[str, Any] | None = None,
234
max_position_embeddings: int = 8192,
235
cache_config: CacheConfig | None = None,
236
quant_config: QuantizationConfig | None = None,
@@ -257,7 +256,6 @@ class MiniCPMAttention(nn.Module):
256
self.q_size = self.num_heads * self.head_dim
257
self.kv_size = self.num_kv_heads * self.head_dim
258
self.scaling = self.head_dim**-0.5
260
- self.rope_theta = rope_theta
259
self.max_position_embeddings = max_position_embeddings
260
261
self.qkv_proj = QKVParallelLinear(
@@ -281,8 +279,7 @@ class MiniCPMAttention(nn.Module):
279
self.head_dim,
280
rotary_dim=self.head_dim,
281
max_position=max_position_embeddings,
284
- base=rope_theta,
285
- rope_scaling=rope_scaling,
282
+ rope_parameters=rope_parameters,
283
)
284
285
self.attn = Attention(
@@ -324,8 +321,6 @@ class MiniCPMDecoderLayer(nn.Module):
321
self.cache_config = cache_config
322
self.quant_config = quant_config
323
self.hidden_size = config.hidden_size
327
- self.rope_theta = getattr(config, "rope_theta", 10000)
328
- self.rope_scaling = getattr(config, "rope_scaling", None)
324
self.max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
325
self.prefix = prefix
326
self._init_attn_block()
@@ -339,8 +334,7 @@ class MiniCPMDecoderLayer(nn.Module):
334
hidden_size=self.hidden_size,
335
num_heads=self.config.num_attention_heads,
336
num_kv_heads=self.config.num_key_value_heads,
342
- rope_theta=self.rope_theta,
343
- rope_scaling=self.rope_scaling,
337
+ rope_parameters=self.config.rope_parameters,
338
max_position_embeddings=self.max_position_embeddings,
339
cache_config=self.cache_config,
340
quant_config=self.quant_config,
vllm/model_executor/models/minicpm3.py
+1
-9
@@ -25,8 +25,6 @@
25
# limitations under the License.
26
"""Inference-only MiniCPM3 model compatible with HuggingFace weights."""
27
28
-from typing import Any
29
-
28
import torch
29
from torch import nn
30
from transformers import PretrainedConfig
@@ -62,8 +60,6 @@ class MiniCPM3Attention(nn.Module):
60
v_head_dim: int,
61
q_lora_rank: int,
62
kv_lora_rank: int,
65
- rope_theta: float = 10000,
66
- rope_scaling: dict[str, Any] | None = None,
63
max_position_embeddings: int = 8192,
64
cache_config: CacheConfig | None = None,
65
quant_config: QuantizationConfig | None = None,
@@ -84,7 +80,6 @@ class MiniCPM3Attention(nn.Module):
80
self.num_local_heads = num_heads // tp_size
81
82
self.scaling = self.qk_head_dim**-0.5
87
- self.rope_theta = rope_theta
83
self.max_position_embeddings = max_position_embeddings
84
85
self.q_a_proj = ReplicatedLinear(
@@ -127,8 +122,7 @@ class MiniCPM3Attention(nn.Module):
122
self.qk_rope_head_dim,
123
rotary_dim=self.qk_rope_head_dim,
124
max_position=max_position_embeddings,
130
- base=rope_theta,
131
- rope_scaling=rope_scaling,
125
+ rope_parameters=config.rope_parameters,
126
)
127
self.attn = Attention(
128
self.num_local_heads,
@@ -204,8 +198,6 @@ class MiniCPM3DecoderLayer(MiniCPMDecoderLayer):
198
v_head_dim=self.config.v_head_dim,
199
q_lora_rank=self.config.q_lora_rank,
200
kv_lora_rank=self.config.kv_lora_rank,
207
- rope_theta=self.rope_theta,
208
- rope_scaling=self.rope_scaling,
201
max_position_embeddings=self.max_position_embeddings,
202
cache_config=self.cache_config,
203
quant_config=self.quant_config,
vllm/model_executor/models/minicpm_eagle.py
+1
-4
@@ -69,8 +69,6 @@ class EagleMiniCPMDecoderLayer(nn.Module):
69
self.cache_config = cache_config
70
self.quant_config = quant_config
71
self.hidden_size = config.hidden_size
72
- self.rope_theta = getattr(config, "rope_theta", 10000)
73
- self.rope_scaling = getattr(config, "rope_scaling", None)
72
self.max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
73
self.prefix = prefix
74
self._init_attn_block()
@@ -84,8 +82,7 @@ class EagleMiniCPMDecoderLayer(nn.Module):
82
hidden_size=self.hidden_size,
83
num_heads=self.config.num_attention_heads,
84
num_kv_heads=self.config.num_key_value_heads,
87
- rope_theta=self.rope_theta,
88
- rope_scaling=self.rope_scaling,
85
+ rope_parameters=self.config.rope_parameters,
86
max_position_embeddings=self.max_position_embeddings,
87
cache_config=self.cache_config,
88
quant_config=self.quant_config,
vllm/model_executor/models/minimax_m2.py
+3
-9
@@ -149,8 +149,7 @@ class MiniMaxM2Attention(nn.Module):
149
num_heads: int,
150
num_kv_heads: int,
151
rotary_dim: int,
152
- rope_theta: float = 10000,
153
- rope_scaling: dict[str, Any] | None = None,
152
+ rope_parameters: dict[str, Any] | None = None,
153
attn_window_size: int | None = None,
154
max_position_embeddings: int = 8192,
155
head_dim: int | None = None,
@@ -180,7 +179,6 @@ class MiniMaxM2Attention(nn.Module):
179
self.q_size = self.num_heads * self.head_dim
180
self.kv_size = self.num_kv_heads * self.head_dim
181
self.scaling = self.head_dim**-0.5
183
- self.rope_theta = rope_theta
182
self.max_position_embeddings = max_position_embeddings
183
184
self.qkv_proj = QKVParallelLinear(
@@ -205,8 +203,7 @@ class MiniMaxM2Attention(nn.Module):
203
self.head_dim,
204
rotary_dim=rotary_dim,
205
max_position=max_position_embeddings,
208
- base=rope_theta,
209
- rope_scaling=rope_scaling,
206
+ rope_parameters=rope_parameters,
207
)
208
self.attn = Attention(
209
self.num_heads,
@@ -252,8 +249,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
249
) -> None:
250
super().__init__()
251
self.hidden_size = config.hidden_size
255
- rope_theta = getattr(config, "rope_theta", 10000)
256
- rope_scaling = getattr(config, "rope_scaling", None)
252
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
253
if hasattr(config, "max_model_len") and isinstance(config.max_model_len, int):
254
max_position_embeddings = max(
@@ -269,8 +264,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
264
num_heads=config.num_attention_heads,
265
num_kv_heads=config.num_key_value_heads,
266
rotary_dim=config.rotary_dim,
272
- rope_theta=rope_theta,
273
- rope_scaling=rope_scaling,
267
+ rope_parameters=config.rope_parameters,
268
max_position_embeddings=max_position_embeddings,
269
rms_norm_eps=config.rms_norm_eps,
270
qkv_bias=getattr(config, "attention_bias", False),
vllm/model_executor/models/minimax_text_01.py
+3
-6
@@ -188,7 +188,7 @@ class MiniMaxText01Attention(nn.Module):
188
num_kv_heads: int,
189
rotary_dim: int,
190
max_position: int = 4096 * 32,
191
- rope_theta: float = 10000,
191
+ rope_parameters: dict | None = None,
192
sliding_window: int | None = None,
193
quant_config: QuantizationConfig | None = None,
194
layer_idx: int = None,
@@ -214,7 +214,6 @@ class MiniMaxText01Attention(nn.Module):
214
self.q_size = self.num_heads * self.head_dim
215
self.kv_size = self.num_kv_heads * self.head_dim
216
self.scaling = self.head_dim**-0.5
217
- self.rope_theta = rope_theta
217
self.sliding_window = sliding_window
218
self.prefix = prefix
219
@@ -247,7 +246,7 @@ class MiniMaxText01Attention(nn.Module):
246
head_size=self.head_dim,
247
rotary_dim=rotary_dim,
248
max_position=max_position,
250
- base=int(rope_theta),
249
+ rope_parameters=rope_parameters,
250
is_neox_style=True,
251
dtype=torch.float32,
252
)
@@ -287,8 +286,6 @@ class MiniMaxText01DecoderLayer(nn.Module):
286
self.hidden_size = config.hidden_size
287
self.expert_num = expert_num
288
290
- rope_theta = getattr(config, "rope_theta", 10000)
291
-
289
head_dim = getattr(config, "head_dim", None)
290
if head_dim is None:
291
head_dim = config.hidden_size // config.num_attention_heads
@@ -328,7 +325,7 @@ class MiniMaxText01DecoderLayer(nn.Module):
325
else head_dim,
326
num_kv_heads=config.num_key_value_heads,
327
max_position=max_position_embeddings,
331
- rope_theta=rope_theta,
328
+ rope_parameters=config.rope_parameters,
329
sliding_window=config.sliding_window,
330
quant_config=quant_config,
331
layer_idx=self._ilayer,
vllm/model_executor/models/mixtral.py
+1
-6
@@ -161,7 +161,6 @@ class MixtralAttention(nn.Module):
161
num_heads: int,
162
num_kv_heads: int,
163
max_position: int = 4096 * 32,
164
- rope_theta: float = 10000,
164
cache_config: CacheConfig | None = None,
165
quant_config: QuantizationConfig | None = None,
166
prefix: str = "",
@@ -189,7 +188,6 @@ class MixtralAttention(nn.Module):
188
self.q_size = self.num_heads * self.head_dim
189
self.kv_size = self.num_kv_heads * self.head_dim
190
self.scaling = self.head_dim**-0.5
192
- self.rope_theta = rope_theta
191
192
self.qkv_proj = QKVParallelLinear(
193
hidden_size,
@@ -211,7 +209,7 @@ class MixtralAttention(nn.Module):
209
self.head_dim,
210
rotary_dim=self.head_dim,
211
max_position=max_position,
214
- base=int(self.rope_theta),
212
+ rope_parameters=config.rope_parameters,
213
is_neox_style=True,
214
)
215
self.attn = Attention(
@@ -248,15 +246,12 @@ class MixtralDecoderLayer(nn.Module):
246
) -> None:
247
super().__init__()
248
self.hidden_size = config.hidden_size
251
- # Requires transformers > 4.32.0
252
- rope_theta = getattr(config, "rope_theta", 10000)
249
self.self_attn = MixtralAttention(
250
config=config,
251
hidden_size=self.hidden_size,
252
num_heads=config.num_attention_heads,
253
max_position=config.max_position_embeddings,
254
num_kv_heads=config.num_key_value_heads,
259
- rope_theta=rope_theta,
255
cache_config=cache_config,
256
quant_config=quant_config,
257
prefix=f"{prefix}.self_attn",
vllm/model_executor/models/mllama4.py
+6
-2
@@ -292,13 +292,17 @@ class Llama4VisionAttention(nn.Module):
292
prefix=f"{prefix}.o_proj",
293
)
294
295
+ rope_parameters = {
296
+ "rope_type": "mllama4",
297
+ "rope_theta": config.rope_parameters["rope_theta"],
298
+ }
299
+
300
self.rotary_emb = get_rope(
301
head_size=self.head_dim,
302
rotary_dim=config.hidden_size // config.num_attention_heads // 2,
303
# number of image patches
304
max_position=(config.image_size // config.patch_size) ** 2,
300
- base=config.rope_theta,
301
- rope_scaling={"rope_type": "mllama4"},
305
+ rope_parameters=rope_parameters,
306
is_neox_style=False,
307
dtype=torch.complex64, # important
308
)
vllm/model_executor/models/molmo.py
+1
-2
@@ -410,7 +410,6 @@ class MolmoAttention(nn.Module):
410
self.q_size = self.num_heads * self.head_dim
411
self.kv_size = self.num_kv_heads * self.head_dim
412
self.max_position_embeddings = config.max_position_embeddings
413
- self.rope_theta = config.rope_theta
413
414
# Attention input projection. Projects x -> (q, k, v)
415
self.qkv_proj = QKVParallelLinear(
@@ -437,7 +436,7 @@ class MolmoAttention(nn.Module):
436
self.head_dim,
437
rotary_dim=self.head_dim,
438
max_position=self.max_position_embeddings,
440
- base=self.rope_theta,
439
+ rope_parameters=config.rope_parameters,
440
)
441
self.scaling = self.head_dim**-0.5
442
self.attn = Attention(
vllm/model_executor/models/nemotron.py
+1
-16
@@ -26,7 +26,6 @@
26
27
from collections.abc import Iterable
28
from itertools import islice
29
-from typing import Any
29
30
import torch
31
from torch import nn
@@ -150,8 +149,6 @@ class NemotronAttention(nn.Module):
149
hidden_size: int,
150
num_heads: int,
151
num_kv_heads: int,
153
- rope_theta: float = 10000,
154
- rope_scaling: dict[str, Any] | None = None,
152
max_position_embeddings: int = 8192,
153
quant_config: QuantizationConfig | None = None,
154
bias: bool = False,
@@ -181,7 +178,6 @@ class NemotronAttention(nn.Module):
178
self.q_size = self.num_heads * self.head_dim
179
self.kv_size = self.num_kv_heads * self.head_dim
180
self.scaling = self.head_dim**-0.5
184
- self.rope_theta = rope_theta
181
self.partial_rotary_factor = config.partial_rotary_factor
182
self.max_position_embeddings = max_position_embeddings
183
@@ -206,8 +202,7 @@ class NemotronAttention(nn.Module):
202
self.head_dim,
203
rotary_dim=self.head_dim,
204
max_position=max_position_embeddings,
209
- base=rope_theta,
210
- rope_scaling=rope_scaling,
205
+ rope_parameters=config.rope_parameters,
206
partial_rotary_factor=self.partial_rotary_factor,
207
)
208
self.attn = Attention(
@@ -243,14 +238,6 @@ class NemotronDecoderLayer(nn.Module):
238
) -> None:
239
super().__init__()
240
self.hidden_size = config.hidden_size
246
- rope_theta = getattr(config, "rope_theta", 10000)
247
- rope_scaling = getattr(config, "rope_scaling", None)
248
- if rope_scaling is not None and getattr(
249
- config, "original_max_position_embeddings", None
250
- ):
251
- rope_scaling["original_max_position_embeddings"] = (
252
- config.original_max_position_embeddings
253
- )
241
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
242
# Support abacusai/Smaug-72B-v0.1 with attention_bias
243
# Support internlm/internlm-7b with bias
@@ -264,8 +251,6 @@ class NemotronDecoderLayer(nn.Module):
251
num_kv_heads=getattr(
252
config, "num_key_value_heads", config.num_attention_heads
253
),
267
- rope_theta=rope_theta,
268
- rope_scaling=rope_scaling,
254
max_position_embeddings=max_position_embeddings,
255
quant_config=quant_config,
256
bias=attention_bias,
vllm/model_executor/models/nemotron_nas.py
+1
-18
@@ -26,7 +26,6 @@
26
27
from collections.abc import Iterable
28
from itertools import islice
29
-from typing import Any
29
30
import torch
31
from torch import nn
@@ -82,8 +81,6 @@ class DeciLMAttention(LlamaAttention):
81
hidden_size: int,
82
num_heads: int,
83
num_kv_heads: int,
85
- rope_theta: float = 10000,
86
- rope_scaling: dict[str, Any] | None = None,
84
max_position_embeddings: int = 8192,
85
quant_config: QuantizationConfig | None = None,
86
bias: bool = False,
@@ -97,8 +94,6 @@ class DeciLMAttention(LlamaAttention):
94
hidden_size,
95
num_heads,
96
num_kv_heads,
100
- rope_theta,
101
- rope_scaling,
97
max_position_embeddings,
98
quant_config,
99
bias,
@@ -111,7 +106,6 @@ class DeciLMAttention(LlamaAttention):
106
def _init_rotary_emb(
107
self,
108
config,
114
- rope_scaling: dict[str, Any] | None,
109
quant_config: QuantizationConfig | None,
110
) -> None:
111
# Enables YARN for Mistral and LLaMA4 derivatives.
@@ -126,8 +120,7 @@ class DeciLMAttention(LlamaAttention):
120
self.head_dim,
121
rotary_dim=self.head_dim,
122
max_position=self.max_position_embeddings,
129
- base=self.rope_theta,
130
- rope_scaling=rope_scaling,
123
+ rope_parameters=config.rope_parameters,
124
is_neox_style=is_neox_style,
125
partial_rotary_factor=self.partial_rotary_factor,
126
)
@@ -148,14 +141,6 @@ class DeciLMDecoderLayer(nn.Module):
141
self._is_no_op_ffn = block_config.ffn.no_op
142
143
self.hidden_size = config.hidden_size
151
- rope_theta = getattr(config, "rope_theta", 10000)
152
- rope_scaling = getattr(config, "rope_scaling", None)
153
- if rope_scaling is not None and getattr(
154
- config, "original_max_position_embeddings", None
155
- ):
156
- rope_scaling["original_max_position_embeddings"] = (
157
- config.original_max_position_embeddings
158
- )
144
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
145
# Support abacusai/Smaug-72B-v0.1 with attention_bias
146
# Support internlm/internlm-7b with bias
@@ -176,8 +161,6 @@ class DeciLMDecoderLayer(nn.Module):
161
hidden_size=self.hidden_size,
162
num_heads=config.num_attention_heads,
163
num_kv_heads=num_kv_heads,
179
- rope_theta=rope_theta,
180
- rope_scaling=rope_scaling,
164
max_position_embeddings=max_position_embeddings,
165
quant_config=quant_config,
166
bias=attention_bias,
vllm/model_executor/models/olmo.py
+1
-2
@@ -87,7 +87,6 @@ class OlmoAttention(nn.Module):
87
self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
88
self.head_dim = self.hidden_size // self.total_num_heads
89
self.max_position_embeddings = config.max_position_embeddings
90
- self.rope_theta = config.rope_theta
90
self.clip_qkv = config.clip_qkv
91
92
# Attention input projection. Projects x -> (q, k, v)
@@ -105,7 +104,7 @@ class OlmoAttention(nn.Module):
104
self.head_dim,
105
rotary_dim=self.head_dim,
106
max_position=self.max_position_embeddings,
108
- base=self.rope_theta,
107
+ rope_parameters=config.rope_parameters,
108
)
109
self.scaling = self.head_dim**-0.5
110
self.attn = Attention(
vllm/model_executor/models/olmo2.py
+7
-6
@@ -99,7 +99,6 @@ class Olmo2Attention(nn.Module):
99
self.q_size = self.num_heads * self.head_dim
100
self.kv_size = self.num_kv_heads * self.head_dim
101
self.max_position_embeddings = self.config.max_position_embeddings
102
- self.rope_theta = self.config.rope_theta
102
103
# Attention input projection. Projects x -> (q, k, v)
104
self.qkv_proj = QKVParallelLinear(
@@ -139,15 +138,17 @@ class Olmo2Attention(nn.Module):
138
prefix=f"{prefix}.attn",
139
)
140
142
- # Rotary embeddings. Rope scaling is only applied on full attention
143
- # layers.
144
- self.rope_scaling = self.config.rope_scaling if sliding_window is None else None
141
+ # Rotary embeddings. Rope scaling is only applied on full attention layers.
142
+ if sliding_window is None:
143
+ rope_parameters = self.config.rope_parameters
144
+ else:
145
+ rope_theta = self.config.rope_parameters["rope_theta"]
146
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
147
self.rotary_emb = get_rope(
148
self.head_dim,
149
rotary_dim=self.head_dim,
150
max_position=self.max_position_embeddings,
149
- base=self.rope_theta, # type: ignore
150
- rope_scaling=self.rope_scaling,
151
+ rope_parameters=rope_parameters,
152
)
153
154
# Attention output projection.
vllm/model_executor/models/olmoe.py
+1
-5
@@ -123,8 +123,6 @@ class OlmoeAttention(nn.Module):
123
quant_config = vllm_config.quant_config
124
125
self.hidden_size = config.hidden_size
126
- rope_theta = getattr(config, "rope_theta", 10000)
127
- rope_scaling = getattr(config, "rope_scaling", None)
126
max_position_embeddings = getattr(config, "max_position_embeddings", 4096)
127
128
num_heads = config.num_attention_heads
@@ -148,7 +146,6 @@ class OlmoeAttention(nn.Module):
146
self.q_size = self.num_heads * self.head_dim
147
self.kv_size = self.num_kv_heads * self.head_dim
148
self.scaling = self.head_dim**-0.5
151
- self.rope_theta = rope_theta
149
self.max_position_embeddings = max_position_embeddings
150
151
self.qkv_proj = QKVParallelLinear(
@@ -176,8 +173,7 @@ class OlmoeAttention(nn.Module):
173
self.head_dim,
174
rotary_dim=self.head_dim,
175
max_position=max_position_embeddings,
179
- base=rope_theta,
180
- rope_scaling=rope_scaling,
176
+ rope_parameters=config.rope_parameters,
177
is_neox_style=True,
178
)
179
self.attn = Attention(
vllm/model_executor/models/openpangu.py
+7
-19
@@ -77,6 +77,7 @@ from vllm.model_executor.models.utils import (
77
sequence_parallel_chunk,
78
)
79
from vllm.sequence import IntermediateTensors
80
+from vllm.transformers_utils.config import set_default_rope_theta
81
82
83
def check_ffn_act_fn(act_fn: str):
@@ -259,7 +260,6 @@ class OpenPanguMLAAttention(nn.Module):
260
v_head_dim: int,
261
q_lora_rank: int | None,
262
kv_lora_rank: int,
262
- rope_theta: float = 10000,
263
max_position_embeddings: int = 8192,
264
cache_config: CacheConfig | None = None,
265
quant_config: QuantizationConfig | None = None,
@@ -274,8 +274,6 @@ class OpenPanguMLAAttention(nn.Module):
274
self.v_head_dim = v_head_dim
275
self.q_lora_rank = q_lora_rank
276
self.kv_lora_rank = kv_lora_rank
277
- self.rope_theta = rope_theta
278
-
277
self.tp_size = get_tensor_model_parallel_world_size()
278
if num_heads % self.tp_size != 0:
279
raise ValueError(
@@ -339,7 +337,9 @@ class OpenPanguMLAAttention(nn.Module):
337
)
338
339
# TODO: remove hard coding
342
- rope_scaling = {
340
+ set_default_rope_theta(config, default_theta=10000)
341
+ rope_parameters = {
342
+ "rope_theta": config.rope_parameters["rope_theta"],
343
"beta_fast": 32,
344
"beta_slow": 1,
345
"factor": 1,
@@ -353,8 +353,7 @@ class OpenPanguMLAAttention(nn.Module):
353
qk_rope_head_dim,
354
rotary_dim=qk_rope_head_dim,
355
max_position=max_position_embeddings,
356
- base=rope_theta,
357
- rope_scaling=rope_scaling,
356
+ rope_parameters=rope_parameters,
357
is_neox_style=False,
358
)
359
@@ -407,8 +406,6 @@ class OpenPanguEmbeddedAttention(nn.Module):
406
hidden_size: int,
407
num_heads: int,
408
num_kv_heads: int,
410
- rope_theta: float = 10000,
411
- rope_scaling: dict[str, Any] | None = None,
409
max_position_embeddings: int = 8192,
410
quant_config: QuantizationConfig | None = None,
411
bias: bool = False,
@@ -454,7 +451,6 @@ class OpenPanguEmbeddedAttention(nn.Module):
451
self.q_size = self.num_heads * self.head_dim
452
self.kv_size = self.num_kv_heads * self.head_dim
453
self.scaling = self.head_dim**-0.5
457
- self.rope_theta = rope_theta
454
self.max_position_embeddings = max_position_embeddings
455
456
self.qkv_proj = QKVParallelLinear(
@@ -475,9 +471,7 @@ class OpenPanguEmbeddedAttention(nn.Module):
471
prefix=f"{prefix}.o_proj",
472
)
473
478
- self._init_rotary_emb(
479
- config, rope_scaling=rope_scaling, quant_config=quant_config
480
- )
474
+ self._init_rotary_emb(config, quant_config=quant_config)
475
476
if hasattr(config, "interleaved_sliding_window"):
477
interleaved_sliding_window = config.interleaved_sliding_window
@@ -521,7 +515,6 @@ class OpenPanguEmbeddedAttention(nn.Module):
515
def _init_rotary_emb(
516
self,
517
config: PretrainedConfig,
524
- rope_scaling: dict[str, Any] | None,
518
quant_config: QuantizationConfig | None,
519
) -> None:
520
is_neox_style = True
@@ -533,8 +526,7 @@ class OpenPanguEmbeddedAttention(nn.Module):
526
self.head_dim,
527
rotary_dim=self.head_dim,
528
max_position=self.max_position_embeddings,
536
- base=self.rope_theta,
537
- rope_scaling=rope_scaling,
529
+ rope_parameters=config.rope_parameters,
530
is_neox_style=is_neox_style,
531
)
532
@@ -555,7 +547,6 @@ class OpenPanguDecoderLayer(nn.Module):
547
parallel_config = vllm_config.parallel_config
548
549
self.hidden_size = config.hidden_size
558
- rope_theta = getattr(config, "rope_theta", 10000)
550
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
551
552
layer_idx = int(prefix.split(sep=".")[-1])
@@ -579,7 +570,6 @@ class OpenPanguDecoderLayer(nn.Module):
570
config.q_lora_rank if hasattr(config, "q_lora_rank") else None
571
),
572
kv_lora_rank=config.kv_lora_rank,
582
- rope_theta=rope_theta,
573
max_position_embeddings=max_position_embeddings,
574
cache_config=cache_config,
575
quant_config=quant_config,
@@ -607,8 +597,6 @@ class OpenPanguDecoderLayer(nn.Module):
597
num_kv_heads=getattr(
598
config, "num_key_value_heads", config.num_attention_heads
599
),
610
- rope_theta=rope_theta,
611
- rope_scaling=getattr(config, "rope_scaling", None),
600
max_position_embeddings=max_position_embeddings,
601
quant_config=quant_config,
602
bias=attention_bias,
vllm/model_executor/models/orion.py
+3
-9
@@ -88,8 +88,7 @@ class OrionAttention(nn.Module):
88
hidden_size: int,
89
num_heads: int,
90
num_kv_heads: int,
91
- rope_theta: float = 10000,
92
- rope_scaling: dict[str, Any] | None = None,
91
+ rope_parameters: dict[str, Any] | None = None,
92
max_position_embeddings: int = 8192,
93
cache_config: CacheConfig | None = None,
94
quant_config: QuantizationConfig | None = None,
@@ -115,7 +114,6 @@ class OrionAttention(nn.Module):
114
self.q_size = self.num_heads * self.head_dim
115
self.kv_size = self.num_kv_heads * self.head_dim
116
self.scaling = self.head_dim**-0.5
118
- self.rope_theta = rope_theta
117
self.max_position_embeddings = max_position_embeddings
118
119
self.qkv_proj = QKVParallelLinear(
@@ -139,8 +137,7 @@ class OrionAttention(nn.Module):
137
self.head_dim,
138
rotary_dim=self.head_dim,
139
max_position=max_position_embeddings,
142
- base=rope_theta,
143
- rope_scaling=rope_scaling,
140
+ rope_parameters=rope_parameters,
141
)
142
self.attn = Attention(
143
self.num_heads,
@@ -175,15 +172,12 @@ class OrionDecoderLayer(nn.Module):
172
) -> None:
173
super().__init__()
174
self.hidden_size = config.hidden_size
178
- rope_theta = getattr(config, "rope_theta", 10000)
179
- rope_scaling = getattr(config, "rope_scaling", None)
175
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
176
self.self_attn = OrionAttention(
177
hidden_size=self.hidden_size,
178
num_heads=config.num_attention_heads,
179
num_kv_heads=config.num_key_value_heads,
185
- rope_theta=rope_theta,
186
- rope_scaling=rope_scaling,
180
+ rope_parameters=config.rope_parameters,
181
max_position_embeddings=max_position_embeddings,
182
cache_config=cache_config,
183
quant_config=quant_config,
vllm/model_executor/models/ouro.py
+1
-10
@@ -112,10 +112,8 @@ class OuroAttention(nn.Module):
112
num_heads: int,
113
num_kv_heads: int,
114
max_position: int = 4096 * 32,
115
- rope_theta: float = 10000,
115
cache_config: CacheConfig | None = None,
116
quant_config: QuantizationConfig | None = None,
118
- rope_scaling: tuple | None = None,
117
prefix: str = "",
118
attn_type: str = AttentionType.DECODER,
119
dual_chunk_attention_config: dict[str, Any] | None = None,
@@ -140,7 +138,6 @@ class OuroAttention(nn.Module):
138
self.q_size = self.num_heads * self.head_dim
139
self.kv_size = self.num_kv_heads * self.head_dim
140
self.scaling = self.head_dim**-0.5
143
- self.rope_theta = rope_theta
141
self.dual_chunk_attention_config = dual_chunk_attention_config
142
143
# Get total_ut_steps from config, default to 4 if not specified
@@ -170,8 +167,7 @@ class OuroAttention(nn.Module):
167
self.head_dim,
168
rotary_dim=self.head_dim,
169
max_position=max_position,
173
- base=self.rope_theta,
174
- rope_scaling=rope_scaling,
170
+ rope_parameters=config.rope_parameters,
171
dual_chunk_attention_config=dual_chunk_attention_config,
172
)
173
self.attn = nn.ModuleList()
@@ -226,9 +222,6 @@ class OuroDecoderLayer(nn.Module):
222
) -> None:
223
super().__init__()
224
self.hidden_size = config.hidden_size
229
- # Requires transformers > 4.32.0
230
- rope_theta = getattr(config, "rope_theta", 1000000)
231
- rope_scaling = getattr(config, "rope_scaling", None)
225
dual_chunk_attention_config = getattr(
226
config, "dual_chunk_attention_config", None
227
)
@@ -244,10 +237,8 @@ class OuroDecoderLayer(nn.Module):
237
num_heads=config.num_attention_heads,
238
max_position=config.max_position_embeddings,
239
num_kv_heads=config.num_key_value_heads,
247
- rope_theta=rope_theta,
240
cache_config=cache_config,
241
quant_config=quant_config,
250
- rope_scaling=rope_scaling,
242
prefix=f"{prefix}.self_attn",
243
attn_type=attn_type,
244
dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/persimmon.py
+1
-2
@@ -106,7 +106,6 @@ class PersimmonAttention(nn.Module):
106
self.num_heads = self.total_num_heads // tensor_parallel_world_size
107
self.head_dim = self.hidden_size // self.total_num_heads
108
self.max_position_embeddings = config.max_position_embeddings
109
- self.rope_theta = config.rope_theta
109
self.partial_rotary_factor = config.partial_rotary_factor
110
self.is_causal = True
111
@@ -138,7 +137,7 @@ class PersimmonAttention(nn.Module):
137
self.head_dim,
138
rotary_dim=self.head_dim,
139
max_position=self.max_position_embeddings,
141
- base=self.rope_theta,
140
+ rope_parameters=config.rope_parameters,
141
partial_rotary_factor=self.partial_rotary_factor,
142
)
143
self.scaling = self.head_dim**-0.5
vllm/model_executor/models/phi.py
+1
-5
@@ -115,16 +115,12 @@ class PhiAttention(nn.Module):
115
)
116
assert rotary_dim % 2 == 0
117
118
- # pylint: disable=C0301
119
- # Refer to:
120
- # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
121
- rope_theta = getattr(config, "rope_theta", 10000.0)
118
max_position_embeddings = getattr(config, "max_position_embeddings", 2048)
119
self.rotary_emb = get_rope(
120
self.head_size,
121
rotary_dim=rotary_dim,
122
max_position=max_position_embeddings,
127
- base=rope_theta,
123
+ rope_parameters=config.rope_parameters,
124
)
125
self.attn = Attention(
126
self.num_heads,
vllm/model_executor/models/phimoe.py
+7
-11
@@ -86,7 +86,7 @@ class PhiMoEConfig(PretrainedConfig):
86
bos_token_id=1,
87
eos_token_id=2,
88
tie_word_embeddings=False,
89
- rope_theta=1e6,
89
+ rope_parameters=None,
90
sliding_window=None,
91
attention_dropout=0.0,
92
num_experts_per_tok=2,
@@ -119,7 +119,9 @@ class PhiMoEConfig(PretrainedConfig):
119
self.initializer_range = initializer_range
120
self.rms_norm_eps = rms_norm_eps
121
self.use_cache = use_cache
122
- self.rope_theta = rope_theta
122
+ if rope_parameters is None:
123
+ rope_theta = kwargs.pop("rope_theta", 1e6)
124
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
125
self.attention_dropout = attention_dropout
126
127
self.num_experts_per_tok = num_experts_per_tok
@@ -302,12 +304,11 @@ class PhiMoEAttention(nn.Module):
304
hidden_size: int,
305
num_heads: int,
306
num_kv_heads: int,
307
+ rope_parameters: dict,
308
head_dim: int | None = None,
309
max_position: int = 4096 * 32,
307
- rope_theta: float = 10000,
310
cache_config: CacheConfig | None = None,
311
quant_config: QuantizationConfig | None = None,
310
- rope_scaling: dict | None = None,
312
prefix: str = "",
313
) -> None:
314
super().__init__()
@@ -332,8 +333,6 @@ class PhiMoEAttention(nn.Module):
333
self.q_size = self.num_heads * self.head_dim
334
self.kv_size = self.num_kv_heads * self.head_dim
335
self.scaling = self.head_dim**-0.5
335
- self.rope_theta = rope_theta
336
- self.rope_scaling = rope_scaling
336
337
self.qkv_proj = QKVParallelLinear(
338
hidden_size,
@@ -355,9 +354,8 @@ class PhiMoEAttention(nn.Module):
354
self.head_dim,
355
rotary_dim=self.head_dim,
356
max_position=max_position,
358
- base=int(self.rope_theta),
357
+ rope_parameters=rope_parameters,
358
is_neox_style=True,
360
- rope_scaling=self.rope_scaling,
359
)
360
self.attn = Attention(
361
self.num_heads,
@@ -393,7 +391,6 @@ class PhiMoEDecoderLayer(nn.Module):
391
super().__init__()
392
self.hidden_size = config.hidden_size
393
# Requires transformers > 4.32.0
396
- rope_theta = getattr(config, "rope_theta", 10000)
394
self.self_attn = PhiMoEAttention(
395
hidden_size=self.hidden_size,
396
num_heads=config.num_attention_heads,
@@ -402,10 +399,9 @@ class PhiMoEDecoderLayer(nn.Module):
399
head_dim=getattr(
400
config, "head_dim", self.hidden_size // config.num_attention_heads
401
),
405
- rope_theta=rope_theta,
402
cache_config=cache_config,
403
quant_config=quant_config,
408
- rope_scaling=config.rope_scaling,
404
+ rope_parameters=config.rope_parameters,
405
prefix=f"{prefix}.self_attn",
406
)
407
self.block_sparse_moe = PhiMoE(
vllm/model_executor/models/plamo2.py
+1
-6
@@ -567,10 +567,6 @@ class Plamo2AttentionMixer(nn.Module):
567
prefix=f"{prefix}.o_proj",
568
)
569
570
- self.rope_theta = config.rope_theta if hasattr(config, "rope_theta") else 10000
571
- self.rope_scaling = (
572
- config.rope_scaling if hasattr(config, "rope_scaling") else None
573
- )
570
max_position = config.max_position_embeddings
571
if hasattr(vllm_config.model_config, "max_model_len") and isinstance(
572
vllm_config.model_config.max_model_len, int
@@ -581,8 +577,7 @@ class Plamo2AttentionMixer(nn.Module):
577
self.head_dim,
578
rotary_dim=self.head_dim,
579
max_position=max_position,
584
- base=self.rope_theta,
585
- rope_scaling=self.rope_scaling,
580
+ rope_parameters=config.rope_parameters,
581
)
582
self.q_norm = RMSNorm(config.hidden_size_per_head, eps=config.rms_norm_eps)
583
self.q_norm.weight = torch.nn.Parameter(
vllm/model_executor/models/qwen.py
+3
-8
@@ -83,8 +83,7 @@ class QWenAttention(nn.Module):
83
hidden_size: int,
84
num_heads: int,
85
max_position_embeddings: int,
86
- rope_theta: float = 10000,
87
- rope_scaling: dict[str, Any] | None = None,
86
+ rope_parameters: dict[str, Any] | None = None,
87
cache_config: CacheConfig | None = None,
88
quant_config: QuantizationConfig | None = None,
89
prefix: str = "",
@@ -117,8 +116,7 @@ class QWenAttention(nn.Module):
116
self.head_dim,
117
rotary_dim=self.head_dim,
118
max_position=max_position_embeddings,
120
- base=rope_theta,
121
- rope_scaling=rope_scaling,
119
+ rope_parameters=rope_parameters,
120
)
121
self.attn = Attention(
122
self.num_heads,
@@ -153,14 +151,11 @@ class QWenBlock(nn.Module):
151
super().__init__()
152
self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
153
156
- rope_theta = getattr(config, "rope_theta", 10000)
157
- rope_scaling = getattr(config, "rope_scaling", None)
154
self.attn = QWenAttention(
155
config.hidden_size,
156
config.num_attention_heads,
157
config.max_position_embeddings,
162
- rope_theta=rope_theta,
163
- rope_scaling=rope_scaling,
158
+ rope_parameters=config.rope_parameters,
159
cache_config=cache_config,
160
quant_config=quant_config,
161
prefix=f"{prefix}.attn",
vllm/model_executor/models/qwen2.py
+5
-11
@@ -57,7 +57,7 @@ from vllm.model_executor.model_loader.weight_utils import (
57
maybe_remap_kv_scale_name,
58
)
59
from vllm.sequence import IntermediateTensors
60
-from vllm.transformers_utils.config import is_interleaved
60
+from vllm.transformers_utils.config import is_interleaved, set_default_rope_theta
61
62
from .interfaces import SupportsEagle3, SupportsLoRA, SupportsPP
63
from .utils import (
@@ -114,11 +114,10 @@ class Qwen2Attention(nn.Module):
114
hidden_size: int,
115
num_heads: int,
116
num_kv_heads: int,
117
+ rope_parameters: dict[str, Any],
118
max_position: int = 4096 * 32,
118
- rope_theta: float = 10000,
119
cache_config: CacheConfig | None = None,
120
quant_config: QuantizationConfig | None = None,
121
- rope_scaling: tuple | None = None,
121
prefix: str = "",
122
attn_type: str = AttentionType.DECODER,
123
dual_chunk_attention_config: dict[str, Any] | None = None,
@@ -143,7 +142,6 @@ class Qwen2Attention(nn.Module):
142
self.q_size = self.num_heads * self.head_dim
143
self.kv_size = self.num_kv_heads * self.head_dim
144
self.scaling = self.head_dim**-0.5
146
- self.rope_theta = rope_theta
145
self.dual_chunk_attention_config = dual_chunk_attention_config
146
147
self.qkv_proj = QKVParallelLinear(
@@ -167,8 +165,7 @@ class Qwen2Attention(nn.Module):
165
self.head_dim,
166
rotary_dim=self.head_dim,
167
max_position=max_position,
170
- base=self.rope_theta,
171
- rope_scaling=rope_scaling,
168
+ rope_parameters=rope_parameters,
169
dual_chunk_attention_config=dual_chunk_attention_config,
170
)
171
attn_cls = (
@@ -216,9 +213,7 @@ class Qwen2DecoderLayer(nn.Module):
213
) -> None:
214
super().__init__()
215
self.hidden_size = config.hidden_size
219
- # Requires transformers > 4.32.0
220
- rope_theta = getattr(config, "rope_theta", 1000000)
221
- rope_scaling = getattr(config, "rope_scaling", None)
216
+ set_default_rope_theta(config, default_theta=1000000)
217
dual_chunk_attention_config = getattr(
218
config, "dual_chunk_attention_config", None
219
)
@@ -237,10 +232,9 @@ class Qwen2DecoderLayer(nn.Module):
232
num_heads=config.num_attention_heads,
233
max_position=config.max_position_embeddings,
234
num_kv_heads=config.num_key_value_heads,
240
- rope_theta=rope_theta,
235
cache_config=cache_config,
236
quant_config=quant_config,
243
- rope_scaling=rope_scaling,
237
+ rope_parameters=config.rope_parameters,
238
prefix=f"{prefix}.self_attn",
239
attn_type=attn_type,
240
dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/qwen2_5_vl.py
-1
@@ -641,7 +641,6 @@ class Qwen2_5_VisionTransformer(nn.Module):
641
head_size=head_dim,
642
rotary_dim=head_dim // 2,
643
max_position=8192,
644
- base=10000.0,
644
is_neox_style=True,
645
)
646
vllm/model_executor/models/qwen2_moe.py
+3
-9
@@ -194,8 +194,7 @@ class Qwen2MoeAttention(nn.Module):
194
hidden_size: int,
195
num_heads: int,
196
num_kv_heads: int,
197
- rope_theta: float = 10000,
198
- rope_scaling: dict[str, Any] | None = None,
197
+ rope_parameters: dict[str, Any] | None = None,
198
max_position_embeddings: int = 8192,
199
cache_config: CacheConfig | None = None,
200
quant_config: QuantizationConfig | None = None,
@@ -222,7 +221,6 @@ class Qwen2MoeAttention(nn.Module):
221
self.q_size = self.num_heads * self.head_dim
222
self.kv_size = self.num_kv_heads * self.head_dim
223
self.scaling = self.head_dim**-0.5
225
- self.rope_theta = rope_theta
224
self.max_position_embeddings = max_position_embeddings
225
self.dual_chunk_attention_config = dual_chunk_attention_config
226
@@ -248,8 +246,7 @@ class Qwen2MoeAttention(nn.Module):
246
self.head_dim,
247
rotary_dim=self.head_dim,
248
max_position=max_position_embeddings,
251
- base=rope_theta,
252
- rope_scaling=rope_scaling,
249
+ rope_parameters=rope_parameters,
250
dual_chunk_attention_config=dual_chunk_attention_config,
251
)
252
self.attn = Attention(
@@ -291,8 +288,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
288
) -> None:
289
super().__init__()
290
self.hidden_size = config.hidden_size
294
- rope_theta = getattr(config, "rope_theta", 10000)
295
- rope_scaling = getattr(config, "rope_scaling", None)
291
dual_chunk_attention_config = getattr(
292
config, "dual_chunk_attention_config", None
293
)
@@ -301,8 +296,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
296
hidden_size=self.hidden_size,
297
num_heads=config.num_attention_heads,
298
num_kv_heads=config.num_key_value_heads,
304
- rope_theta=rope_theta,
305
- rope_scaling=rope_scaling,
299
+ rope_parameters=config.rope_parameters,
300
max_position_embeddings=max_position_embeddings,
301
cache_config=cache_config,
302
quant_config=quant_config,
vllm/model_executor/models/qwen2_vl.py
-1
@@ -643,7 +643,6 @@ class Qwen2VisionTransformer(nn.Module):
643
head_size=head_dim,
644
rotary_dim=head_dim // 2,
645
max_position=8192,
646
- base=10000.0,
646
is_neox_style=True,
647
)
648
vllm/model_executor/models/qwen3.py
+5
-10
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
44
from vllm.sequence import IntermediateTensors
45
+from vllm.transformers_utils.config import set_default_rope_theta
46
47
from .interfaces import SupportsEagle3, SupportsLoRA, SupportsPP
48
from .qwen2 import Qwen2MLP as Qwen3MLP
@@ -57,14 +58,13 @@ class Qwen3Attention(nn.Module):
58
hidden_size: int,
59
num_heads: int,
60
num_kv_heads: int,
61
+ rope_parameters: dict,
62
max_position: int = 4096 * 32,
63
head_dim: int | None = None,
64
rms_norm_eps: float = 1e-06,
65
qkv_bias: bool = False,
64
- rope_theta: float = 10000,
66
cache_config: CacheConfig | None = None,
67
quant_config: QuantizationConfig | None = None,
67
- rope_scaling: tuple | None = None,
68
prefix: str = "",
69
attn_type: str = AttentionType.DECODER,
70
dual_chunk_attention_config: dict[str, Any] | None = None,
@@ -89,7 +89,6 @@ class Qwen3Attention(nn.Module):
89
self.q_size = self.num_heads * self.head_dim
90
self.kv_size = self.num_kv_heads * self.head_dim
91
self.scaling = self.head_dim**-0.5
92
- self.rope_theta = rope_theta
92
self.dual_chunk_attention_config = dual_chunk_attention_config
93
94
self.qkv_proj = QKVParallelLinear(
@@ -113,8 +112,7 @@ class Qwen3Attention(nn.Module):
112
self.head_dim,
113
rotary_dim=self.head_dim,
114
max_position=max_position,
116
- base=self.rope_theta,
117
- rope_scaling=rope_scaling,
115
+ rope_parameters=rope_parameters,
116
dual_chunk_attention_config=dual_chunk_attention_config,
117
)
118
self.attn = Attention(
@@ -166,9 +164,7 @@ class Qwen3DecoderLayer(nn.Module):
164
) -> None:
165
super().__init__()
166
self.hidden_size = config.hidden_size
169
- # Requires transformers > 4.32.0
170
- rope_theta = getattr(config, "rope_theta", 1000000)
171
- rope_scaling = getattr(config, "rope_scaling", None)
167
+ set_default_rope_theta(config, default_theta=1000000)
168
dual_chunk_attention_config = getattr(
169
config, "dual_chunk_attention_config", None
170
)
@@ -187,13 +183,12 @@ class Qwen3DecoderLayer(nn.Module):
183
num_heads=config.num_attention_heads,
184
max_position=config.max_position_embeddings,
185
num_kv_heads=config.num_key_value_heads,
190
- rope_theta=rope_theta,
186
rms_norm_eps=config.rms_norm_eps,
187
qkv_bias=getattr(config, "attention_bias", False),
188
head_dim=getattr(config, "head_dim", None),
189
cache_config=cache_config,
190
quant_config=quant_config,
196
- rope_scaling=rope_scaling,
191
+ rope_parameters=config.rope_parameters,
192
prefix=f"{prefix}.self_attn",
193
attn_type=attn_type,
194
dual_chunk_attention_config=dual_chunk_attention_config,
vllm/model_executor/models/qwen3_moe.py
+3
-9
@@ -216,8 +216,7 @@ class Qwen3MoeAttention(nn.Module):
216
hidden_size: int,
217
num_heads: int,
218
num_kv_heads: int,
219
- rope_theta: float = 10000,
220
- rope_scaling: dict[str, Any] | None = None,
219
+ rope_parameters: dict[str, Any],
220
max_position_embeddings: int = 8192,
221
head_dim: int | None = None,
222
rms_norm_eps: float = 1e-06,
@@ -247,7 +246,6 @@ class Qwen3MoeAttention(nn.Module):
246
self.q_size = self.num_heads * self.head_dim
247
self.kv_size = self.num_kv_heads * self.head_dim
248
self.scaling = self.head_dim**-0.5
250
- self.rope_theta = rope_theta
249
self.max_position_embeddings = max_position_embeddings
250
self.dual_chunk_attention_config = dual_chunk_attention_config
251
@@ -273,8 +271,7 @@ class Qwen3MoeAttention(nn.Module):
271
self.head_dim,
272
rotary_dim=self.head_dim,
273
max_position=max_position_embeddings,
276
- base=rope_theta,
277
- rope_scaling=rope_scaling,
274
+ rope_parameters=rope_parameters,
275
dual_chunk_attention_config=dual_chunk_attention_config,
276
)
277
self.attn = Attention(
@@ -326,8 +323,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
323
quant_config = vllm_config.quant_config
324
325
self.hidden_size = config.hidden_size
329
- rope_theta = getattr(config, "rope_theta", 10000)
330
- rope_scaling = getattr(config, "rope_scaling", None)
326
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
327
dual_chunk_attention_config = getattr(
328
config, "dual_chunk_attention_config", None
@@ -336,8 +331,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
331
hidden_size=self.hidden_size,
332
num_heads=config.num_attention_heads,
333
num_kv_heads=config.num_key_value_heads,
339
- rope_theta=rope_theta,
340
- rope_scaling=rope_scaling,
334
+ rope_parameters=config.rope_parameters,
335
max_position_embeddings=max_position_embeddings,
336
rms_norm_eps=config.rms_norm_eps,
337
qkv_bias=getattr(config, "attention_bias", False),
vllm/model_executor/models/qwen3_next.py
+1
-2
@@ -748,8 +748,7 @@ class Qwen3NextAttention(nn.Module):
748
head_size=self.head_dim,
749
rotary_dim=self.head_dim,
750
max_position=config.max_position_embeddings,
751
- base=config.rope_theta,
752
- rope_scaling=config.rope_scaling,
751
+ rope_parameters=config.rope_parameters,
752
partial_rotary_factor=config.partial_rotary_factor,
753
dual_chunk_attention_config=self.dual_chunk_attention_config,
754
)
vllm/model_executor/models/qwen3_omni_moe_thinker.py
-1
@@ -338,7 +338,6 @@ class Qwen3Omni_VisionTransformer(nn.Module):
338
head_size=head_dim,
339
rotary_dim=head_dim // 2,
340
max_position=8192,
341
- base=10000.0,
341
is_neox_style=True,
342
)
343
vllm/model_executor/models/qwen3_vl.py
-1
@@ -345,7 +345,6 @@ class Qwen3_VisionTransformer(nn.Module):
345
head_size=head_dim,
346
rotary_dim=head_dim // 2,
347
max_position=8192,
348
- base=10000.0,
348
is_neox_style=True,
349
)
350
vllm/model_executor/models/seed_oss.py
+5
-10
@@ -54,6 +54,7 @@ from vllm.model_executor.model_loader.weight_utils import (
54
maybe_remap_kv_scale_name,
55
)
56
from vllm.sequence import IntermediateTensors
57
+from vllm.transformers_utils.config import set_default_rope_theta
58
59
from .interfaces import SupportsLoRA, SupportsPP
60
from .utils import (
@@ -112,11 +113,10 @@ class SeedOssAttention(nn.Module):
113
num_heads: int,
114
num_kv_heads: int,
115
head_dim: int,
116
+ rope_parameters: dict,
117
max_position: int = 4096 * 32,
116
- rope_theta: float = 10000,
118
cache_config: CacheConfig | None = None,
119
quant_config: QuantizationConfig | None = None,
119
- rope_scaling: tuple | None = None,
120
prefix: str = "",
121
attn_type: str = AttentionType.DECODER,
122
) -> None:
@@ -140,7 +140,6 @@ class SeedOssAttention(nn.Module):
140
self.q_size = self.num_heads * self.head_dim
141
self.kv_size = self.num_kv_heads * self.head_dim
142
self.scaling = self.head_dim**-0.5
143
- self.rope_theta = rope_theta
143
144
self.qkv_proj = QKVParallelLinear(
145
hidden_size,
@@ -163,8 +162,7 @@ class SeedOssAttention(nn.Module):
162
self.head_dim,
163
rotary_dim=self.head_dim,
164
max_position=max_position,
166
- base=self.rope_theta,
167
- rope_scaling=rope_scaling,
165
+ rope_parameters=rope_parameters,
166
)
167
self.attn = Attention(
168
self.num_heads,
@@ -200,9 +198,7 @@ class SeedOssDecoderLayer(nn.Module):
198
) -> None:
199
super().__init__()
200
self.hidden_size = config.hidden_size
203
- # Requires transformers > 4.32.0
204
- rope_theta = getattr(config, "rope_theta", 1000000)
205
- rope_scaling = getattr(config, "rope_scaling", None)
201
+ set_default_rope_theta(config, default_theta=1000000)
202
203
# By default, SeedOss uses causal attention as it is a
204
# decoder-only model.
@@ -219,10 +215,9 @@ class SeedOssDecoderLayer(nn.Module):
215
max_position=config.max_position_embeddings,
216
num_kv_heads=config.num_key_value_heads,
217
head_dim=config.head_dim,
222
- rope_theta=rope_theta,
218
cache_config=cache_config,
219
quant_config=quant_config,
225
- rope_scaling=rope_scaling,
220
+ rope_parameters=config.rope_parameters,
221
prefix=f"{prefix}.self_attn",
222
attn_type=attn_type,
223
)
vllm/model_executor/models/solar.py
+1
-17
@@ -25,7 +25,6 @@
25
"""Inference-only Solar model compatible with HuggingFace weights."""
26
27
from collections.abc import Iterable
28
-from typing import Any
28
29
import torch
30
from torch import nn
@@ -111,8 +110,6 @@ class SolarAttention(nn.Module):
110
hidden_size: int,
111
num_heads: int,
112
num_kv_heads: int,
114
- rope_theta: float = 10000,
115
- rope_scaling: dict[str, Any] | None = None,
113
max_position_embeddings: int = 8192,
114
quant_config: QuantizationConfig | None = None,
115
bias: bool = False,
@@ -142,7 +139,6 @@ class SolarAttention(nn.Module):
139
self.q_size = self.num_heads * self.head_dim
140
self.kv_size = self.num_kv_heads * self.head_dim
141
self.scaling = self.head_dim**-0.5
145
- self.rope_theta = rope_theta
142
self.max_position_embeddings = max_position_embeddings
143
144
self.qkv_proj = QKVParallelLinear(
@@ -166,8 +162,7 @@ class SolarAttention(nn.Module):
162
self.head_dim,
163
rotary_dim=self.head_dim,
164
max_position=max_position_embeddings,
169
- base=rope_theta,
170
- rope_scaling=rope_scaling,
165
+ rope_parameters=config.rope_parameters,
166
)
167
self.attn = Attention(
168
self.num_heads,
@@ -202,15 +197,6 @@ class SolarDecoderLayer(nn.Module):
197
) -> None:
198
super().__init__()
199
self.hidden_size = config.hidden_size
205
- rope_theta = getattr(config, "rope_theta", 10000)
206
- rope_scaling = getattr(config, "rope_scaling", None)
207
-
208
- if rope_scaling is not None and getattr(
209
- config, "original_max_position_embeddings", None
210
- ):
211
- rope_scaling["original_max_position_embeddings"] = (
212
- config.original_max_position_embeddings
213
- )
200
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
201
# Support abacusai/Smaug-72B-v0.1 with attention_bias
202
# Support internlm/internlm-7b with bias
@@ -224,8 +210,6 @@ class SolarDecoderLayer(nn.Module):
210
num_kv_heads=getattr(
211
config, "num_key_value_heads", config.num_attention_heads
212
),
227
- rope_theta=rope_theta,
228
- rope_scaling=rope_scaling,
213
max_position_embeddings=max_position_embeddings,
214
quant_config=quant_config,
215
bias=attention_bias,
vllm/model_executor/models/stablelm.py
+1
-1
@@ -153,7 +153,7 @@ class StablelmAttention(nn.Module):
153
self.head_dim,
154
rotary_dim=self.head_dim,
155
max_position=self.config.max_position_embeddings,
156
- base=self.config.rope_theta,
156
+ rope_parameters=self.config.rope_parameters,
157
partial_rotary_factor=self.partial_rotary_factor,
158
)
159
self.attn = Attention(
vllm/model_executor/models/starcoder2.py
+1
-2
@@ -91,7 +91,6 @@ class Starcoder2Attention(nn.Module):
91
self.q_size = self.num_heads * self.head_dim
92
self.kv_size = self.num_kv_heads * self.head_dim
93
self.scaling = self.head_dim**-0.5
94
- self.rope_theta = config.rope_theta
94
self.max_position_embeddings = config.max_position_embeddings
95
self.use_bias = config.use_bias
96
@@ -115,7 +114,7 @@ class Starcoder2Attention(nn.Module):
114
self.head_dim,
115
rotary_dim=self.head_dim,
116
max_position=self.max_position_embeddings,
118
- base=int(self.rope_theta),
117
+ rope_parameters=config.rope_parameters,
118
is_neox_style=True,
119
)
120
self.attn = Attention(
vllm/model_executor/models/step3_text.py
+6
-10
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36
)
37
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
from vllm.sequence import IntermediateTensors
39
+from vllm.transformers_utils.configs.step3_vl import Step3TextConfig
40
41
from .interfaces import SupportsPP
42
from .utils import (
@@ -144,9 +145,8 @@ class Step3TextAttention(nn.Module):
145
num_heads: int,
146
num_kv_heads: int,
147
norm_eps: float,
147
- rope_theta: int,
148
+ rope_parameters: dict[str, Any],
149
share_q_dim: int | None = None,
149
- rope_scaling: dict[str, Any] | None = None,
150
max_position_embedding: int = 8192,
151
head_dim: int = 256,
152
cache_config: CacheConfig | None = None,
@@ -198,8 +198,7 @@ class Step3TextAttention(nn.Module):
198
self.head_dim,
199
rotary_dim=self.head_dim,
200
max_position=max_position_embedding,
201
- base=rope_theta,
202
- rope_scaling=rope_scaling,
201
+ rope_parameters=rope_parameters,
202
)
203
scaling = self.head_dim**-0.5
204
self.attn = Attention(
@@ -227,15 +226,13 @@ class Step3TextAttention(nn.Module):
226
class Step3TextDecoderLayer(nn.Module):
227
def __init__(
228
self,
230
- config: ModelConfig,
229
+ config: Step3TextConfig,
230
cache_config: CacheConfig | None = None,
231
quant_config: QuantizationConfig | None = None,
232
prefix: str = "",
233
) -> None:
234
super().__init__()
236
- config = config.hf_config
235
self.hidden_size = config.hidden_size
238
- rope_scaling = getattr(config, "rope_scaling", None)
236
237
self.self_attn = Step3TextAttention(
238
hidden_size=self.hidden_size,
@@ -247,8 +244,7 @@ class Step3TextDecoderLayer(nn.Module):
244
max_position_embedding=config.max_position_embedding,
245
head_dim=config.head_dim,
246
share_q_dim=config.share_q_dim,
250
- rope_theta=config.rope_theta,
251
- rope_scaling=rope_scaling,
247
+ rope_parameters=config.rope_parameters,
248
prefix=f"{prefix}.self_attn",
249
)
250
@@ -338,7 +334,7 @@ class Step3TextModel(nn.Module):
334
self.start_layer, self.end_layer, self.layers = make_layers(
335
config.num_hidden_layers,
336
lambda prefix: Step3TextDecoderLayer(
341
- config=vllm_config.model_config,
337
+ config=config,
338
cache_config=cache_config,
339
quant_config=quant_config,
340
prefix=prefix,
vllm/model_executor/models/transformers/utils.py
+8
-2
@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Literal
22
23
import torch
24
from torch import nn
25
+from transformers.configuration_utils import ALLOWED_LAYER_TYPES
26
27
from vllm.config.utils import getattr_iter
28
from vllm.logger import init_logger
@@ -203,5 +204,10 @@ def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
204
"""
205
text_config = vllm_config.model_config.hf_config.get_text_config()
206
# Dynamic rope scaling is not compatible with torch.compile
206
- rope_scaling: dict = getattr(text_config, "rope_scaling", None) or {}
207
- return rope_scaling.get("rope_type") != "dynamic"
207
+ rope_parameters: dict | None = getattr(text_config, "rope_parameters", None) or {}
208
+ if rope_parameters:
209
+ # Nest rope_parameters if not nested already to simplify logic
210
+ if not set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
211
+ rope_parameters = {"": rope_parameters}
212
+ return all(rp["rope_type"] != "dynamic" for rp in rope_parameters.values())
213
+ return True
vllm/model_executor/models/zamba2.py
+1
-3
@@ -128,7 +128,6 @@ class Zamba2Attention(nn.Module):
128
tp_size = get_tensor_model_parallel_world_size()
129
self.config = config
130
self.num_hybrid_layers = num_hybrid_layers
131
- self.rope_theta = config.rope_theta
131
132
self.attention_hidden_size = config.attention_hidden_size
133
self.total_num_attention_heads = config.num_attention_heads
@@ -233,8 +232,7 @@ class Zamba2Attention(nn.Module):
232
head_size=self.attention_head_dim,
233
rotary_dim=self.attention_head_dim,
234
max_position=config.max_position_embeddings,
236
- base=self.rope_theta,
237
- rope_scaling=None,
235
+ rope_parameters=config.rope_parameters,
236
is_neox_style=True,
237
)
238
vllm/transformers_utils/config.py
+75
-25
@@ -7,8 +7,9 @@ import time
7
from collections.abc import Callable
8
from dataclasses import asdict
9
from functools import cache, partial
10
+from importlib.metadata import version
11
from pathlib import Path
11
-from typing import Any, Literal, TypeVar
12
+from typing import Any, Literal, TypeAlias, TypeVar
13
14
import huggingface_hub
15
from huggingface_hub import (
@@ -24,7 +25,9 @@ from huggingface_hub.utils import (
25
RepositoryNotFoundError,
26
RevisionNotFoundError,
27
)
28
+from packaging.version import Version
29
from transformers import DeepseekV3Config, GenerationConfig, PretrainedConfig
30
+from transformers.configuration_utils import ALLOWED_LAYER_TYPES
31
from transformers.models.auto.image_processing_auto import get_image_processor_config
32
from transformers.models.auto.modeling_auto import (
33
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
@@ -390,21 +393,61 @@ def file_or_path_exists(
393
)
394
395
393
-def patch_rope_scaling(config: PretrainedConfig) -> None:
396
+def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
397
+ """Some models may have no rope_theta in their config but still use RoPE.
398
+ This function sets a default rope_theta if it's missing."""
399
+ if getattr(config, "rope_parameters", None) is None:
400
+ config.rope_parameters = {"rope_type": "default"}
401
+ if "rope_theta" not in config.rope_parameters:
402
+ config.rope_parameters["rope_theta"] = default_theta
403
+
404
+
405
+def patch_rope_parameters(config: PretrainedConfig) -> None:
406
"""Provide backwards compatibility for RoPE."""
395
- text_config = getattr(config, "text_config", None)
396
- if text_config is not None:
397
- patch_rope_scaling(text_config)
407
+ # Retrieve rope_parameters differently based on Transformers version
408
+ if Version(version("transformers")) >= Version("5.0.0.dev0"):
409
+ from transformers.modeling_rope_utils import RopeParameters
410
399
- rope_scaling = getattr(config, "rope_scaling", None)
400
- if rope_scaling is not None:
401
- patch_rope_scaling_dict(rope_scaling)
411
+ rope_parameters: RopeParameters | dict[str, RopeParameters] | None = getattr(
412
+ config, "rope_parameters", None
413
+ )
414
+ elif hasattr(config, "rope_parameters"):
415
+ # We are in Transformers v4 and rope_parameters
416
+ # has already been patched for this config
417
+ return
418
+ else:
419
+ # Convert Transformers v4 rope_theta and rope_scaling into rope_parameters
420
+ rope_theta: float | None = getattr(config, "rope_theta", None)
421
+ rope_scaling: dict | None = getattr(config, "rope_scaling", None)
422
+ rope_parameters = rope_scaling
423
+ # Move rope_theta into rope_parameters
424
+ if rope_theta is not None:
425
+ rope_parameters = rope_parameters or {"rope_type": "default"}
426
+ rope_parameters["rope_theta"] = rope_theta
427
+ # Add original_max_position_embeddings if present
428
+ if rope_parameters and (
429
+ ompe := getattr(config, "original_max_position_embeddings", None)
430
+ ):
431
+ rope_parameters["original_max_position_embeddings"] = ompe
432
+ # Write back to config
433
+ config.rope_parameters = rope_parameters
434
+
435
+ # No RoPE parameters to patch
436
+ if rope_parameters is None:
437
+ return
438
+
439
+ # Handle nested rope_parameters in interleaved sliding attention models
440
+ if set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
441
+ for rope_parameters_layer_type in rope_parameters.values():
442
+ patch_rope_parameters_dict(rope_parameters_layer_type)
443
+ else:
444
+ patch_rope_parameters_dict(rope_parameters)
445
446
404
-def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
405
- if "rope_type" in rope_scaling and "type" in rope_scaling:
406
- rope_type = rope_scaling["rope_type"]
407
- rope_type_legacy = rope_scaling["type"]
447
+def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
448
+ if "rope_type" in rope_parameters and "type" in rope_parameters:
449
+ rope_type = rope_parameters["rope_type"]
450
+ rope_type_legacy = rope_parameters["type"]
451
if rope_type != rope_type_legacy:
452
raise ValueError(
453
f"Found conflicts between 'rope_type={rope_type}' (modern "
@@ -412,28 +455,28 @@ def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
455
"You should only specify one of them."
456
)
457
415
- if "rope_type" not in rope_scaling and "type" in rope_scaling:
416
- rope_scaling["rope_type"] = rope_scaling["type"]
458
+ if "rope_type" not in rope_parameters and "type" in rope_parameters:
459
+ rope_parameters["rope_type"] = rope_parameters["type"]
460
logger.info("Replacing legacy 'type' key with 'rope_type'")
461
419
- if "rope_type" not in rope_scaling:
420
- raise ValueError("rope_scaling should have a 'rope_type' key")
462
+ if "rope_type" not in rope_parameters:
463
+ raise ValueError("rope_parameters should have a 'rope_type' key")
464
422
- if rope_scaling["rope_type"] == "su":
423
- rope_scaling["rope_type"] = "longrope"
465
+ if rope_parameters["rope_type"] == "su":
466
+ rope_parameters["rope_type"] = "longrope"
467
logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
425
- elif rope_scaling["rope_type"] == "mrope":
426
- assert "mrope_section" in rope_scaling
427
- rope_scaling["rope_type"] = "default"
468
+ elif rope_parameters["rope_type"] == "mrope":
469
+ assert "mrope_section" in rope_parameters
470
+ rope_parameters["rope_type"] = "default"
471
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
472
473
474
def _uses_mrope(config: PretrainedConfig) -> bool:
432
- rope_scaling = getattr(config, "rope_scaling", None)
433
- if rope_scaling is None:
475
+ rope_parameters = getattr(config, "rope_parameters", None)
476
+ if rope_parameters is None:
477
return False
478
436
- return "mrope_section" in rope_scaling
479
+ return "mrope_section" in rope_parameters
480
481
482
def uses_mrope(config: PretrainedConfig) -> bool:
@@ -690,7 +733,14 @@ def get_config(
733
logger.debug("Overriding HF config with %s", hf_overrides_fn)
734
config = hf_overrides_fn(config)
735
693
- patch_rope_scaling(config)
736
+ # Exhaustively patch RoPE parameters everywhere they might be
737
+ patch_rope_parameters(config)
738
+ patch_rope_parameters(config.get_text_config())
739
+ SubConfigs: TypeAlias = dict[str, PretrainedConfig]
740
+ sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
741
+ if sub_configs:
742
+ for sub_config in sub_configs:
743
+ patch_rope_parameters(getattr(config, sub_config))
744
745
if trust_remote_code:
746
maybe_register_config_serialize_by_value()
vllm/transformers_utils/configs/afmoe.py
+5
-2
@@ -24,7 +24,7 @@ class AfmoeConfig(PretrainedConfig):
24
rms_norm_eps: float = 1e-5,
25
use_cache: bool = True,
26
tie_word_embeddings: bool = False,
27
- rope_theta: float = 10000.0,
27
+ rope_parameters: dict | None = None,
28
rope_scaling: dict | None = None,
29
num_experts: int = 64,
30
num_experts_per_tok: int = 6,
@@ -56,7 +56,10 @@ class AfmoeConfig(PretrainedConfig):
56
self.initializer_range = initializer_range
57
self.rms_norm_eps = rms_norm_eps
58
self.use_cache = use_cache
59
- self.rope_theta = rope_theta
59
+ rope_theta = kwargs.pop("rope_theta", 10000.0)
60
+ if rope_parameters is None:
61
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
62
+ self.rope_parameters = rope_parameters
63
self.rope_scaling = rope_scaling
64
65
self.moe_intermediate_size = moe_intermediate_size
vllm/transformers_utils/configs/arctic.py
+14
-4
@@ -85,8 +85,15 @@ class ArcticConfig(PretrainedConfig):
85
The id of the "end-of-sequence" token.
86
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
87
Whether the model's input and output word embeddings should be tied.
88
- rope_theta (`float`, *optional*, defaults to 1000000.0):
89
- The base period of the RoPE embeddings.
88
+ rope_parameters (`dict`, *optional*):
89
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
90
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
91
+ accordingly.
92
+ Expected contents:
93
+ `rope_theta` (`float`): The base period of the RoPE embeddings.
94
+ `rope_type` (`str`):
95
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
96
+ 'llama3'], with 'default' being the original RoPE implementation.
97
sliding_window (`int`, *optional*):
98
Sliding window attention window size. If not specified, will default to `4096`.
99
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -132,7 +139,7 @@ class ArcticConfig(PretrainedConfig):
139
bos_token_id=1,
140
eos_token_id=2,
141
tie_word_embeddings=False,
135
- rope_theta=1e6,
142
+ rope_parameters: dict[str, Any] | None = None,
143
sliding_window=None,
144
attention_dropout=0.0,
145
num_experts_per_tok=1,
@@ -165,7 +172,10 @@ class ArcticConfig(PretrainedConfig):
172
self.initializer_range = initializer_range
173
self.rms_norm_eps = rms_norm_eps
174
self.use_cache = use_cache
168
- self.rope_theta = rope_theta
175
+ rope_theta = kwargs.pop("rope_theta", 1e6)
176
+ if rope_parameters is None:
177
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
178
+ self.rope_parameters = rope_parameters
179
self.attention_dropout = attention_dropout
180
181
self.num_experts_per_tok = num_experts_per_tok
vllm/transformers_utils/configs/flex_olmo.py
+11
-6
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
5
from transformers.configuration_utils import PretrainedConfig
6
@@ -25,8 +26,7 @@ class FlexOlmoConfig(PretrainedConfig):
26
bos_token_id=None,
27
eos_token_id=100257,
28
tie_word_embeddings=False,
28
- rope_theta=500000.0,
29
- rope_scaling=None,
29
+ rope_parameters: dict[str, Any] | None = None,
30
attention_bias=False,
31
attention_dropout=0.0,
32
num_experts_per_tok=5,
@@ -62,8 +62,13 @@ class FlexOlmoConfig(PretrainedConfig):
62
self.initializer_range = initializer_range
63
self.rms_norm_eps = rms_norm_eps
64
self.use_cache = use_cache
65
- self.rope_theta = rope_theta
66
- self.rope_scaling = rope_scaling
65
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
66
+ rope_scaling = kwargs.pop("rope_scaling", None)
67
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
68
+ rope_theta = kwargs.pop("rope_theta", 500000.0)
69
+ if "rope_theta" not in rope_parameters:
70
+ rope_parameters["rope_theta"] = rope_theta
71
+ self.rope_parameters = rope_parameters
72
self.attention_bias = attention_bias
73
self.attention_dropout = attention_dropout
74
self.num_experts_per_tok = num_experts_per_tok
@@ -73,5 +78,5 @@ class FlexOlmoConfig(PretrainedConfig):
78
self.norm_topk_prob = norm_topk_prob
79
# Validate the correctness of rotary position embeddings parameters
80
# BC: if there is a 'type' field, move it to 'rope_type'.
76
- if self.rope_scaling is not None and "type" in self.rope_scaling:
77
- self.rope_scaling["rope_type"] = self.rope_scaling["type"]
81
+ if self.rope_parameters is not None and "type" in self.rope_parameters:
82
+ self.rope_parameters["rope_type"] = self.rope_parameters["type"]
vllm/transformers_utils/configs/kimi_linear.py
+8
-4
@@ -29,8 +29,7 @@ class KimiLinearConfig(PretrainedConfig):
29
pad_token_id=0,
30
bos_token_id=1,
31
eos_token_id=2,
32
- rope_theta=10000.0,
33
- rope_scaling=None,
32
+ rope_parameters=None,
33
tie_word_embeddings=False,
34
moe_intermediate_size: int | None = None,
35
moe_renormalize: bool = True,
@@ -73,8 +72,13 @@ class KimiLinearConfig(PretrainedConfig):
72
self.initializer_range = initializer_range
73
self.rms_norm_eps = rms_norm_eps
74
self.use_cache = use_cache
76
- self.rope_theta = rope_theta
77
- self.rope_scaling = rope_scaling
75
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
76
+ rope_scaling = kwargs.pop("rope_scaling", None)
77
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
78
+ rope_theta = kwargs.pop("rope_theta", 10000.0)
79
+ if "rope_theta" not in rope_parameters:
80
+ rope_parameters["rope_theta"] = rope_theta
81
+ self.rope_parameters = rope_parameters
82
83
self.q_lora_rank = q_lora_rank
84
self.kv_lora_rank = kv_lora_rank
vllm/transformers_utils/configs/lfm2_moe.py
+8
-4
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
5
from transformers.configuration_utils import PretrainedConfig
6
@@ -35,8 +36,8 @@ class Lfm2MoeConfig(PretrainedConfig):
36
End of stream token id.
37
tie_word_embeddings (`bool`, *optional*, defaults to `True`):
38
Whether to tie weight embeddings
38
- rope_theta (`float`, *optional*, defaults to 1000000.0):
39
- The base period of the RoPE embeddings.
39
+ rope_parameters (`dict`, *optional*):
40
+ The parameters of the RoPE embeddings.
41
max_position_embeddings (`int`, *optional*, defaults to 128000):
42
The maximum sequence length that this model might ever be used with.
43
use_cache (`bool`, *optional*, defaults to `True`):
@@ -100,7 +101,7 @@ class Lfm2MoeConfig(PretrainedConfig):
101
bos_token_id: int = 1,
102
eos_token_id: int = 2,
103
tie_word_embeddings: bool = True,
103
- rope_theta: float = 1000000.0,
104
+ rope_parameters: dict[str, Any] | None = None,
105
max_position_embeddings: int = 128_000,
106
use_cache: bool = True,
107
norm_eps: float = 0.00001,
@@ -121,7 +122,10 @@ class Lfm2MoeConfig(PretrainedConfig):
122
self.hidden_size = hidden_size
123
self.intermediate_size = intermediate_size
124
self.num_hidden_layers = num_hidden_layers
124
- self.rope_theta = rope_theta
125
+ rope_theta = kwargs.pop("rope_theta", 1000000.0)
126
+ if rope_parameters is None:
127
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
128
+ self.rope_parameters = rope_parameters
129
self.max_position_embeddings = max_position_embeddings
130
self.use_cache = use_cache
131
self.norm_eps = norm_eps
vllm/transformers_utils/configs/midashenglm.py
+1
-1
@@ -98,6 +98,6 @@ class MiDashengLMConfig(PretrainedConfig):
98
if text_config
99
else Qwen2_5OmniTextConfig()
100
)
101
- self.text_config.rope_scaling = None # uses_mrope is false
101
+ self.text_config.rope_parameters = None # uses_mrope is false
102
self.audio_token_id = audio_token_id
103
super().__init__(**kwargs)
vllm/transformers_utils/configs/mistral.py
+2
-2
@@ -86,13 +86,13 @@ def _remap_mistral_yarn_args(config: dict) -> dict:
86
"apply_scale": "apply_yarn_scaling",
87
}
88
yarn_config = config.get("yarn") or {}
89
- config["rope_scaling"] = {
89
+ config["rope_parameters"] = {
90
"rope_type": "yarn",
91
"mscale_all_dim": 1,
92
}
93
for old_name, new_name in yarn_config_map.items():
94
if old_name in yarn_config:
95
- config["rope_scaling"][new_name] = yarn_config.pop(old_name)
95
+ config["rope_parameters"][new_name] = yarn_config.pop(old_name)
96
97
assert len(yarn_config) == 0, f"Unparsed yarn config: {yarn_config}"
98
vllm/transformers_utils/configs/nemotron.py
+31
-29
@@ -88,8 +88,8 @@ class NemotronConfig(PretrainedConfig):
88
End of stream token id.
89
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
90
Whether to tie weight embeddings
91
- rope_theta (`float`, *optional*, defaults to 10000.0):
92
- The base period of the RoPE embeddings.
91
+ rope_parameters (`dict`, *optional*):
92
+ The parameters of the RoPE embeddings.
93
partial_rotary_factor (`float`, *optional*, defaults to 0.5):
94
Percentage of the query and keys which will have rotary embedding.
95
attention_bias (`bool`, *optional*, defaults to `False`):
@@ -132,8 +132,7 @@ class NemotronConfig(PretrainedConfig):
132
bos_token_id=2,
133
eos_token_id=3,
134
tie_word_embeddings=False,
135
- rope_theta=10000.0,
136
- rope_scaling=None,
135
+ rope_parameters=None,
136
partial_rotary_factor=0.5,
137
attention_bias=False,
138
attention_dropout=0.0,
@@ -160,8 +159,13 @@ class NemotronConfig(PretrainedConfig):
159
self.initializer_range = initializer_range
160
self.norm_eps = norm_eps
161
self.use_cache = use_cache
163
- self.rope_theta = rope_theta
164
- self.rope_scaling = rope_scaling
162
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
163
+ rope_scaling = kwargs.pop("rope_scaling", None)
164
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
165
+ rope_theta = kwargs.pop("rope_theta", 10000.0)
166
+ if "rope_theta" not in rope_parameters:
167
+ rope_parameters["rope_theta"] = rope_theta
168
+ self.rope_parameters = rope_parameters
169
# for backward compatibility
170
partial_rotary_factor = (
171
kwargs.get("rope_percent")
@@ -169,7 +173,7 @@ class NemotronConfig(PretrainedConfig):
173
or partial_rotary_factor
174
)
175
self.partial_rotary_factor = partial_rotary_factor
172
- self._rope_scaling_validation()
176
+ self._rope_parameters_validation()
177
self.attention_bias = attention_bias
178
self.attention_dropout = attention_dropout
179
self.mlp_bias = mlp_bias
@@ -182,31 +186,29 @@ class NemotronConfig(PretrainedConfig):
186
**kwargs,
187
)
188
185
- def _rope_scaling_validation(self):
189
+ def _rope_parameters_validation(self):
190
"""
187
- Validate the `rope_scaling` configuration.
191
+ Validate the `rope_parameters` configuration.
192
"""
189
- if self.rope_scaling is None:
193
+ if self.rope_parameters is None:
194
return
195
192
- if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
193
- raise ValueError(
194
- "`rope_scaling` must be a dictionary with two fields, "
195
- f"`type` and `factor`, got {self.rope_scaling}"
196
- )
197
- rope_scaling_type = self.rope_scaling.get("type", None)
198
- rope_scaling_factor = self.rope_scaling.get("factor", None)
199
- if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
200
- raise ValueError(
201
- "`rope_scaling`'s type field must be one of ['linear', "
202
- f"'dynamic'], got {rope_scaling_type}"
203
- )
204
- if (
205
- rope_scaling_factor is None
206
- or not isinstance(rope_scaling_factor, float)
207
- or rope_scaling_factor <= 1.0
208
- ):
196
+ rope_type: str | None = self.rope_parameters.get("rope_type", None)
197
+ factor: float | None = self.rope_parameters.get("factor", None)
198
+
199
+ if rope_type not in {"default", "linear", "dynamic"}:
200
raise ValueError(
210
- "`rope_scaling`'s factor field must be a float > 1, got "
211
- f"{rope_scaling_factor}"
201
+ "`rope_type` must be one of ['default', 'linear', 'dynamic'], "
202
+ f"got {rope_type}"
203
)
204
+ if rope_type != "default":
205
+ if factor is None:
206
+ raise ValueError(
207
+ "If `rope_type` is not 'default', `rope_parameters` "
208
+ "must include a `factor` field. Got `None`."
209
+ )
210
+ if not isinstance(factor, float) or factor <= 1.0:
211
+ raise ValueError(
212
+ "`rope_parameters`'s factor field must be a float > 1, got "
213
+ f"{factor}"
214
+ )
vllm/transformers_utils/configs/olmo3.py
+8
-4
@@ -24,8 +24,7 @@ class Olmo3Config(PretrainedConfig):
24
bos_token_id=None,
25
eos_token_id=50279,
26
tie_word_embeddings=False,
27
- rope_theta=10000.0,
28
- rope_scaling=None,
27
+ rope_parameters=None,
28
attention_bias=False,
29
attention_dropout=0.0,
30
rms_norm_eps=1e-5,
@@ -63,8 +62,13 @@ class Olmo3Config(PretrainedConfig):
62
self.hidden_act = hidden_act
63
self.initializer_range = initializer_range
64
self.use_cache = use_cache
66
- self.rope_theta = rope_theta
67
- self.rope_scaling = rope_scaling
65
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
66
+ rope_scaling = kwargs.pop("rope_scaling", None)
67
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
68
+ rope_theta = kwargs.pop("rope_theta", 10000.0)
69
+ if "rope_theta" not in rope_parameters:
70
+ rope_parameters["rope_theta"] = rope_theta
71
+ self.rope_parameters = rope_parameters
72
self.attention_bias = attention_bias
73
self.attention_dropout = attention_dropout
74
vllm/transformers_utils/configs/qwen3_next.py
+10
-7
@@ -66,13 +66,12 @@ class Qwen3NextConfig(PretrainedConfig):
66
relevant if `config.is_decoder=True`.
67
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
68
Whether the model's input and output word embeddings should be tied.
69
- rope_theta (`float`, *optional*, defaults to 10000.0):
70
- The base period of the RoPE embeddings.
71
- rope_scaling (`Dict`, *optional*):
69
+ rope_parameters (`dict`, *optional*):
70
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
71
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
72
accordingly.
73
Expected contents:
74
+ `rope_theta` (`float`): The base period of the RoPE embeddings.
75
`rope_type` (`str`):
76
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
77
'llama3'], with 'default' being the original RoPE implementation.
@@ -199,8 +198,7 @@ class Qwen3NextConfig(PretrainedConfig):
198
rms_norm_eps=1e-6,
199
use_cache=True,
200
tie_word_embeddings=False,
202
- rope_theta=10000.0,
203
- rope_scaling=None,
201
+ rope_parameters=None,
202
partial_rotary_factor=0.25,
203
attention_bias=False,
204
attention_dropout=0.0,
@@ -236,8 +234,13 @@ class Qwen3NextConfig(PretrainedConfig):
234
self.initializer_range = initializer_range
235
self.rms_norm_eps = rms_norm_eps
236
self.use_cache = use_cache
239
- self.rope_theta = rope_theta
240
- self.rope_scaling = rope_scaling
237
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
238
+ rope_scaling = kwargs.pop("rope_scaling", None)
239
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
240
+ rope_theta = kwargs.pop("rope_theta", 10000.0)
241
+ if "rope_theta" not in rope_parameters:
242
+ rope_parameters["rope_theta"] = rope_theta
243
+ self.rope_parameters = rope_parameters
244
self.partial_rotary_factor = partial_rotary_factor
245
self.attention_bias = attention_bias
246
self.attention_dropout = attention_dropout
vllm/transformers_utils/configs/step3_vl.py
+8
-4
@@ -52,8 +52,7 @@ class Step3TextConfig(PretrainedConfig):
52
moe_intermediate_size: int = 5120,
53
moe_num_experts: int = 48,
54
moe_top_k: int = 3,
55
- rope_theta: float = 500000,
56
- rope_scaling: dict[str, Any] | None = None,
55
+ rope_parameters: dict[str, Any] | None = None,
56
max_position_embedding: int = 65536,
57
share_expert_dim: int = 5120,
58
share_q_dim: int = 2048,
@@ -130,8 +129,13 @@ class Step3TextConfig(PretrainedConfig):
129
self.moe_intermediate_size = moe_intermediate_size
130
self.moe_num_experts = moe_num_experts
131
self.moe_top_k = moe_top_k
133
- self.rope_theta = rope_theta
134
- self.rope_scaling = rope_scaling
132
+ # Try to set `rope_scaling` if available, otherwise use `rope_parameters`
133
+ rope_scaling = kwargs.pop("rope_scaling", None)
134
+ rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
135
+ rope_theta = kwargs.pop("rope_theta", 500000.0)
136
+ if "rope_theta" not in rope_parameters:
137
+ rope_parameters["rope_theta"] = rope_theta
138
+ self.rope_parameters = rope_parameters
139
self.max_position_embedding = max_position_embedding
140
self.share_expert_dim = share_expert_dim
141
self.share_q_dim = share_q_dim