A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 19:45 UTC
[Bugfix] Fix Plamo3 rope handling (#29092)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Cyrus Leung committed
Nov 21, 2025 at 11:38 UTC
0e741c12e3dc45093b2ddab8a31310703aa27002
1 file changed
+18
-8
vllm/model_executor/models/plamo3.py
+18
-8
@@ -62,7 +62,7 @@ class Plamo3Config(PretrainedConfig): # type: ignore
62
# if `sliding_window` is list
63
interleaved_sliding_window: list[int | None]
64
sliding_window_pattern: int
65
- rope_theta: int
65
+ rope_parameters: dict[str, Any]
66
rope_local_theta: int
67
# MLP
68
intermediate_size: int
@@ -153,13 +153,24 @@ class Plamo3AttentionMixer(nn.Module):
153
quant_config=quant_config,
154
prefix=f"{prefix}.o_proj",
155
)
156
+
157
layer_idx = extract_layer_index(prefix)
157
- full_attn = config.interleaved_sliding_window[layer_idx] is None
158
+ layer_type = config.layer_types[layer_idx]
159
+ is_sliding = layer_type == "sliding_attention"
160
159
- self.rope_theta = config.rope_theta if full_attn else config.rope_local_theta
160
- self.rope_scaling = (
161
- config.rope_scaling if hasattr(config, "rope_scaling") else None
162
- )
161
+ # Initialize the rotary embedding.
162
+ if layer_type in config.rope_parameters:
163
+ # Transformers v5 rope config.
164
+ rope_parameters = config.rope_parameters[layer_type]
165
+ else:
166
+ # Transformers v4 rope config.
167
+ # Global attention. Use the values in config.json.
168
+ rope_parameters = config.rope_parameters
169
+ # Local attention. Override the values in config.json.
170
+ if is_sliding:
171
+ rope_parameters = dict(
172
+ rope_type="default", rope_theta=config.rope_local_theta
173
+ )
174
max_position = config.max_position_embeddings
175
if hasattr(vllm_config.model_config, "max_model_len") and isinstance(
176
vllm_config.model_config.max_model_len, int
@@ -170,8 +181,7 @@ class Plamo3AttentionMixer(nn.Module):
181
self.head_dim,
182
rotary_dim=self.head_dim,
183
max_position=max_position,
173
- base=self.rope_theta,
174
- rope_scaling=self.rope_scaling,
184
+ rope_parameters=rope_parameters,
185
)
186
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
187
set_weight_attrs(