A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 13:30 UTC
Fix RoPE related failures in Transformers nightly tests (#29333)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Nov 25, 2025 at 16:23 UTC
0353d2e162cbda776d9dbfe026e65303204a7f1f
5 files changed
+37
-33
vllm/model_executor/models/baichuan.py
+1
-1
@@ -233,7 +233,7 @@ class BaiChuanDecoderLayer(nn.Module):
233
hidden_size=self.hidden_size,
234
num_heads=config.num_attention_heads,
235
position_embedding=position_embedding,
236
- rope_parameters=config.rope_parameters,
236
+ rope_parameters=getattr(config, "rope_parameters", None),
237
max_position_embeddings=max_position_embeddings,
238
cache_config=cache_config,
239
quant_config=quant_config,
vllm/model_executor/models/gpt_j.py
+1
-1
@@ -100,7 +100,7 @@ class GPTJAttention(nn.Module):
100
self.head_size,
101
rotary_dim=config.rotary_dim,
102
max_position=max_position_embeddings,
103
- rope_parameters=config.rope_parameters,
103
+ rope_parameters=getattr(config, "rope_parameters", None),
104
is_neox_style=False,
105
)
106
self.attn = Attention(
vllm/model_executor/models/grok1.py
+1
-1
@@ -239,7 +239,7 @@ class Grok1DecoderLayer(nn.Module):
239
num_heads=config.num_attention_heads,
240
max_position=config.max_position_embeddings,
241
num_kv_heads=config.num_key_value_heads,
242
- rope_parameters=config.rope_parameters,
242
+ rope_parameters=getattr(config, "rope_parameters", None),
243
cache_config=cache_config,
244
quant_config=quant_config,
245
prefix=f"{prefix}.attn",
vllm/model_executor/models/llama.py
+1
-1
@@ -262,7 +262,7 @@ class LlamaAttention(nn.Module):
262
self.head_dim,
263
rotary_dim=self.head_dim,
264
max_position=self.max_position_embeddings,
265
- rope_parameters=config.rope_parameters,
265
+ rope_parameters=getattr(config, "rope_parameters", None),
266
is_neox_style=is_neox_style,
267
partial_rotary_factor=self.partial_rotary_factor,
268
)
vllm/transformers_utils/config.py
+33
-29
@@ -456,51 +456,55 @@ def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> No
456
457
def patch_rope_parameters(config: PretrainedConfig) -> None:
458
"""Provide backwards compatibility for RoPE."""
459
- # Retrieve rope_parameters differently based on Transformers version
459
+ # Patch rope_parameters differently based on Transformers version
460
if Version(version("transformers")) >= Version("5.0.0.dev0"):
461
- from transformers.modeling_rope_utils import RopeParameters
462
-
463
- rope_parameters: RopeParameters | dict[str, RopeParameters] | None = getattr(
464
- config, "rope_parameters", None
461
+ from transformers.modeling_rope_utils import (
462
+ rope_config_validation,
463
+ standardize_rope_params,
464
)
466
- elif hasattr(config, "rope_parameters"):
467
- # We are in Transformers v4 and rope_parameters
468
- # has already been patched for this config
469
- return
465
+
466
+ # When Transformers v5 is installed, legacy rope_theta may be present
467
+ # when using custom code models written for Transformers v4
468
+ if (rope_theta := getattr(config, "rope_theta", None)) is not None:
469
+ standardize_rope_params(config, rope_theta=rope_theta)
470
+ rope_config_validation(config)
471
+ # Delete rope_theta to avoid confusion in downstream code
472
+ del config.rope_theta
473
else:
471
- # Convert Transformers v4 rope_theta and rope_scaling into rope_parameters
472
- rope_theta: float | None = getattr(config, "rope_theta", None)
473
- rope_scaling: dict | None = getattr(config, "rope_scaling", None)
474
- rope_parameters = rope_scaling
475
- # Move rope_theta into rope_parameters
476
- if rope_theta is not None:
477
- rope_parameters = rope_parameters or {"rope_type": "default"}
478
- rope_parameters["rope_theta"] = rope_theta
479
- # Add original_max_position_embeddings if present
480
- if rope_parameters and (
481
- ompe := getattr(config, "original_max_position_embeddings", None)
482
- ):
483
- rope_parameters["original_max_position_embeddings"] = ompe
484
- # Write back to config
485
- config.rope_parameters = rope_parameters
474
+ # When Transformers v4 is installed, legacy rope_scaling may be present
475
+ if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
476
+ config.rope_parameters = rope_scaling
477
+ # When Transformers v4 is installed, legacy rope_theta may be present
478
+ if (rope_theta := getattr(config, "rope_theta", None)) is not None:
479
+ if not hasattr(config, "rope_parameters"):
480
+ config.rope_parameters = {"rope_type": "default"}
481
+ config.rope_parameters["rope_theta"] = rope_theta
482
483
# No RoPE parameters to patch
488
- if rope_parameters is None:
484
+ if not hasattr(config, "rope_parameters"):
485
return
486
487
+ # Add original_max_position_embeddings if present
488
+ if ompe := getattr(config, "original_max_position_embeddings", None):
489
+ config.rope_parameters["original_max_position_embeddings"] = ompe
490
+
491
# Handle nested rope_parameters in interleaved sliding attention models
492
- if set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
493
- for rope_parameters_layer_type in rope_parameters.values():
492
+ if set(config.rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
493
+ for rope_parameters_layer_type in config.rope_parameters.values():
494
patch_rope_parameters_dict(rope_parameters_layer_type)
495
else:
496
- patch_rope_parameters_dict(rope_parameters)
496
+ patch_rope_parameters_dict(config.rope_parameters)
497
498
499
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
500
if "rope_type" in rope_parameters and "type" in rope_parameters:
501
rope_type = rope_parameters["rope_type"]
502
rope_type_legacy = rope_parameters["type"]
503
- if rope_type != rope_type_legacy:
503
+ if (rope_type_legacy == "su" and rope_type == "longrope") or (
504
+ rope_type_legacy == "mrope" and rope_type == "default"
505
+ ):
506
+ pass # No action needed
507
+ elif rope_type != rope_type_legacy:
508
raise ValueError(
509
f"Found conflicts between 'rope_type={rope_type}' (modern "
510
f"field) and 'type={rope_type_legacy}' (legacy field). "