A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
Add util function for checking nesting of rope parameters (#31146)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Dec 23, 2025 at 11:41 UTC
b10d47e0e099b29aecc20740daaaa1afe3476c59
3 files changed
+23
-8
vllm/config/model.py
+2
-4
@@ -11,7 +11,6 @@ import torch
11
from pydantic import ConfigDict, Field, field_validator, model_validator
12
from pydantic.dataclasses import dataclass
13
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
14
-from transformers.configuration_utils import ALLOWED_LAYER_TYPES
14
15
import vllm.envs as envs
16
from vllm.attention.backends.registry import AttentionBackendEnum
@@ -29,6 +28,7 @@ from vllm.transformers_utils.config import (
28
get_pooling_config,
29
get_sentence_transformer_tokenizer_config,
30
is_encoder_decoder,
31
+ is_rope_parameters_nested,
32
try_get_dense_modules,
33
try_get_generation_config,
34
try_get_safetensors_metadata,
@@ -2125,9 +2125,7 @@ def _get_and_verify_max_len(
2125
# In Transformers v5 rope_parameters could be TypedDict or dict[str, TypedDict].
2126
# To simplify the verification, we convert it to dict[str, TypedDict].
2127
rope_parameters = getattr(hf_config, "rope_parameters", None)
2128
- if rope_parameters and not set(rope_parameters.keys()).issubset(
2129
- ALLOWED_LAYER_TYPES
2130
- ):
2128
+ if rope_parameters and not is_rope_parameters_nested(rope_parameters):
2129
rope_parameters = {"": rope_parameters}
2130
2131
# NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE
vllm/model_executor/models/transformers/utils.py
+2
-2
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Literal
22
23
import torch
24
from torch import nn
25
-from transformers.configuration_utils import ALLOWED_LAYER_TYPES
25
26
from vllm.config.utils import getattr_iter
27
from vllm.logger import init_logger
@@ -32,6 +31,7 @@ from vllm.model_executor.layers.linear import (
31
ReplicatedLinear,
32
RowParallelLinear,
33
)
34
+from vllm.transformers_utils.config import is_rope_parameters_nested
35
36
if TYPE_CHECKING:
37
from vllm.config import VllmConfig
@@ -207,7 +207,7 @@ def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
207
rope_parameters: dict | None = getattr(text_config, "rope_parameters", None) or {}
208
if rope_parameters:
209
# Nest rope_parameters if not nested already to simplify logic
210
- if not set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
210
+ if not is_rope_parameters_nested(rope_parameters):
211
rope_parameters = {"": rope_parameters}
212
return all(rp["rope_type"] != "dynamic" for rp in rope_parameters.values())
213
return True
vllm/transformers_utils/config.py
+19
-2
@@ -15,7 +15,6 @@ from huggingface_hub import (
15
)
16
from packaging.version import Version
17
from transformers import GenerationConfig, PretrainedConfig
18
-from transformers.configuration_utils import ALLOWED_LAYER_TYPES
18
from transformers.models.auto.image_processing_auto import get_image_processor_config
19
from transformers.models.auto.modeling_auto import (
20
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
@@ -44,6 +43,16 @@ from .repo_utils import (
43
with_retry,
44
)
45
46
+try:
47
+ # Transformers v5
48
+ from transformers.configuration_utils import ALLOWED_ATTENTION_LAYER_TYPES
49
+except ImportError:
50
+ # Transformers v4
51
+ from transformers.configuration_utils import (
52
+ ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
53
+ )
54
+
55
+
56
if envs.VLLM_USE_MODELSCOPE:
57
from modelscope import AutoConfig
58
else:
@@ -104,6 +113,14 @@ _AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
113
}
114
115
116
+def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool:
117
+ """Check if rope_parameters is nested by layer types."""
118
+ # Cannot be nested if rope_parameters is empty
119
+ if not rope_parameters:
120
+ return False
121
+ return set(rope_parameters.keys()).issubset(ALLOWED_ATTENTION_LAYER_TYPES)
122
+
123
+
124
class HFConfigParser(ConfigParserBase):
125
def parse(
126
self,
@@ -346,7 +363,7 @@ def patch_rope_parameters(config: PretrainedConfig) -> None:
363
config.rope_parameters["original_max_position_embeddings"] = ompe
364
365
# Handle nested rope_parameters in interleaved sliding attention models
349
- if set(config.rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
366
+ if is_rope_parameters_nested(config.rope_parameters):
367
for rope_parameters_layer_type in config.rope_parameters.values():
368
patch_rope_parameters_dict(rope_parameters_layer_type)
369
else: