A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 20:00 UTC
Fix sarvam forward compatibility with transformers v5 (#38804)
Signed-off-by: vikrantpalle <vikrantpalle@gmail.com>
Flame committed
Jun 5, 2026 at 21:21 UTC
91e17d43152b6209c7bf444e08df4e3a803012d9
1 file changed
+34
-1
vllm/transformers_utils/config.py
+34
-1
@@ -5,7 +5,7 @@ import os
5
from collections.abc import Callable, Iterator
6
from contextlib import contextmanager
7
from dataclasses import asdict
8
-from functools import cache, partial
8
+from functools import cache, partial, wraps
9
from importlib.metadata import version
10
from pathlib import Path
11
from typing import Any, Literal, TypeAlias
@@ -141,6 +141,8 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
141
142
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"}
143
144
+_PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}
145
+
146
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
147
"llm_config": "text_config",
148
}
@@ -173,6 +175,34 @@ def _mistral_patch_hf_hub_constants() -> Iterator[None]:
175
constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file
176
177
178
+def _patch_hf_transformers_validate_rope():
179
+ """Transformers v5 moved the ignore_keys option from the method signature of
180
+ validate_rope and replaced it with the ignore_keys_at_rope_validation parameter
181
+ in the PreTrainedConfig class. This is a patch to make older versions of
182
+ validate_rope() with the ignore_keys parameter work with newer versions of
183
+ hf transformers (from v5 onwards)
184
+ """
185
+
186
+ if Version(version("transformers")) >= Version("5.0.0"):
187
+ if hasattr(PretrainedConfig.validate_rope, "__vllm_patched__"):
188
+ return
189
+
190
+ _original_validate_rope = PretrainedConfig.validate_rope
191
+
192
+ @wraps(_original_validate_rope)
193
+ def patched_validate_rope(self, *args, **kwargs):
194
+ ignore_keys_param = kwargs.pop("ignore_keys", None)
195
+ original_ignore_keys = self.ignore_keys_at_rope_validation
196
+ self.ignore_keys_at_rope_validation = (
197
+ original_ignore_keys or ignore_keys_param
198
+ )
199
+ result = _original_validate_rope(self, *args, **kwargs)
200
+ return result
201
+
202
+ patched_validate_rope.__vllm_patched__ = True # type: ignore[attr-defined]
203
+ PretrainedConfig.validate_rope = patched_validate_rope
204
+
205
+
206
class HFConfigParser(ConfigParserBase):
207
def parse(
208
self,
@@ -212,6 +242,9 @@ class HFConfigParser(ConfigParserBase):
242
dummy_model_type = hf_overrides(dummy_config).model_type
243
model_type = dummy_model_type.removeprefix("dummy_")
244
245
+ if model_type in _PATCH_HF_VALIDATE_ROPE:
246
+ _patch_hf_transformers_validate_rope()
247
+
248
if model_type in _SPECULATIVE_DECODING_CONFIGS:
249
config_class = _CONFIG_REGISTRY[model_type]
250
config = config_class.from_pretrained(