A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] Fix: add patch_rope_scaling after hf override (#20857)
Signed-off-by: Wang Siyuan <wsy0227@sjtu.edu.cn> Signed-off-by: Wang Siyuan <sywang0227@gmail.com>
Wang Siyuan committed
Jul 13, 2025 at 15:13 UTC
247102f07f1dda117fef06493292e1925a5fcd31
2 files changed
+17
-11
vllm/config.py
+7
-11
@@ -532,16 +532,12 @@ class ModelConfig:
532
self.config_format = ConfigFormat(self.config_format)
533
534
hf_config = get_config(self.hf_config_path or self.model,
535
- self.trust_remote_code, self.revision,
536
- self.code_revision, self.config_format)
537
-
538
- if hf_overrides_kw:
539
- logger.debug("Overriding HF config with %s", hf_overrides_kw)
540
- hf_config.update(hf_overrides_kw)
541
- if hf_overrides_fn:
542
- logger.debug("Overriding HF config with %s", hf_overrides_fn)
543
- hf_config = hf_overrides_fn(hf_config)
544
-
535
+ self.trust_remote_code,
536
+ self.revision,
537
+ self.code_revision,
538
+ self.config_format,
539
+ hf_overrides_kw=hf_overrides_kw,
540
+ hf_overrides_fn=hf_overrides_fn)
541
self.hf_config = hf_config
542
543
self.hf_text_config = get_hf_text_config(self.hf_config)
@@ -5052,4 +5048,4 @@ class SpeechToTextConfig:
5048
5049
@property
5050
def allow_audio_chunking(self) -> bool:
5055
- return self.min_energy_split_window_size is not None
\ No newline at end of file
5051
+ return self.min_energy_split_window_size is not None
vllm/transformers_utils/config.py
+10
@@ -305,6 +305,9 @@ def get_config(
305
revision: Optional[str] = None,
306
code_revision: Optional[str] = None,
307
config_format: ConfigFormat = ConfigFormat.AUTO,
308
+ hf_overrides_kw: Optional[dict[str, Any]] = None,
309
+ hf_overrides_fn: Optional[Callable[[PretrainedConfig],
310
+ PretrainedConfig]] = None,
311
**kwargs,
312
) -> PretrainedConfig:
313
# Separate model folder from file path for GGUF models
@@ -423,6 +426,13 @@ def get_config(
426
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
427
config.update({"architectures": [model_type]})
428
429
+ if hf_overrides_kw:
430
+ logger.debug("Overriding HF config with %s", hf_overrides_kw)
431
+ config.update(hf_overrides_kw)
432
+ if hf_overrides_fn:
433
+ logger.debug("Overriding HF config with %s", hf_overrides_fn)
434
+ config = hf_overrides_fn(config)
435
+
436
patch_rope_scaling(config)
437
438
if trust_remote_code: