A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix failing GGUF models test (#22174)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Aug 4, 2025 at 16:27 UTC
fed5849d3fd7a5e7454cf87f101a18c2bad0436f
1 file changed
+15
-6
vllm/transformers_utils/config.py
+15
-6
@@ -290,20 +290,29 @@ def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
290
291
292
def maybe_override_with_speculators_target_model(
293
- model: str,
294
- tokenizer: str,
295
- trust_remote_code: bool,
296
- revision: Optional[str] = None) -> tuple[str, str]:
293
+ model: str,
294
+ tokenizer: str,
295
+ trust_remote_code: bool,
296
+ revision: Optional[str] = None,
297
+ **kwargs,
298
+) -> tuple[str, str]:
299
"""
300
If running a speculators config, override running model with target model
301
"""
302
+ is_gguf = check_gguf_file(model)
303
+ if is_gguf:
304
+ kwargs["gguf_file"] = Path(model).name
305
+ gguf_model_repo = Path(model).parent
306
+ else:
307
+ gguf_model_repo = None
308
config_dict, _ = PretrainedConfig.get_config_dict(
301
- model,
309
+ model if gguf_model_repo is None else gguf_model_repo,
310
revision=revision,
311
trust_remote_code=trust_remote_code,
312
token=_get_hf_token(),
313
+ **kwargs,
314
)
306
- spec_config = config_dict.get("speculators_config")
315
+ spec_config = config_dict.get("speculators_config", None)
316
# Return the target model
317
if spec_config is not None:
318
model = tokenizer = spec_config["verifier"]["name_or_path"]