A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
[Bugfix] Pass hf_token through config loading paths for gated model support (#37920)
Signed-off-by: javierdejesusda <javier.dejesusj9@gmail.com>
Javier De Jesus committed
Mar 24, 2026 at 20:22 UTC
54b0578adacd0413b18c0f59948dabc7533a6524
3 files changed
+10
vllm/config/model.py
+3
@@ -488,6 +488,7 @@ class ModelConfig: # type: ignore[misc]
488
self.config_format,
489
hf_overrides_kw=hf_overrides_kw,
490
hf_overrides_fn=hf_overrides_fn,
491
+ token=self.hf_token,
492
)
493
hf_config = maybe_patch_hf_config_from_gguf(
494
self.model,
@@ -1341,12 +1342,14 @@ class ModelConfig: # type: ignore[misc]
1342
trust_remote_code=self.trust_remote_code,
1343
revision=self.revision,
1344
config_format=self.config_format,
1345
+ hf_token=self.hf_token,
1346
)
1347
else:
1348
config = try_get_generation_config(
1349
self.generation_config,
1350
trust_remote_code=self.trust_remote_code,
1351
config_format=self.config_format,
1352
+ hf_token=self.hf_token,
1353
)
1354
1355
if config is None:
vllm/engine/arg_utils.py
+1
@@ -1530,6 +1530,7 @@ class EngineArgs:
1530
revision=self.revision,
1531
trust_remote_code=self.trust_remote_code,
1532
vllm_speculative_config=self.speculative_config,
1533
+ hf_token=self.hf_token,
1534
)
1535
)
1536
vllm/transformers_utils/config.py
+6
@@ -542,6 +542,7 @@ def maybe_override_with_speculators(
542
trust_remote_code: bool,
543
revision: str | None = None,
544
vllm_speculative_config: dict[str, Any] | None = None,
545
+ hf_token: bool | str | None = None,
546
**kwargs,
547
) -> tuple[str, str | None, dict[str, Any] | None]:
548
"""
@@ -556,6 +557,7 @@ def maybe_override_with_speculators(
557
trust_remote_code: Whether to trust remote code
558
revision: Model revision
559
vllm_speculative_config: Existing vLLM speculative config
560
+ hf_token: HuggingFace token for authenticated model access
561
562
Returns:
563
Tuple of (resolved_model, resolved_tokenizer, speculative_config)
@@ -572,6 +574,7 @@ def maybe_override_with_speculators(
574
config_dict, _ = PretrainedConfig.get_config_dict(
575
model if gguf_model_repo is None else gguf_model_repo,
576
revision=revision,
577
+ token=hf_token,
578
**without_trust_remote_code(kwargs),
579
)
580
speculators_config = config_dict.get("speculators_config")
@@ -1054,6 +1057,7 @@ def try_get_generation_config(
1057
trust_remote_code: bool,
1058
revision: str | None = None,
1059
config_format: str | ConfigFormat = "auto",
1060
+ hf_token: bool | str | None = None,
1061
) -> GenerationConfig | None:
1062
# GGUF files don't have generation_config.json - their config is embedded
1063
# in the file header. Skip all filesystem lookups to avoid re-reading the
@@ -1066,6 +1070,7 @@ def try_get_generation_config(
1070
return GenerationConfig.from_pretrained(
1071
model,
1072
revision=revision,
1073
+ token=hf_token,
1074
)
1075
except OSError: # Not found
1076
try:
@@ -1074,6 +1079,7 @@ def try_get_generation_config(
1079
trust_remote_code=trust_remote_code,
1080
revision=revision,
1081
config_format=config_format,
1082
+ token=hf_token,
1083
)
1084
return GenerationConfig.from_model_config(config)
1085
except OSError: # Not found