A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Pass config_format via try_get_generation_config (#25912)
acisseJZhong committed
Sep 29, 2025 at 22:09 UTC
e47433b3c175ef6c29254637e12a52c26c4c25b2
2 files changed
+4
vllm/config/model.py
+2
@@ -1334,11 +1334,13 @@ class ModelConfig:
1334
self.hf_config_path or self.model,
1335
trust_remote_code=self.trust_remote_code,
1336
revision=self.revision,
1337
+ config_format=self.config_format,
1338
)
1339
else:
1340
config = try_get_generation_config(
1341
self.generation_config,
1342
trust_remote_code=self.trust_remote_code,
1343
+ config_format=self.config_format,
1344
)
1345
1346
if config is None:
vllm/transformers_utils/config.py
+2
@@ -949,6 +949,7 @@ def try_get_generation_config(
949
model: str,
950
trust_remote_code: bool,
951
revision: Optional[str] = None,
952
+ config_format: Union[str, ConfigFormat] = "auto",
953
) -> Optional[GenerationConfig]:
954
try:
955
return GenerationConfig.from_pretrained(
@@ -961,6 +962,7 @@ def try_get_generation_config(
962
model,
963
trust_remote_code=trust_remote_code,
964
revision=revision,
965
+ config_format=config_format,
966
)
967
return GenerationConfig.from_model_config(config)
968
except OSError: # Not found