A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
Improve documentation of `ModelConfig.try_get_generation_config` to prevent future confusion (#21526)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Aug 1, 2025 at 17:32 UTC
326a1b001db10afc2dc5b2bfcb60a3b8f8bcb2ac
1 file changed
+22
-6
vllm/config.py
+22
-6
@@ -1575,7 +1575,18 @@ class ModelConfig:
1575
return self.multimodal_config
1576
1577
def try_get_generation_config(self) -> dict[str, Any]:
1578
- if self.generation_config in ("auto", "vllm"):
1578
+ """
1579
+ This method attempts to retrieve the non-default values of the
1580
+ generation config for this model.
1581
+
1582
+ The generation config can contain information about special tokens, as
1583
+ well as sampling parameters. Which is why this method exists separately
1584
+ to `get_diff_sampling_param`.
1585
+
1586
+ Returns:
1587
+ A dictionary containing the non-default generation config.
1588
+ """
1589
+ if self.generation_config in {"auto", "vllm"}:
1590
config = try_get_generation_config(
1591
self.hf_config_path or self.model,
1592
trust_remote_code=self.trust_remote_code,
@@ -1594,13 +1605,18 @@ class ModelConfig:
1605
1606
def get_diff_sampling_param(self) -> dict[str, Any]:
1607
"""
1597
- This method returns a dictionary containing the parameters
1598
- that differ from the default sampling parameters. If
1599
- `generation_config` is `"vllm"`, an empty dictionary is returned.
1608
+ This method returns a dictionary containing the non-default sampling
1609
+ parameters with `override_generation_config` applied.
1610
+
1611
+ The default sampling parameters are:
1612
+
1613
+ - vLLM's neutral defaults if `self.generation_config="vllm"`
1614
+ - the model's defaults if `self.generation_config="auto"`
1615
+ - as defined in `generation_config.json` if
1616
+ `self.generation_config="path/to/generation_config/dir"`
1617
1618
Returns:
1602
- dict[str, Any]: A dictionary with the differing sampling
1603
- parameters, if `generation_config` is `"vllm"` an empty dictionary.
1619
+ A dictionary containing the non-default sampling parameters.
1620
"""
1621
if self.generation_config == "vllm":
1622
config = {}