A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[core] add `extra_args` to `SamplingParams` (#13300)
Signed-off-by: Aviv Keshet <akeshet@scaledcognition.com>
Aviv Keshet committed
Mar 7, 2025 at 22:41 UTC
4aae667668e59509ab152b08c5eb617b8ae1187a
1 file changed
+8
-1
vllm/sampling_params.py
+8
-1
@@ -184,6 +184,9 @@ class SamplingParams(
184
allowed_token_ids: If provided, the engine will construct a logits
185
processor which only retains scores for the given token ids.
186
Defaults to None.
187
+ extra_args: Arbitrary additional args, that can be used by custom
188
+ sampling implementations. Not used by any in-tree sampling
189
+ implementations.
190
"""
191
192
n: int = 1
@@ -227,6 +230,7 @@ class SamplingParams(
230
guided_decoding: Optional[GuidedDecodingParams] = None
231
logit_bias: Optional[dict[int, float]] = None
232
allowed_token_ids: Optional[list[int]] = None
233
+ extra_args: Optional[dict[str, Any]] = None
234
235
@staticmethod
236
def from_optional(
@@ -259,6 +263,7 @@ class SamplingParams(
263
guided_decoding: Optional[GuidedDecodingParams] = None,
264
logit_bias: Optional[Union[dict[int, float], dict[str, float]]] = None,
265
allowed_token_ids: Optional[list[int]] = None,
266
+ extra_args: Optional[dict[str, Any]] = None,
267
) -> "SamplingParams":
268
if logit_bias is not None:
269
# Convert token_id to integer
@@ -300,6 +305,7 @@ class SamplingParams(
305
guided_decoding=guided_decoding,
306
logit_bias=logit_bias,
307
allowed_token_ids=allowed_token_ids,
308
+ extra_args=extra_args,
309
)
310
311
def __post_init__(self) -> None:
@@ -509,7 +515,8 @@ class SamplingParams(
515
"spaces_between_special_tokens="
516
f"{self.spaces_between_special_tokens}, "
517
f"truncate_prompt_tokens={self.truncate_prompt_tokens}, "
512
- f"guided_decoding={self.guided_decoding})")
518
+ f"guided_decoding={self.guided_decoding}, "
519
+ f"extra_args={self.extra_args})")
520
521
522
class BeamSearchParams(