A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Avoid creating guided decoding LogitsProcessor unnecessarily (#9521)
Nick Hill committed
Oct 19, 2024 at 04:21 UTC
1325872ec8c97d797c18f490bdb6be7f4def5aa8
1 file changed
+5
-2
vllm/sampling_params.py
+5
-2
@@ -49,14 +49,17 @@ class GuidedDecodingParams:
49
50
@staticmethod
51
def from_optional(
52
- json: Optional[Union[Dict, BaseModel, str]],
52
+ json: Optional[Union[Dict, BaseModel, str]] = None,
53
regex: Optional[str] = None,
54
choice: Optional[List[str]] = None,
55
grammar: Optional[str] = None,
56
json_object: Optional[bool] = None,
57
backend: Optional[str] = None,
58
whitespace_pattern: Optional[str] = None,
59
- ) -> "GuidedDecodingParams":
59
+ ) -> Optional["GuidedDecodingParams"]:
60
+ if all(arg is None
61
+ for arg in (json, regex, choice, grammar, json_object)):
62
+ return None
63
# Extract json schemas from pydantic models
64
if isinstance(json, (BaseModel, type(BaseModel))):
65
json = json.model_json_schema()