@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1325872ec8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Avoid creating guided decoding LogitsProcessor unnecessarily (#9521)

Nick Hill committed Oct 19, 2024 at 04:21 UTC 1325872ec8c97d797c18f490bdb6be7f4def5aa8
1 file changed +5 -2
vllm/sampling_params.py
+5 -2
@@ -49,14 +49,17 @@ class GuidedDecodingParams:
49
50 @staticmethod
51 def from_optional(
52 - json: Optional[Union[Dict, BaseModel, str]],
52 + json: Optional[Union[Dict, BaseModel, str]] = None,
53 regex: Optional[str] = None,
54 choice: Optional[List[str]] = None,
55 grammar: Optional[str] = None,
56 json_object: Optional[bool] = None,
57 backend: Optional[str] = None,
58 whitespace_pattern: Optional[str] = None,
59 - ) -> "GuidedDecodingParams":
59 + ) -> Optional["GuidedDecodingParams"]:
60 + if all(arg is None
61 + for arg in (json, regex, choice, grammar, json_object)):
62 + return None
63 # Extract json schemas from pydantic models
64 if isinstance(json, (BaseModel, type(BaseModel))):
65 json = json.model_json_schema()