@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5574081c49
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Added early stopping to completion APIs (#2939)

Mustafa Eyceoz committed Feb 21, 2024 at 21:24 UTC 5574081c49c9a5ac51662981aff80250119a97bd
1 file changed +4
vllm/entrypoints/openai/protocol.py
+4
@@ -72,6 +72,7 @@ class ChatCompletionRequest(BaseModel):
72 top_k: Optional[int] = -1
73 ignore_eos: Optional[bool] = False
74 use_beam_search: Optional[bool] = False
75 + early_stopping: Optional[bool] = False
76 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
77 skip_special_tokens: Optional[bool] = True
78 spaces_between_special_tokens: Optional[bool] = True
@@ -99,6 +100,7 @@ class ChatCompletionRequest(BaseModel):
100 top_k=self.top_k,
101 ignore_eos=self.ignore_eos,
102 use_beam_search=self.use_beam_search,
103 + early_stopping=self.early_stopping,
104 skip_special_tokens=self.skip_special_tokens,
105 spaces_between_special_tokens=self.spaces_between_special_tokens,
106 include_stop_str_in_output=self.include_stop_str_in_output,
@@ -129,6 +131,7 @@ class CompletionRequest(BaseModel):
131 top_k: Optional[int] = -1
132 ignore_eos: Optional[bool] = False
133 use_beam_search: Optional[bool] = False
134 + early_stopping: Optional[bool] = False
135 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
136 skip_special_tokens: Optional[bool] = True
137 spaces_between_special_tokens: Optional[bool] = True
@@ -157,6 +160,7 @@ class CompletionRequest(BaseModel):
160 max_tokens=self.max_tokens if not echo_without_generation else 1,
161 logprobs=self.logprobs,
162 use_beam_search=self.use_beam_search,
163 + early_stopping=self.early_stopping,
164 prompt_logprobs=self.logprobs if self.echo else None,
165 skip_special_tokens=self.skip_special_tokens,
166 spaces_between_special_tokens=(self.spaces_between_special_tokens),