A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Added early stopping to completion APIs (#2939)
Mustafa Eyceoz committed
Feb 21, 2024 at 21:24 UTC
5574081c49c9a5ac51662981aff80250119a97bd
1 file changed
+4
vllm/entrypoints/openai/protocol.py
+4
@@ -72,6 +72,7 @@ class ChatCompletionRequest(BaseModel):
72
top_k: Optional[int] = -1
73
ignore_eos: Optional[bool] = False
74
use_beam_search: Optional[bool] = False
75
+ early_stopping: Optional[bool] = False
76
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
77
skip_special_tokens: Optional[bool] = True
78
spaces_between_special_tokens: Optional[bool] = True
@@ -99,6 +100,7 @@ class ChatCompletionRequest(BaseModel):
100
top_k=self.top_k,
101
ignore_eos=self.ignore_eos,
102
use_beam_search=self.use_beam_search,
103
+ early_stopping=self.early_stopping,
104
skip_special_tokens=self.skip_special_tokens,
105
spaces_between_special_tokens=self.spaces_between_special_tokens,
106
include_stop_str_in_output=self.include_stop_str_in_output,
@@ -129,6 +131,7 @@ class CompletionRequest(BaseModel):
131
top_k: Optional[int] = -1
132
ignore_eos: Optional[bool] = False
133
use_beam_search: Optional[bool] = False
134
+ early_stopping: Optional[bool] = False
135
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
136
skip_special_tokens: Optional[bool] = True
137
spaces_between_special_tokens: Optional[bool] = True
@@ -157,6 +160,7 @@ class CompletionRequest(BaseModel):
160
max_tokens=self.max_tokens if not echo_without_generation else 1,
161
logprobs=self.logprobs,
162
use_beam_search=self.use_beam_search,
163
+ early_stopping=self.early_stopping,
164
prompt_logprobs=self.logprobs if self.echo else None,
165
skip_special_tokens=self.skip_special_tokens,
166
spaces_between_special_tokens=(self.spaces_between_special_tokens),