A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Added `include_stop_str_in_output` and `length_penalty` parameters to OpenAI API (#2562)
Federico Galatolo committed
Jan 24, 2024 at 19:21 UTC
f1f6cc10c77d6ee40b3ce769124cb2760428dc48
1 file changed
+8
vllm/entrypoints/openai/protocol.py
+8
@@ -78,6 +78,8 @@ class ChatCompletionRequest(BaseModel):
78
echo: Optional[bool] = False
79
repetition_penalty: Optional[float] = 1.0
80
min_p: Optional[float] = 0.0
81
+ include_stop_str_in_output: Optional[bool] = False
82
+ length_penalty: Optional[float] = 1.0
83
84
def to_sampling_params(self) -> SamplingParams:
85
return SamplingParams(
@@ -97,6 +99,8 @@ class ChatCompletionRequest(BaseModel):
99
use_beam_search=self.use_beam_search,
100
skip_special_tokens=self.skip_special_tokens,
101
spaces_between_special_tokens=self.spaces_between_special_tokens,
102
+ include_stop_str_in_output=self.include_stop_str_in_output,
103
+ length_penalty=self.length_penalty,
104
)
105
106
@@ -127,6 +131,8 @@ class CompletionRequest(BaseModel):
131
spaces_between_special_tokens: Optional[bool] = True
132
repetition_penalty: Optional[float] = 1.0
133
min_p: Optional[float] = 0.0
134
+ include_stop_str_in_output: Optional[bool] = False
135
+ length_penalty: Optional[float] = 1.0
136
137
def to_sampling_params(self):
138
echo_without_generation = self.echo and self.max_tokens == 0
@@ -150,6 +156,8 @@ class CompletionRequest(BaseModel):
156
prompt_logprobs=self.logprobs if self.echo else None,
157
skip_special_tokens=self.skip_special_tokens,
158
spaces_between_special_tokens=(self.spaces_between_special_tokens),
159
+ include_stop_str_in_output=self.include_stop_str_in_output,
160
+ length_penalty=self.length_penalty,
161
)
162
163