@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f1f6cc10c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Added `include_stop_str_in_output` and `length_penalty` parameters to OpenAI API (#2562)

Federico Galatolo committed Jan 24, 2024 at 19:21 UTC f1f6cc10c77d6ee40b3ce769124cb2760428dc48
1 file changed +8
vllm/entrypoints/openai/protocol.py
+8
@@ -78,6 +78,8 @@ class ChatCompletionRequest(BaseModel):
78 echo: Optional[bool] = False
79 repetition_penalty: Optional[float] = 1.0
80 min_p: Optional[float] = 0.0
81 + include_stop_str_in_output: Optional[bool] = False
82 + length_penalty: Optional[float] = 1.0
83
84 def to_sampling_params(self) -> SamplingParams:
85 return SamplingParams(
@@ -97,6 +99,8 @@ class ChatCompletionRequest(BaseModel):
99 use_beam_search=self.use_beam_search,
100 skip_special_tokens=self.skip_special_tokens,
101 spaces_between_special_tokens=self.spaces_between_special_tokens,
102 + include_stop_str_in_output=self.include_stop_str_in_output,
103 + length_penalty=self.length_penalty,
104 )
105
106
@@ -127,6 +131,8 @@ class CompletionRequest(BaseModel):
131 spaces_between_special_tokens: Optional[bool] = True
132 repetition_penalty: Optional[float] = 1.0
133 min_p: Optional[float] = 0.0
134 + include_stop_str_in_output: Optional[bool] = False
135 + length_penalty: Optional[float] = 1.0
136
137 def to_sampling_params(self):
138 echo_without_generation = self.echo and self.max_tokens == 0
@@ -150,6 +156,8 @@ class CompletionRequest(BaseModel):
156 prompt_logprobs=self.logprobs if self.echo else None,
157 skip_special_tokens=self.skip_special_tokens,
158 spaces_between_special_tokens=(self.spaces_between_special_tokens),
159 + include_stop_str_in_output=self.include_stop_str_in_output,
160 + length_penalty=self.length_penalty,
161 )
162
163