A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
add custom server params (#1868)
Roy committed
Dec 4, 2023 at 04:59 UTC
60dc62dc9e53428912953276e0d12a034b353fb6
3 files changed
+9
vllm/entrypoints/openai/api_server.py
+4
@@ -253,8 +253,10 @@ async def create_chat_completion(request: ChatCompletionRequest,
253
n=request.n,
254
presence_penalty=request.presence_penalty,
255
frequency_penalty=request.frequency_penalty,
256
+ repetition_penalty=request.repetition_penalty,
257
temperature=request.temperature,
258
top_p=request.top_p,
259
+ min_p=request.min_p,
260
stop=request.stop,
261
stop_token_ids=request.stop_token_ids,
262
max_tokens=request.max_tokens,
@@ -497,9 +499,11 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
499
best_of=request.best_of,
500
presence_penalty=request.presence_penalty,
501
frequency_penalty=request.frequency_penalty,
502
+ repetition_penalty=request.repetition_penalty,
503
temperature=request.temperature,
504
top_p=request.top_p,
505
top_k=request.top_k,
506
+ min_p=request.min_p,
507
stop=request.stop,
508
stop_token_ids=request.stop_token_ids,
509
ignore_eos=request.ignore_eos,
vllm/entrypoints/openai/protocol.py
+4
@@ -75,6 +75,8 @@ class ChatCompletionRequest(BaseModel):
75
spaces_between_special_tokens: Optional[bool] = True
76
add_generation_prompt: Optional[bool] = True
77
echo: Optional[bool] = False
78
+ repetition_penalty: Optional[float] = 1.0
79
+ min_p: Optional[float] = 0.0
80
81
82
class CompletionRequest(BaseModel):
@@ -102,6 +104,8 @@ class CompletionRequest(BaseModel):
104
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
105
skip_special_tokens: Optional[bool] = True
106
spaces_between_special_tokens: Optional[bool] = True
107
+ repetition_penalty: Optional[float] = 1.0
108
+ min_p: Optional[float] = 0.0
109
110
111
class LogProbs(BaseModel):
vllm/sampling_params.py
+1
@@ -149,6 +149,7 @@ class SamplingParams:
149
# Zero temperature means greedy sampling.
150
self.top_p = 1.0
151
self.top_k = -1
152
+ self.min_p = 0.0
153
self._verify_greedy_sampling()
154
155
def _verify_args(self) -> None: