@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 60dc62dc9e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

add custom server params (#1868)

Roy committed Dec 4, 2023 at 04:59 UTC 60dc62dc9e53428912953276e0d12a034b353fb6
3 files changed +9
vllm/entrypoints/openai/api_server.py
+4
@@ -253,8 +253,10 @@ async def create_chat_completion(request: ChatCompletionRequest,
253 n=request.n,
254 presence_penalty=request.presence_penalty,
255 frequency_penalty=request.frequency_penalty,
256 + repetition_penalty=request.repetition_penalty,
257 temperature=request.temperature,
258 top_p=request.top_p,
259 + min_p=request.min_p,
260 stop=request.stop,
261 stop_token_ids=request.stop_token_ids,
262 max_tokens=request.max_tokens,
@@ -497,9 +499,11 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
499 best_of=request.best_of,
500 presence_penalty=request.presence_penalty,
501 frequency_penalty=request.frequency_penalty,
502 + repetition_penalty=request.repetition_penalty,
503 temperature=request.temperature,
504 top_p=request.top_p,
505 top_k=request.top_k,
506 + min_p=request.min_p,
507 stop=request.stop,
508 stop_token_ids=request.stop_token_ids,
509 ignore_eos=request.ignore_eos,
vllm/entrypoints/openai/protocol.py
+4
@@ -75,6 +75,8 @@ class ChatCompletionRequest(BaseModel):
75 spaces_between_special_tokens: Optional[bool] = True
76 add_generation_prompt: Optional[bool] = True
77 echo: Optional[bool] = False
78 + repetition_penalty: Optional[float] = 1.0
79 + min_p: Optional[float] = 0.0
80
81
82 class CompletionRequest(BaseModel):
@@ -102,6 +104,8 @@ class CompletionRequest(BaseModel):
104 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
105 skip_special_tokens: Optional[bool] = True
106 spaces_between_special_tokens: Optional[bool] = True
107 + repetition_penalty: Optional[float] = 1.0
108 + min_p: Optional[float] = 0.0
109
110
111 class LogProbs(BaseModel):
vllm/sampling_params.py
+1
@@ -149,6 +149,7 @@ class SamplingParams:
149 # Zero temperature means greedy sampling.
150 self.top_p = 1.0
151 self.top_k = -1
152 + self.min_p = 0.0
153 self._verify_greedy_sampling()
154
155 def _verify_args(self) -> None: