A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] logprobs is not compatible with the OpenAI spec #4795 (#5031)
Itay Etelis committed
May 30, 2024 at 02:13 UTC
7c3604fb68031da36567151a9bdfe69e04de44b8
2 files changed
+4
-5
vllm/entrypoints/openai/protocol.py
+2
-3
@@ -109,7 +109,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
109
frequency_penalty: Optional[float] = 0.0
110
logit_bias: Optional[Dict[str, float]] = None
111
logprobs: Optional[bool] = False
112
- top_logprobs: Optional[int] = None
112
+ top_logprobs: Optional[int] = 0
113
max_tokens: Optional[int] = None
114
n: Optional[int] = 1
115
presence_penalty: Optional[float] = 0.0
@@ -192,8 +192,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
192
# doc: end-chat-completion-extra-params
193
194
def to_sampling_params(self) -> SamplingParams:
195
- if self.logprobs and not self.top_logprobs:
196
- raise ValueError("Top logprobs must be set when logprobs is.")
195
+ # We now allow logprobs being true without top_logrobs.
196
197
logits_processors = None
198
if self.logit_bias:
vllm/entrypoints/openai/serving_chat.py
+2
-2
@@ -286,7 +286,7 @@ class OpenAIServingChat(OpenAIServing):
286
logprobs = self._create_logprobs(
287
token_ids=delta_token_ids,
288
top_logprobs=top_logprobs,
289
- num_output_top_logprobs=request.logprobs,
289
+ num_output_top_logprobs=request.top_logprobs,
290
initial_text_offset=len(previous_texts[i]),
291
)
292
else:
@@ -373,7 +373,7 @@ class OpenAIServingChat(OpenAIServing):
373
logprobs = self._create_logprobs(
374
token_ids=token_ids,
375
top_logprobs=top_logprobs,
376
- num_output_top_logprobs=request.logprobs,
376
+ num_output_top_logprobs=request.top_logprobs,
377
)
378
else:
379
logprobs = None