@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c3604fb68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] logprobs is not compatible with the OpenAI spec #4795 (#5031)

Itay Etelis committed May 30, 2024 at 02:13 UTC 7c3604fb68031da36567151a9bdfe69e04de44b8
2 files changed +4 -5
vllm/entrypoints/openai/protocol.py
+2 -3
@@ -109,7 +109,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
109 frequency_penalty: Optional[float] = 0.0
110 logit_bias: Optional[Dict[str, float]] = None
111 logprobs: Optional[bool] = False
112 - top_logprobs: Optional[int] = None
112 + top_logprobs: Optional[int] = 0
113 max_tokens: Optional[int] = None
114 n: Optional[int] = 1
115 presence_penalty: Optional[float] = 0.0
@@ -192,8 +192,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
192 # doc: end-chat-completion-extra-params
193
194 def to_sampling_params(self) -> SamplingParams:
195 - if self.logprobs and not self.top_logprobs:
196 - raise ValueError("Top logprobs must be set when logprobs is.")
195 + # We now allow logprobs being true without top_logrobs.
196
197 logits_processors = None
198 if self.logit_bias:
vllm/entrypoints/openai/serving_chat.py
+2 -2
@@ -286,7 +286,7 @@ class OpenAIServingChat(OpenAIServing):
286 logprobs = self._create_logprobs(
287 token_ids=delta_token_ids,
288 top_logprobs=top_logprobs,
289 - num_output_top_logprobs=request.logprobs,
289 + num_output_top_logprobs=request.top_logprobs,
290 initial_text_offset=len(previous_texts[i]),
291 )
292 else:
@@ -373,7 +373,7 @@ class OpenAIServingChat(OpenAIServing):
373 logprobs = self._create_logprobs(
374 token_ids=token_ids,
375 top_logprobs=top_logprobs,
376 - num_output_top_logprobs=request.logprobs,
376 + num_output_top_logprobs=request.top_logprobs,
377 )
378 else:
379 logprobs = None