@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 351d5e7b82
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] OpenAI entrypoint limits logprobs while ignoring server defined --max-logprobs (#5312)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

maor-ps committed Jun 11, 2024 at 05:30 UTC 351d5e7b8253d754b2a951152cd48927c4c1629d
4 files changed +12 -9
tests/entrypoints/test_openai_server.py
+6 -2
@@ -264,7 +264,9 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
264 prompt=[0, 0, 0, 0, 0],
265 max_tokens=5,
266 temperature=0.0,
267 - logprobs=6,
267 + # vLLM has higher default max_logprobs (20 instead of 5) to support
268 + # both Completion API and Chat Completion API
269 + logprobs=21,
270 )
271 ...
272 with pytest.raises(
@@ -274,7 +276,9 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
276 prompt=[0, 0, 0, 0, 0],
277 max_tokens=5,
278 temperature=0.0,
277 - logprobs=6,
279 + # vLLM has higher default max_logprobs (20 instead of 5) to support
280 + # both Completion API and Chat Completion API
281 + logprobs=30,
282 stream=True,
283 )
284 async for chunk in stream:
vllm/config.py
+1 -1
@@ -100,7 +100,7 @@ class ModelConfig:
100 enforce_eager: bool = False,
101 max_context_len_to_capture: Optional[int] = None,
102 max_seq_len_to_capture: Optional[int] = None,
103 - max_logprobs: int = 5,
103 + max_logprobs: int = 20,
104 disable_sliding_window: bool = False,
105 skip_tokenizer_init: bool = False,
106 served_model_name: Optional[Union[str, List[str]]] = None,
vllm/engine/arg_utils.py
+1 -1
@@ -48,7 +48,7 @@ class EngineArgs:
48 gpu_memory_utilization: float = 0.90
49 max_num_batched_tokens: Optional[int] = None
50 max_num_seqs: int = 256
51 - max_logprobs: int = 5 # OpenAI default value
51 + max_logprobs: int = 20 # Default value for OpenAI Chat Completions API
52 disable_log_stats: bool = False
53 revision: Optional[str] = None
54 code_revision: Optional[str] = None
vllm/entrypoints/openai/protocol.py
+4 -5
@@ -322,9 +322,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
322 raise ValueError(
323 "when using `top_logprobs`, `logprobs` must be set to true."
324 )
325 - elif not 0 <= data["top_logprobs"] <= 20:
325 + elif data["top_logprobs"] < 0:
326 raise ValueError(
327 - "`top_logprobs` must be a value in the interval [0, 20].")
327 + "`top_logprobs` must be a value a positive value.")
328 return data
329
330
@@ -478,9 +478,8 @@ class CompletionRequest(OpenAIBaseModel):
478 @classmethod
479 def check_logprobs(cls, data):
480 if "logprobs" in data and data[
481 - "logprobs"] is not None and not 0 <= data["logprobs"] <= 5:
482 - raise ValueError(("if passed, `logprobs` must be a value",
483 - " in the interval [0, 5]."))
481 + "logprobs"] is not None and not data["logprobs"] >= 0:
482 + raise ValueError("if passed, `logprobs` must be a positive value.")
483 return data
484
485 @model_validator(mode="before")