A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] OpenAI entrypoint limits logprobs while ignoring server defined --max-logprobs (#5312)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
maor-ps committed
Jun 11, 2024 at 05:30 UTC
351d5e7b8253d754b2a951152cd48927c4c1629d
4 files changed
+12
-9
tests/entrypoints/test_openai_server.py
+6
-2
@@ -264,7 +264,9 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
264
prompt=[0, 0, 0, 0, 0],
265
max_tokens=5,
266
temperature=0.0,
267
- logprobs=6,
267
+ # vLLM has higher default max_logprobs (20 instead of 5) to support
268
+ # both Completion API and Chat Completion API
269
+ logprobs=21,
270
)
271
...
272
with pytest.raises(
@@ -274,7 +276,9 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
276
prompt=[0, 0, 0, 0, 0],
277
max_tokens=5,
278
temperature=0.0,
277
- logprobs=6,
279
+ # vLLM has higher default max_logprobs (20 instead of 5) to support
280
+ # both Completion API and Chat Completion API
281
+ logprobs=30,
282
stream=True,
283
)
284
async for chunk in stream:
vllm/config.py
+1
-1
@@ -100,7 +100,7 @@ class ModelConfig:
100
enforce_eager: bool = False,
101
max_context_len_to_capture: Optional[int] = None,
102
max_seq_len_to_capture: Optional[int] = None,
103
- max_logprobs: int = 5,
103
+ max_logprobs: int = 20,
104
disable_sliding_window: bool = False,
105
skip_tokenizer_init: bool = False,
106
served_model_name: Optional[Union[str, List[str]]] = None,
vllm/engine/arg_utils.py
+1
-1
@@ -48,7 +48,7 @@ class EngineArgs:
48
gpu_memory_utilization: float = 0.90
49
max_num_batched_tokens: Optional[int] = None
50
max_num_seqs: int = 256
51
- max_logprobs: int = 5 # OpenAI default value
51
+ max_logprobs: int = 20 # Default value for OpenAI Chat Completions API
52
disable_log_stats: bool = False
53
revision: Optional[str] = None
54
code_revision: Optional[str] = None
vllm/entrypoints/openai/protocol.py
+4
-5
@@ -322,9 +322,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
322
raise ValueError(
323
"when using `top_logprobs`, `logprobs` must be set to true."
324
)
325
- elif not 0 <= data["top_logprobs"] <= 20:
325
+ elif data["top_logprobs"] < 0:
326
raise ValueError(
327
- "`top_logprobs` must be a value in the interval [0, 20].")
327
+ "`top_logprobs` must be a value a positive value.")
328
return data
329
330
@@ -478,9 +478,8 @@ class CompletionRequest(OpenAIBaseModel):
478
@classmethod
479
def check_logprobs(cls, data):
480
if "logprobs" in data and data[
481
- "logprobs"] is not None and not 0 <= data["logprobs"] <= 5:
482
- raise ValueError(("if passed, `logprobs` must be a value",
483
- " in the interval [0, 5]."))
481
+ "logprobs"] is not None and not data["logprobs"] >= 0:
482
+ raise ValueError("if passed, `logprobs` must be a positive value.")
483
return data
484
485
@model_validator(mode="before")