@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bbbf86565f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Align `max_tokens` behavior with openai (#852)

Wen Sun committed Sep 24, 2023 at 09:10 UTC bbbf86565f2fb2bab0cf6675f9ebefcd449390bd
2 files changed +3 -1
vllm/entrypoints/openai/api_server.py
+2
@@ -130,6 +130,8 @@ async def check_length(
130 input_ids = tokenizer(prompt).input_ids
131 token_num = len(input_ids)
132
133 + if request.max_tokens is None:
134 + request.max_tokens = max_model_len - token_num
135 if token_num + request.max_tokens > max_model_len:
136 return input_ids, create_error_response(
137 HTTPStatus.BAD_REQUEST,
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -58,7 +58,7 @@ class ChatCompletionRequest(BaseModel):
58 temperature: Optional[float] = 0.7
59 top_p: Optional[float] = 1.0
60 n: Optional[int] = 1
61 - max_tokens: Optional[int] = 16
61 + max_tokens: Optional[int] = None
62 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
63 stream: Optional[bool] = False
64 presence_penalty: Optional[float] = 0.0