A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Align `max_tokens` behavior with openai (#852)
Wen Sun committed
Sep 24, 2023 at 09:10 UTC
bbbf86565f2fb2bab0cf6675f9ebefcd449390bd
2 files changed
+3
-1
vllm/entrypoints/openai/api_server.py
+2
@@ -130,6 +130,8 @@ async def check_length(
130
input_ids = tokenizer(prompt).input_ids
131
token_num = len(input_ids)
132
133
+ if request.max_tokens is None:
134
+ request.max_tokens = max_model_len - token_num
135
if token_num + request.max_tokens > max_model_len:
136
return input_ids, create_error_response(
137
HTTPStatus.BAD_REQUEST,
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -58,7 +58,7 @@ class ChatCompletionRequest(BaseModel):
58
temperature: Optional[float] = 0.7
59
top_p: Optional[float] = 1.0
60
n: Optional[int] = 1
61
- max_tokens: Optional[int] = 16
61
+ max_tokens: Optional[int] = None
62
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
63
stream: Optional[bool] = False
64
presence_penalty: Optional[float] = 0.0