A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fixing max token error message for openai compatible server (#4016)
Jack Gordley committed
Apr 23, 2024 at 12:06 UTC
d3c8180ac4143f4affd2ef26855058e96b72b5f5
1 file changed
+6
vllm/entrypoints/openai/serving_engine.py
+6
@@ -206,6 +206,12 @@ class OpenAIServing:
206
token_num = len(input_ids)
207
208
if request.max_tokens is None:
209
+ if token_num >= self.max_model_len:
210
+ raise ValueError(
211
+ f"This model's maximum context length is "
212
+ f"{self.max_model_len} tokens. However, you requested "
213
+ f"{token_num} tokens in the messages, "
214
+ f"Please reduce the length of the messages.", )
215
request.max_tokens = self.max_model_len - token_num
216
217
if token_num + request.max_tokens > self.max_model_len: