A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] clear and separate error messages for input too long and input + max-tokens too long (#22803)
Signed-off-by: Roger Wang <hey@rogerw.me>
Roger Wang committed
Aug 13, 2025 at 07:22 UTC
da2705198fa19030a25d0bea437f7be6547d47d4
1 file changed
+18
-13
vllm/entrypoints/openai/serving_engine.py
+18
-13
@@ -585,6 +585,8 @@ class OpenAIServing:
585
(EmbeddingChatRequest, EmbeddingCompletionRequest,
586
ScoreRequest, RerankRequest, ClassificationRequest)):
587
588
+ # Note: input length can be up to the entire model context length
589
+ # since these requests don't generate tokens.
590
if token_num > self.max_model_len:
591
operations: dict[type[AnyRequest], str] = {
592
ScoreRequest: "score",
@@ -613,21 +615,24 @@ class OpenAIServing:
615
max_tokens = request.max_completion_tokens or request.max_tokens
616
else:
617
max_tokens = getattr(request, "max_tokens", None)
616
- if max_tokens is None:
617
- if token_num >= self.max_model_len:
618
- raise ValueError(
619
- f"This model's maximum context length is "
620
- f"{self.max_model_len} tokens. However, you requested "
621
- f"{token_num} tokens in the messages, "
622
- f"Please reduce the length of the messages.")
623
- elif token_num + max_tokens > self.max_model_len:
618
+
619
+ # Note: input length can be up to model context length - 1 for
620
+ # completion-like requests.
621
+ if token_num >= self.max_model_len:
622
raise ValueError(
623
f"This model's maximum context length is "
626
- f"{self.max_model_len} tokens. However, you requested "
627
- f"{max_tokens + token_num} tokens "
628
- f"({token_num} in the messages, "
629
- f"{max_tokens} in the completion). "
630
- f"Please reduce the length of the messages or completion.")
624
+ f"{self.max_model_len} tokens. However, your request has "
625
+ f"{token_num} input tokens. Please reduce the length of "
626
+ "the input messages.")
627
+
628
+ if max_tokens is not None and \
629
+ token_num + max_tokens > self.max_model_len:
630
+ raise ValueError(
631
+ "'max_tokens' or 'max_completion_tokens' is too large: "
632
+ f"{max_tokens}. This model's maximum context length is "
633
+ f"{self.max_model_len} tokens and your request has "
634
+ f"{token_num} input tokens ({max_tokens} > {self.max_model_len}"
635
+ f" - {token_num}).")
636
637
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
638