A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 04:00 UTC
[Bugfix] Fix misleading context length error messages (#36197)
Signed-off-by: AjAnubolu <anuboluajay@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Ajay Anubolu committed
Mar 5, 2026 at 22:15 UTC
43f10573c9701df093f6523da43cc1a2fac1b3b3
4 files changed
+35
-23
tests/entrypoints/llm/test_chat.py
+1
-1
@@ -200,7 +200,7 @@ def test_chat_batch_failure_cleanup(llm_for_failure_test):
200
batch_2 = [valid_msg, valid_msg]
201
sampling_params = SamplingParams(temperature=0, max_tokens=10)
202
203
- with pytest.raises(ValueError, match="context length is only"):
203
+ with pytest.raises(ValueError, match="maximum context length is"):
204
llm.chat(batch_1, sampling_params=sampling_params)
205
assert llm.llm_engine.get_num_unfinished_requests() == 0
206
tests/renderers/test_completions.py
+3
-3
@@ -271,7 +271,7 @@ class TestRenderPrompt:
271
272
with pytest.raises(
273
ValueError,
274
- match="input characters and requested .* context length is only",
274
+ match="maximum context length is",
275
):
276
renderer.tokenize_prompts(
277
prompts,
@@ -292,7 +292,7 @@ class TestRenderPrompt:
292
293
with pytest.raises(
294
ValueError,
295
- match="input tokens and requested .* context length is only",
295
+ match="maximum context length is",
296
):
297
renderer.tokenize_prompts(
298
prompts,
@@ -313,7 +313,7 @@ class TestRenderPrompt:
313
314
with pytest.raises(
315
ValueError,
316
- match="input tokens and requested .* context length is only",
316
+ match="maximum context length is",
317
):
318
renderer.tokenize_prompts(
319
prompts,
vllm/entrypoints/openai/engine/serving.py
+9
-5
@@ -791,11 +791,15 @@ class OpenAIServing:
791
792
if max_tokens is not None and token_num + max_tokens > max_model_len:
793
raise VLLMValidationError(
794
- "'max_tokens' or 'max_completion_tokens' is too large: "
795
- f"{max_tokens}. This model's maximum context length is "
796
- f"{max_model_len} tokens and your request has "
797
- f"{token_num} input tokens ({max_tokens} > {max_model_len}"
798
- f" - {token_num}).",
794
+ f"This model's maximum context length is "
795
+ f"{max_model_len} tokens. However, you requested "
796
+ f"{max_tokens} output tokens and your prompt contains "
797
+ f"{token_num} input tokens, for a total of "
798
+ f"{token_num + max_tokens} tokens "
799
+ f"({token_num} + {max_tokens} = "
800
+ f"{token_num + max_tokens} > {max_model_len}). "
801
+ f"Please reduce the length of the input prompt or the "
802
+ f"number of requested output tokens.",
803
parameter="max_tokens",
804
value=max_tokens,
805
)
vllm/renderers/params.py
+22
-14
@@ -253,13 +253,14 @@ class TokenizeParams:
253
# To save resources, fail the request outright without even
254
# attempting tokenization
255
raise VLLMValidationError(
256
- f"You passed {len(text)} input characters "
257
- f"and requested {self.max_output_tokens} output tokens. "
258
- f"However, the model's context length is only "
259
- f"{self.max_total_tokens} tokens, resulting in a maximum "
260
- f"input length of {max_input_tokens} tokens "
261
- f"(at most {max_input_chars} characters). "
262
- f"Please reduce the length of the input prompt.",
256
+ f"This model's maximum context length is "
257
+ f"{self.max_total_tokens} tokens. However, you requested "
258
+ f"{self.max_output_tokens} output tokens and your prompt "
259
+ f"contains {len(text)} characters (more than "
260
+ f"{max_input_chars} characters, which is the upper bound "
261
+ f"for {max_input_tokens} input tokens). "
262
+ f"Please reduce the length of the input prompt or the "
263
+ f"number of requested output tokens.",
264
parameter="input_text",
265
value=len(text),
266
)
@@ -334,15 +335,22 @@ class TokenizeParams:
335
return tokens
336
337
if len(tokens) > max_input_tokens:
338
+ token_count = len(tokens)
339
+ # The tokenizer may have truncated the prompt to
340
+ # max_input_tokens + 1 (see get_encode_kwargs), so the
341
+ # actual prompt length could be larger.
342
+ qualifier = "at least " if token_count == max_input_tokens + 1 else ""
343
+ total = token_count + self.max_output_tokens
344
raise VLLMValidationError(
338
- f"You passed {len(tokens)} input tokens "
339
- f"and requested {self.max_output_tokens} output tokens. "
340
- f"However, the model's context length is only "
341
- f"{self.max_total_tokens} tokens, resulting in a maximum "
342
- f"input length of {max_input_tokens} tokens. "
343
- f"Please reduce the length of the input prompt.",
345
+ f"This model's maximum context length is "
346
+ f"{self.max_total_tokens} tokens. However, you requested "
347
+ f"{self.max_output_tokens} output tokens and your prompt "
348
+ f"contains {qualifier}{token_count} input tokens, "
349
+ f"for a total of {qualifier}{total} tokens. "
350
+ f"Please reduce the length of the input prompt or the "
351
+ f"number of requested output tokens.",
352
parameter="input_tokens",
345
- value=len(tokens),
353
+ value=token_count,
354
)
355
356
return tokens