A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 07:00 UTC
[Bugfix] Fix validate model input for decoder models (#27099)
Signed-off-by: Yannick Schnider <yannick.schnider1@ibm.com> Signed-off-by: Yannick Schnider <Yannick.Schnider1@ibm.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Yannick Schnider committed
Nov 13, 2025 at 19:18 UTC
119c4927b33f78cb8bb2283a57ee0e3a14021777
2 files changed
+78
tests/v1/e2e/test_context_length.py
new
+63
@@ -0,0 +1,63 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Tests for vLLM `vllm/v1/engine/processor.Processor._validate_model_input()`
5
+handling of maximum context length for decoder models.
6
+
7
+This test ensures:
8
+- A prompt that is one token shorter than the model's maximum context length
9
+ can be processed successfully when requesting one additional token.
10
+- A prompt that reaches the model's maximum context length throws a
11
+ `ValueError` when requesting at least one additional token.
12
+"""
13
+
14
+import pytest
15
+
16
+from tests.conftest import VllmRunner
17
+from tests.utils import create_new_process_for_each_test
18
+
19
+
20
+@create_new_process_for_each_test()
21
+@pytest.mark.parametrize("model, max_model_len", [("JackFram/llama-160m", 2048)])
22
+@pytest.mark.parametrize(
23
+ "prompt_len, max_tokens",
24
+ [
25
+ (2047, 1), # prompt_len = max_model_len - 1 -> allowed
26
+ (2048, 1), # prompt_len = max_model_len -> not allowed
27
+ ],
28
+)
29
+def test_decoder_max_context_length_validation(
30
+ model: str,
31
+ max_model_len: int,
32
+ vllm_runner: type[VllmRunner],
33
+ prompt_len: int,
34
+ max_tokens: int,
35
+) -> None:
36
+ """Check vLLM decoder model input validation for edge cases where
37
+ the prompt length is (almost) equal to the max model length."""
38
+
39
+ prompt_ids = [[43] * prompt_len]
40
+
41
+ with vllm_runner(
42
+ model_name=model,
43
+ tokenizer_name=model,
44
+ max_model_len=max_model_len,
45
+ max_num_seqs=1,
46
+ tensor_parallel_size=1,
47
+ ) as vllm_model:
48
+ if prompt_len + max_tokens <= max_model_len:
49
+ # Should succeed as constraints are met
50
+ vllm_model.generate_greedy(prompt_ids, max_tokens)
51
+ else:
52
+ # Should raise the ValueError defined in
53
+ # vllm/v1/engine/processor.Processor_validate_model_input()
54
+ expected_msg = (
55
+ f"The decoder prompt (length {prompt_len}) plus the number of "
56
+ f"requested output tokens (at least 1) is longer than "
57
+ f"the maximum model length of {max_model_len}. "
58
+ "Make sure that `max_model_len` is no smaller than the number of "
59
+ "text tokens (prompt + requested output tokens)."
60
+ )
61
+ with pytest.raises(ValueError) as excinfo:
62
+ vllm_model.generate_greedy(prompt_ids, max_tokens)
63
+ assert expected_msg in str(excinfo.value)
vllm/v1/engine/processor.py
+15
@@ -575,6 +575,21 @@ class Processor:
575
# check that chunked prefill does not truncate them
576
# max_batch_len = self.scheduler_config.max_num_batched_tokens
577
578
+ if (
579
+ prompt_len == max_prompt_len
580
+ and prompt_type == "decoder"
581
+ and not model_config.is_multimodal_model
582
+ ):
583
+ suggestion = (
584
+ "Make sure that `max_model_len` is no smaller than the "
585
+ "number of text tokens (prompt + requested output tokens)."
586
+ )
587
+ raise ValueError(
588
+ f"The {prompt_type} prompt (length {prompt_len}) plus the number of "
589
+ f"requested output tokens (at least 1) is longer than the maximum "
590
+ f"model length of {max_prompt_len}. {suggestion}"
591
+ )
592
+
593
def stat_mm_cache(self) -> MultiModalCacheStats | None:
594
return self.input_preprocessor.stat_mm_cache()
595