A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix][Core] Prevent token lengths exceeding `max_model_len` in V0 (#19348)
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com>
22quinn committed
Jun 9, 2025 at 08:01 UTC
c1c7dbbeeb6d4f0155d25b673f2063bfb14b37b9
2 files changed
+23
-1
tests/entrypoints/llm/test_generate.py
+22
@@ -25,6 +25,12 @@ TOKEN_IDS = [
25
]
26
27
28
+@pytest.fixture(autouse=True)
29
+def v1(run_with_both_engines):
30
+ """We can run both engines for this test."""
31
+ pass
32
+
33
+
34
@pytest.fixture(scope="module")
35
def llm():
36
# pytest caches the fixture so we use weakref.proxy to
@@ -104,3 +110,19 @@ def test_multiple_sampling_params(llm: LLM):
110
# sampling_params is None, default params should be applied
111
outputs = llm.generate(PROMPTS, sampling_params=None)
112
assert len(PROMPTS) == len(outputs)
113
+
114
+
115
+def test_max_model_len():
116
+ max_model_len = 20
117
+ llm = LLM(
118
+ model=MODEL_NAME,
119
+ max_model_len=max_model_len,
120
+ gpu_memory_utilization=0.10,
121
+ enforce_eager=True, # reduce test time
122
+ )
123
+ sampling_params = SamplingParams(max_tokens=max_model_len + 10)
124
+ outputs = llm.generate(PROMPTS, sampling_params)
125
+ for output in outputs:
126
+ num_total_tokens = len(output.prompt_token_ids) + len(
127
+ output.outputs[0].token_ids)
128
+ assert num_total_tokens == max_model_len
vllm/engine/output_processor/stop_checker.py
+1
-1
@@ -82,7 +82,7 @@ class StopChecker:
82
return
83
84
# Check if the sequence has reached max_model_len.
85
- if seq.get_len() > self._get_max_model_len(lora_req):
85
+ if seq.get_len() >= self._get_max_model_len(lora_req):
86
seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
87
return
88