A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Revert #24446 and #26168 (#26332)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Oct 8, 2025 at 00:38 UTC
31a4b3e6c40278025664169eafbc8165e1d0c393
5 files changed
+10
-117
tests/entrypoints/llm/test_generate.py
+2
-3
@@ -85,11 +85,10 @@ def test_max_model_len():
85
num_total_tokens = len(output.prompt_token_ids) + len(
86
output.outputs[0].token_ids
87
)
88
- # Total tokens must not exceed max_model_len + 1 (the last token can be
89
- # generated with the context length equal to the max model length)
88
+ # Total tokens must not exceed max_model_len.
89
# It can be less if generation finishes due to other reasons (e.g., EOS)
90
# before reaching the absolute model length limit.
92
- assert num_total_tokens <= max_model_len + 1
91
+ assert num_total_tokens <= max_model_len
92
93
94
def test_log_stats():
tests/v1/e2e/test_context_length.py
deleted
-90
@@ -1,90 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-end-to-end tests for context length corner cases of vLLM v1 model runner
5
-versus HuggingFace's transformers.
6
-
7
-This test verifies the following behavior: allow prefill and decodes on the
8
-model's maximum context length ``max_model_len`` and get one more token.
9
-
10
-Test strategy
11
-- Build a prompt consisting of exactly ``prompt_len`` tokens.
12
-- Run vLLM generation requesting ``max_tokens`` new tokens.
13
-- Run HF generation on the same prompt requesting the same number of tokens.
14
-- Assert both return the same number of generated tokens and the same ids.
15
-
16
-Test cases
17
-- Prefill a prompt of ``max_model_len`` (2048) and request a single token which
18
-will be sampled after the prefill (context length ``max_model_len``).
19
-- Prefill a prompt of ``max_model_len`` - 1 (2047) and request two tokens where
20
-the 1st will be sampled after the prefill and the 2nd after the first decode
21
-(context length ``max_model_len``).
22
-
23
-"""
24
-
25
-import pytest
26
-
27
-from tests.conftest import HfRunner, VllmRunner
28
-from tests.models.utils import check_outputs_equal
29
-from tests.utils import create_new_process_for_each_test
30
-
31
-
32
-@create_new_process_for_each_test()
33
-@pytest.mark.parametrize("model", ["JackFram/llama-160m"])
34
-@pytest.mark.parametrize(
35
- "prompt_len, max_tokens",
36
- [
37
- (2048, 1), # prompt_len = max_model_len
38
- (2047, 2), # prompt_len = max_model_len - 1
39
- ],
40
-)
41
-def test_max_context_length(
42
- model: str,
43
- vllm_runner: type[VllmRunner],
44
- hf_runner: type[HfRunner],
45
- prompt_len: int,
46
- max_tokens: int,
47
-) -> None:
48
- """Compare vLLM and HuggingFace when the prompt already fills the
49
- model's maximum context length and we request a single new token.
50
-
51
- The test ensures vLLM does not raise the "Sampled token IDs exceed the
52
- max model length" assertion and that both vLLM and HF produce the same
53
- single token when given the same inputs.
54
- """
55
-
56
- # Construct a prompt of size prompt_len
57
- prompt_ids = [[43] * prompt_len]
58
-
59
- # --- vLLM generation ---
60
- with vllm_runner(
61
- model_name=model,
62
- tokenizer_name=model,
63
- max_model_len=2048,
64
- max_num_seqs=1,
65
- tensor_parallel_size=1,
66
- ) as vllm_model:
67
- # Generate max_tokens new tokens deterministically.
68
- vllm_outputs = vllm_model.generate_greedy(prompt_ids, max_tokens)
69
-
70
- # --- HuggingFace generation ---
71
- with hf_runner(
72
- model_name=model,
73
- ) as hf_model:
74
- hf_outputs = hf_model.generate_greedy(prompt_ids, max_tokens)
75
-
76
- # vLLM and HF runners return prompt + generated tokens. Slice off the prompt.
77
- vllm_output_ids = vllm_outputs[0][0][prompt_len:]
78
- hf_output_ids = hf_outputs[0][0][prompt_len:]
79
-
80
- # check that exactly max_tokens tokens were generated with vLLM and HF
81
- assert len(vllm_output_ids) == len(hf_output_ids) == max_tokens
82
-
83
- # check that vLLM outputs (token ids) match HF outputs
84
- # Note: for simplicity don't pass detokenized string
85
- check_outputs_equal(
86
- outputs_0_lst=[(hf_output_ids, "")],
87
- outputs_1_lst=[(vllm_output_ids, "")],
88
- name_0="hf",
89
- name_1="vllm",
90
- )
vllm/v1/core/sched/scheduler.py
+1
-1
@@ -223,7 +223,7 @@ class Scheduler(SchedulerInterface):
223
# Make sure the input position does not exceed the max model len.
224
# This is necessary when using spec decoding.
225
num_new_tokens = min(
226
- num_new_tokens, self.max_model_len - request.num_computed_tokens
226
+ num_new_tokens, self.max_model_len - 1 - request.num_computed_tokens
227
)
228
229
# Schedule encoder inputs.
vllm/v1/core/sched/utils.py
+1
-1
@@ -44,7 +44,7 @@ def check_stop(
44
request: Request, max_model_len: int, pooler_output: Optional[torch.Tensor] = None
45
) -> bool:
46
if (
47
- request.num_tokens > max_model_len
47
+ request.num_tokens >= max_model_len
48
or request.num_output_tokens >= request.max_tokens
49
):
50
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
vllm/v1/worker/gpu_model_runner.py
+6
-22
@@ -2317,30 +2317,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2317
2318
start_idx = self.input_batch.num_tokens_no_spec[req_idx]
2319
end_idx = start_idx + len(sampled_ids)
2320
- assert end_idx <= self.max_model_len + 1, (
2321
- "Sampled token IDs exceed the max model length + 1. "
2322
- f"Total number of tokens: {end_idx} > max_model_len + 1: "
2323
- f"{self.max_model_len + 1}"
2320
+ assert end_idx <= self.max_model_len, (
2321
+ "Sampled token IDs exceed the max model length. "
2322
+ f"Total number of tokens: {end_idx} > max_model_len: "
2323
+ f"{self.max_model_len}"
2324
)
2325
2326
- n_tokens_cache = len(sampled_ids)
2327
-
2328
- # Sampled token IDs exceed the max model length by 1. This is
2329
- # legitimate as we can still sample 1 last token when the context
2330
- # length equals the max model length. Note that we do not need to
2331
- # cache this token ID as the sequence finishes after this step.
2332
- # Additionally, the buffers token_ids_cpu and is_token_ids are of
2333
- # size max model length only.
2334
- if end_idx == self.max_model_len + 1:
2335
- n_tokens_cache -= 1
2336
-
2337
- self.input_batch.token_ids_cpu[
2338
- req_idx, start_idx : (start_idx + n_tokens_cache)
2339
- ] = sampled_ids[:n_tokens_cache]
2340
- self.input_batch.is_token_ids[
2341
- req_idx, start_idx : (start_idx + n_tokens_cache)
2342
- ] = True
2343
-
2326
+ self.input_batch.token_ids_cpu[req_idx, start_idx:end_idx] = sampled_ids
2327
+ self.input_batch.is_token_ids[req_idx, start_idx:end_idx] = True
2328
self.input_batch.num_tokens_no_spec[req_idx] = end_idx
2329
self.input_batch.num_tokens[req_idx] = end_idx
2330