A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 10:15 UTC
[Cleanup] Remove obsolete spec decoding compatibility logic (#32003)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Jan 8, 2026 at 21:44 UTC
29ce48221c327d4d10f18dfa8fd06a6cecd358fd
8 files changed
+45
-75
benchmarks/benchmark_ngram_proposer.py
-3
@@ -135,7 +135,6 @@ def benchmark_batched_propose(args):
135
block_sizes=[16],
136
)
137
dummy_input_batch._req_ids = list(str(id) for id in range(args.num_req))
138
- dummy_input_batch.spec_decode_unsupported_reqs = ()
138
dummy_input_batch.num_tokens_no_spec = [args.num_token] * args.num_req
139
dummy_input_batch.token_ids_cpu = np.random.randint(
140
0, 20, (args.num_req, args.num_token)
@@ -151,10 +150,8 @@ def benchmark_batched_propose(args):
150
start = time.time()
151
runner.drafter.propose(
152
sampled_token_ids,
154
- dummy_input_batch.req_ids,
153
dummy_input_batch.num_tokens_no_spec,
154
dummy_input_batch.token_ids_cpu,
157
- dummy_input_batch.spec_decode_unsupported_reqs,
155
)
156
end = time.time()
157
print(f"Iteration time (s): {end - start}")
tests/v1/sample/test_logprobs.py
+45
-13
@@ -940,27 +940,62 @@ def test_correct_decoded_token_preserves_valid_tokens():
940
(
941
"eagle",
942
"meta-llama/Llama-3.2-1B-Instruct",
943
- "nm-testing/Llama3_2_1B_speculator.eagle3",
943
+ {
944
+ "method": "eagle",
945
+ "model": "nm-testing/Llama3_2_1B_speculator.eagle3",
946
+ "num_speculative_tokens": 3,
947
+ },
948
+ 0,
949
),
950
marks=large_gpu_mark(min_gb=32),
951
+ id="eagle0",
952
+ ),
953
+ pytest.param(
954
+ (
955
+ "eagle",
956
+ "meta-llama/Llama-3.2-1B-Instruct",
957
+ {
958
+ "method": "eagle",
959
+ "model": "nm-testing/Llama3_2_1B_speculator.eagle3",
960
+ "num_speculative_tokens": 3,
961
+ },
962
+ 3,
963
+ ),
964
+ marks=large_gpu_mark(min_gb=32),
965
+ id="eagle3",
966
+ ),
967
+ pytest.param(
968
+ (
969
+ "ngram",
970
+ "meta-llama/Llama-3.2-1B-Instruct",
971
+ {
972
+ "method": "ngram",
973
+ "prompt_lookup_max": 5,
974
+ "prompt_lookup_min": 3,
975
+ "num_speculative_tokens": 3,
976
+ },
977
+ 3,
978
+ ),
979
+ marks=large_gpu_mark(min_gb=32),
980
+ id="ngram",
981
),
982
],
983
)
949
-@pytest.mark.parametrize("top_logprobs", [0, 3])
984
def test_spec_decode_logprobs(
985
logprobs_mode: LogprobsMode,
952
- model_setup: tuple[str, str, str],
953
- top_logprobs: int,
986
+ model_setup: tuple[str, str, dict, int],
987
):
988
"""Spec decode logprobs should match those of the base model.
989
990
Args:
991
logprobs_mode: logprobs mode.
959
- model_setup: Spec decode method, base model name, and
960
- draft model name.
992
+ model_setup: Tuple of (method, base model name,
993
+ speculative_config dict, top_logprobs).
994
"""
995
from vllm import LLM
996
997
+ method, model_name, spec_config, top_logprobs = model_setup
998
+
999
prompt = "Hello world " * 50
1000
sampling_params = SamplingParams(
1001
temperature=0, logprobs=top_logprobs, max_tokens=10, ignore_eos=False
@@ -972,7 +1007,7 @@ def test_spec_decode_logprobs(
1007
ignore_eos=False,
1008
presence_penalty=-1.0,
1009
)
975
- method, model_name, spec_model_name = model_setup
1010
+
1011
max_model_len = 256
1012
1013
# Run base LLM.
@@ -999,14 +1034,11 @@ def test_spec_decode_logprobs(
1034
cleanup_dist_env_and_memory()
1035
1036
# Run spec decode LLM.
1037
+ # Add max_model_len to spec_config if not present
1038
+ spec_config_with_len = {**spec_config, "max_model_len": max_model_len}
1039
spec_llm = LLM(
1040
model_name,
1004
- speculative_config={
1005
- "method": method,
1006
- "model": spec_model_name,
1007
- "num_speculative_tokens": 3,
1008
- "max_model_len": max_model_len,
1009
- },
1041
+ speculative_config=spec_config_with_len,
1042
max_logprobs=5,
1043
max_model_len=max_model_len,
1044
seed=42,
tests/v1/spec_decode/test_ngram.py
-20
@@ -82,10 +82,8 @@ def test_ngram_proposer():
82
token_ids_cpu = np.array([[1, 2, 3, 4, 5]])
83
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
84
sampled_token_ids=[[0]],
85
- req_ids=["0"],
85
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
86
token_ids_cpu=token_ids_cpu,
88
- spec_decode_unsupported_reqs=(),
87
)
88
assert len(result[0]) == 0
89
@@ -93,10 +91,8 @@ def test_ngram_proposer():
91
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
92
result = get_ngram_proposer(min_n=4, max_n=4, k=2).propose(
93
sampled_token_ids=[[0]],
96
- req_ids=["0"],
94
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
95
token_ids_cpu=token_ids_cpu,
99
- spec_decode_unsupported_reqs=(),
96
)
97
assert len(result[0]) == 0
98
@@ -104,10 +100,8 @@ def test_ngram_proposer():
100
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
101
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
102
sampled_token_ids=[[0]],
107
- req_ids=["0"],
103
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
104
token_ids_cpu=token_ids_cpu,
110
- spec_decode_unsupported_reqs=(),
105
)
106
assert np.array_equal(result, np.array([[4, 1]]))
107
@@ -116,10 +110,8 @@ def test_ngram_proposer():
110
token_ids_cpu = np.array([[2, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 4]])
111
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
112
sampled_token_ids=[[0]],
119
- req_ids=["0"],
113
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
114
token_ids_cpu=token_ids_cpu,
122
- spec_decode_unsupported_reqs=(),
115
)
116
assert np.array_equal(result, np.array([[1, 2]])) # Not [5, 1]]
117
@@ -127,10 +119,8 @@ def test_ngram_proposer():
119
token_ids_cpu = np.array([[3, 4, 5, 2, 3, 4, 1, 2, 3, 4]])
120
result = get_ngram_proposer(min_n=2, max_n=4, k=2).propose(
121
sampled_token_ids=[[0]],
130
- req_ids=["0"],
122
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
123
token_ids_cpu=token_ids_cpu,
133
- spec_decode_unsupported_reqs=(),
124
)
125
assert np.array_equal(result, np.array([[1, 2]])) # Not [5, 2]]
126
@@ -138,10 +128,8 @@ def test_ngram_proposer():
128
token_ids_cpu = np.array([[1, 2, 3, 100, 1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]])
129
result = get_ngram_proposer(min_n=3, max_n=3, k=2).propose(
130
sampled_token_ids=[[0]],
141
- req_ids=["0"],
131
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
132
token_ids_cpu=token_ids_cpu,
144
- spec_decode_unsupported_reqs=(),
133
)
134
assert np.array_equal(result, np.array([[100, 1]]))
135
@@ -149,10 +137,8 @@ def test_ngram_proposer():
137
token_ids_cpu = np.array([[]])
138
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
139
sampled_token_ids=[[0]],
152
- req_ids=["0"],
140
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
141
token_ids_cpu=token_ids_cpu,
155
- spec_decode_unsupported_reqs=(),
142
)
143
assert len(result[0]) == 0
144
@@ -162,10 +148,8 @@ def test_ngram_proposer():
148
token_ids_cpu = np.array([[1, 2, 3, 1, 2], [4, 5, 6, -1, -1]])
149
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
150
sampled_token_ids=[[0], [1]],
165
- req_ids=["0", "1"],
151
num_tokens_no_spec=np.array([5, 3]),
152
token_ids_cpu=token_ids_cpu,
168
- spec_decode_unsupported_reqs=(),
153
)
154
assert len(result[0]) == 2
155
assert np.array_equal(result[0], np.array([3, 1]))
@@ -183,10 +167,8 @@ def test_ngram_proposer():
167
sampled_token_ids = [[2], [], [8]] # Empty list for request 1 simulates prefill
168
result = proposer.propose(
169
sampled_token_ids=sampled_token_ids,
186
- req_ids=["0", "1", "2"],
170
num_tokens_no_spec=num_tokens_no_spec,
171
token_ids_cpu=token_ids_cpu,
189
- spec_decode_unsupported_reqs=(),
172
)
173
assert len(result) == 3
174
assert np.array_equal(result[0], [3, 1])
@@ -214,10 +196,8 @@ def test_ngram_proposer():
196
token_ids_cpu = np.array([input_1, input_2])
197
result = ngram_proposer.propose(
198
sampled_token_ids=[[0], [1]],
217
- req_ids=["0", "1"],
199
num_tokens_no_spec=np.array([len(input_1), 3]),
200
token_ids_cpu=token_ids_cpu,
220
- spec_decode_unsupported_reqs=(),
201
)
202
assert len(result[0]) == 2
203
assert np.array_equal(result[0], np.array([middle_integer + 2, middle_integer + 3]))
vllm/v1/spec_decode/ngram_proposer.py
-10
@@ -55,10 +55,8 @@ class NgramProposer:
55
# This usually takes less than 1 second.
56
self.propose(
57
[[]] * 1024,
58
- [""] * 1024,
58
np.zeros(1024, dtype=np.int32),
59
np.zeros((1024, self.max_model_len), dtype=np.int32),
61
- set(),
60
)
61
62
def batch_propose(
@@ -132,10 +130,8 @@ class NgramProposer:
130
def propose(
131
self,
132
sampled_token_ids: list[list[int]],
135
- req_ids: list[str],
133
num_tokens_no_spec: np.ndarray,
134
token_ids_cpu: np.ndarray,
138
- spec_decode_unsupported_reqs: set,
135
) -> list[list[int]]:
136
# find which requests need ngram proposals
137
valid_ngram_requests = []
@@ -145,12 +141,6 @@ class NgramProposer:
141
# Skip speculative decoding.
142
continue
143
148
- # Skip requests that require sampling parameters that are not
149
- # supported with speculative decoding.
150
- req_id = req_ids[i]
151
- if req_id in spec_decode_unsupported_reqs:
152
- continue
153
-
144
num_tokens = num_tokens_no_spec[i]
145
if num_tokens >= self.max_model_len:
146
# Skip requests that have already reached the max model length.
vllm/v1/spec_decode/suffix_decoding.py
-6
@@ -46,13 +46,7 @@ class SuffixDecodingProposer:
46
draft_token_ids.append([])
47
continue
48
49
- # Skip requests that require sampling parameters that are not
50
- # supported with speculative decoding.
49
req_id = input_batch.req_ids[i]
52
- if req_id in input_batch.spec_decode_unsupported_reqs:
53
- draft_token_ids.append([])
54
- continue
55
-
50
num_tokens = input_batch.num_tokens_no_spec[i]
51
if num_tokens >= self.max_model_len:
52
# Skip requests that have already reached the max model length.
vllm/v1/spec_decode/utils.py
-14
@@ -1,21 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from vllm.sampling_params import SamplingParams
3
from vllm.triton_utils import tl, triton
4
6
-_SAMPLING_EPS = 1e-5
7
-
8
-
9
-def is_spec_decode_unsupported(sampling_params: SamplingParams) -> bool:
10
- """True if request is incompatible with speculative decoding"""
11
- return (
12
- sampling_params.frequency_penalty != 0.0
13
- or sampling_params.presence_penalty != 0.0
14
- or sampling_params.repetition_penalty != 1.0
15
- or sampling_params.min_p > _SAMPLING_EPS
16
- or sampling_params.logprobs is not None
17
- )
18
-
5
6
@triton.jit
7
def eagle_prepare_inputs_padded_kernel(
vllm/v1/worker/gpu_input_batch.py
-7
@@ -22,7 +22,6 @@ from vllm.v1.sample.logits_processor import (
22
MoveDirectionality,
23
)
24
from vllm.v1.sample.metadata import SamplingMetadata
25
-from vllm.v1.spec_decode.utils import is_spec_decode_unsupported
25
from vllm.v1.utils import copy_slice
26
from vllm.v1.worker.block_table import MultiGroupBlockTable
27
@@ -176,9 +175,6 @@ class InputBatch:
175
self.top_k_cpu = self.top_k_cpu_tensor.numpy()
176
self.top_k_reqs: set[str] = set()
177
179
- # IDs of requests which do not support spec decoding
180
- self.spec_decode_unsupported_reqs: set[str] = set()
181
-
178
# Frequency penalty related data structures
179
self.frequency_penalties = torch.empty(
180
(max_num_reqs,), dtype=torch.float, device=device
@@ -346,8 +342,6 @@ class InputBatch:
342
self.block_table.add_row(request.block_ids, req_index)
343
344
if sampling_params := request.sampling_params:
349
- if self.is_spec_decode and is_spec_decode_unsupported(sampling_params):
350
- self.spec_decode_unsupported_reqs.add(req_id)
345
if sampling_params.sampling_type == SamplingType.GREEDY:
346
# Should avoid division by zero later when apply_temperature.
347
self.temperature_cpu[req_index] = 0.0
@@ -510,7 +504,6 @@ class InputBatch:
504
self.random_reqs.discard(req_id)
505
self.top_p_reqs.discard(req_id)
506
self.top_k_reqs.discard(req_id)
513
- self.spec_decode_unsupported_reqs.discard(req_id)
507
self.frequency_penalties_reqs.discard(req_id)
508
self.presence_penalties_reqs.discard(req_id)
509
self.repetition_penalties_reqs.discard(req_id)
vllm/v1/worker/gpu_model_runner.py
-2
@@ -3620,10 +3620,8 @@ class GPUModelRunner(
3620
assert isinstance(self.drafter, NgramProposer)
3621
draft_token_ids = self.drafter.propose(
3622
sampled_token_ids,
3623
- self.input_batch.req_ids,
3623
self.input_batch.num_tokens_no_spec,
3624
self.input_batch.token_ids_cpu,
3626
- self.input_batch.spec_decode_unsupported_reqs,
3625
)
3626
elif spec_config.method == "suffix":
3627
assert isinstance(sampled_token_ids, list)