A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Speculative decoding 6/9] Integrate speculative decoding with LLMEngine (#3894)
Cade Daniel committed
Apr 16, 2024 at 13:09 UTC
e95cd879598b834f85e70ebcd23db316ae430540
31 files changed
+1347
-407
tests/core/block/e2e/test_correctness.py
+70
@@ -230,6 +230,76 @@ def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
230
assert baseline_token_ids == test_token_ids
231
232
233
+@pytest.mark.parametrize(
234
+ "common_llm_kwargs",
235
+ [
236
+ {
237
+ # Use a small model for a fast test.
238
+ "model": "facebook/opt-125m",
239
+
240
+ # skip cuda graph creation for fast test.
241
+ "enforce_eager": True,
242
+ "enable_chunked_prefill": True,
243
+ "max_num_batched_tokens": 2,
244
+ "max_num_seqs": 2,
245
+ },
246
+ ])
247
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
248
+@pytest.mark.parametrize("baseline_llm_kwargs", [
249
+ {
250
+ "use_v2_block_manager": False,
251
+ },
252
+])
253
+@pytest.mark.parametrize("test_llm_kwargs", [
254
+ {
255
+ "use_v2_block_manager": True,
256
+ "num_lookahead_slots": 0,
257
+ },
258
+ {
259
+ "use_v2_block_manager": True,
260
+ "num_lookahead_slots": 5,
261
+ },
262
+])
263
+@pytest.mark.parametrize("batch_size", [4])
264
+@pytest.mark.parametrize("seed", [1])
265
+def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
266
+ test_llm_generator, batch_size):
267
+ """Verify that chunked prefill works with BlockManagerV2, with and without
268
+ lookahead scheduling.
269
+ """
270
+ output_len = 32
271
+ temperature = 0.0
272
+
273
+ prompts = [
274
+ "Hello, my name is",
275
+ "The president of the United States is",
276
+ "The capital of France is",
277
+ "The future of AI is",
278
+ ]
279
+
280
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
281
+
282
+ sampling_params = SamplingParams(
283
+ max_tokens=output_len,
284
+ ignore_eos=True,
285
+ temperature=temperature,
286
+ )
287
+
288
+ print('Getting token ids with BlockManagerV1')
289
+ baseline_token_ids = get_token_ids_from_llm_generator(
290
+ baseline_llm_generator, prompts, sampling_params)
291
+
292
+ print('Getting token ids with BlockManagerV2')
293
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
294
+ prompts, sampling_params)
295
+
296
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
297
+ test_token_ids):
298
+ assert expected_token_ids == actual_token_ids
299
+
300
+ assert baseline_token_ids == test_token_ids
301
+
302
+
303
def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
304
for llm in llm_generator:
305
outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
tests/core/utils.py
+10
-7
@@ -1,5 +1,5 @@
1
import time
2
-from typing import Optional, Tuple
2
+from typing import Iterable, Optional, Tuple
3
4
from vllm import SamplingParams
5
from vllm.lora.request import LoRARequest
@@ -31,14 +31,17 @@ def create_dummy_prompt(
31
32
33
def create_seq_group(
34
- seq_prompt_len=1024,
35
- seq_output_lens=(128, ),
36
- request_id='0',
37
- seq_id_start=0,
38
-) -> SequenceGroup:
34
+ seq_prompt_len: int = 1024,
35
+ seq_output_lens: Iterable[int] = (128, ),
36
+ request_id: str = '0',
37
+ seq_id_start: int = 0,
38
+ sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
39
40
assert len(seq_output_lens) > 0
41
42
+ if sampling_params is None:
43
+ sampling_params = SamplingParams()
44
+
45
prompt_token_ids = [0] * seq_prompt_len
46
47
seqs = []
@@ -60,7 +63,7 @@ def create_seq_group(
63
seq_group = SequenceGroup(
64
request_id=request_id,
65
seqs=seqs,
63
- sampling_params=SamplingParams(),
66
+ sampling_params=sampling_params,
67
arrival_time=time.time(),
68
)
69
tests/engine/output_processor/test_multi_step.py
new
+270
@@ -0,0 +1,270 @@
1
+import random
2
+from unittest.mock import MagicMock
3
+
4
+import pytest
5
+from transformers import PreTrainedTokenizer
6
+
7
+from tests.core.utils import create_seq_group
8
+from vllm.core.scheduler import Scheduler
9
+from vllm.engine.output_processor.multi_step import MultiStepOutputProcessor
10
+from vllm.engine.output_processor.stop_checker import StopChecker
11
+from vllm.sampling_params import SamplingParams
12
+from vllm.sequence import (Logprob, SequenceGroupOutput, SequenceOutput,
13
+ SequenceStatus)
14
+from vllm.transformers_utils.detokenizer import Detokenizer
15
+from vllm.utils import Counter
16
+
17
+
18
+@pytest.mark.parametrize("seq_output_len", [128])
19
+@pytest.mark.parametrize("num_new_tokens", [1, 12])
20
+@pytest.mark.skip_global_cleanup
21
+def test_appends_token_ids(num_new_tokens: int, seq_output_len: int):
22
+ """Verify multi-step decoding appends token ids correctly.
23
+
24
+ We append token ids and verify all the token ids were appended correctly.
25
+ Note that ignore_eos=True.
26
+ """
27
+ detokenizer = MagicMock(spec=Detokenizer)
28
+ scheduler = MagicMock(spec=Scheduler)
29
+ stop_checker = MagicMock(spec=StopChecker)
30
+ seq_counter = Counter()
31
+
32
+ output_processor = MultiStepOutputProcessor(
33
+ detokenizer=detokenizer,
34
+ scheduler=scheduler,
35
+ seq_counter=seq_counter,
36
+ get_tokenizer_for_seq=lambda _: mock_tokenizer(),
37
+ stop_checker=stop_checker,
38
+ )
39
+
40
+ seq_group = create_seq_group(
41
+ seq_prompt_len=1024,
42
+ seq_output_lens=[seq_output_len],
43
+ sampling_params=SamplingParams(max_tokens=seq_output_len +
44
+ num_new_tokens,
45
+ ignore_eos=True),
46
+ )
47
+
48
+ seq = seq_group.get_seqs()[0]
49
+ seq.status = SequenceStatus.RUNNING
50
+
51
+ new_token_ids = list(range(num_new_tokens))
52
+
53
+ outputs = [
54
+ SequenceGroupOutput(
55
+ samples=[
56
+ SequenceOutput(
57
+ parent_seq_id=seq.seq_id,
58
+ output_token=output_token,
59
+ logprobs={output_token: Logprob(0.0)},
60
+ )
61
+ ],
62
+ prompt_logprobs=None,
63
+ ) for output_token in new_token_ids
64
+ ]
65
+
66
+ assert seq.get_token_ids()[-len(new_token_ids):] != new_token_ids
67
+ output_processor.process_outputs(seq_group, outputs)
68
+ assert seq.get_token_ids()[-len(new_token_ids):] == new_token_ids
69
+
70
+
71
+@pytest.mark.parametrize("seq_prompt_len", [1024])
72
+@pytest.mark.parametrize("seq_output_len", [128])
73
+@pytest.mark.parametrize("num_new_tokens", [5, 6, 7, 8])
74
+@pytest.mark.parametrize("max_tokens", [128 + 3])
75
+@pytest.mark.skip_global_cleanup
76
+def test_respects_max_tokens(num_new_tokens: int, seq_prompt_len: int,
77
+ seq_output_len: int, max_tokens: int):
78
+ """Verify tokens after max_tokens are dropped and not appended to the
79
+ sequence.
80
+ """
81
+ detokenizer = MagicMock(spec=Detokenizer)
82
+ scheduler = MagicMock(spec=Scheduler)
83
+ stop_checker = MagicMock(spec=StopChecker)
84
+ seq_counter = Counter()
85
+
86
+ output_processor = MultiStepOutputProcessor(
87
+ detokenizer=detokenizer,
88
+ scheduler=scheduler,
89
+ seq_counter=seq_counter,
90
+ get_tokenizer_for_seq=lambda _: mock_tokenizer(),
91
+ stop_checker=stop_checker,
92
+ )
93
+
94
+ seq_group = create_seq_group(
95
+ seq_prompt_len=seq_prompt_len,
96
+ seq_output_lens=[seq_output_len],
97
+ sampling_params=SamplingParams(max_tokens=max_tokens, ),
98
+ )
99
+
100
+ seq = seq_group.get_seqs()[0]
101
+ seq.status = SequenceStatus.RUNNING
102
+
103
+ new_token_ids = list(range(num_new_tokens))
104
+
105
+ outputs = [
106
+ SequenceGroupOutput(
107
+ samples=[
108
+ SequenceOutput(
109
+ parent_seq_id=seq.seq_id,
110
+ output_token=output_token,
111
+ logprobs={output_token: Logprob(0.0)},
112
+ )
113
+ ],
114
+ prompt_logprobs=None,
115
+ ) for output_token in new_token_ids
116
+ ]
117
+
118
+ assert seq.get_len() == seq_prompt_len + seq_output_len
119
+ output_processor.process_outputs(seq_group, outputs)
120
+
121
+ # Expect the processed sequence to not go over max tokens in len.
122
+ assert seq.get_len() == seq_prompt_len + max_tokens
123
+
124
+ # Expect the correct tokens were appended.
125
+ expected_appended_tokens = new_token_ids[:max_tokens - seq_output_len]
126
+ assert seq.get_token_ids(
127
+ )[-len(expected_appended_tokens):] == expected_appended_tokens
128
+
129
+
130
+@pytest.mark.parametrize("seq_prompt_len", [1024])
131
+@pytest.mark.parametrize("seq_output_len", [128])
132
+@pytest.mark.parametrize("num_new_tokens", [12])
133
+@pytest.mark.parametrize("seed", list(range(6)))
134
+@pytest.mark.skip_global_cleanup
135
+def test_respects_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
136
+ seq_output_len: int, seed: int):
137
+ """Verify the eos token id is included in the sequence, but subsequent
138
+ tokens are dropped (not appended to sequence).
139
+ """
140
+ random.seed(seed)
141
+ detokenizer = MagicMock(spec=Detokenizer)
142
+ scheduler = MagicMock(spec=Scheduler)
143
+ stop_checker = MagicMock(spec=StopChecker)
144
+ seq_counter = Counter()
145
+
146
+ eos_token_id = 100
147
+
148
+ output_processor = MultiStepOutputProcessor(
149
+ detokenizer=detokenizer,
150
+ scheduler=scheduler,
151
+ seq_counter=seq_counter,
152
+ get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
153
+ stop_checker=stop_checker,
154
+ )
155
+
156
+ seq_group = create_seq_group(
157
+ seq_prompt_len=seq_prompt_len,
158
+ seq_output_lens=[seq_output_len],
159
+ sampling_params=SamplingParams(
160
+ # Ensure enough space.
161
+ max_tokens=seq_output_len + num_new_tokens, ),
162
+ )
163
+
164
+ seq = seq_group.get_seqs()[0]
165
+ seq.status = SequenceStatus.RUNNING
166
+
167
+ new_token_ids = list(range(num_new_tokens))
168
+ assert eos_token_id not in new_token_ids
169
+ eos_index = random.randint(0, len(new_token_ids) - 1)
170
+ new_token_ids[eos_index] = eos_token_id
171
+
172
+ outputs = [
173
+ SequenceGroupOutput(
174
+ samples=[
175
+ SequenceOutput(
176
+ parent_seq_id=seq.seq_id,
177
+ output_token=output_token,
178
+ logprobs={output_token: Logprob(0.0)},
179
+ )
180
+ ],
181
+ prompt_logprobs=None,
182
+ ) for output_token in new_token_ids
183
+ ]
184
+
185
+ assert seq.get_len() == seq_prompt_len + seq_output_len
186
+ output_processor.process_outputs(seq_group, outputs)
187
+
188
+ # Expect the processed sequence to not go beyond provided eos.
189
+ assert seq.get_len() == seq_prompt_len + seq_output_len + (eos_index + 1)
190
+
191
+ # Expect the correct tokens were appended.
192
+ expected_appended_tokens = new_token_ids[:eos_index + 1]
193
+ assert seq.get_token_ids(
194
+ )[-len(expected_appended_tokens):] == expected_appended_tokens
195
+
196
+
197
+@pytest.mark.parametrize("seq_prompt_len", [1024])
198
+@pytest.mark.parametrize("seq_output_len", [128])
199
+@pytest.mark.parametrize("num_new_tokens", [12])
200
+@pytest.mark.parametrize("seed", list(range(6)))
201
+@pytest.mark.skip_global_cleanup
202
+def test_ignores_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
203
+ seq_output_len: int, seed: int):
204
+ """When sampling parameters dictate that we should ignore the eos token id,
205
+ ensure all token ids are appended even if the eos token id is emitted.
206
+ """
207
+ random.seed(seed)
208
+ detokenizer = MagicMock(spec=Detokenizer)
209
+ scheduler = MagicMock(spec=Scheduler)
210
+ stop_checker = MagicMock(spec=StopChecker)
211
+ seq_counter = Counter()
212
+
213
+ eos_token_id = 100
214
+
215
+ output_processor = MultiStepOutputProcessor(
216
+ detokenizer=detokenizer,
217
+ scheduler=scheduler,
218
+ seq_counter=seq_counter,
219
+ get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
220
+ stop_checker=stop_checker,
221
+ )
222
+
223
+ seq_group = create_seq_group(
224
+ seq_prompt_len=seq_prompt_len,
225
+ seq_output_lens=[seq_output_len],
226
+ sampling_params=SamplingParams(
227
+ # Ensure enough space.
228
+ max_tokens=seq_output_len + num_new_tokens,
229
+ ignore_eos=True,
230
+ ),
231
+ )
232
+
233
+ seq = seq_group.get_seqs()[0]
234
+ seq.status = SequenceStatus.RUNNING
235
+
236
+ new_token_ids = list(range(num_new_tokens))
237
+ assert eos_token_id not in new_token_ids
238
+ eos_index = random.randint(0, len(new_token_ids) - 1)
239
+ new_token_ids[eos_index] = eos_token_id
240
+
241
+ outputs = [
242
+ SequenceGroupOutput(
243
+ samples=[
244
+ SequenceOutput(
245
+ parent_seq_id=seq.seq_id,
246
+ output_token=output_token,
247
+ logprobs={output_token: Logprob(0.0)},
248
+ )
249
+ ],
250
+ prompt_logprobs=None,
251
+ ) for output_token in new_token_ids
252
+ ]
253
+
254
+ assert seq.get_len() == seq_prompt_len + seq_output_len
255
+ output_processor.process_outputs(seq_group, outputs)
256
+
257
+ # Expect the processed sequence to go beyond eos.
258
+ assert seq.get_len() == seq_prompt_len + seq_output_len + num_new_tokens
259
+
260
+ # Expect the correct tokens were appended.
261
+ expected_appended_tokens = new_token_ids[:seq_output_len + num_new_tokens -
262
+ seq_output_len]
263
+ assert seq.get_token_ids(
264
+ )[-len(expected_appended_tokens):] == expected_appended_tokens
265
+
266
+
267
+def mock_tokenizer(eos_token_id=1000):
268
+ tokenizer = MagicMock(spec=PreTrainedTokenizer)
269
+ tokenizer.eos_token_id = eos_token_id
270
+ return tokenizer
tests/spec_decode/e2e/test_correctness.py
+114
-13
@@ -1,4 +1,8 @@
1
+from itertools import cycle
2
+from typing import List, Tuple
3
+
4
import pytest
5
+from transformers import AutoTokenizer
6
7
from vllm import SamplingParams
8
@@ -7,18 +11,47 @@ from vllm import SamplingParams
11
"common_llm_kwargs",
12
[{
13
# Use a small model for a fast test.
10
- "model": "facebook/opt-125m",
11
- "speculative_model": "facebook/opt-125m",
12
- "num_speculative_tokens": 5,
14
+ # Note this is repeated in the test body; to initialize a tokenizer.
15
+ "model": "JackFram/llama-68m",
16
+
17
+ # Skip real loading for fast test.
18
+ "load_format": "dummy",
19
+
20
+ # Skip cuda graph recording for fast test.
21
+ "enforce_eager": True,
22
23
# Required for spec decode.
24
"use_v2_block_manager": True
25
}])
17
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
26
+@pytest.mark.parametrize(
27
+ "per_test_common_llm_kwargs",
28
+ [
29
+ {
30
+ "speculative_model": "JackFram/llama-68m",
31
+ "num_speculative_tokens": 5,
32
+ },
33
+ {
34
+ "speculative_model": "JackFram/llama-68m",
35
+ "num_speculative_tokens": 1,
36
+ },
37
+ {
38
+ # No spec decode.
39
+ },
40
+ ])
41
@pytest.mark.parametrize("test_llm_kwargs", [{}])
42
+@pytest.mark.parametrize("batch_size", [1])
43
+# NOTE: We should run more permutations of this test (more BS, more seeds). But
44
+# because our spec decode generates gibberish token ids, the likelihood of
45
+# emitting an invalid token combination is nontrivial. This causes divergence in
46
+# behavior of vLLM detokenization vs. hf tokenizer, for example when two "utf-
47
+# start" bytes are emitted.
48
@pytest.mark.parametrize("seed", [1])
20
-def test_spec_decode_config(test_llm_generator):
21
- output_len = 1024
49
+def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
50
+ """Run generation with speculative decoding on a batch. Verify the engine
51
+ generates the correct number of tokens (via ignore_eos=True), and that the
52
+ detokenization matches HF transformers.
53
+ """
54
+ output_len = 32
55
temperature = 0.0
56
57
prompts = [
@@ -28,23 +61,91 @@ def test_spec_decode_config(test_llm_generator):
61
"The future of AI is",
62
]
63
64
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
65
+
66
+ sampling_params = SamplingParams(
67
+ max_tokens=output_len,
68
+ ignore_eos=True,
69
+ temperature=temperature,
70
+ skip_special_tokens=True,
71
+ spaces_between_special_tokens=False,
72
+ )
73
+
74
+ batch_tokens, batch_token_ids = get_output_from_llm_generator(
75
+ test_llm_generator, prompts, sampling_params)
76
+
77
+ # Expect a generation for each prompt in the batch.
78
+ assert len(batch_token_ids) == len(prompts)
79
+
80
+ # Expect each generation to have expected number of tokens (note
81
+ # ignore_eos=True).
82
+ assert all(len(token_ids) == output_len for token_ids in batch_token_ids)
83
+
84
+ # Expect detokenized string to match.
85
+ tok = AutoTokenizer.from_pretrained("JackFram/llama-68m")
86
+ for actual_tokens, actual_token_ids in zip(batch_tokens, batch_token_ids):
87
+ expected_tokens = tok.decode(actual_token_ids)
88
+ print(f"{actual_token_ids=}")
89
+ assert actual_tokens.strip() == expected_tokens.strip()
90
+
91
+
92
+@pytest.mark.parametrize(
93
+ "common_llm_kwargs",
94
+ [{
95
+ # Use a small model for a fast test.
96
+ "model": "JackFram/llama-68m",
97
+ "speculative_model": "JackFram/llama-68m",
98
+ "num_speculative_tokens": 5,
99
+
100
+ # Skip real loading for fast test.
101
+ "load_format": "dummy",
102
+
103
+ # Skip cuda graph recording for fast test.
104
+ "enforce_eager": True,
105
+
106
+ # Required for spec decode.
107
+ "use_v2_block_manager": True
108
+ }])
109
+@pytest.mark.parametrize(
110
+ "per_test_common_llm_kwargs",
111
+ [
112
+ {
113
+ # Expect failure as spec decode not supported by
114
+ # Ray backend.
115
+ "worker_use_ray": True,
116
+ },
117
+ ])
118
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
119
+@pytest.mark.parametrize("seed", [1])
120
+def test_spec_decode_xfail(test_llm_generator):
121
+ """Verify that speculative decoding with Ray fails.
122
+ """
123
+ output_len = 128
124
+ temperature = 0.0
125
+
126
+ prompts = [
127
+ "Hello, my name is",
128
+ ]
129
+
130
sampling_params = SamplingParams(
131
max_tokens=output_len,
132
ignore_eos=True,
133
temperature=temperature,
134
)
135
37
- with pytest.raises(
38
- AssertionError,
39
- match="Speculative decoding not yet supported for GPU backend"):
40
- get_token_ids_from_llm_generator(test_llm_generator, prompts,
41
- sampling_params)
136
+ with pytest.raises(AssertionError,
137
+ match="Speculative decoding not yet supported for "):
138
+ get_output_from_llm_generator(test_llm_generator, prompts,
139
+ sampling_params)
140
141
44
-def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
142
+def get_output_from_llm_generator(
143
+ llm_generator, prompts,
144
+ sampling_params) -> Tuple[List[str], List[List[int]]]:
145
for llm in llm_generator:
146
outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
147
token_ids = [output.outputs[0].token_ids for output in outputs]
148
+ tokens = [output.outputs[0].text for output in outputs]
149
del llm
150
50
- return token_ids
151
+ return tokens, token_ids
tests/spec_decode/test_multi_step_worker.py
+2
-2
@@ -125,7 +125,7 @@ def test_same_output_for_single_step():
125
zero_kv_cache(worker.cache_engine)
126
set_random_seed(seed)
127
expected_output = worker.execute_model(
128
- **single_step_execute_model_data.to_dict(), )
128
+ **single_step_execute_model_data.to_dict(), )[0]
129
130
actual_token_ids = [
131
output.samples[0].output_token for output in actual_output
@@ -219,7 +219,7 @@ def test_same_output_for_multi_step():
219
continuations=continuations,
220
final_seq_lens=final_seq_lens))
221
222
- single_step_output.append(
222
+ single_step_output.extend(
223
worker.execute_model(**execute_model_data.to_dict(), ))
224
225
# Append output tokens to new sequence data.
tests/spec_decode/test_spec_decode_worker.py
+20
-12
@@ -6,6 +6,7 @@ import torch
6
7
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8
from vllm.model_executor.utils import set_random_seed
9
+from vllm.sequence import SamplerOutput
10
from vllm.spec_decode.interfaces import SpeculativeProposals
11
from vllm.spec_decode.metrics import (AsyncMetricsCollector,
12
SpecDecodeWorkerMetrics)
@@ -37,7 +38,8 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
38
execute_model_data, _, _ = create_batch(batch_size, k)
39
40
with pytest.raises(ValueError, match=exception_secret):
40
- worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
41
+ worker.execute_model(**execute_model_data.to_dict(),
42
+ num_lookahead_slots=k)
43
44
call_args_list = draft_worker.get_spec_proposals.call_args_list
45
assert len(call_args_list) == 1
@@ -102,7 +104,8 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
104
target_worker.execute_model.side_effect = ValueError(exception_secret)
105
106
with pytest.raises(ValueError, match=exception_secret):
105
- worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
107
+ worker.execute_model(**execute_model_data.to_dict(),
108
+ num_lookahead_slots=k)
109
110
seen_contexts = []
111
@@ -189,13 +192,14 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
192
target_output = create_sampler_output_list(target_token_ids,
193
target_token_probs)
194
192
- target_worker.execute_model.return_value = target_output[0]
195
+ target_worker.execute_model.return_value = [target_output[0]]
196
197
exception_secret = 'artifical stop'
198
rejection_sampler.side_effect = ValueError(exception_secret)
199
200
with pytest.raises(ValueError, match=exception_secret):
198
- worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
201
+ worker.execute_model(**execute_model_data.to_dict(),
202
+ num_lookahead_slots=k)
203
204
assert len(rejection_sampler.call_args_list) == 1
205
args, _ = rejection_sampler.call_args_list[0]
@@ -268,7 +272,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
272
target_output = create_sampler_output_list(target_token_ids,
273
target_token_probs)
274
271
- target_worker.execute_model.return_value = target_output[0]
275
+ target_worker.execute_model.return_value = [target_output[0]]
276
277
rejection_sampler_output = torch.randint(low=0,
278
high=vocab_size,
@@ -283,7 +287,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
287
rejection_sampler.return_value = rejection_sampler_output
288
289
output = worker.execute_model(**execute_model_data.to_dict(),
286
- num_spec_tokens=k)
290
+ num_lookahead_slots=k)
291
292
expected_output = create_sampler_output_list(
293
rejection_sampler_output.transpose(0, 1), [None for _ in range(k + 1)])
@@ -380,7 +384,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
384
target_output = create_sampler_output_list(target_token_ids,
385
target_token_probs)
386
383
- target_worker.execute_model.return_value = target_output[0]
387
+ target_worker.execute_model.return_value = [target_output[0]]
388
389
rejection_sampler_output = torch.randint(low=0,
390
high=vocab_size,
@@ -400,7 +404,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
404
mock_rejsample_metrics)
405
406
output = worker.execute_model(**execute_model_data.to_dict(),
403
- num_spec_tokens=k)
407
+ num_lookahead_slots=k)
408
assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
409
410
call_args_list = (
@@ -423,6 +427,8 @@ def test_k_equals_zero(k: int, batch_size: int):
427
rejection_sampler.token_id_dtype = torch.int64
428
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
429
430
+ target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
431
+
432
draft_worker.device = 'cuda'
433
target_worker.device = 'cuda'
434
@@ -435,7 +441,7 @@ def test_k_equals_zero(k: int, batch_size: int):
441
batch_size, k, prev_output_token_len=0)
442
443
out = worker.execute_model(**execute_model_data.to_dict(),
438
- num_spec_tokens=k)
444
+ num_lookahead_slots=k)
445
446
assert len(out) == 1, f"expected only one token output when {k=}"
447
assert out[0].probs is None, "expect gpu tensor references to be None"
@@ -443,7 +449,7 @@ def test_k_equals_zero(k: int, batch_size: int):
449
0].sampled_tokens is None, "expect gpu tensor references to be None"
450
451
draft_worker.execute_model.assert_called_once_with(
446
- **execute_model_data.to_dict(), return_python_output=False)
452
+ **execute_model_data.to_dict())
453
target_worker.execute_model.assert_called_once_with(
454
**execute_model_data.to_dict())
455
@@ -462,6 +468,8 @@ def test_empty_input_batch(k: int, batch_size: int):
468
rejection_sampler.token_id_dtype = torch.int64
469
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
470
471
+ target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
472
+
473
draft_worker.device = 'cuda'
474
target_worker.device = 'cuda'
475
@@ -474,7 +482,7 @@ def test_empty_input_batch(k: int, batch_size: int):
482
batch_size, k, prev_output_token_len=0)
483
484
out = worker.execute_model(**execute_model_data.to_dict(),
477
- num_spec_tokens=k)
485
+ num_lookahead_slots=k)
486
487
assert len(out) == 1, f"expected only one token output when {k=}"
488
assert out[0].probs is None, "expect gpu tensor references to be None"
@@ -482,7 +490,7 @@ def test_empty_input_batch(k: int, batch_size: int):
490
0].sampled_tokens is None, "expect gpu tensor references to be None"
491
492
draft_worker.execute_model.assert_called_once_with(
485
- **execute_model_data.to_dict(), return_python_output=False)
493
+ **execute_model_data.to_dict())
494
target_worker.execute_model.assert_called_once_with(
495
**execute_model_data.to_dict())
496
tests/spec_decode/utils.py
+1
-1
@@ -212,7 +212,7 @@ def create_sampler_output_list(
212
SequenceOutput(
213
output_token=token_id,
214
parent_seq_id=seq_ids[seq_index],
215
- logprobs={token_id: 0},
215
+ logprobs={token_id: Logprob(0)},
216
)
217
],
218
prompt_logprobs=None,
vllm/core/block/block_table.py
-1
@@ -104,7 +104,6 @@ class BlockTable:
104
token_ids (List[int]): The sequence of token IDs to be appended.
105
"""
106
assert self._is_allocated
107
- assert token_ids, "can't append empty token ids"
107
108
self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
109
num_lookahead_slots)
vllm/core/scheduler.py
+2
-6
@@ -762,9 +762,7 @@ class Scheduler:
762
blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
763
swapped_in.blocks_to_copy),
764
ignored_seq_groups=prefills.ignored_seq_groups,
765
- num_lookahead_slots=(prefills.num_lookahead_slots +
766
- running_scheduled.num_lookahead_slots +
767
- swapped_in.num_lookahead_slots),
765
+ num_lookahead_slots=running_scheduled.num_lookahead_slots,
766
)
767
768
def _schedule_chunked_prefill(self):
@@ -850,9 +848,7 @@ class Scheduler:
848
blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
849
swapped_in.blocks_to_copy),
850
ignored_seq_groups=prefills.ignored_seq_groups,
853
- num_lookahead_slots=(prefills.num_lookahead_slots +
854
- running_scheduled.num_lookahead_slots +
855
- swapped_in.num_lookahead_slots),
851
+ num_lookahead_slots=running_scheduled.num_lookahead_slots,
852
)
853
854
def _schedule(self) -> SchedulerOutputs:
vllm/engine/async_llm_engine.py
+3
-1
@@ -217,7 +217,9 @@ class _AsyncLLMEngine(LLMEngine):
217
else:
218
output = []
219
220
- return self._process_model_outputs(output, scheduler_outputs)
220
+ return self._process_model_outputs(
221
+ output, scheduler_outputs.scheduled_seq_groups,
222
+ scheduler_outputs.ignored_seq_groups)
223
224
async def encode_request_async(
225
self,
vllm/engine/llm_engine.py
+53
-318
@@ -1,5 +1,5 @@
1
import time
2
-from typing import Iterable, List, Optional, Tuple, Type, Union
2
+from typing import Iterable, List, Optional, Type, Union
3
4
from transformers import PreTrainedTokenizer
5
@@ -11,6 +11,10 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
11
from vllm.core.scheduler import Scheduler, SchedulerOutputs
12
from vllm.engine.arg_utils import EngineArgs
13
from vllm.engine.metrics import StatLogger, Stats
14
+from vllm.engine.output_processor.interfaces import (
15
+ SequenceGroupOutputProcessor)
16
+from vllm.engine.output_processor.stop_checker import StopChecker
17
+from vllm.engine.output_processor.util import create_output_by_sequence_group
18
from vllm.engine.ray_utils import initialize_ray_cluster
19
from vllm.executor.executor_base import ExecutorBase
20
from vllm.logger import init_logger
@@ -18,8 +22,7 @@ from vllm.lora.request import LoRARequest
22
from vllm.outputs import RequestOutput
23
from vllm.sampling_params import SamplingParams
24
from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
21
- SequenceGroup, SequenceGroupOutput, SequenceOutput,
22
- SequenceStatus)
25
+ SequenceGroup)
26
from vllm.transformers_utils.detokenizer import Detokenizer
27
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
28
get_tokenizer_group)
@@ -187,6 +190,21 @@ class LLMEngine:
190
labels=dict(model_name=model_config.model))
191
self.stat_logger.info("cache_config", self.cache_config)
192
193
+ # Create sequence output processor, e.g. for beam search or
194
+ # speculative decoding.
195
+ self.output_processor = (
196
+ SequenceGroupOutputProcessor.create_output_processor(
197
+ self.scheduler_config,
198
+ self.detokenizer,
199
+ self.scheduler,
200
+ self.seq_counter,
201
+ self.get_tokenizer_for_seq,
202
+ stop_checker=StopChecker(
203
+ self.scheduler_config.max_model_len,
204
+ self.get_tokenizer_for_seq,
205
+ ),
206
+ ))
207
+
208
def _initialize_kv_caches(self) -> None:
209
"""Initialize the KV cache in the worker(s).
210
@@ -412,240 +430,32 @@ class LLMEngine:
430
"""Returns True if there are unfinished requests."""
431
return self.scheduler.has_unfinished_seqs()
432
415
- def _check_beam_search_early_stopping(
416
- self,
417
- early_stopping: Union[bool, str],
418
- sampling_params: SamplingParams,
419
- best_running_seq: Sequence,
420
- current_worst_seq: Sequence,
421
- ) -> bool:
422
- assert sampling_params.use_beam_search
423
- length_penalty = sampling_params.length_penalty
424
- if early_stopping is True:
425
- return True
426
-
427
- current_worst_score = current_worst_seq.get_beam_search_score(
428
- length_penalty=length_penalty,
429
- eos_token_id=current_worst_seq.eos_token_id)
430
- if early_stopping is False:
431
- highest_attainable_score = best_running_seq.get_beam_search_score(
432
- length_penalty=length_penalty,
433
- eos_token_id=best_running_seq.eos_token_id)
434
- else:
435
- assert early_stopping == "never"
436
- if length_penalty > 0.0:
437
- # If length_penalty > 0.0, beam search will prefer longer
438
- # sequences. The highest attainable score calculation is
439
- # based on the longest possible sequence length in this case.
440
- max_possible_length = max(
441
- best_running_seq.get_prompt_len() +
442
- sampling_params.max_tokens,
443
- self.scheduler_config.max_model_len)
444
- highest_attainable_score = (
445
- best_running_seq.get_beam_search_score(
446
- length_penalty=length_penalty,
447
- eos_token_id=best_running_seq.eos_token_id,
448
- seq_len=max_possible_length))
449
- else:
450
- # Otherwise, beam search will prefer shorter sequences. The
451
- # highest attainable score calculation is based on the current
452
- # sequence length.
453
- highest_attainable_score = (
454
- best_running_seq.get_beam_search_score(
455
- length_penalty=length_penalty,
456
- eos_token_id=best_running_seq.eos_token_id))
457
- return current_worst_score >= highest_attainable_score
458
-
459
- def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
460
- outputs: SequenceGroupOutput) -> None:
461
-
462
- # Process prompt logprobs
463
- prompt_logprobs = outputs.prompt_logprobs
464
- if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
465
- self.detokenizer.decode_prompt_logprobs_inplace(
466
- seq_group, prompt_logprobs)
467
- seq_group.prompt_logprobs = prompt_logprobs
468
-
469
- # Process samples
470
- samples = outputs.samples
471
- parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
472
- existing_finished_seqs = seq_group.get_finished_seqs()
473
- parent_child_dict = {
474
- parent_seq.seq_id: []
475
- for parent_seq in parent_seqs
476
- }
477
- for sample in samples:
478
- parent_child_dict[sample.parent_seq_id].append(sample)
479
- # List of (child, parent)
480
- child_seqs: List[Tuple[Sequence, Sequence]] = []
481
-
482
- # Process the child samples for each parent sequence
483
- for parent in parent_seqs:
484
- child_samples: List[SequenceOutput] = parent_child_dict[
485
- parent.seq_id]
486
- if len(child_samples) == 0:
487
- # This parent sequence has no children samples. Remove
488
- # the parent sequence from the sequence group since it will
489
- # not be used in the future iterations.
490
- parent.status = SequenceStatus.FINISHED_ABORTED
491
- seq_group.remove(parent.seq_id)
492
- self.scheduler.free_seq(parent)
493
- continue
494
- # Fork the parent sequence if there are multiple child samples.
495
- for child_sample in child_samples[:-1]:
496
- new_child_seq_id = next(self.seq_counter)
497
- child = parent.fork(new_child_seq_id)
498
- child.append_token_id(child_sample.output_token,
499
- child_sample.logprobs)
500
- child_seqs.append((child, parent))
501
- # Continue the parent sequence for the last child sample.
502
- # We reuse the parent sequence here to reduce redundant memory
503
- # copies, especially when using non-beam search sampling methods.
504
- last_child_sample = child_samples[-1]
505
- parent.append_token_id(last_child_sample.output_token,
506
- last_child_sample.logprobs)
507
- child_seqs.append((parent, parent))
508
-
509
- for seq, _ in child_seqs:
510
- if seq_group.sampling_params.detokenize:
511
- new_char_count = self.detokenizer.decode_sequence_inplace(
512
- seq, seq_group.sampling_params)
513
- else:
514
- new_char_count = 0
515
- self._check_stop(seq, new_char_count, seq_group.sampling_params)
516
-
517
- # Non-beam search case
518
- if not seq_group.sampling_params.use_beam_search:
519
- # For newly created child sequences, add them to the sequence group
520
- # and fork them in block manager if they are not finished.
521
- for seq, parent in child_seqs:
522
- if seq is not parent:
523
- seq_group.add(seq)
524
- if not seq.is_finished():
525
- self.scheduler.fork_seq(parent, seq)
526
-
527
- # Free the finished and selected parent sequences' memory in block
528
- # manager. Keep them in the sequence group as candidate output.
529
- # NOTE: we need to fork the new sequences before freeing the
530
- # old sequences.
531
- for seq, parent in child_seqs:
532
- if seq is parent and seq.is_finished():
533
- self.scheduler.free_seq(seq)
534
- return
535
-
536
- # Beam search case
537
- # Select the child sequences to keep in the sequence group.
538
- selected_child_seqs = []
539
- unselected_child_seqs = []
540
- beam_width = seq_group.sampling_params.best_of
541
- length_penalty = seq_group.sampling_params.length_penalty
542
-
543
- # Select the newly finished sequences with the highest scores
544
- # to replace existing finished sequences.
545
- # Tuple of (seq, parent, is_new)
546
- existing_finished_seqs = [(seq, None, False)
547
- for seq in existing_finished_seqs]
548
- new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
549
- if seq.is_finished()]
550
- all_finished_seqs = existing_finished_seqs + new_finished_seqs
551
- # Sort the finished sequences by their scores.
552
- all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
553
- length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
554
- reverse=True)
555
- for seq, parent, is_new in all_finished_seqs[:beam_width]:
556
- if is_new:
557
- # A newly generated child sequence finishes and has a high
558
- # score, so we will add it into the sequence group.
559
- selected_child_seqs.append((seq, parent))
560
- for seq, parent, is_new in all_finished_seqs[beam_width:]:
561
- if is_new:
562
- # A newly generated child sequence finishes but has a low
563
- # score, so we will not add it into the sequence group.
564
- # Additionally, if this sequence is a continuation of a
565
- # parent sequence, we will need remove the parent sequence
566
- # from the sequence group.
567
- unselected_child_seqs.append((seq, parent))
568
- else:
569
- # An existing finished sequence has a low score, so we will
570
- # remove it from the sequence group.
571
- seq_group.remove(seq.seq_id)
572
-
573
- # select the top beam_width sequences from the running
574
- # sequences for the next iteration to continue the beam
575
- # search.
576
- running_child_seqs = [(seq, parent) for seq, parent in child_seqs
577
- if not seq.is_finished()]
578
- # Sort the running sequences by their scores.
579
- running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
580
- length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
581
- reverse=True)
582
-
583
- # Check if we can stop the beam search.
584
- if len(running_child_seqs) == 0:
585
- # No running sequences, stop the beam search.
586
- stop_beam_search = True
587
- elif len(all_finished_seqs) < beam_width:
588
- # Not enough finished sequences, continue the beam search.
589
- stop_beam_search = False
590
- else:
591
- # Check the early stopping criteria
592
- best_running_seq = running_child_seqs[0][0]
593
- current_worst_seq = all_finished_seqs[beam_width - 1][0]
594
- stop_beam_search = self._check_beam_search_early_stopping(
595
- seq_group.sampling_params.early_stopping,
596
- seq_group.sampling_params, best_running_seq, current_worst_seq)
597
-
598
- if stop_beam_search:
599
- # Stop the beam search and remove all the running sequences from
600
- # the sequence group.
601
- unselected_child_seqs.extend(running_child_seqs)
602
- else:
603
- # Continue the beam search and select the top beam_width sequences
604
- # to continue the beam search.
605
- selected_child_seqs.extend(running_child_seqs[:beam_width])
606
- # The remaining running sequences will not be used in the next
607
- # iteration. Again, if these sequences are continuations of
608
- # parent sequences, we will need to remove the parent sequences
609
- # from the sequence group.
610
- unselected_child_seqs.extend(running_child_seqs[beam_width:])
611
-
612
- # For newly created child sequences, add them to the sequence group
613
- # and fork them in block manager if they are not finished.
614
- for seq, parent in selected_child_seqs:
615
- if seq is not parent:
616
- seq_group.add(seq)
617
- if not seq.is_finished():
618
- self.scheduler.fork_seq(parent, seq)
619
-
620
- # Free the finished and selected parent sequences' memory in block
621
- # manager. Keep them in the sequence group as candidate output.
622
- for seq, parent in selected_child_seqs:
623
- if seq is parent and seq.is_finished():
624
- self.scheduler.free_seq(seq)
625
-
626
- # Remove the unselected parent sequences from the sequence group and
627
- # free their memory in block manager.
628
- for seq, parent in unselected_child_seqs:
629
- if seq is parent:
630
- # Remove the parent sequence if it is not selected for next
631
- # iteration
632
- seq_group.remove(seq.seq_id)
633
- self.scheduler.free_seq(seq)
634
-
433
def _process_model_outputs(
636
- self, output: SamplerOutput,
637
- scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
434
+ self, output: List[SamplerOutput],
435
+ scheduled_seq_groups: List[SequenceGroup],
436
+ ignored_seq_groups: List[SequenceGroup]) -> List[RequestOutput]:
437
+ """Apply the model output to the sequences in the scheduled seq groups.
438
+
439
+ Returns RequestOutputs that can be returned to the client.
440
+ """
441
+
442
now = time.time()
443
+
444
+ # Organize outputs by [sequence group][step] instead of
445
+ # [step][sequence group].
446
+ output_by_sequence_group = create_output_by_sequence_group(
447
+ sampler_outputs=output, num_seq_groups=len(scheduled_seq_groups))
448
+
449
# Update the scheduled sequence groups with the model outputs.
640
- scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
641
- for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
450
+ for scheduled_seq_group, outputs in zip(scheduled_seq_groups,
451
+ output_by_sequence_group):
452
seq_group = scheduled_seq_group.seq_group
453
seq_group.update_num_computed_tokens(
454
scheduled_seq_group.token_chunk_size)
455
# If uncomputed tokens > 0, it means prefill is chunked.
456
# We don't need to process outputs in that case.
457
if seq_group.get_num_uncomputed_tokens() == 0:
648
- self._process_sequence_group_outputs(seq_group, outputs)
458
+ self.output_processor.process_outputs(seq_group, outputs)
459
460
# Free the finished sequence groups.
461
self.scheduler.free_finished_seq_groups()
@@ -657,13 +467,9 @@ class LLMEngine:
467
seq_group.maybe_set_first_token_time(now)
468
request_output = RequestOutput.from_seq_group(seq_group)
469
request_outputs.append(request_output)
660
- for seq_group in scheduler_outputs.ignored_seq_groups:
470
+ for seq_group in ignored_seq_groups:
471
request_output = RequestOutput.from_seq_group(seq_group)
472
request_outputs.append(request_output)
663
-
664
- # Log stats.
665
- if self.log_stats:
666
- self.stat_logger.log(self._get_stats(scheduler_outputs))
473
return request_outputs
474
475
def step(self) -> List[RequestOutput]:
@@ -721,13 +527,23 @@ class LLMEngine:
527
528
if not scheduler_outputs.is_empty():
529
output = self.model_executor.execute_model(
724
- seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
725
- scheduler_outputs.blocks_to_swap_out,
726
- scheduler_outputs.blocks_to_copy)
530
+ seq_group_metadata_list=seq_group_metadata_list,
531
+ blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
532
+ blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
533
+ blocks_to_copy=scheduler_outputs.blocks_to_copy,
534
+ num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
535
else:
536
output = []
537
730
- return self._process_model_outputs(output, scheduler_outputs)
538
+ request_outputs = self._process_model_outputs(
539
+ output, scheduler_outputs.scheduled_seq_groups,
540
+ scheduler_outputs.ignored_seq_groups)
541
+
542
+ # Log stats.
543
+ if self.log_stats:
544
+ self.stat_logger.log(self._get_stats(scheduler_outputs))
545
+
546
+ return request_outputs
547
548
def do_log_stats(self) -> None:
549
"""Forced log when no requests active."""
@@ -807,87 +623,6 @@ class LLMEngine:
623
time_e2e_requests=time_e2e_requests,
624
)
625
810
- def _check_stop(self, seq: Sequence, new_char_count: int,
811
- sampling_params: SamplingParams) -> None:
812
- """Stop the finished sequences.
813
-
814
- new_char_count is the number of chars added to the
815
- sequence's output text for the newly generated token
816
- """
817
-
818
- # Check if the minimum number of tokens has been generated yet;
819
- # skip the stop string/token checks if not
820
- if seq.get_output_len() < sampling_params.min_tokens:
821
- return
822
-
823
- # Check if the sequence has generated the EOS token.
824
- if ((not sampling_params.ignore_eos)
825
- and seq.get_last_token_id() == seq.eos_token_id):
826
- seq.status = SequenceStatus.FINISHED_STOPPED
827
- return
828
-
829
- # Check if a stop token was encountered.
830
- # This assumes a single token produced per step.
831
- last_token_id = seq.get_last_token_id()
832
- if last_token_id in sampling_params.stop_token_ids:
833
- if new_char_count and (
834
- not sampling_params.include_stop_str_in_output):
835
- # Remove last token
836
- seq.output_text = seq.output_text[:-new_char_count]
837
- seq.status = SequenceStatus.FINISHED_STOPPED
838
- seq.stop_reason = last_token_id
839
- return
840
-
841
- # Check if any stop strings are matched.
842
- stop_str = self._check_stop_strings(seq, new_char_count,
843
- sampling_params)
844
- if stop_str is not None:
845
- seq.status = SequenceStatus.FINISHED_STOPPED
846
- seq.stop_reason = stop_str
847
- return
848
-
849
- # Check if the sequence has reached max_model_len.
850
- if seq.get_len() > self.scheduler_config.max_model_len:
851
- seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
852
- return
853
-
854
- # Check if the sequence has reached max_tokens.
855
- if seq.get_output_len() == sampling_params.max_tokens:
856
- seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
857
- return
858
-
859
- @staticmethod
860
- def _check_stop_strings(seq: Sequence, new_char_count: int,
861
- sampling_params: SamplingParams) -> Optional[str]:
862
- """Check if any stop strings are matched and truncate sequence
863
- output text accordingly.
864
-
865
- Returns the stop string if matched or else None.
866
- """
867
- if not new_char_count:
868
- return None
869
-
870
- for stop_str in sampling_params.stop:
871
- stop_string_len = len(stop_str)
872
- # Avoid searching already-searched text.
873
- stop_index = seq.output_text.find(
874
- stop_str, -new_char_count - stop_string_len)
875
- if stop_index == -1:
876
- continue
877
-
878
- if sampling_params.include_stop_str_in_output:
879
- # Truncate to end of stop string.
880
- stop_index += stop_string_len
881
- if stop_index >= len(seq.output_text):
882
- # No truncation required.
883
- return stop_str
884
-
885
- # Truncate the output text to either the beginning
886
- # or end of the stop string.
887
- seq.output_text = seq.output_text[:stop_index]
888
- return stop_str
889
- return None
890
-
626
def add_lora(self, lora_request: LoRARequest) -> bool:
627
return self.model_executor.add_lora(lora_request)
628
vllm/engine/output_processor/__init__.py
vllm/engine/output_processor/interfaces.py
new
+69
@@ -0,0 +1,69 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Callable, Iterable, List
3
+
4
+from transformers import PreTrainedTokenizer
5
+
6
+from vllm.config import SchedulerConfig
7
+from vllm.core.scheduler import Scheduler
8
+from vllm.engine.output_processor.stop_checker import StopChecker
9
+from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
10
+from vllm.transformers_utils.detokenizer import Detokenizer
11
+
12
+
13
+class SequenceGroupOutputProcessor(ABC):
14
+ """Interface for logic that processes new token ids in sequence groups,
15
+ managing detokenization, stop checking, and freeing/forking sequences with
16
+ the scheduler.
17
+
18
+ This is highly coupled with the LLMEngine and should be seen as an extension
19
+ of it. The logic is separated to simplify the LLMEngine class and allow
20
+ separate implementations for single-step decoding (which supports beam
21
+ search sequence forking) and multi-step decoding (which does not support
22
+ beam search, but does support speculative decoding).
23
+ """
24
+
25
+ @staticmethod
26
+ def create_output_processor(
27
+ scheduler_config: SchedulerConfig,
28
+ detokenizer: Detokenizer,
29
+ scheduler: Scheduler,
30
+ seq_counter: Iterable[int],
31
+ get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
32
+ stop_checker: "StopChecker",
33
+ ):
34
+ """Create an output processor.
35
+
36
+ This returns a single-step output processor if num_lookahead_slots is
37
+ zero, else returns a multi-step output processor.
38
+ """
39
+ if scheduler_config.num_lookahead_slots == 0:
40
+ # Importing here to avoid cycle.
41
+ from vllm.engine.output_processor.single_step import (
42
+ SingleStepOutputProcessor)
43
+ return SingleStepOutputProcessor(
44
+ scheduler_config,
45
+ detokenizer,
46
+ scheduler,
47
+ seq_counter,
48
+ stop_checker,
49
+ )
50
+ else:
51
+ # Importing here to avoid cycle.
52
+ from vllm.engine.output_processor.multi_step import (
53
+ MultiStepOutputProcessor)
54
+ return MultiStepOutputProcessor(
55
+ detokenizer,
56
+ scheduler,
57
+ seq_counter,
58
+ get_tokenizer_for_seq,
59
+ stop_checker,
60
+ )
61
+
62
+ @abstractmethod
63
+ def process_outputs(self, sequence_group: SequenceGroup,
64
+ outputs: List[SequenceGroupOutput]) -> None:
65
+ """Process new token ids for the sequence group. Handles logic such as
66
+ detokenization, stop checking, and freeing/forking sequences in the
67
+ scheduler.
68
+ """
69
+ pass
vllm/engine/output_processor/multi_step.py
new
+126
@@ -0,0 +1,126 @@
1
+from typing import Callable, Iterable, List
2
+
3
+from transformers import PreTrainedTokenizer
4
+
5
+from vllm.core.scheduler import Scheduler
6
+from vllm.engine.output_processor.interfaces import (
7
+ SequenceGroupOutputProcessor)
8
+from vllm.engine.output_processor.stop_checker import StopChecker
9
+from vllm.logger import init_logger
10
+from vllm.sampling_params import SamplingParams
11
+from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12
+ SequenceGroupOutput, SequenceOutput, SequenceStatus)
13
+from vllm.transformers_utils.detokenizer import Detokenizer
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
19
+ """SequenceGroupOutputProcessor which handles logic related to
20
+ detokenization and stopping conditions. It specializes to "multi-step
21
+ decoding", where vLLM's worker may generate multiple tokens per invocation.
22
+ This is currently mutually exclusive with advanced sampling techniques like
23
+ beam search, which motivates the separation of this logic from the single
24
+ step output processor.
25
+
26
+ This class is responsible for things such as correctly appending all new
27
+ token ids to their sequence, detokenizing new token ids, truncating new
28
+ output tokens after an eos token, and correctly handling the case where the
29
+ number of new output tokens per sequence differs in a single batch.
30
+ """
31
+
32
+ def __init__(
33
+ self,
34
+ detokenizer: Detokenizer,
35
+ scheduler: Scheduler,
36
+ seq_counter: Iterable[int],
37
+ get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
38
+ stop_checker: StopChecker,
39
+ ):
40
+ self.detokenizer = detokenizer
41
+ self.scheduler = scheduler
42
+ self.seq_counter = seq_counter
43
+ self.get_tokenizer_for_seq = get_tokenizer_for_seq
44
+ self.stop_checker = stop_checker
45
+
46
+ def process_outputs(self, sequence_group: SequenceGroup,
47
+ outputs: List[SequenceGroupOutput]) -> None:
48
+ """Append new tokens in the outputs to sequences in the sequence group.
49
+
50
+ This only supports sequence groups of size 1. It supports greater than
51
+ one new token per sequence.
52
+
53
+ This applies logic like stop condition checking and detokenization,
54
+ including freeing finished sequences. It also handles cases where there
55
+ are tokens emitted after the EOS token.
56
+ """
57
+ seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)
58
+
59
+ assert seqs, "expected running sequences"
60
+ assert len(seqs) == 1, (
61
+ "Beam search not supported in multi-step decoding.")
62
+ seq = seqs[0]
63
+
64
+ # Since there's only one sequence per sequence group, we can take the
65
+ # first sample.
66
+ samples = [outputs[step].samples[0] for step in range(len(outputs))]
67
+
68
+ # -1 means the output token is not valid (eg. due to spec decode
69
+ # rejecting tokens).
70
+ valid_samples = [
71
+ sample for sample in samples if sample.output_token != -1
72
+ ]
73
+ assert valid_samples
74
+
75
+ self._process_seq_outputs(seq, valid_samples,
76
+ sequence_group.sampling_params)
77
+
78
+ def _process_seq_outputs(self, seq: Sequence,
79
+ valid_samples: List[SequenceOutput],
80
+ sampling_params: SamplingParams) -> None:
81
+ output_token_ids = [sample.output_token for sample in valid_samples]
82
+
83
+ # Truncate to max_tokens if necessary.
84
+ remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() +
85
+ len(output_token_ids))
86
+ if remaining_tokens < 0:
87
+ valid_samples = valid_samples[:remaining_tokens]
88
+ output_token_ids = output_token_ids[:remaining_tokens]
89
+
90
+ # Truncate any tokens after EOS. This is required as spec decode
91
+ # generates a fixed number of tokens without evaluating stopping
92
+ # conditions within the block. This can cause an eos token to be
93
+ # unintentionally ignored.
94
+ if not sampling_params.ignore_eos:
95
+ eos_token_id = self.get_tokenizer_for_seq(seq).eos_token_id
96
+ # Avoiding .index calls as exception throwing in the happy path
97
+ # is expensive.
98
+ for i in range(len(output_token_ids)):
99
+ if output_token_ids[i] == eos_token_id:
100
+ output_token_ids = output_token_ids[:i + 1]
101
+ valid_samples = valid_samples[:i + 1]
102
+ break
103
+
104
+ # Incrementally append tokens to the sequence, as if we had only one new
105
+ # token.
106
+ for output_token_id in output_token_ids:
107
+ seq.append_token_id(
108
+ token_id=output_token_id,
109
+ # TODO emit logprobs in multi-step decoding.
110
+ logprobs={output_token_id: Logprob(0.0)},
111
+ )
112
+
113
+ new_char_count = 0
114
+ if sampling_params.detokenize:
115
+ new_char_count = self.detokenizer.decode_sequence_inplace(
116
+ seq, sampling_params)
117
+
118
+ self.stop_checker.maybe_stop_sequence(
119
+ seq,
120
+ new_char_count=new_char_count,
121
+ sampling_params=sampling_params)
122
+ if seq.is_finished():
123
+ break
124
+
125
+ if seq.is_finished():
126
+ self.scheduler.free_seq(seq)
vllm/engine/output_processor/single_step.py
new
+276
@@ -0,0 +1,276 @@
1
+from typing import Iterable, List, Tuple, Union
2
+
3
+from vllm.config import SchedulerConfig
4
+from vllm.core.scheduler import Scheduler
5
+from vllm.engine.output_processor.interfaces import (
6
+ SequenceGroupOutputProcessor)
7
+from vllm.engine.output_processor.stop_checker import StopChecker
8
+from vllm.logger import init_logger
9
+from vllm.sampling_params import SamplingParams
10
+from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
11
+ SequenceOutput, SequenceStatus)
12
+from vllm.transformers_utils.detokenizer import Detokenizer
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
18
+ """SequenceGroupOutputProcessor which handles "output processing" logic,
19
+ which happens after the model returns generated token ids and before
20
+ scheduling of the next batch. Output processing logic includes
21
+ detokenization, and determining if a sequence is finished (e.g. via max len
22
+ or eos token).
23
+
24
+ The SingleStepOutputProcessor is specialized to the case where the model
25
+ emits at most a single token per invocation, which precludes configurations
26
+ such as speculative decoding or multi-step decoding. This enables beam
27
+ search sampling, which requires forking/finishing/freeing sequences in a way
28
+ that is currently difficult to schedule multiple steps ahead of time.
29
+ """
30
+
31
+ def __init__(
32
+ self,
33
+ scheduler_config: SchedulerConfig,
34
+ detokenizer: Detokenizer,
35
+ scheduler: Scheduler,
36
+ seq_counter: Iterable[int],
37
+ stop_checker: StopChecker,
38
+ ):
39
+ self.scheduler_config = scheduler_config
40
+ self.detokenizer = detokenizer
41
+ self.scheduler = scheduler
42
+ self.seq_counter = seq_counter
43
+ self.stop_checker = stop_checker
44
+
45
+ def process_outputs(self, sequence_group: SequenceGroup,
46
+ outputs: List[SequenceGroupOutput]) -> None:
47
+ """Append all new tokens to sequences in the sequence group. Fork any
48
+ surviving beam candidates; free any unsurviving ones.
49
+
50
+ Invokes detokenizer to detokenize new tokens, and also marks sequences
51
+ as finished if they meet stop conditions.
52
+ """
53
+ assert (len(outputs) == 1
54
+ ), f"{type(self)} does not support multiple outputs per step"
55
+ return self._process_sequence_group_outputs(sequence_group, outputs[0])
56
+
57
+ def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
58
+ outputs: SequenceGroupOutput) -> None:
59
+
60
+ # Process prompt logprobs
61
+ prompt_logprobs = outputs.prompt_logprobs
62
+ if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
63
+ self.detokenizer.decode_prompt_logprobs_inplace(
64
+ seq_group, prompt_logprobs)
65
+ seq_group.prompt_logprobs = prompt_logprobs
66
+
67
+ # Process samples
68
+ samples = outputs.samples
69
+ parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
70
+ existing_finished_seqs = seq_group.get_finished_seqs()
71
+ parent_child_dict = {
72
+ parent_seq.seq_id: []
73
+ for parent_seq in parent_seqs
74
+ }
75
+ for sample in samples:
76
+ parent_child_dict[sample.parent_seq_id].append(sample)
77
+ # List of (child, parent)
78
+ child_seqs: List[Tuple[Sequence, Sequence]] = []
79
+
80
+ # Process the child samples for each parent sequence
81
+ for parent in parent_seqs:
82
+ child_samples: List[SequenceOutput] = parent_child_dict[
83
+ parent.seq_id]
84
+ if len(child_samples) == 0:
85
+ # This parent sequence has no children samples. Remove
86
+ # the parent sequence from the sequence group since it will
87
+ # not be used in the future iterations.
88
+ parent.status = SequenceStatus.FINISHED_ABORTED
89
+ seq_group.remove(parent.seq_id)
90
+ self.scheduler.free_seq(parent)
91
+ continue
92
+ # Fork the parent sequence if there are multiple child samples.
93
+ for child_sample in child_samples[:-1]:
94
+ new_child_seq_id = next(self.seq_counter)
95
+ child = parent.fork(new_child_seq_id)
96
+ child.append_token_id(child_sample.output_token,
97
+ child_sample.logprobs)
98
+ child_seqs.append((child, parent))
99
+ # Continue the parent sequence for the last child sample.
100
+ # We reuse the parent sequence here to reduce redundant memory
101
+ # copies, especially when using non-beam search sampling methods.
102
+ last_child_sample = child_samples[-1]
103
+ parent.append_token_id(last_child_sample.output_token,
104
+ last_child_sample.logprobs)
105
+ child_seqs.append((parent, parent))
106
+
107
+ for seq, _ in child_seqs:
108
+ if seq_group.sampling_params.detokenize:
109
+ new_char_count = self.detokenizer.decode_sequence_inplace(
110
+ seq, seq_group.sampling_params)
111
+ else:
112
+ new_char_count = 0
113
+ self.stop_checker.maybe_stop_sequence(seq, new_char_count,
114
+ seq_group.sampling_params)
115
+
116
+ # Non-beam search case
117
+ if not seq_group.sampling_params.use_beam_search:
118
+ # For newly created child sequences, add them to the sequence group
119
+ # and fork them in block manager if they are not finished.
120
+ for seq, parent in child_seqs:
121
+ if seq is not parent:
122
+ seq_group.add(seq)
123
+ if not seq.is_finished():
124
+ self.scheduler.fork_seq(parent, seq)
125
+
126
+ # Free the finished and selected parent sequences' memory in block
127
+ # manager. Keep them in the sequence group as candidate output.
128
+ # NOTE: we need to fork the new sequences before freeing the
129
+ # old sequences.
130
+ for seq, parent in child_seqs:
131
+ if seq is parent and seq.is_finished():
132
+ self.scheduler.free_seq(seq)
133
+ return
134
+
135
+ # Beam search case
136
+ # Select the child sequences to keep in the sequence group.
137
+ selected_child_seqs = []
138
+ unselected_child_seqs = []
139
+ beam_width = seq_group.sampling_params.best_of
140
+ length_penalty = seq_group.sampling_params.length_penalty
141
+
142
+ # Select the newly finished sequences with the highest scores
143
+ # to replace existing finished sequences.
144
+ # Tuple of (seq, parent, is_new)
145
+ existing_finished_seqs = [(seq, None, False)
146
+ for seq in existing_finished_seqs]
147
+ new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
148
+ if seq.is_finished()]
149
+ all_finished_seqs = existing_finished_seqs + new_finished_seqs
150
+ # Sort the finished sequences by their scores.
151
+ all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
152
+ length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
153
+ reverse=True)
154
+ for seq, parent, is_new in all_finished_seqs[:beam_width]:
155
+ if is_new:
156
+ # A newly generated child sequence finishes and has a high
157
+ # score, so we will add it into the sequence group.
158
+ selected_child_seqs.append((seq, parent))
159
+ for seq, parent, is_new in all_finished_seqs[beam_width:]:
160
+ if is_new:
161
+ # A newly generated child sequence finishes but has a low
162
+ # score, so we will not add it into the sequence group.
163
+ # Additionally, if this sequence is a continuation of a
164
+ # parent sequence, we will need remove the parent sequence
165
+ # from the sequence group.
166
+ unselected_child_seqs.append((seq, parent))
167
+ else:
168
+ # An existing finished sequence has a low score, so we will
169
+ # remove it from the sequence group.
170
+ seq_group.remove(seq.seq_id)
171
+
172
+ # select the top beam_width sequences from the running
173
+ # sequences for the next iteration to continue the beam
174
+ # search.
175
+ running_child_seqs = [(seq, parent) for seq, parent in child_seqs
176
+ if not seq.is_finished()]
177
+ # Sort the running sequences by their scores.
178
+ running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
179
+ length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
180
+ reverse=True)
181
+
182
+ # Check if we can stop the beam search.
183
+ if len(running_child_seqs) == 0:
184
+ # No running sequences, stop the beam search.
185
+ stop_beam_search = True
186
+ elif len(all_finished_seqs) < beam_width:
187
+ # Not enough finished sequences, continue the beam search.
188
+ stop_beam_search = False
189
+ else:
190
+ # Check the early stopping criteria
191
+ best_running_seq = running_child_seqs[0][0]
192
+ current_worst_seq = all_finished_seqs[beam_width - 1][0]
193
+ stop_beam_search = self._check_beam_search_early_stopping(
194
+ seq_group.sampling_params.early_stopping,
195
+ seq_group.sampling_params, best_running_seq, current_worst_seq)
196
+
197
+ if stop_beam_search:
198
+ # Stop the beam search and remove all the running sequences from
199
+ # the sequence group.
200
+ unselected_child_seqs.extend(running_child_seqs)
201
+ else:
202
+ # Continue the beam search and select the top beam_width sequences
203
+ # to continue the beam search.
204
+ selected_child_seqs.extend(running_child_seqs[:beam_width])
205
+ # The remaining running sequences will not be used in the next
206
+ # iteration. Again, if these sequences are continuations of
207
+ # parent sequences, we will need to remove the parent sequences
208
+ # from the sequence group.
209
+ unselected_child_seqs.extend(running_child_seqs[beam_width:])
210
+
211
+ # For newly created child sequences, add them to the sequence group
212
+ # and fork them in block manager if they are not finished.
213
+ for seq, parent in selected_child_seqs:
214
+ if seq is not parent:
215
+ seq_group.add(seq)
216
+ if not seq.is_finished():
217
+ self.scheduler.fork_seq(parent, seq)
218
+
219
+ # Free the finished and selected parent sequences' memory in block
220
+ # manager. Keep them in the sequence group as candidate output.
221
+ for seq, parent in selected_child_seqs:
222
+ if seq is parent and seq.is_finished():
223
+ self.scheduler.free_seq(seq)
224
+
225
+ # Remove the unselected parent sequences from the sequence group and
226
+ # free their memory in block manager.
227
+ for seq, parent in unselected_child_seqs:
228
+ if seq is parent:
229
+ # Remove the parent sequence if it is not selected for next
230
+ # iteration
231
+ seq_group.remove(seq.seq_id)
232
+ self.scheduler.free_seq(seq)
233
+
234
+ def _check_beam_search_early_stopping(
235
+ self,
236
+ early_stopping: Union[bool, str],
237
+ sampling_params: SamplingParams,
238
+ best_running_seq: Sequence,
239
+ current_worst_seq: Sequence,
240
+ ) -> bool:
241
+ assert sampling_params.use_beam_search
242
+ length_penalty = sampling_params.length_penalty
243
+ if early_stopping is True:
244
+ return True
245
+
246
+ current_worst_score = current_worst_seq.get_beam_search_score(
247
+ length_penalty=length_penalty,
248
+ eos_token_id=current_worst_seq.eos_token_id)
249
+ if early_stopping is False:
250
+ highest_attainable_score = best_running_seq.get_beam_search_score(
251
+ length_penalty=length_penalty,
252
+ eos_token_id=best_running_seq.eos_token_id)
253
+ else:
254
+ assert early_stopping == "never"
255
+ if length_penalty > 0.0:
256
+ # If length_penalty > 0.0, beam search will prefer longer
257
+ # sequences. The highest attainable score calculation is
258
+ # based on the longest possible sequence length in this case.
259
+ max_possible_length = max(
260
+ best_running_seq.get_prompt_len() +
261
+ sampling_params.max_tokens,
262
+ self.scheduler_config.max_model_len)
263
+ highest_attainable_score = (
264
+ best_running_seq.get_beam_search_score(
265
+ length_penalty=length_penalty,
266
+ eos_token_id=best_running_seq.eos_token_id,
267
+ seq_len=max_possible_length))
268
+ else:
269
+ # Otherwise, beam search will prefer shorter sequences. The
270
+ # highest attainable score calculation is based on the current
271
+ # sequence length.
272
+ highest_attainable_score = (
273
+ best_running_seq.get_beam_search_score(
274
+ length_penalty=length_penalty,
275
+ eos_token_id=best_running_seq.eos_token_id))
276
+ return current_worst_score >= highest_attainable_score
vllm/engine/output_processor/stop_checker.py
new
+101
@@ -0,0 +1,101 @@
1
+from typing import Callable, Optional
2
+
3
+from transformers import PreTrainedTokenizer
4
+
5
+from vllm.sampling_params import SamplingParams
6
+from vllm.sequence import Sequence, SequenceStatus
7
+
8
+
9
+class StopChecker:
10
+ """LLMEngine helper class which separates out the logic involving stop
11
+ checking. This checks things such as: whether the eos token was emitted,
12
+ whether the max_tokens has been consumed, whether a stop string has been
13
+ emitted, or if we have exceeded the max model len.
14
+ """
15
+
16
+ def __init__(self, max_model_len: int,
17
+ get_tokenizer_for_seq: Callable[[Sequence],
18
+ PreTrainedTokenizer]):
19
+ self.max_model_len = max_model_len
20
+ self.get_tokenizer_for_seq = get_tokenizer_for_seq
21
+
22
+ def maybe_stop_sequence(self, seq: Sequence, new_char_count: int,
23
+ sampling_params: SamplingParams) -> None:
24
+ """Stop the finished sequences.
25
+
26
+ new_char_count is the number of chars added to the
27
+ sequence's output text for the newly generated token
28
+ """
29
+
30
+ # Check if the minimum number of tokens has been generated yet;
31
+ # skip the stop string/token checks if not
32
+ if seq.get_output_len() < sampling_params.min_tokens:
33
+ return
34
+
35
+ # Check if the sequence has generated the EOS token.
36
+ if ((not sampling_params.ignore_eos)
37
+ and seq.get_last_token_id() == seq.eos_token_id):
38
+ seq.status = SequenceStatus.FINISHED_STOPPED
39
+ return
40
+
41
+ # Check if a stop token was encountered.
42
+ # This assumes a single token produced per step.
43
+ last_token_id = seq.get_last_token_id()
44
+ if last_token_id in sampling_params.stop_token_ids:
45
+ if new_char_count and (
46
+ not sampling_params.include_stop_str_in_output):
47
+ # Remove last token
48
+ seq.output_text = seq.output_text[:-new_char_count]
49
+ seq.status = SequenceStatus.FINISHED_STOPPED
50
+ seq.stop_reason = last_token_id
51
+ return
52
+
53
+ # Check if any stop strings are matched.
54
+ stop_str = self._check_stop_strings(seq, new_char_count,
55
+ sampling_params)
56
+ if stop_str is not None:
57
+ seq.status = SequenceStatus.FINISHED_STOPPED
58
+ seq.stop_reason = stop_str
59
+ return
60
+
61
+ # Check if the sequence has reached max_model_len.
62
+ if seq.get_len() > self.max_model_len:
63
+ seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
64
+ return
65
+
66
+ # Check if the sequence has reached max_tokens.
67
+ if seq.get_output_len() == sampling_params.max_tokens:
68
+ seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
69
+ return
70
+
71
+ @staticmethod
72
+ def _check_stop_strings(seq: Sequence, new_char_count: int,
73
+ sampling_params: SamplingParams) -> Optional[str]:
74
+ """Check if any stop strings are matched and truncate sequence
75
+ output text accordingly.
76
+
77
+ Returns the stop string if matched or else None.
78
+ """
79
+ if not new_char_count:
80
+ return None
81
+
82
+ for stop_str in sampling_params.stop:
83
+ stop_string_len = len(stop_str)
84
+ # Avoid searching already-searched text.
85
+ stop_index = seq.output_text.find(
86
+ stop_str, -new_char_count - stop_string_len)
87
+ if stop_index == -1:
88
+ continue
89
+
90
+ if sampling_params.include_stop_str_in_output:
91
+ # Truncate to end of stop string.
92
+ stop_index += stop_string_len
93
+ if stop_index >= len(seq.output_text):
94
+ # No truncation required.
95
+ return stop_str
96
+
97
+ # Truncate the output text to either the beginning
98
+ # or end of the stop string.
99
+ seq.output_text = seq.output_text[:stop_index]
100
+ return stop_str
101
+ return None
vllm/engine/output_processor/util.py
new
+16
@@ -0,0 +1,16 @@
1
+from typing import List
2
+
3
+from vllm.sequence import SamplerOutput
4
+
5
+
6
+def create_output_by_sequence_group(sampler_outputs: List[SamplerOutput],
7
+ num_seq_groups: int):
8
+ """Helper method which transforms a 2d list organized by
9
+ [step][sequence group] into [sequence group][step].
10
+ """
11
+ output_by_sequence_group = [[] for _ in range(num_seq_groups)]
12
+ for step in sampler_outputs:
13
+ for i, sequence_group_output in enumerate(step):
14
+ output_by_sequence_group[i].append(sequence_group_output)
15
+
16
+ return output_by_sequence_group
vllm/executor/cpu_executor.py
+2
-1
@@ -74,7 +74,8 @@ class CPUExecutor(ExecutorBase):
74
seq_group_metadata_list: List[SequenceGroupMetadata],
75
blocks_to_swap_in: Dict[int, int],
76
blocks_to_swap_out: Dict[int, int],
77
- blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
77
+ blocks_to_copy: Dict[int, List[int]],
78
+ num_lookahead_slots: int) -> List[SamplerOutput]:
79
output = self.driver_worker.execute_model(
80
seq_group_metadata_list=seq_group_metadata_list,
81
blocks_to_swap_in=blocks_to_swap_in,
vllm/executor/executor_base.py
+3
-2
@@ -72,8 +72,9 @@ class ExecutorBase(ABC):
72
seq_group_metadata_list: List[SequenceGroupMetadata],
73
blocks_to_swap_in: Dict[int, int],
74
blocks_to_swap_out: Dict[int, int],
75
- blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
76
- """Executes one model step on the given sequences."""
75
+ blocks_to_copy: Dict[int, List[int]],
76
+ num_lookahead_slots: int) -> List[SamplerOutput]:
77
+ """Executes at least one model step on the given sequences."""
78
raise NotImplementedError
79
80
@abstractmethod
vllm/executor/gpu_executor.py
+69
-10
@@ -13,13 +13,17 @@ logger = init_logger(__name__)
13
class GPUExecutor(ExecutorBase):
14
15
def _init_executor(self) -> None:
16
- assert (not self.speculative_config
17
- ), "Speculative decoding not yet supported for GPU backend"
16
+ """Initialize the worker and load the model.
17
19
- # Instantiate the worker and load the model to GPU.
20
- self._init_worker()
18
+ If speculative decoding is enabled, we instead create the speculative
19
+ worker.
20
+ """
21
+ if self.speculative_config is None:
22
+ self._init_non_spec_worker()
23
+ else:
24
+ self._init_spec_worker()
25
22
- def _init_worker(self):
26
+ def _init_non_spec_worker(self):
27
# Lazy import the Worker to avoid importing torch.cuda/xformers
28
# before CUDA_VISIBLE_DEVICES is set in the Worker
29
from vllm.worker.worker import Worker
@@ -46,6 +50,57 @@ class GPUExecutor(ExecutorBase):
50
self.driver_worker.init_device()
51
self.driver_worker.load_model()
52
53
+ def _init_spec_worker(self):
54
+ """Initialize a SpecDecodeWorker, using a draft model for proposals.
55
+ """
56
+ assert self.speculative_config is not None
57
+
58
+ from vllm.spec_decode.multi_step_worker import MultiStepWorker
59
+ from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
60
+ from vllm.worker.worker import Worker
61
+
62
+ distributed_init_method = get_distributed_init_method(
63
+ get_ip(), get_open_port())
64
+
65
+ target_worker = Worker(
66
+ model_config=self.model_config,
67
+ parallel_config=self.parallel_config,
68
+ scheduler_config=self.scheduler_config,
69
+ device_config=self.device_config,
70
+ cache_config=self.cache_config,
71
+ local_rank=0,
72
+ rank=0,
73
+ distributed_init_method=distributed_init_method,
74
+ lora_config=self.lora_config,
75
+ vision_language_config=self.vision_language_config,
76
+ is_driver_worker=True,
77
+ )
78
+
79
+ draft_worker = MultiStepWorker(
80
+ model_config=self.speculative_config.draft_model_config,
81
+ parallel_config=self.speculative_config.draft_parallel_config,
82
+ scheduler_config=self.scheduler_config,
83
+ device_config=self.device_config,
84
+ cache_config=self.cache_config,
85
+ local_rank=0,
86
+ rank=0,
87
+ distributed_init_method=distributed_init_method,
88
+ lora_config=self.lora_config,
89
+ vision_language_config=self.vision_language_config,
90
+ is_driver_worker=True,
91
+ )
92
+
93
+ spec_decode_worker = SpecDecodeWorker.from_workers(
94
+ proposer_worker=draft_worker, scorer_worker=target_worker)
95
+
96
+ assert self.parallel_config.world_size == 1, (
97
+ "GPUExecutor only supports single GPU.")
98
+
99
+ self.driver_worker = spec_decode_worker
100
+
101
+ # Load model handled in spec decode worker.
102
+ self.driver_worker.init_device()
103
+
104
def determine_num_available_blocks(self) -> Tuple[int, int]:
105
"""Determine the number of available KV blocks by invoking the
106
underlying worker.
@@ -63,16 +118,20 @@ class GPUExecutor(ExecutorBase):
118
119
self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
120
66
- def execute_model(self,
67
- seq_group_metadata_list: List[SequenceGroupMetadata],
68
- blocks_to_swap_in: Dict[int, int],
69
- blocks_to_swap_out: Dict[int, int],
70
- blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
121
+ def execute_model(
122
+ self,
123
+ seq_group_metadata_list: List[SequenceGroupMetadata],
124
+ blocks_to_swap_in: Dict[int, int],
125
+ blocks_to_swap_out: Dict[int, int],
126
+ blocks_to_copy: Dict[int, List[int]],
127
+ num_lookahead_slots: int,
128
+ ) -> List[SamplerOutput]:
129
output = self.driver_worker.execute_model(
130
seq_group_metadata_list=seq_group_metadata_list,
131
blocks_to_swap_in=blocks_to_swap_in,
132
blocks_to_swap_out=blocks_to_swap_out,
133
blocks_to_copy=blocks_to_copy,
134
+ num_lookahead_slots=num_lookahead_slots,
135
)
136
return output
137
vllm/executor/neuron_executor.py
+4
-1
@@ -48,10 +48,13 @@ class NeuronExecutor(ExecutorBase):
48
seq_group_metadata_list: List[SequenceGroupMetadata],
49
blocks_to_swap_in: Dict[int, int],
50
blocks_to_swap_out: Dict[int, int],
51
- blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
51
+ blocks_to_copy: Dict[int, List[int]],
52
+ num_lookahead_slots: int) -> List[SamplerOutput]:
53
assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
54
and blocks_to_copy == {}), (
55
"Cache operations are not supported for Neuron backend.")
56
+ assert num_lookahead_slots == 0, (
57
+ "lookahead not supported for Neuron backend.")
58
59
output = self.driver_worker.execute_model(
60
seq_group_metadata_list=seq_group_metadata_list)
vllm/executor/ray_gpu_executor.py
+2
-1
@@ -242,7 +242,8 @@ class RayGPUExecutor(ExecutorBase):
242
seq_group_metadata_list: List[SequenceGroupMetadata],
243
blocks_to_swap_in: Dict[int, int],
244
blocks_to_swap_out: Dict[int, int],
245
- blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
245
+ blocks_to_copy: Dict[int, List[int]],
246
+ num_lookahead_slots: int = 0) -> SamplerOutput:
247
all_outputs = self._run_workers(
248
"execute_model",
249
driver_kwargs={
vllm/sequence.py
+13
@@ -693,3 +693,16 @@ class SamplerOutput:
693
def __eq__(self, other: object):
694
return isinstance(other,
695
self.__class__) and self.outputs == other.outputs
696
+
697
+ def __repr__(self) -> str:
698
+ """Show the shape of a tensor instead of its values to reduce noise.
699
+ """
700
+ sampled_token_probs_repr = ("None" if self.sampled_token_probs is None
701
+ else self.sampled_token_probs.shape)
702
+ sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else
703
+ self.sampled_token_ids.shape)
704
+ return (
705
+ f"SamplerOutput(outputs={self.outputs}, "
706
+ f"sampled_token_probs={sampled_token_probs_repr}, "
707
+ f"sampled_token_ids={sampled_token_ids_repr}, "
708
+ f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
vllm/spec_decode/batch_expansion.py
+18
-5
@@ -6,10 +6,10 @@ import torch
6
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeScorer, SpeculativeScores)
9
-from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
10
- sampler_output_to_torch,
9
+from vllm.spec_decode.util import (get_all_seq_ids, maybe_mock_device_tensors,
10
+ nvtx_range, sampler_output_to_torch,
11
split_batch_by_proposal_len)
12
-from vllm.worker.worker import Worker
12
+from vllm.worker.worker_base import WorkerBase
13
14
SeqId = int
15
TargetSeqId = int
@@ -31,7 +31,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
31
of topk/tree.
32
"""
33
34
- def __init__(self, scorer_worker: Worker, device: str, vocab_size: int):
34
+ def __init__(self, scorer_worker: WorkerBase, device: str,
35
+ vocab_size: int):
36
self._scorer_worker = scorer_worker
37
self._device = device
38
self._vocab_size = vocab_size
@@ -83,7 +84,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
84
blocks_to_swap_in=blocks_to_swap_in,
85
blocks_to_swap_out=blocks_to_swap_out,
86
blocks_to_copy=blocks_to_copy,
86
- return_python_output=False)
87
+ )
88
+ assert len(target_sampler_output) == 1, "expected single-step output"
89
+ target_sampler_output = target_sampler_output[0]
90
91
all_tokens, all_probs = self._contract_batch(
92
original_bs=len(seq_group_metadata_list),
@@ -142,6 +145,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
145
This maps the scores of speculative tokens back to their original
146
sequences.
147
"""
148
+
149
+ # We mock the device tensors until PR 7/9 is merged (e2e correctness).
150
+ # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
151
+ maybe_mock_device_tensors(
152
+ sampler_output=target_sampler_output,
153
+ batch_size=len(non_spec_indices) + num_scoring_tokens,
154
+ vocab_size=self._vocab_size,
155
+ device=self._device,
156
+ )
157
+
158
(target_token_ids, target_probs, non_spec_target_token_ids,
159
non_spec_target_probs) = self._split_scoring_output(
160
target_sampler_output, num_scoring_tokens)
vllm/spec_decode/multi_step_worker.py
+15
-1
@@ -6,7 +6,8 @@ import torch
6
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeProposer)
9
-from vllm.spec_decode.util import sampler_output_to_torch
9
+from vllm.spec_decode.util import (maybe_mock_device_tensors,
10
+ sampler_output_to_torch)
11
from vllm.worker.worker import Worker
12
13
@@ -69,6 +70,9 @@ class MultiStepWorker(Worker):
70
blocks_to_swap_out=blocks_to_swap_out,
71
blocks_to_copy=blocks_to_copy,
72
)
73
+ assert (len(model_output) == 1
74
+ ), "composing multistep workers not supported"
75
+ model_output = model_output[0]
76
77
self._append_new_tokens(model_output,
78
copied_seq_group_metadata_list)
@@ -341,6 +345,16 @@ class DraftModelTop1Proposer(SpeculativeProposer):
345
346
sampler_output = maybe_sampler_output
347
348
+ # We mock the device tensors until PR 7/9 is merged (e2e correctness).
349
+ # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
350
+ for step_output in sampler_output:
351
+ maybe_mock_device_tensors(
352
+ sampler_output=step_output,
353
+ batch_size=len(proposal_lens),
354
+ vocab_size=self._vocab_size,
355
+ device=self._device,
356
+ )
357
+
358
proposal_tokens, proposal_probs = sampler_output_to_torch(
359
sampler_output)
360
vllm/spec_decode/spec_decode_worker.py
+35
-9
@@ -3,8 +3,9 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
+from vllm.logger import init_logger
7
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7
-from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
8
+from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
9
SequenceGroupOutput, SequenceOutput)
10
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
11
from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -13,8 +14,9 @@ from vllm.spec_decode.metrics import AsyncMetricsCollector
14
from vllm.spec_decode.multi_step_worker import MultiStepWorker
15
from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
16
split_batch_by_proposal_len)
16
-from vllm.worker.worker import Worker
17
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
17
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
18
+
19
+logger = init_logger(__name__)
20
21
22
class SpecDecodeWorker(LoraNotSupportedWorkerBase):
@@ -45,10 +47,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
47
More info here https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit.
48
"""
49
50
+ @classmethod
51
+ def from_workers(cls, proposer_worker: MultiStepWorker,
52
+ scorer_worker: WorkerBase) -> "SpecDecodeWorker":
53
+ return SpecDecodeWorker(
54
+ proposer_worker,
55
+ scorer_worker,
56
+ # TODO(cade) disable strict mode for speedup.
57
+ rejection_sampler=RejectionSampler(strict_mode=True),
58
+ )
59
+
60
def __init__(
61
self,
62
proposer_worker: MultiStepWorker,
51
- scorer_worker: Worker,
63
+ scorer_worker: WorkerBase,
64
rejection_sampler: RejectionSampler,
65
metrics_collector: Optional[AsyncMetricsCollector] = None,
66
):
@@ -87,6 +99,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
99
self.scorer_worker.init_device()
100
self.proposer_worker.init_device()
101
102
+ # NOTE(cade): load_model is not part of the WorkerBase interface.
103
+ self.scorer_worker.load_model()
104
+ self.proposer_worker.load_model()
105
+
106
self._metrics.init_gpu_tensors(self.rank)
107
self.rejection_sampler.init_gpu_tensors(self.rank)
108
self.scorer = BatchExpansionTop1Scorer(
@@ -131,7 +147,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
147
blocks_to_swap_in: Optional[Dict[int, int]],
148
blocks_to_swap_out: Optional[Dict[int, int]],
149
blocks_to_copy: Optional[Dict[int, List[int]]],
134
- num_spec_tokens: int,
150
+ num_lookahead_slots: int,
151
) -> List[SamplerOutput]:
152
"""Perform speculative decoding on the input batch.
153
"""
@@ -140,9 +156,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
156
"speculative decoding "
157
"requires non-None seq_group_metadata_list")
158
159
+ logger.info(f"spec_decode_worker.execute_model {num_lookahead_slots=}")
160
+
161
# If no spec tokens, call the proposer and scorer workers normally.
162
# Used for prefill.
145
- if num_spec_tokens == 0 or len(seq_group_metadata_list) == 0:
163
+ if num_lookahead_slots == 0 or len(seq_group_metadata_list) == 0:
164
return self._run_no_spec(
165
seq_group_metadata_list=seq_group_metadata_list,
166
blocks_to_swap_in=blocks_to_swap_in,
@@ -155,7 +173,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
173
blocks_to_swap_in=blocks_to_swap_in,
174
blocks_to_swap_out=blocks_to_swap_out,
175
blocks_to_copy=blocks_to_copy,
158
- k=num_spec_tokens,
176
+ k=num_lookahead_slots,
177
)
178
179
@nvtx_range("spec_decode_worker._run_no_spec")
@@ -170,20 +188,24 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
188
proposer and scorer model so that the KV cache is consistent between the
189
two.
190
"""
191
+ logger.info("run proposer worker no spec")
192
193
self.proposer_worker.execute_model(
194
seq_group_metadata_list=seq_group_metadata_list,
195
blocks_to_swap_in=blocks_to_swap_in,
196
blocks_to_swap_out=blocks_to_swap_out,
197
blocks_to_copy=blocks_to_copy,
179
- return_python_output=False)
198
+ )
199
200
+ logger.info("run target worker no spec")
201
sampler_output = self.scorer_worker.execute_model(
202
seq_group_metadata_list=seq_group_metadata_list,
203
blocks_to_swap_in=blocks_to_swap_in,
204
blocks_to_swap_out=blocks_to_swap_out,
205
blocks_to_copy=blocks_to_copy,
206
)
207
+ assert len(sampler_output) == 1
208
+ sampler_output = sampler_output[0]
209
210
# Clear device tensors from sampler output. This reduces communication
211
# overhead when the engine runs in a different process than the workers.
@@ -209,11 +231,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
231
sequence.
232
"""
233
234
+ logger.info("get spec proposals")
235
# Generate proposals using draft worker.
236
proposals = self.proposer_worker.get_spec_proposals(
237
seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
238
blocks_to_copy, k)
239
240
+ logger.info("score proposals")
241
proposal_scores = self.scorer.score_proposals(
242
seq_group_metadata_list,
243
blocks_to_swap_in,
@@ -223,9 +247,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
247
proposals,
248
)
249
250
+ logger.info("verify proposals")
251
accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
252
proposal_scores, proposals, k)
253
254
+ logger.info("create output list")
255
return self._create_output_sampler_list(seq_group_metadata_list,
256
accepted_token_ids, k)
257
@@ -311,7 +337,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
337
parent_seq_id=seq_id,
338
output_token=token_id,
339
# TODO Add verifier logprobs.
314
- logprobs={token_id: 0.0},
340
+ logprobs={token_id: Logprob(0.0)},
341
)
342
],
343
prompt_logprobs=None,
vllm/spec_decode/util.py
+26
@@ -82,6 +82,32 @@ def sampler_output_to_torch(
82
return sampled_token_ids, sampled_token_probs
83
84
85
+def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int,
86
+ vocab_size: int, device: str) -> None:
87
+ """Helper method which mocks out the GPU tensors in SamplerOutput with dummy
88
+ values. This will be removed in PR 7/9.
89
+ https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
90
+ """
91
+ values = [
92
+ sampler_output.sampled_token_probs, sampler_output.sampled_token_ids
93
+ ]
94
+ assert all(v is None for v in values) or not any(v is None for v in values)
95
+ if not any(v is None for v in values):
96
+ # Do nothing if the tensors are already created (usually in unit tests).
97
+ return
98
+
99
+ # Softmax to ensure valid probs.
100
+ sampler_output.sampled_token_probs = torch.nn.functional.softmax(
101
+ torch.rand(batch_size, vocab_size, dtype=torch.float32, device=device),
102
+ dim=-1)
103
+
104
+ sampler_output.sampled_token_ids = torch.randint(low=10,
105
+ high=100,
106
+ size=(batch_size, ),
107
+ dtype=torch.long,
108
+ device=device)
109
+
110
+
111
@contextmanager
112
def nvtx_range(msg, *args, **kwargs):
113
"""
vllm/worker/cpu_worker.py
+5
-3
@@ -251,7 +251,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
251
blocks_to_swap_in: Optional[Dict[int, int]] = None,
252
blocks_to_swap_out: Optional[Dict[int, int]] = None,
253
blocks_to_copy: Optional[Dict[int, List[int]]] = None,
254
- ) -> Optional[SamplerOutput]:
254
+ ) -> List[SamplerOutput]:
255
if self.is_driver_worker:
256
assert seq_group_metadata_list is not None
257
num_seq_groups = len(seq_group_metadata_list)
@@ -274,11 +274,13 @@ class CPUWorker(LoraNotSupportedWorkerBase):
274
275
# If there is no input, we don't need to execute the model.
276
if num_seq_groups == 0:
277
- return {}
277
+ return []
278
279
output = self.model_runner.execute_model(seq_group_metadata_list,
280
self.cpu_cache)
281
- return output
281
+
282
+ # CPU worker only supports single-step execution.
283
+ return [output]
284
285
def init_distributed_environment(self) -> None:
286
"""Initialize the distributed environment."""
vllm/worker/neuron_worker.py
+7
-4
@@ -1,5 +1,5 @@
1
"""A Neuron worker class."""
2
-from typing import List, Optional, Tuple
2
+from typing import List, Tuple
3
4
import torch
5
import torch.distributed
@@ -73,15 +73,18 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
73
def execute_model(
74
self,
75
seq_group_metadata_list: List[SequenceGroupMetadata],
76
- ) -> Optional[SamplerOutput]:
76
+ ) -> List[SamplerOutput]:
77
num_seq_groups = len(seq_group_metadata_list)
78
79
# If there is no input, we don't need to execute the model.
80
if num_seq_groups == 0:
81
- return {}
81
+ return []
82
83
output = self.model_runner.execute_model(seq_group_metadata_list)
84
- return output
84
+
85
+ # Neuron worker only supports single-step output. Wrap the output in a
86
+ # list to conform to interface.
87
+ return [output]
88
89
def get_cache_block_size_bytes(self) -> int:
90
"""Determine the size in bytes of a cache block.
vllm/worker/worker.py
+8
-3
@@ -210,7 +210,9 @@ class Worker(WorkerBase):
210
blocks_to_swap_in: Optional[Dict[int, int]] = None,
211
blocks_to_swap_out: Optional[Dict[int, int]] = None,
212
blocks_to_copy: Optional[Dict[int, List[int]]] = None,
213
- ) -> Optional[SamplerOutput]:
213
+ num_lookahead_slots: int = 0,
214
+ ) -> List[SamplerOutput]:
215
+
216
if self.is_driver_worker:
217
assert seq_group_metadata_list is not None
218
num_seq_groups = len(seq_group_metadata_list)
@@ -235,11 +237,14 @@ class Worker(WorkerBase):
237
238
# If there is no input, we don't need to execute the model.
239
if num_seq_groups == 0:
238
- return {}
240
+ return []
241
242
output = self.model_runner.execute_model(seq_group_metadata_list,
243
self.gpu_cache)
242
- return output
244
+
245
+ # Worker only supports single-step execution. Wrap the output in a list
246
+ # to conform to interface.
247
+ return [output]
248
249
def add_lora(self, lora_request: LoRARequest) -> bool:
250
return self.model_runner.add_lora(lora_request)
vllm/worker/worker_base.py
+7
-6
@@ -40,12 +40,13 @@ class WorkerBase(ABC):
40
raise NotImplementedError
41
42
@abstractmethod
43
- def execute_model(self,
44
- seq_group_metadata_list: List[SequenceGroupMetadata],
45
- blocks_to_swap_in: Dict[int, int],
46
- blocks_to_swap_out: Dict[int, int],
47
- blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
48
- """Executes one model step on the given sequences."""
43
+ def execute_model(
44
+ self, seq_group_metadata_list: List[SequenceGroupMetadata],
45
+ blocks_to_swap_in: Dict[int, int], blocks_to_swap_out: Dict[int,
46
+ int],
47
+ blocks_to_copy: Dict[int, List[int]]) -> List[SamplerOutput]:
48
+ """Executes at least one model step on the given sequences, unless no
49
+ sequences are provided."""
50
raise NotImplementedError
51
52
@abstractmethod