@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e95cd87959
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Speculative decoding 6/9] Integrate speculative decoding with LLMEngine (#3894)

Cade Daniel committed Apr 16, 2024 at 13:09 UTC e95cd879598b834f85e70ebcd23db316ae430540
31 files changed +1347 -407
tests/core/block/e2e/test_correctness.py
+70
@@ -230,6 +230,76 @@ def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
230 assert baseline_token_ids == test_token_ids
231
232
233 +@pytest.mark.parametrize(
234 + "common_llm_kwargs",
235 + [
236 + {
237 + # Use a small model for a fast test.
238 + "model": "facebook/opt-125m",
239 +
240 + # skip cuda graph creation for fast test.
241 + "enforce_eager": True,
242 + "enable_chunked_prefill": True,
243 + "max_num_batched_tokens": 2,
244 + "max_num_seqs": 2,
245 + },
246 + ])
247 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
248 +@pytest.mark.parametrize("baseline_llm_kwargs", [
249 + {
250 + "use_v2_block_manager": False,
251 + },
252 +])
253 +@pytest.mark.parametrize("test_llm_kwargs", [
254 + {
255 + "use_v2_block_manager": True,
256 + "num_lookahead_slots": 0,
257 + },
258 + {
259 + "use_v2_block_manager": True,
260 + "num_lookahead_slots": 5,
261 + },
262 +])
263 +@pytest.mark.parametrize("batch_size", [4])
264 +@pytest.mark.parametrize("seed", [1])
265 +def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
266 + test_llm_generator, batch_size):
267 + """Verify that chunked prefill works with BlockManagerV2, with and without
268 + lookahead scheduling.
269 + """
270 + output_len = 32
271 + temperature = 0.0
272 +
273 + prompts = [
274 + "Hello, my name is",
275 + "The president of the United States is",
276 + "The capital of France is",
277 + "The future of AI is",
278 + ]
279 +
280 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
281 +
282 + sampling_params = SamplingParams(
283 + max_tokens=output_len,
284 + ignore_eos=True,
285 + temperature=temperature,
286 + )
287 +
288 + print('Getting token ids with BlockManagerV1')
289 + baseline_token_ids = get_token_ids_from_llm_generator(
290 + baseline_llm_generator, prompts, sampling_params)
291 +
292 + print('Getting token ids with BlockManagerV2')
293 + test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
294 + prompts, sampling_params)
295 +
296 + for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
297 + test_token_ids):
298 + assert expected_token_ids == actual_token_ids
299 +
300 + assert baseline_token_ids == test_token_ids
301 +
302 +
303 def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
304 for llm in llm_generator:
305 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
tests/core/utils.py
+10 -7
@@ -1,5 +1,5 @@
1 import time
2 -from typing import Optional, Tuple
2 +from typing import Iterable, Optional, Tuple
3
4 from vllm import SamplingParams
5 from vllm.lora.request import LoRARequest
@@ -31,14 +31,17 @@ def create_dummy_prompt(
31
32
33 def create_seq_group(
34 - seq_prompt_len=1024,
35 - seq_output_lens=(128, ),
36 - request_id='0',
37 - seq_id_start=0,
38 -) -> SequenceGroup:
34 + seq_prompt_len: int = 1024,
35 + seq_output_lens: Iterable[int] = (128, ),
36 + request_id: str = '0',
37 + seq_id_start: int = 0,
38 + sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
39
40 assert len(seq_output_lens) > 0
41
42 + if sampling_params is None:
43 + sampling_params = SamplingParams()
44 +
45 prompt_token_ids = [0] * seq_prompt_len
46
47 seqs = []
@@ -60,7 +63,7 @@ def create_seq_group(
63 seq_group = SequenceGroup(
64 request_id=request_id,
65 seqs=seqs,
63 - sampling_params=SamplingParams(),
66 + sampling_params=sampling_params,
67 arrival_time=time.time(),
68 )
69
tests/engine/output_processor/test_multi_step.py new
+270
@@ -0,0 +1,270 @@
1 +import random
2 +from unittest.mock import MagicMock
3 +
4 +import pytest
5 +from transformers import PreTrainedTokenizer
6 +
7 +from tests.core.utils import create_seq_group
8 +from vllm.core.scheduler import Scheduler
9 +from vllm.engine.output_processor.multi_step import MultiStepOutputProcessor
10 +from vllm.engine.output_processor.stop_checker import StopChecker
11 +from vllm.sampling_params import SamplingParams
12 +from vllm.sequence import (Logprob, SequenceGroupOutput, SequenceOutput,
13 + SequenceStatus)
14 +from vllm.transformers_utils.detokenizer import Detokenizer
15 +from vllm.utils import Counter
16 +
17 +
18 +@pytest.mark.parametrize("seq_output_len", [128])
19 +@pytest.mark.parametrize("num_new_tokens", [1, 12])
20 +@pytest.mark.skip_global_cleanup
21 +def test_appends_token_ids(num_new_tokens: int, seq_output_len: int):
22 + """Verify multi-step decoding appends token ids correctly.
23 +
24 + We append token ids and verify all the token ids were appended correctly.
25 + Note that ignore_eos=True.
26 + """
27 + detokenizer = MagicMock(spec=Detokenizer)
28 + scheduler = MagicMock(spec=Scheduler)
29 + stop_checker = MagicMock(spec=StopChecker)
30 + seq_counter = Counter()
31 +
32 + output_processor = MultiStepOutputProcessor(
33 + detokenizer=detokenizer,
34 + scheduler=scheduler,
35 + seq_counter=seq_counter,
36 + get_tokenizer_for_seq=lambda _: mock_tokenizer(),
37 + stop_checker=stop_checker,
38 + )
39 +
40 + seq_group = create_seq_group(
41 + seq_prompt_len=1024,
42 + seq_output_lens=[seq_output_len],
43 + sampling_params=SamplingParams(max_tokens=seq_output_len +
44 + num_new_tokens,
45 + ignore_eos=True),
46 + )
47 +
48 + seq = seq_group.get_seqs()[0]
49 + seq.status = SequenceStatus.RUNNING
50 +
51 + new_token_ids = list(range(num_new_tokens))
52 +
53 + outputs = [
54 + SequenceGroupOutput(
55 + samples=[
56 + SequenceOutput(
57 + parent_seq_id=seq.seq_id,
58 + output_token=output_token,
59 + logprobs={output_token: Logprob(0.0)},
60 + )
61 + ],
62 + prompt_logprobs=None,
63 + ) for output_token in new_token_ids
64 + ]
65 +
66 + assert seq.get_token_ids()[-len(new_token_ids):] != new_token_ids
67 + output_processor.process_outputs(seq_group, outputs)
68 + assert seq.get_token_ids()[-len(new_token_ids):] == new_token_ids
69 +
70 +
71 +@pytest.mark.parametrize("seq_prompt_len", [1024])
72 +@pytest.mark.parametrize("seq_output_len", [128])
73 +@pytest.mark.parametrize("num_new_tokens", [5, 6, 7, 8])
74 +@pytest.mark.parametrize("max_tokens", [128 + 3])
75 +@pytest.mark.skip_global_cleanup
76 +def test_respects_max_tokens(num_new_tokens: int, seq_prompt_len: int,
77 + seq_output_len: int, max_tokens: int):
78 + """Verify tokens after max_tokens are dropped and not appended to the
79 + sequence.
80 + """
81 + detokenizer = MagicMock(spec=Detokenizer)
82 + scheduler = MagicMock(spec=Scheduler)
83 + stop_checker = MagicMock(spec=StopChecker)
84 + seq_counter = Counter()
85 +
86 + output_processor = MultiStepOutputProcessor(
87 + detokenizer=detokenizer,
88 + scheduler=scheduler,
89 + seq_counter=seq_counter,
90 + get_tokenizer_for_seq=lambda _: mock_tokenizer(),
91 + stop_checker=stop_checker,
92 + )
93 +
94 + seq_group = create_seq_group(
95 + seq_prompt_len=seq_prompt_len,
96 + seq_output_lens=[seq_output_len],
97 + sampling_params=SamplingParams(max_tokens=max_tokens, ),
98 + )
99 +
100 + seq = seq_group.get_seqs()[0]
101 + seq.status = SequenceStatus.RUNNING
102 +
103 + new_token_ids = list(range(num_new_tokens))
104 +
105 + outputs = [
106 + SequenceGroupOutput(
107 + samples=[
108 + SequenceOutput(
109 + parent_seq_id=seq.seq_id,
110 + output_token=output_token,
111 + logprobs={output_token: Logprob(0.0)},
112 + )
113 + ],
114 + prompt_logprobs=None,
115 + ) for output_token in new_token_ids
116 + ]
117 +
118 + assert seq.get_len() == seq_prompt_len + seq_output_len
119 + output_processor.process_outputs(seq_group, outputs)
120 +
121 + # Expect the processed sequence to not go over max tokens in len.
122 + assert seq.get_len() == seq_prompt_len + max_tokens
123 +
124 + # Expect the correct tokens were appended.
125 + expected_appended_tokens = new_token_ids[:max_tokens - seq_output_len]
126 + assert seq.get_token_ids(
127 + )[-len(expected_appended_tokens):] == expected_appended_tokens
128 +
129 +
130 +@pytest.mark.parametrize("seq_prompt_len", [1024])
131 +@pytest.mark.parametrize("seq_output_len", [128])
132 +@pytest.mark.parametrize("num_new_tokens", [12])
133 +@pytest.mark.parametrize("seed", list(range(6)))
134 +@pytest.mark.skip_global_cleanup
135 +def test_respects_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
136 + seq_output_len: int, seed: int):
137 + """Verify the eos token id is included in the sequence, but subsequent
138 + tokens are dropped (not appended to sequence).
139 + """
140 + random.seed(seed)
141 + detokenizer = MagicMock(spec=Detokenizer)
142 + scheduler = MagicMock(spec=Scheduler)
143 + stop_checker = MagicMock(spec=StopChecker)
144 + seq_counter = Counter()
145 +
146 + eos_token_id = 100
147 +
148 + output_processor = MultiStepOutputProcessor(
149 + detokenizer=detokenizer,
150 + scheduler=scheduler,
151 + seq_counter=seq_counter,
152 + get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
153 + stop_checker=stop_checker,
154 + )
155 +
156 + seq_group = create_seq_group(
157 + seq_prompt_len=seq_prompt_len,
158 + seq_output_lens=[seq_output_len],
159 + sampling_params=SamplingParams(
160 + # Ensure enough space.
161 + max_tokens=seq_output_len + num_new_tokens, ),
162 + )
163 +
164 + seq = seq_group.get_seqs()[0]
165 + seq.status = SequenceStatus.RUNNING
166 +
167 + new_token_ids = list(range(num_new_tokens))
168 + assert eos_token_id not in new_token_ids
169 + eos_index = random.randint(0, len(new_token_ids) - 1)
170 + new_token_ids[eos_index] = eos_token_id
171 +
172 + outputs = [
173 + SequenceGroupOutput(
174 + samples=[
175 + SequenceOutput(
176 + parent_seq_id=seq.seq_id,
177 + output_token=output_token,
178 + logprobs={output_token: Logprob(0.0)},
179 + )
180 + ],
181 + prompt_logprobs=None,
182 + ) for output_token in new_token_ids
183 + ]
184 +
185 + assert seq.get_len() == seq_prompt_len + seq_output_len
186 + output_processor.process_outputs(seq_group, outputs)
187 +
188 + # Expect the processed sequence to not go beyond provided eos.
189 + assert seq.get_len() == seq_prompt_len + seq_output_len + (eos_index + 1)
190 +
191 + # Expect the correct tokens were appended.
192 + expected_appended_tokens = new_token_ids[:eos_index + 1]
193 + assert seq.get_token_ids(
194 + )[-len(expected_appended_tokens):] == expected_appended_tokens
195 +
196 +
197 +@pytest.mark.parametrize("seq_prompt_len", [1024])
198 +@pytest.mark.parametrize("seq_output_len", [128])
199 +@pytest.mark.parametrize("num_new_tokens", [12])
200 +@pytest.mark.parametrize("seed", list(range(6)))
201 +@pytest.mark.skip_global_cleanup
202 +def test_ignores_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
203 + seq_output_len: int, seed: int):
204 + """When sampling parameters dictate that we should ignore the eos token id,
205 + ensure all token ids are appended even if the eos token id is emitted.
206 + """
207 + random.seed(seed)
208 + detokenizer = MagicMock(spec=Detokenizer)
209 + scheduler = MagicMock(spec=Scheduler)
210 + stop_checker = MagicMock(spec=StopChecker)
211 + seq_counter = Counter()
212 +
213 + eos_token_id = 100
214 +
215 + output_processor = MultiStepOutputProcessor(
216 + detokenizer=detokenizer,
217 + scheduler=scheduler,
218 + seq_counter=seq_counter,
219 + get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
220 + stop_checker=stop_checker,
221 + )
222 +
223 + seq_group = create_seq_group(
224 + seq_prompt_len=seq_prompt_len,
225 + seq_output_lens=[seq_output_len],
226 + sampling_params=SamplingParams(
227 + # Ensure enough space.
228 + max_tokens=seq_output_len + num_new_tokens,
229 + ignore_eos=True,
230 + ),
231 + )
232 +
233 + seq = seq_group.get_seqs()[0]
234 + seq.status = SequenceStatus.RUNNING
235 +
236 + new_token_ids = list(range(num_new_tokens))
237 + assert eos_token_id not in new_token_ids
238 + eos_index = random.randint(0, len(new_token_ids) - 1)
239 + new_token_ids[eos_index] = eos_token_id
240 +
241 + outputs = [
242 + SequenceGroupOutput(
243 + samples=[
244 + SequenceOutput(
245 + parent_seq_id=seq.seq_id,
246 + output_token=output_token,
247 + logprobs={output_token: Logprob(0.0)},
248 + )
249 + ],
250 + prompt_logprobs=None,
251 + ) for output_token in new_token_ids
252 + ]
253 +
254 + assert seq.get_len() == seq_prompt_len + seq_output_len
255 + output_processor.process_outputs(seq_group, outputs)
256 +
257 + # Expect the processed sequence to go beyond eos.
258 + assert seq.get_len() == seq_prompt_len + seq_output_len + num_new_tokens
259 +
260 + # Expect the correct tokens were appended.
261 + expected_appended_tokens = new_token_ids[:seq_output_len + num_new_tokens -
262 + seq_output_len]
263 + assert seq.get_token_ids(
264 + )[-len(expected_appended_tokens):] == expected_appended_tokens
265 +
266 +
267 +def mock_tokenizer(eos_token_id=1000):
268 + tokenizer = MagicMock(spec=PreTrainedTokenizer)
269 + tokenizer.eos_token_id = eos_token_id
270 + return tokenizer
tests/spec_decode/e2e/test_correctness.py
+114 -13
@@ -1,4 +1,8 @@
1 +from itertools import cycle
2 +from typing import List, Tuple
3 +
4 import pytest
5 +from transformers import AutoTokenizer
6
7 from vllm import SamplingParams
8
@@ -7,18 +11,47 @@ from vllm import SamplingParams
11 "common_llm_kwargs",
12 [{
13 # Use a small model for a fast test.
10 - "model": "facebook/opt-125m",
11 - "speculative_model": "facebook/opt-125m",
12 - "num_speculative_tokens": 5,
14 + # Note this is repeated in the test body; to initialize a tokenizer.
15 + "model": "JackFram/llama-68m",
16 +
17 + # Skip real loading for fast test.
18 + "load_format": "dummy",
19 +
20 + # Skip cuda graph recording for fast test.
21 + "enforce_eager": True,
22
23 # Required for spec decode.
24 "use_v2_block_manager": True
25 }])
17 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
26 +@pytest.mark.parametrize(
27 + "per_test_common_llm_kwargs",
28 + [
29 + {
30 + "speculative_model": "JackFram/llama-68m",
31 + "num_speculative_tokens": 5,
32 + },
33 + {
34 + "speculative_model": "JackFram/llama-68m",
35 + "num_speculative_tokens": 1,
36 + },
37 + {
38 + # No spec decode.
39 + },
40 + ])
41 @pytest.mark.parametrize("test_llm_kwargs", [{}])
42 +@pytest.mark.parametrize("batch_size", [1])
43 +# NOTE: We should run more permutations of this test (more BS, more seeds). But
44 +# because our spec decode generates gibberish token ids, the likelihood of
45 +# emitting an invalid token combination is nontrivial. This causes divergence in
46 +# behavior of vLLM detokenization vs. hf tokenizer, for example when two "utf-
47 +# start" bytes are emitted.
48 @pytest.mark.parametrize("seed", [1])
20 -def test_spec_decode_config(test_llm_generator):
21 - output_len = 1024
49 +def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
50 + """Run generation with speculative decoding on a batch. Verify the engine
51 + generates the correct number of tokens (via ignore_eos=True), and that the
52 + detokenization matches HF transformers.
53 + """
54 + output_len = 32
55 temperature = 0.0
56
57 prompts = [
@@ -28,23 +61,91 @@ def test_spec_decode_config(test_llm_generator):
61 "The future of AI is",
62 ]
63
64 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
65 +
66 + sampling_params = SamplingParams(
67 + max_tokens=output_len,
68 + ignore_eos=True,
69 + temperature=temperature,
70 + skip_special_tokens=True,
71 + spaces_between_special_tokens=False,
72 + )
73 +
74 + batch_tokens, batch_token_ids = get_output_from_llm_generator(
75 + test_llm_generator, prompts, sampling_params)
76 +
77 + # Expect a generation for each prompt in the batch.
78 + assert len(batch_token_ids) == len(prompts)
79 +
80 + # Expect each generation to have expected number of tokens (note
81 + # ignore_eos=True).
82 + assert all(len(token_ids) == output_len for token_ids in batch_token_ids)
83 +
84 + # Expect detokenized string to match.
85 + tok = AutoTokenizer.from_pretrained("JackFram/llama-68m")
86 + for actual_tokens, actual_token_ids in zip(batch_tokens, batch_token_ids):
87 + expected_tokens = tok.decode(actual_token_ids)
88 + print(f"{actual_token_ids=}")
89 + assert actual_tokens.strip() == expected_tokens.strip()
90 +
91 +
92 +@pytest.mark.parametrize(
93 + "common_llm_kwargs",
94 + [{
95 + # Use a small model for a fast test.
96 + "model": "JackFram/llama-68m",
97 + "speculative_model": "JackFram/llama-68m",
98 + "num_speculative_tokens": 5,
99 +
100 + # Skip real loading for fast test.
101 + "load_format": "dummy",
102 +
103 + # Skip cuda graph recording for fast test.
104 + "enforce_eager": True,
105 +
106 + # Required for spec decode.
107 + "use_v2_block_manager": True
108 + }])
109 +@pytest.mark.parametrize(
110 + "per_test_common_llm_kwargs",
111 + [
112 + {
113 + # Expect failure as spec decode not supported by
114 + # Ray backend.
115 + "worker_use_ray": True,
116 + },
117 + ])
118 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
119 +@pytest.mark.parametrize("seed", [1])
120 +def test_spec_decode_xfail(test_llm_generator):
121 + """Verify that speculative decoding with Ray fails.
122 + """
123 + output_len = 128
124 + temperature = 0.0
125 +
126 + prompts = [
127 + "Hello, my name is",
128 + ]
129 +
130 sampling_params = SamplingParams(
131 max_tokens=output_len,
132 ignore_eos=True,
133 temperature=temperature,
134 )
135
37 - with pytest.raises(
38 - AssertionError,
39 - match="Speculative decoding not yet supported for GPU backend"):
40 - get_token_ids_from_llm_generator(test_llm_generator, prompts,
41 - sampling_params)
136 + with pytest.raises(AssertionError,
137 + match="Speculative decoding not yet supported for "):
138 + get_output_from_llm_generator(test_llm_generator, prompts,
139 + sampling_params)
140
141
44 -def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
142 +def get_output_from_llm_generator(
143 + llm_generator, prompts,
144 + sampling_params) -> Tuple[List[str], List[List[int]]]:
145 for llm in llm_generator:
146 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
147 token_ids = [output.outputs[0].token_ids for output in outputs]
148 + tokens = [output.outputs[0].text for output in outputs]
149 del llm
150
50 - return token_ids
151 + return tokens, token_ids
tests/spec_decode/test_multi_step_worker.py
+2 -2
@@ -125,7 +125,7 @@ def test_same_output_for_single_step():
125 zero_kv_cache(worker.cache_engine)
126 set_random_seed(seed)
127 expected_output = worker.execute_model(
128 - **single_step_execute_model_data.to_dict(), )
128 + **single_step_execute_model_data.to_dict(), )[0]
129
130 actual_token_ids = [
131 output.samples[0].output_token for output in actual_output
@@ -219,7 +219,7 @@ def test_same_output_for_multi_step():
219 continuations=continuations,
220 final_seq_lens=final_seq_lens))
221
222 - single_step_output.append(
222 + single_step_output.extend(
223 worker.execute_model(**execute_model_data.to_dict(), ))
224
225 # Append output tokens to new sequence data.
tests/spec_decode/test_spec_decode_worker.py
+20 -12
@@ -6,6 +6,7 @@ import torch
6
7 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8 from vllm.model_executor.utils import set_random_seed
9 +from vllm.sequence import SamplerOutput
10 from vllm.spec_decode.interfaces import SpeculativeProposals
11 from vllm.spec_decode.metrics import (AsyncMetricsCollector,
12 SpecDecodeWorkerMetrics)
@@ -37,7 +38,8 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
38 execute_model_data, _, _ = create_batch(batch_size, k)
39
40 with pytest.raises(ValueError, match=exception_secret):
40 - worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
41 + worker.execute_model(**execute_model_data.to_dict(),
42 + num_lookahead_slots=k)
43
44 call_args_list = draft_worker.get_spec_proposals.call_args_list
45 assert len(call_args_list) == 1
@@ -102,7 +104,8 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
104 target_worker.execute_model.side_effect = ValueError(exception_secret)
105
106 with pytest.raises(ValueError, match=exception_secret):
105 - worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
107 + worker.execute_model(**execute_model_data.to_dict(),
108 + num_lookahead_slots=k)
109
110 seen_contexts = []
111
@@ -189,13 +192,14 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
192 target_output = create_sampler_output_list(target_token_ids,
193 target_token_probs)
194
192 - target_worker.execute_model.return_value = target_output[0]
195 + target_worker.execute_model.return_value = [target_output[0]]
196
197 exception_secret = 'artifical stop'
198 rejection_sampler.side_effect = ValueError(exception_secret)
199
200 with pytest.raises(ValueError, match=exception_secret):
198 - worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
201 + worker.execute_model(**execute_model_data.to_dict(),
202 + num_lookahead_slots=k)
203
204 assert len(rejection_sampler.call_args_list) == 1
205 args, _ = rejection_sampler.call_args_list[0]
@@ -268,7 +272,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
272 target_output = create_sampler_output_list(target_token_ids,
273 target_token_probs)
274
271 - target_worker.execute_model.return_value = target_output[0]
275 + target_worker.execute_model.return_value = [target_output[0]]
276
277 rejection_sampler_output = torch.randint(low=0,
278 high=vocab_size,
@@ -283,7 +287,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
287 rejection_sampler.return_value = rejection_sampler_output
288
289 output = worker.execute_model(**execute_model_data.to_dict(),
286 - num_spec_tokens=k)
290 + num_lookahead_slots=k)
291
292 expected_output = create_sampler_output_list(
293 rejection_sampler_output.transpose(0, 1), [None for _ in range(k + 1)])
@@ -380,7 +384,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
384 target_output = create_sampler_output_list(target_token_ids,
385 target_token_probs)
386
383 - target_worker.execute_model.return_value = target_output[0]
387 + target_worker.execute_model.return_value = [target_output[0]]
388
389 rejection_sampler_output = torch.randint(low=0,
390 high=vocab_size,
@@ -400,7 +404,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
404 mock_rejsample_metrics)
405
406 output = worker.execute_model(**execute_model_data.to_dict(),
403 - num_spec_tokens=k)
407 + num_lookahead_slots=k)
408 assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
409
410 call_args_list = (
@@ -423,6 +427,8 @@ def test_k_equals_zero(k: int, batch_size: int):
427 rejection_sampler.token_id_dtype = torch.int64
428 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
429
430 + target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
431 +
432 draft_worker.device = 'cuda'
433 target_worker.device = 'cuda'
434
@@ -435,7 +441,7 @@ def test_k_equals_zero(k: int, batch_size: int):
441 batch_size, k, prev_output_token_len=0)
442
443 out = worker.execute_model(**execute_model_data.to_dict(),
438 - num_spec_tokens=k)
444 + num_lookahead_slots=k)
445
446 assert len(out) == 1, f"expected only one token output when {k=}"
447 assert out[0].probs is None, "expect gpu tensor references to be None"
@@ -443,7 +449,7 @@ def test_k_equals_zero(k: int, batch_size: int):
449 0].sampled_tokens is None, "expect gpu tensor references to be None"
450
451 draft_worker.execute_model.assert_called_once_with(
446 - **execute_model_data.to_dict(), return_python_output=False)
452 + **execute_model_data.to_dict())
453 target_worker.execute_model.assert_called_once_with(
454 **execute_model_data.to_dict())
455
@@ -462,6 +468,8 @@ def test_empty_input_batch(k: int, batch_size: int):
468 rejection_sampler.token_id_dtype = torch.int64
469 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
470
471 + target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
472 +
473 draft_worker.device = 'cuda'
474 target_worker.device = 'cuda'
475
@@ -474,7 +482,7 @@ def test_empty_input_batch(k: int, batch_size: int):
482 batch_size, k, prev_output_token_len=0)
483
484 out = worker.execute_model(**execute_model_data.to_dict(),
477 - num_spec_tokens=k)
485 + num_lookahead_slots=k)
486
487 assert len(out) == 1, f"expected only one token output when {k=}"
488 assert out[0].probs is None, "expect gpu tensor references to be None"
@@ -482,7 +490,7 @@ def test_empty_input_batch(k: int, batch_size: int):
490 0].sampled_tokens is None, "expect gpu tensor references to be None"
491
492 draft_worker.execute_model.assert_called_once_with(
485 - **execute_model_data.to_dict(), return_python_output=False)
493 + **execute_model_data.to_dict())
494 target_worker.execute_model.assert_called_once_with(
495 **execute_model_data.to_dict())
496
tests/spec_decode/utils.py
+1 -1
@@ -212,7 +212,7 @@ def create_sampler_output_list(
212 SequenceOutput(
213 output_token=token_id,
214 parent_seq_id=seq_ids[seq_index],
215 - logprobs={token_id: 0},
215 + logprobs={token_id: Logprob(0)},
216 )
217 ],
218 prompt_logprobs=None,
vllm/core/block/block_table.py
-1
@@ -104,7 +104,6 @@ class BlockTable:
104 token_ids (List[int]): The sequence of token IDs to be appended.
105 """
106 assert self._is_allocated
107 - assert token_ids, "can't append empty token ids"
107
108 self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
109 num_lookahead_slots)
vllm/core/scheduler.py
+2 -6
@@ -762,9 +762,7 @@ class Scheduler:
762 blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
763 swapped_in.blocks_to_copy),
764 ignored_seq_groups=prefills.ignored_seq_groups,
765 - num_lookahead_slots=(prefills.num_lookahead_slots +
766 - running_scheduled.num_lookahead_slots +
767 - swapped_in.num_lookahead_slots),
765 + num_lookahead_slots=running_scheduled.num_lookahead_slots,
766 )
767
768 def _schedule_chunked_prefill(self):
@@ -850,9 +848,7 @@ class Scheduler:
848 blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
849 swapped_in.blocks_to_copy),
850 ignored_seq_groups=prefills.ignored_seq_groups,
853 - num_lookahead_slots=(prefills.num_lookahead_slots +
854 - running_scheduled.num_lookahead_slots +
855 - swapped_in.num_lookahead_slots),
851 + num_lookahead_slots=running_scheduled.num_lookahead_slots,
852 )
853
854 def _schedule(self) -> SchedulerOutputs:
vllm/engine/async_llm_engine.py
+3 -1
@@ -217,7 +217,9 @@ class _AsyncLLMEngine(LLMEngine):
217 else:
218 output = []
219
220 - return self._process_model_outputs(output, scheduler_outputs)
220 + return self._process_model_outputs(
221 + output, scheduler_outputs.scheduled_seq_groups,
222 + scheduler_outputs.ignored_seq_groups)
223
224 async def encode_request_async(
225 self,
vllm/engine/llm_engine.py
+53 -318
@@ -1,5 +1,5 @@
1 import time
2 -from typing import Iterable, List, Optional, Tuple, Type, Union
2 +from typing import Iterable, List, Optional, Type, Union
3
4 from transformers import PreTrainedTokenizer
5
@@ -11,6 +11,10 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
11 from vllm.core.scheduler import Scheduler, SchedulerOutputs
12 from vllm.engine.arg_utils import EngineArgs
13 from vllm.engine.metrics import StatLogger, Stats
14 +from vllm.engine.output_processor.interfaces import (
15 + SequenceGroupOutputProcessor)
16 +from vllm.engine.output_processor.stop_checker import StopChecker
17 +from vllm.engine.output_processor.util import create_output_by_sequence_group
18 from vllm.engine.ray_utils import initialize_ray_cluster
19 from vllm.executor.executor_base import ExecutorBase
20 from vllm.logger import init_logger
@@ -18,8 +22,7 @@ from vllm.lora.request import LoRARequest
22 from vllm.outputs import RequestOutput
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
21 - SequenceGroup, SequenceGroupOutput, SequenceOutput,
22 - SequenceStatus)
25 + SequenceGroup)
26 from vllm.transformers_utils.detokenizer import Detokenizer
27 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
28 get_tokenizer_group)
@@ -187,6 +190,21 @@ class LLMEngine:
190 labels=dict(model_name=model_config.model))
191 self.stat_logger.info("cache_config", self.cache_config)
192
193 + # Create sequence output processor, e.g. for beam search or
194 + # speculative decoding.
195 + self.output_processor = (
196 + SequenceGroupOutputProcessor.create_output_processor(
197 + self.scheduler_config,
198 + self.detokenizer,
199 + self.scheduler,
200 + self.seq_counter,
201 + self.get_tokenizer_for_seq,
202 + stop_checker=StopChecker(
203 + self.scheduler_config.max_model_len,
204 + self.get_tokenizer_for_seq,
205 + ),
206 + ))
207 +
208 def _initialize_kv_caches(self) -> None:
209 """Initialize the KV cache in the worker(s).
210
@@ -412,240 +430,32 @@ class LLMEngine:
430 """Returns True if there are unfinished requests."""
431 return self.scheduler.has_unfinished_seqs()
432
415 - def _check_beam_search_early_stopping(
416 - self,
417 - early_stopping: Union[bool, str],
418 - sampling_params: SamplingParams,
419 - best_running_seq: Sequence,
420 - current_worst_seq: Sequence,
421 - ) -> bool:
422 - assert sampling_params.use_beam_search
423 - length_penalty = sampling_params.length_penalty
424 - if early_stopping is True:
425 - return True
426 -
427 - current_worst_score = current_worst_seq.get_beam_search_score(
428 - length_penalty=length_penalty,
429 - eos_token_id=current_worst_seq.eos_token_id)
430 - if early_stopping is False:
431 - highest_attainable_score = best_running_seq.get_beam_search_score(
432 - length_penalty=length_penalty,
433 - eos_token_id=best_running_seq.eos_token_id)
434 - else:
435 - assert early_stopping == "never"
436 - if length_penalty > 0.0:
437 - # If length_penalty > 0.0, beam search will prefer longer
438 - # sequences. The highest attainable score calculation is
439 - # based on the longest possible sequence length in this case.
440 - max_possible_length = max(
441 - best_running_seq.get_prompt_len() +
442 - sampling_params.max_tokens,
443 - self.scheduler_config.max_model_len)
444 - highest_attainable_score = (
445 - best_running_seq.get_beam_search_score(
446 - length_penalty=length_penalty,
447 - eos_token_id=best_running_seq.eos_token_id,
448 - seq_len=max_possible_length))
449 - else:
450 - # Otherwise, beam search will prefer shorter sequences. The
451 - # highest attainable score calculation is based on the current
452 - # sequence length.
453 - highest_attainable_score = (
454 - best_running_seq.get_beam_search_score(
455 - length_penalty=length_penalty,
456 - eos_token_id=best_running_seq.eos_token_id))
457 - return current_worst_score >= highest_attainable_score
458 -
459 - def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
460 - outputs: SequenceGroupOutput) -> None:
461 -
462 - # Process prompt logprobs
463 - prompt_logprobs = outputs.prompt_logprobs
464 - if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
465 - self.detokenizer.decode_prompt_logprobs_inplace(
466 - seq_group, prompt_logprobs)
467 - seq_group.prompt_logprobs = prompt_logprobs
468 -
469 - # Process samples
470 - samples = outputs.samples
471 - parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
472 - existing_finished_seqs = seq_group.get_finished_seqs()
473 - parent_child_dict = {
474 - parent_seq.seq_id: []
475 - for parent_seq in parent_seqs
476 - }
477 - for sample in samples:
478 - parent_child_dict[sample.parent_seq_id].append(sample)
479 - # List of (child, parent)
480 - child_seqs: List[Tuple[Sequence, Sequence]] = []
481 -
482 - # Process the child samples for each parent sequence
483 - for parent in parent_seqs:
484 - child_samples: List[SequenceOutput] = parent_child_dict[
485 - parent.seq_id]
486 - if len(child_samples) == 0:
487 - # This parent sequence has no children samples. Remove
488 - # the parent sequence from the sequence group since it will
489 - # not be used in the future iterations.
490 - parent.status = SequenceStatus.FINISHED_ABORTED
491 - seq_group.remove(parent.seq_id)
492 - self.scheduler.free_seq(parent)
493 - continue
494 - # Fork the parent sequence if there are multiple child samples.
495 - for child_sample in child_samples[:-1]:
496 - new_child_seq_id = next(self.seq_counter)
497 - child = parent.fork(new_child_seq_id)
498 - child.append_token_id(child_sample.output_token,
499 - child_sample.logprobs)
500 - child_seqs.append((child, parent))
501 - # Continue the parent sequence for the last child sample.
502 - # We reuse the parent sequence here to reduce redundant memory
503 - # copies, especially when using non-beam search sampling methods.
504 - last_child_sample = child_samples[-1]
505 - parent.append_token_id(last_child_sample.output_token,
506 - last_child_sample.logprobs)
507 - child_seqs.append((parent, parent))
508 -
509 - for seq, _ in child_seqs:
510 - if seq_group.sampling_params.detokenize:
511 - new_char_count = self.detokenizer.decode_sequence_inplace(
512 - seq, seq_group.sampling_params)
513 - else:
514 - new_char_count = 0
515 - self._check_stop(seq, new_char_count, seq_group.sampling_params)
516 -
517 - # Non-beam search case
518 - if not seq_group.sampling_params.use_beam_search:
519 - # For newly created child sequences, add them to the sequence group
520 - # and fork them in block manager if they are not finished.
521 - for seq, parent in child_seqs:
522 - if seq is not parent:
523 - seq_group.add(seq)
524 - if not seq.is_finished():
525 - self.scheduler.fork_seq(parent, seq)
526 -
527 - # Free the finished and selected parent sequences' memory in block
528 - # manager. Keep them in the sequence group as candidate output.
529 - # NOTE: we need to fork the new sequences before freeing the
530 - # old sequences.
531 - for seq, parent in child_seqs:
532 - if seq is parent and seq.is_finished():
533 - self.scheduler.free_seq(seq)
534 - return
535 -
536 - # Beam search case
537 - # Select the child sequences to keep in the sequence group.
538 - selected_child_seqs = []
539 - unselected_child_seqs = []
540 - beam_width = seq_group.sampling_params.best_of
541 - length_penalty = seq_group.sampling_params.length_penalty
542 -
543 - # Select the newly finished sequences with the highest scores
544 - # to replace existing finished sequences.
545 - # Tuple of (seq, parent, is_new)
546 - existing_finished_seqs = [(seq, None, False)
547 - for seq in existing_finished_seqs]
548 - new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
549 - if seq.is_finished()]
550 - all_finished_seqs = existing_finished_seqs + new_finished_seqs
551 - # Sort the finished sequences by their scores.
552 - all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
553 - length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
554 - reverse=True)
555 - for seq, parent, is_new in all_finished_seqs[:beam_width]:
556 - if is_new:
557 - # A newly generated child sequence finishes and has a high
558 - # score, so we will add it into the sequence group.
559 - selected_child_seqs.append((seq, parent))
560 - for seq, parent, is_new in all_finished_seqs[beam_width:]:
561 - if is_new:
562 - # A newly generated child sequence finishes but has a low
563 - # score, so we will not add it into the sequence group.
564 - # Additionally, if this sequence is a continuation of a
565 - # parent sequence, we will need remove the parent sequence
566 - # from the sequence group.
567 - unselected_child_seqs.append((seq, parent))
568 - else:
569 - # An existing finished sequence has a low score, so we will
570 - # remove it from the sequence group.
571 - seq_group.remove(seq.seq_id)
572 -
573 - # select the top beam_width sequences from the running
574 - # sequences for the next iteration to continue the beam
575 - # search.
576 - running_child_seqs = [(seq, parent) for seq, parent in child_seqs
577 - if not seq.is_finished()]
578 - # Sort the running sequences by their scores.
579 - running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
580 - length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
581 - reverse=True)
582 -
583 - # Check if we can stop the beam search.
584 - if len(running_child_seqs) == 0:
585 - # No running sequences, stop the beam search.
586 - stop_beam_search = True
587 - elif len(all_finished_seqs) < beam_width:
588 - # Not enough finished sequences, continue the beam search.
589 - stop_beam_search = False
590 - else:
591 - # Check the early stopping criteria
592 - best_running_seq = running_child_seqs[0][0]
593 - current_worst_seq = all_finished_seqs[beam_width - 1][0]
594 - stop_beam_search = self._check_beam_search_early_stopping(
595 - seq_group.sampling_params.early_stopping,
596 - seq_group.sampling_params, best_running_seq, current_worst_seq)
597 -
598 - if stop_beam_search:
599 - # Stop the beam search and remove all the running sequences from
600 - # the sequence group.
601 - unselected_child_seqs.extend(running_child_seqs)
602 - else:
603 - # Continue the beam search and select the top beam_width sequences
604 - # to continue the beam search.
605 - selected_child_seqs.extend(running_child_seqs[:beam_width])
606 - # The remaining running sequences will not be used in the next
607 - # iteration. Again, if these sequences are continuations of
608 - # parent sequences, we will need to remove the parent sequences
609 - # from the sequence group.
610 - unselected_child_seqs.extend(running_child_seqs[beam_width:])
611 -
612 - # For newly created child sequences, add them to the sequence group
613 - # and fork them in block manager if they are not finished.
614 - for seq, parent in selected_child_seqs:
615 - if seq is not parent:
616 - seq_group.add(seq)
617 - if not seq.is_finished():
618 - self.scheduler.fork_seq(parent, seq)
619 -
620 - # Free the finished and selected parent sequences' memory in block
621 - # manager. Keep them in the sequence group as candidate output.
622 - for seq, parent in selected_child_seqs:
623 - if seq is parent and seq.is_finished():
624 - self.scheduler.free_seq(seq)
625 -
626 - # Remove the unselected parent sequences from the sequence group and
627 - # free their memory in block manager.
628 - for seq, parent in unselected_child_seqs:
629 - if seq is parent:
630 - # Remove the parent sequence if it is not selected for next
631 - # iteration
632 - seq_group.remove(seq.seq_id)
633 - self.scheduler.free_seq(seq)
634 -
433 def _process_model_outputs(
636 - self, output: SamplerOutput,
637 - scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
434 + self, output: List[SamplerOutput],
435 + scheduled_seq_groups: List[SequenceGroup],
436 + ignored_seq_groups: List[SequenceGroup]) -> List[RequestOutput]:
437 + """Apply the model output to the sequences in the scheduled seq groups.
438 +
439 + Returns RequestOutputs that can be returned to the client.
440 + """
441 +
442 now = time.time()
443 +
444 + # Organize outputs by [sequence group][step] instead of
445 + # [step][sequence group].
446 + output_by_sequence_group = create_output_by_sequence_group(
447 + sampler_outputs=output, num_seq_groups=len(scheduled_seq_groups))
448 +
449 # Update the scheduled sequence groups with the model outputs.
640 - scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
641 - for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
450 + for scheduled_seq_group, outputs in zip(scheduled_seq_groups,
451 + output_by_sequence_group):
452 seq_group = scheduled_seq_group.seq_group
453 seq_group.update_num_computed_tokens(
454 scheduled_seq_group.token_chunk_size)
455 # If uncomputed tokens > 0, it means prefill is chunked.
456 # We don't need to process outputs in that case.
457 if seq_group.get_num_uncomputed_tokens() == 0:
648 - self._process_sequence_group_outputs(seq_group, outputs)
458 + self.output_processor.process_outputs(seq_group, outputs)
459
460 # Free the finished sequence groups.
461 self.scheduler.free_finished_seq_groups()
@@ -657,13 +467,9 @@ class LLMEngine:
467 seq_group.maybe_set_first_token_time(now)
468 request_output = RequestOutput.from_seq_group(seq_group)
469 request_outputs.append(request_output)
660 - for seq_group in scheduler_outputs.ignored_seq_groups:
470 + for seq_group in ignored_seq_groups:
471 request_output = RequestOutput.from_seq_group(seq_group)
472 request_outputs.append(request_output)
663 -
664 - # Log stats.
665 - if self.log_stats:
666 - self.stat_logger.log(self._get_stats(scheduler_outputs))
473 return request_outputs
474
475 def step(self) -> List[RequestOutput]:
@@ -721,13 +527,23 @@ class LLMEngine:
527
528 if not scheduler_outputs.is_empty():
529 output = self.model_executor.execute_model(
724 - seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
725 - scheduler_outputs.blocks_to_swap_out,
726 - scheduler_outputs.blocks_to_copy)
530 + seq_group_metadata_list=seq_group_metadata_list,
531 + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
532 + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
533 + blocks_to_copy=scheduler_outputs.blocks_to_copy,
534 + num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
535 else:
536 output = []
537
730 - return self._process_model_outputs(output, scheduler_outputs)
538 + request_outputs = self._process_model_outputs(
539 + output, scheduler_outputs.scheduled_seq_groups,
540 + scheduler_outputs.ignored_seq_groups)
541 +
542 + # Log stats.
543 + if self.log_stats:
544 + self.stat_logger.log(self._get_stats(scheduler_outputs))
545 +
546 + return request_outputs
547
548 def do_log_stats(self) -> None:
549 """Forced log when no requests active."""
@@ -807,87 +623,6 @@ class LLMEngine:
623 time_e2e_requests=time_e2e_requests,
624 )
625
810 - def _check_stop(self, seq: Sequence, new_char_count: int,
811 - sampling_params: SamplingParams) -> None:
812 - """Stop the finished sequences.
813 -
814 - new_char_count is the number of chars added to the
815 - sequence's output text for the newly generated token
816 - """
817 -
818 - # Check if the minimum number of tokens has been generated yet;
819 - # skip the stop string/token checks if not
820 - if seq.get_output_len() < sampling_params.min_tokens:
821 - return
822 -
823 - # Check if the sequence has generated the EOS token.
824 - if ((not sampling_params.ignore_eos)
825 - and seq.get_last_token_id() == seq.eos_token_id):
826 - seq.status = SequenceStatus.FINISHED_STOPPED
827 - return
828 -
829 - # Check if a stop token was encountered.
830 - # This assumes a single token produced per step.
831 - last_token_id = seq.get_last_token_id()
832 - if last_token_id in sampling_params.stop_token_ids:
833 - if new_char_count and (
834 - not sampling_params.include_stop_str_in_output):
835 - # Remove last token
836 - seq.output_text = seq.output_text[:-new_char_count]
837 - seq.status = SequenceStatus.FINISHED_STOPPED
838 - seq.stop_reason = last_token_id
839 - return
840 -
841 - # Check if any stop strings are matched.
842 - stop_str = self._check_stop_strings(seq, new_char_count,
843 - sampling_params)
844 - if stop_str is not None:
845 - seq.status = SequenceStatus.FINISHED_STOPPED
846 - seq.stop_reason = stop_str
847 - return
848 -
849 - # Check if the sequence has reached max_model_len.
850 - if seq.get_len() > self.scheduler_config.max_model_len:
851 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
852 - return
853 -
854 - # Check if the sequence has reached max_tokens.
855 - if seq.get_output_len() == sampling_params.max_tokens:
856 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
857 - return
858 -
859 - @staticmethod
860 - def _check_stop_strings(seq: Sequence, new_char_count: int,
861 - sampling_params: SamplingParams) -> Optional[str]:
862 - """Check if any stop strings are matched and truncate sequence
863 - output text accordingly.
864 -
865 - Returns the stop string if matched or else None.
866 - """
867 - if not new_char_count:
868 - return None
869 -
870 - for stop_str in sampling_params.stop:
871 - stop_string_len = len(stop_str)
872 - # Avoid searching already-searched text.
873 - stop_index = seq.output_text.find(
874 - stop_str, -new_char_count - stop_string_len)
875 - if stop_index == -1:
876 - continue
877 -
878 - if sampling_params.include_stop_str_in_output:
879 - # Truncate to end of stop string.
880 - stop_index += stop_string_len
881 - if stop_index >= len(seq.output_text):
882 - # No truncation required.
883 - return stop_str
884 -
885 - # Truncate the output text to either the beginning
886 - # or end of the stop string.
887 - seq.output_text = seq.output_text[:stop_index]
888 - return stop_str
889 - return None
890 -
626 def add_lora(self, lora_request: LoRARequest) -> bool:
627 return self.model_executor.add_lora(lora_request)
628
vllm/engine/output_processor/__init__.py
vllm/engine/output_processor/interfaces.py new
+69
@@ -0,0 +1,69 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Callable, Iterable, List
3 +
4 +from transformers import PreTrainedTokenizer
5 +
6 +from vllm.config import SchedulerConfig
7 +from vllm.core.scheduler import Scheduler
8 +from vllm.engine.output_processor.stop_checker import StopChecker
9 +from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
10 +from vllm.transformers_utils.detokenizer import Detokenizer
11 +
12 +
13 +class SequenceGroupOutputProcessor(ABC):
14 + """Interface for logic that processes new token ids in sequence groups,
15 + managing detokenization, stop checking, and freeing/forking sequences with
16 + the scheduler.
17 +
18 + This is highly coupled with the LLMEngine and should be seen as an extension
19 + of it. The logic is separated to simplify the LLMEngine class and allow
20 + separate implementations for single-step decoding (which supports beam
21 + search sequence forking) and multi-step decoding (which does not support
22 + beam search, but does support speculative decoding).
23 + """
24 +
25 + @staticmethod
26 + def create_output_processor(
27 + scheduler_config: SchedulerConfig,
28 + detokenizer: Detokenizer,
29 + scheduler: Scheduler,
30 + seq_counter: Iterable[int],
31 + get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
32 + stop_checker: "StopChecker",
33 + ):
34 + """Create an output processor.
35 +
36 + This returns a single-step output processor if num_lookahead_slots is
37 + zero, else returns a multi-step output processor.
38 + """
39 + if scheduler_config.num_lookahead_slots == 0:
40 + # Importing here to avoid cycle.
41 + from vllm.engine.output_processor.single_step import (
42 + SingleStepOutputProcessor)
43 + return SingleStepOutputProcessor(
44 + scheduler_config,
45 + detokenizer,
46 + scheduler,
47 + seq_counter,
48 + stop_checker,
49 + )
50 + else:
51 + # Importing here to avoid cycle.
52 + from vllm.engine.output_processor.multi_step import (
53 + MultiStepOutputProcessor)
54 + return MultiStepOutputProcessor(
55 + detokenizer,
56 + scheduler,
57 + seq_counter,
58 + get_tokenizer_for_seq,
59 + stop_checker,
60 + )
61 +
62 + @abstractmethod
63 + def process_outputs(self, sequence_group: SequenceGroup,
64 + outputs: List[SequenceGroupOutput]) -> None:
65 + """Process new token ids for the sequence group. Handles logic such as
66 + detokenization, stop checking, and freeing/forking sequences in the
67 + scheduler.
68 + """
69 + pass
vllm/engine/output_processor/multi_step.py new
+126
@@ -0,0 +1,126 @@
1 +from typing import Callable, Iterable, List
2 +
3 +from transformers import PreTrainedTokenizer
4 +
5 +from vllm.core.scheduler import Scheduler
6 +from vllm.engine.output_processor.interfaces import (
7 + SequenceGroupOutputProcessor)
8 +from vllm.engine.output_processor.stop_checker import StopChecker
9 +from vllm.logger import init_logger
10 +from vllm.sampling_params import SamplingParams
11 +from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12 + SequenceGroupOutput, SequenceOutput, SequenceStatus)
13 +from vllm.transformers_utils.detokenizer import Detokenizer
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
19 + """SequenceGroupOutputProcessor which handles logic related to
20 + detokenization and stopping conditions. It specializes to "multi-step
21 + decoding", where vLLM's worker may generate multiple tokens per invocation.
22 + This is currently mutually exclusive with advanced sampling techniques like
23 + beam search, which motivates the separation of this logic from the single
24 + step output processor.
25 +
26 + This class is responsible for things such as correctly appending all new
27 + token ids to their sequence, detokenizing new token ids, truncating new
28 + output tokens after an eos token, and correctly handling the case where the
29 + number of new output tokens per sequence differs in a single batch.
30 + """
31 +
32 + def __init__(
33 + self,
34 + detokenizer: Detokenizer,
35 + scheduler: Scheduler,
36 + seq_counter: Iterable[int],
37 + get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
38 + stop_checker: StopChecker,
39 + ):
40 + self.detokenizer = detokenizer
41 + self.scheduler = scheduler
42 + self.seq_counter = seq_counter
43 + self.get_tokenizer_for_seq = get_tokenizer_for_seq
44 + self.stop_checker = stop_checker
45 +
46 + def process_outputs(self, sequence_group: SequenceGroup,
47 + outputs: List[SequenceGroupOutput]) -> None:
48 + """Append new tokens in the outputs to sequences in the sequence group.
49 +
50 + This only supports sequence groups of size 1. It supports greater than
51 + one new token per sequence.
52 +
53 + This applies logic like stop condition checking and detokenization,
54 + including freeing finished sequences. It also handles cases where there
55 + are tokens emitted after the EOS token.
56 + """
57 + seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)
58 +
59 + assert seqs, "expected running sequences"
60 + assert len(seqs) == 1, (
61 + "Beam search not supported in multi-step decoding.")
62 + seq = seqs[0]
63 +
64 + # Since there's only one sequence per sequence group, we can take the
65 + # first sample.
66 + samples = [outputs[step].samples[0] for step in range(len(outputs))]
67 +
68 + # -1 means the output token is not valid (eg. due to spec decode
69 + # rejecting tokens).
70 + valid_samples = [
71 + sample for sample in samples if sample.output_token != -1
72 + ]
73 + assert valid_samples
74 +
75 + self._process_seq_outputs(seq, valid_samples,
76 + sequence_group.sampling_params)
77 +
78 + def _process_seq_outputs(self, seq: Sequence,
79 + valid_samples: List[SequenceOutput],
80 + sampling_params: SamplingParams) -> None:
81 + output_token_ids = [sample.output_token for sample in valid_samples]
82 +
83 + # Truncate to max_tokens if necessary.
84 + remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() +
85 + len(output_token_ids))
86 + if remaining_tokens < 0:
87 + valid_samples = valid_samples[:remaining_tokens]
88 + output_token_ids = output_token_ids[:remaining_tokens]
89 +
90 + # Truncate any tokens after EOS. This is required as spec decode
91 + # generates a fixed number of tokens without evaluating stopping
92 + # conditions within the block. This can cause an eos token to be
93 + # unintentionally ignored.
94 + if not sampling_params.ignore_eos:
95 + eos_token_id = self.get_tokenizer_for_seq(seq).eos_token_id
96 + # Avoiding .index calls as exception throwing in the happy path
97 + # is expensive.
98 + for i in range(len(output_token_ids)):
99 + if output_token_ids[i] == eos_token_id:
100 + output_token_ids = output_token_ids[:i + 1]
101 + valid_samples = valid_samples[:i + 1]
102 + break
103 +
104 + # Incrementally append tokens to the sequence, as if we had only one new
105 + # token.
106 + for output_token_id in output_token_ids:
107 + seq.append_token_id(
108 + token_id=output_token_id,
109 + # TODO emit logprobs in multi-step decoding.
110 + logprobs={output_token_id: Logprob(0.0)},
111 + )
112 +
113 + new_char_count = 0
114 + if sampling_params.detokenize:
115 + new_char_count = self.detokenizer.decode_sequence_inplace(
116 + seq, sampling_params)
117 +
118 + self.stop_checker.maybe_stop_sequence(
119 + seq,
120 + new_char_count=new_char_count,
121 + sampling_params=sampling_params)
122 + if seq.is_finished():
123 + break
124 +
125 + if seq.is_finished():
126 + self.scheduler.free_seq(seq)
vllm/engine/output_processor/single_step.py new
+276
@@ -0,0 +1,276 @@
1 +from typing import Iterable, List, Tuple, Union
2 +
3 +from vllm.config import SchedulerConfig
4 +from vllm.core.scheduler import Scheduler
5 +from vllm.engine.output_processor.interfaces import (
6 + SequenceGroupOutputProcessor)
7 +from vllm.engine.output_processor.stop_checker import StopChecker
8 +from vllm.logger import init_logger
9 +from vllm.sampling_params import SamplingParams
10 +from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
11 + SequenceOutput, SequenceStatus)
12 +from vllm.transformers_utils.detokenizer import Detokenizer
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
18 + """SequenceGroupOutputProcessor which handles "output processing" logic,
19 + which happens after the model returns generated token ids and before
20 + scheduling of the next batch. Output processing logic includes
21 + detokenization, and determining if a sequence is finished (e.g. via max len
22 + or eos token).
23 +
24 + The SingleStepOutputProcessor is specialized to the case where the model
25 + emits at most a single token per invocation, which precludes configurations
26 + such as speculative decoding or multi-step decoding. This enables beam
27 + search sampling, which requires forking/finishing/freeing sequences in a way
28 + that is currently difficult to schedule multiple steps ahead of time.
29 + """
30 +
31 + def __init__(
32 + self,
33 + scheduler_config: SchedulerConfig,
34 + detokenizer: Detokenizer,
35 + scheduler: Scheduler,
36 + seq_counter: Iterable[int],
37 + stop_checker: StopChecker,
38 + ):
39 + self.scheduler_config = scheduler_config
40 + self.detokenizer = detokenizer
41 + self.scheduler = scheduler
42 + self.seq_counter = seq_counter
43 + self.stop_checker = stop_checker
44 +
45 + def process_outputs(self, sequence_group: SequenceGroup,
46 + outputs: List[SequenceGroupOutput]) -> None:
47 + """Append all new tokens to sequences in the sequence group. Fork any
48 + surviving beam candidates; free any unsurviving ones.
49 +
50 + Invokes detokenizer to detokenize new tokens, and also marks sequences
51 + as finished if they meet stop conditions.
52 + """
53 + assert (len(outputs) == 1
54 + ), f"{type(self)} does not support multiple outputs per step"
55 + return self._process_sequence_group_outputs(sequence_group, outputs[0])
56 +
57 + def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
58 + outputs: SequenceGroupOutput) -> None:
59 +
60 + # Process prompt logprobs
61 + prompt_logprobs = outputs.prompt_logprobs
62 + if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
63 + self.detokenizer.decode_prompt_logprobs_inplace(
64 + seq_group, prompt_logprobs)
65 + seq_group.prompt_logprobs = prompt_logprobs
66 +
67 + # Process samples
68 + samples = outputs.samples
69 + parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
70 + existing_finished_seqs = seq_group.get_finished_seqs()
71 + parent_child_dict = {
72 + parent_seq.seq_id: []
73 + for parent_seq in parent_seqs
74 + }
75 + for sample in samples:
76 + parent_child_dict[sample.parent_seq_id].append(sample)
77 + # List of (child, parent)
78 + child_seqs: List[Tuple[Sequence, Sequence]] = []
79 +
80 + # Process the child samples for each parent sequence
81 + for parent in parent_seqs:
82 + child_samples: List[SequenceOutput] = parent_child_dict[
83 + parent.seq_id]
84 + if len(child_samples) == 0:
85 + # This parent sequence has no children samples. Remove
86 + # the parent sequence from the sequence group since it will
87 + # not be used in the future iterations.
88 + parent.status = SequenceStatus.FINISHED_ABORTED
89 + seq_group.remove(parent.seq_id)
90 + self.scheduler.free_seq(parent)
91 + continue
92 + # Fork the parent sequence if there are multiple child samples.
93 + for child_sample in child_samples[:-1]:
94 + new_child_seq_id = next(self.seq_counter)
95 + child = parent.fork(new_child_seq_id)
96 + child.append_token_id(child_sample.output_token,
97 + child_sample.logprobs)
98 + child_seqs.append((child, parent))
99 + # Continue the parent sequence for the last child sample.
100 + # We reuse the parent sequence here to reduce redundant memory
101 + # copies, especially when using non-beam search sampling methods.
102 + last_child_sample = child_samples[-1]
103 + parent.append_token_id(last_child_sample.output_token,
104 + last_child_sample.logprobs)
105 + child_seqs.append((parent, parent))
106 +
107 + for seq, _ in child_seqs:
108 + if seq_group.sampling_params.detokenize:
109 + new_char_count = self.detokenizer.decode_sequence_inplace(
110 + seq, seq_group.sampling_params)
111 + else:
112 + new_char_count = 0
113 + self.stop_checker.maybe_stop_sequence(seq, new_char_count,
114 + seq_group.sampling_params)
115 +
116 + # Non-beam search case
117 + if not seq_group.sampling_params.use_beam_search:
118 + # For newly created child sequences, add them to the sequence group
119 + # and fork them in block manager if they are not finished.
120 + for seq, parent in child_seqs:
121 + if seq is not parent:
122 + seq_group.add(seq)
123 + if not seq.is_finished():
124 + self.scheduler.fork_seq(parent, seq)
125 +
126 + # Free the finished and selected parent sequences' memory in block
127 + # manager. Keep them in the sequence group as candidate output.
128 + # NOTE: we need to fork the new sequences before freeing the
129 + # old sequences.
130 + for seq, parent in child_seqs:
131 + if seq is parent and seq.is_finished():
132 + self.scheduler.free_seq(seq)
133 + return
134 +
135 + # Beam search case
136 + # Select the child sequences to keep in the sequence group.
137 + selected_child_seqs = []
138 + unselected_child_seqs = []
139 + beam_width = seq_group.sampling_params.best_of
140 + length_penalty = seq_group.sampling_params.length_penalty
141 +
142 + # Select the newly finished sequences with the highest scores
143 + # to replace existing finished sequences.
144 + # Tuple of (seq, parent, is_new)
145 + existing_finished_seqs = [(seq, None, False)
146 + for seq in existing_finished_seqs]
147 + new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
148 + if seq.is_finished()]
149 + all_finished_seqs = existing_finished_seqs + new_finished_seqs
150 + # Sort the finished sequences by their scores.
151 + all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
152 + length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
153 + reverse=True)
154 + for seq, parent, is_new in all_finished_seqs[:beam_width]:
155 + if is_new:
156 + # A newly generated child sequence finishes and has a high
157 + # score, so we will add it into the sequence group.
158 + selected_child_seqs.append((seq, parent))
159 + for seq, parent, is_new in all_finished_seqs[beam_width:]:
160 + if is_new:
161 + # A newly generated child sequence finishes but has a low
162 + # score, so we will not add it into the sequence group.
163 + # Additionally, if this sequence is a continuation of a
164 + # parent sequence, we will need remove the parent sequence
165 + # from the sequence group.
166 + unselected_child_seqs.append((seq, parent))
167 + else:
168 + # An existing finished sequence has a low score, so we will
169 + # remove it from the sequence group.
170 + seq_group.remove(seq.seq_id)
171 +
172 + # select the top beam_width sequences from the running
173 + # sequences for the next iteration to continue the beam
174 + # search.
175 + running_child_seqs = [(seq, parent) for seq, parent in child_seqs
176 + if not seq.is_finished()]
177 + # Sort the running sequences by their scores.
178 + running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
179 + length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
180 + reverse=True)
181 +
182 + # Check if we can stop the beam search.
183 + if len(running_child_seqs) == 0:
184 + # No running sequences, stop the beam search.
185 + stop_beam_search = True
186 + elif len(all_finished_seqs) < beam_width:
187 + # Not enough finished sequences, continue the beam search.
188 + stop_beam_search = False
189 + else:
190 + # Check the early stopping criteria
191 + best_running_seq = running_child_seqs[0][0]
192 + current_worst_seq = all_finished_seqs[beam_width - 1][0]
193 + stop_beam_search = self._check_beam_search_early_stopping(
194 + seq_group.sampling_params.early_stopping,
195 + seq_group.sampling_params, best_running_seq, current_worst_seq)
196 +
197 + if stop_beam_search:
198 + # Stop the beam search and remove all the running sequences from
199 + # the sequence group.
200 + unselected_child_seqs.extend(running_child_seqs)
201 + else:
202 + # Continue the beam search and select the top beam_width sequences
203 + # to continue the beam search.
204 + selected_child_seqs.extend(running_child_seqs[:beam_width])
205 + # The remaining running sequences will not be used in the next
206 + # iteration. Again, if these sequences are continuations of
207 + # parent sequences, we will need to remove the parent sequences
208 + # from the sequence group.
209 + unselected_child_seqs.extend(running_child_seqs[beam_width:])
210 +
211 + # For newly created child sequences, add them to the sequence group
212 + # and fork them in block manager if they are not finished.
213 + for seq, parent in selected_child_seqs:
214 + if seq is not parent:
215 + seq_group.add(seq)
216 + if not seq.is_finished():
217 + self.scheduler.fork_seq(parent, seq)
218 +
219 + # Free the finished and selected parent sequences' memory in block
220 + # manager. Keep them in the sequence group as candidate output.
221 + for seq, parent in selected_child_seqs:
222 + if seq is parent and seq.is_finished():
223 + self.scheduler.free_seq(seq)
224 +
225 + # Remove the unselected parent sequences from the sequence group and
226 + # free their memory in block manager.
227 + for seq, parent in unselected_child_seqs:
228 + if seq is parent:
229 + # Remove the parent sequence if it is not selected for next
230 + # iteration
231 + seq_group.remove(seq.seq_id)
232 + self.scheduler.free_seq(seq)
233 +
234 + def _check_beam_search_early_stopping(
235 + self,
236 + early_stopping: Union[bool, str],
237 + sampling_params: SamplingParams,
238 + best_running_seq: Sequence,
239 + current_worst_seq: Sequence,
240 + ) -> bool:
241 + assert sampling_params.use_beam_search
242 + length_penalty = sampling_params.length_penalty
243 + if early_stopping is True:
244 + return True
245 +
246 + current_worst_score = current_worst_seq.get_beam_search_score(
247 + length_penalty=length_penalty,
248 + eos_token_id=current_worst_seq.eos_token_id)
249 + if early_stopping is False:
250 + highest_attainable_score = best_running_seq.get_beam_search_score(
251 + length_penalty=length_penalty,
252 + eos_token_id=best_running_seq.eos_token_id)
253 + else:
254 + assert early_stopping == "never"
255 + if length_penalty > 0.0:
256 + # If length_penalty > 0.0, beam search will prefer longer
257 + # sequences. The highest attainable score calculation is
258 + # based on the longest possible sequence length in this case.
259 + max_possible_length = max(
260 + best_running_seq.get_prompt_len() +
261 + sampling_params.max_tokens,
262 + self.scheduler_config.max_model_len)
263 + highest_attainable_score = (
264 + best_running_seq.get_beam_search_score(
265 + length_penalty=length_penalty,
266 + eos_token_id=best_running_seq.eos_token_id,
267 + seq_len=max_possible_length))
268 + else:
269 + # Otherwise, beam search will prefer shorter sequences. The
270 + # highest attainable score calculation is based on the current
271 + # sequence length.
272 + highest_attainable_score = (
273 + best_running_seq.get_beam_search_score(
274 + length_penalty=length_penalty,
275 + eos_token_id=best_running_seq.eos_token_id))
276 + return current_worst_score >= highest_attainable_score
vllm/engine/output_processor/stop_checker.py new
+101
@@ -0,0 +1,101 @@
1 +from typing import Callable, Optional
2 +
3 +from transformers import PreTrainedTokenizer
4 +
5 +from vllm.sampling_params import SamplingParams
6 +from vllm.sequence import Sequence, SequenceStatus
7 +
8 +
9 +class StopChecker:
10 + """LLMEngine helper class which separates out the logic involving stop
11 + checking. This checks things such as: whether the eos token was emitted,
12 + whether the max_tokens has been consumed, whether a stop string has been
13 + emitted, or if we have exceeded the max model len.
14 + """
15 +
16 + def __init__(self, max_model_len: int,
17 + get_tokenizer_for_seq: Callable[[Sequence],
18 + PreTrainedTokenizer]):
19 + self.max_model_len = max_model_len
20 + self.get_tokenizer_for_seq = get_tokenizer_for_seq
21 +
22 + def maybe_stop_sequence(self, seq: Sequence, new_char_count: int,
23 + sampling_params: SamplingParams) -> None:
24 + """Stop the finished sequences.
25 +
26 + new_char_count is the number of chars added to the
27 + sequence's output text for the newly generated token
28 + """
29 +
30 + # Check if the minimum number of tokens has been generated yet;
31 + # skip the stop string/token checks if not
32 + if seq.get_output_len() < sampling_params.min_tokens:
33 + return
34 +
35 + # Check if the sequence has generated the EOS token.
36 + if ((not sampling_params.ignore_eos)
37 + and seq.get_last_token_id() == seq.eos_token_id):
38 + seq.status = SequenceStatus.FINISHED_STOPPED
39 + return
40 +
41 + # Check if a stop token was encountered.
42 + # This assumes a single token produced per step.
43 + last_token_id = seq.get_last_token_id()
44 + if last_token_id in sampling_params.stop_token_ids:
45 + if new_char_count and (
46 + not sampling_params.include_stop_str_in_output):
47 + # Remove last token
48 + seq.output_text = seq.output_text[:-new_char_count]
49 + seq.status = SequenceStatus.FINISHED_STOPPED
50 + seq.stop_reason = last_token_id
51 + return
52 +
53 + # Check if any stop strings are matched.
54 + stop_str = self._check_stop_strings(seq, new_char_count,
55 + sampling_params)
56 + if stop_str is not None:
57 + seq.status = SequenceStatus.FINISHED_STOPPED
58 + seq.stop_reason = stop_str
59 + return
60 +
61 + # Check if the sequence has reached max_model_len.
62 + if seq.get_len() > self.max_model_len:
63 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
64 + return
65 +
66 + # Check if the sequence has reached max_tokens.
67 + if seq.get_output_len() == sampling_params.max_tokens:
68 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
69 + return
70 +
71 + @staticmethod
72 + def _check_stop_strings(seq: Sequence, new_char_count: int,
73 + sampling_params: SamplingParams) -> Optional[str]:
74 + """Check if any stop strings are matched and truncate sequence
75 + output text accordingly.
76 +
77 + Returns the stop string if matched or else None.
78 + """
79 + if not new_char_count:
80 + return None
81 +
82 + for stop_str in sampling_params.stop:
83 + stop_string_len = len(stop_str)
84 + # Avoid searching already-searched text.
85 + stop_index = seq.output_text.find(
86 + stop_str, -new_char_count - stop_string_len)
87 + if stop_index == -1:
88 + continue
89 +
90 + if sampling_params.include_stop_str_in_output:
91 + # Truncate to end of stop string.
92 + stop_index += stop_string_len
93 + if stop_index >= len(seq.output_text):
94 + # No truncation required.
95 + return stop_str
96 +
97 + # Truncate the output text to either the beginning
98 + # or end of the stop string.
99 + seq.output_text = seq.output_text[:stop_index]
100 + return stop_str
101 + return None
vllm/engine/output_processor/util.py new
+16
@@ -0,0 +1,16 @@
1 +from typing import List
2 +
3 +from vllm.sequence import SamplerOutput
4 +
5 +
6 +def create_output_by_sequence_group(sampler_outputs: List[SamplerOutput],
7 + num_seq_groups: int):
8 + """Helper method which transforms a 2d list organized by
9 + [step][sequence group] into [sequence group][step].
10 + """
11 + output_by_sequence_group = [[] for _ in range(num_seq_groups)]
12 + for step in sampler_outputs:
13 + for i, sequence_group_output in enumerate(step):
14 + output_by_sequence_group[i].append(sequence_group_output)
15 +
16 + return output_by_sequence_group
vllm/executor/cpu_executor.py
+2 -1
@@ -74,7 +74,8 @@ class CPUExecutor(ExecutorBase):
74 seq_group_metadata_list: List[SequenceGroupMetadata],
75 blocks_to_swap_in: Dict[int, int],
76 blocks_to_swap_out: Dict[int, int],
77 - blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
77 + blocks_to_copy: Dict[int, List[int]],
78 + num_lookahead_slots: int) -> List[SamplerOutput]:
79 output = self.driver_worker.execute_model(
80 seq_group_metadata_list=seq_group_metadata_list,
81 blocks_to_swap_in=blocks_to_swap_in,
vllm/executor/executor_base.py
+3 -2
@@ -72,8 +72,9 @@ class ExecutorBase(ABC):
72 seq_group_metadata_list: List[SequenceGroupMetadata],
73 blocks_to_swap_in: Dict[int, int],
74 blocks_to_swap_out: Dict[int, int],
75 - blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
76 - """Executes one model step on the given sequences."""
75 + blocks_to_copy: Dict[int, List[int]],
76 + num_lookahead_slots: int) -> List[SamplerOutput]:
77 + """Executes at least one model step on the given sequences."""
78 raise NotImplementedError
79
80 @abstractmethod
vllm/executor/gpu_executor.py
+69 -10
@@ -13,13 +13,17 @@ logger = init_logger(__name__)
13 class GPUExecutor(ExecutorBase):
14
15 def _init_executor(self) -> None:
16 - assert (not self.speculative_config
17 - ), "Speculative decoding not yet supported for GPU backend"
16 + """Initialize the worker and load the model.
17
19 - # Instantiate the worker and load the model to GPU.
20 - self._init_worker()
18 + If speculative decoding is enabled, we instead create the speculative
19 + worker.
20 + """
21 + if self.speculative_config is None:
22 + self._init_non_spec_worker()
23 + else:
24 + self._init_spec_worker()
25
22 - def _init_worker(self):
26 + def _init_non_spec_worker(self):
27 # Lazy import the Worker to avoid importing torch.cuda/xformers
28 # before CUDA_VISIBLE_DEVICES is set in the Worker
29 from vllm.worker.worker import Worker
@@ -46,6 +50,57 @@ class GPUExecutor(ExecutorBase):
50 self.driver_worker.init_device()
51 self.driver_worker.load_model()
52
53 + def _init_spec_worker(self):
54 + """Initialize a SpecDecodeWorker, using a draft model for proposals.
55 + """
56 + assert self.speculative_config is not None
57 +
58 + from vllm.spec_decode.multi_step_worker import MultiStepWorker
59 + from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
60 + from vllm.worker.worker import Worker
61 +
62 + distributed_init_method = get_distributed_init_method(
63 + get_ip(), get_open_port())
64 +
65 + target_worker = Worker(
66 + model_config=self.model_config,
67 + parallel_config=self.parallel_config,
68 + scheduler_config=self.scheduler_config,
69 + device_config=self.device_config,
70 + cache_config=self.cache_config,
71 + local_rank=0,
72 + rank=0,
73 + distributed_init_method=distributed_init_method,
74 + lora_config=self.lora_config,
75 + vision_language_config=self.vision_language_config,
76 + is_driver_worker=True,
77 + )
78 +
79 + draft_worker = MultiStepWorker(
80 + model_config=self.speculative_config.draft_model_config,
81 + parallel_config=self.speculative_config.draft_parallel_config,
82 + scheduler_config=self.scheduler_config,
83 + device_config=self.device_config,
84 + cache_config=self.cache_config,
85 + local_rank=0,
86 + rank=0,
87 + distributed_init_method=distributed_init_method,
88 + lora_config=self.lora_config,
89 + vision_language_config=self.vision_language_config,
90 + is_driver_worker=True,
91 + )
92 +
93 + spec_decode_worker = SpecDecodeWorker.from_workers(
94 + proposer_worker=draft_worker, scorer_worker=target_worker)
95 +
96 + assert self.parallel_config.world_size == 1, (
97 + "GPUExecutor only supports single GPU.")
98 +
99 + self.driver_worker = spec_decode_worker
100 +
101 + # Load model handled in spec decode worker.
102 + self.driver_worker.init_device()
103 +
104 def determine_num_available_blocks(self) -> Tuple[int, int]:
105 """Determine the number of available KV blocks by invoking the
106 underlying worker.
@@ -63,16 +118,20 @@ class GPUExecutor(ExecutorBase):
118
119 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
120
66 - def execute_model(self,
67 - seq_group_metadata_list: List[SequenceGroupMetadata],
68 - blocks_to_swap_in: Dict[int, int],
69 - blocks_to_swap_out: Dict[int, int],
70 - blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
121 + def execute_model(
122 + self,
123 + seq_group_metadata_list: List[SequenceGroupMetadata],
124 + blocks_to_swap_in: Dict[int, int],
125 + blocks_to_swap_out: Dict[int, int],
126 + blocks_to_copy: Dict[int, List[int]],
127 + num_lookahead_slots: int,
128 + ) -> List[SamplerOutput]:
129 output = self.driver_worker.execute_model(
130 seq_group_metadata_list=seq_group_metadata_list,
131 blocks_to_swap_in=blocks_to_swap_in,
132 blocks_to_swap_out=blocks_to_swap_out,
133 blocks_to_copy=blocks_to_copy,
134 + num_lookahead_slots=num_lookahead_slots,
135 )
136 return output
137
vllm/executor/neuron_executor.py
+4 -1
@@ -48,10 +48,13 @@ class NeuronExecutor(ExecutorBase):
48 seq_group_metadata_list: List[SequenceGroupMetadata],
49 blocks_to_swap_in: Dict[int, int],
50 blocks_to_swap_out: Dict[int, int],
51 - blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
51 + blocks_to_copy: Dict[int, List[int]],
52 + num_lookahead_slots: int) -> List[SamplerOutput]:
53 assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
54 and blocks_to_copy == {}), (
55 "Cache operations are not supported for Neuron backend.")
56 + assert num_lookahead_slots == 0, (
57 + "lookahead not supported for Neuron backend.")
58
59 output = self.driver_worker.execute_model(
60 seq_group_metadata_list=seq_group_metadata_list)
vllm/executor/ray_gpu_executor.py
+2 -1
@@ -242,7 +242,8 @@ class RayGPUExecutor(ExecutorBase):
242 seq_group_metadata_list: List[SequenceGroupMetadata],
243 blocks_to_swap_in: Dict[int, int],
244 blocks_to_swap_out: Dict[int, int],
245 - blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
245 + blocks_to_copy: Dict[int, List[int]],
246 + num_lookahead_slots: int = 0) -> SamplerOutput:
247 all_outputs = self._run_workers(
248 "execute_model",
249 driver_kwargs={
vllm/sequence.py
+13
@@ -693,3 +693,16 @@ class SamplerOutput:
693 def __eq__(self, other: object):
694 return isinstance(other,
695 self.__class__) and self.outputs == other.outputs
696 +
697 + def __repr__(self) -> str:
698 + """Show the shape of a tensor instead of its values to reduce noise.
699 + """
700 + sampled_token_probs_repr = ("None" if self.sampled_token_probs is None
701 + else self.sampled_token_probs.shape)
702 + sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else
703 + self.sampled_token_ids.shape)
704 + return (
705 + f"SamplerOutput(outputs={self.outputs}, "
706 + f"sampled_token_probs={sampled_token_probs_repr}, "
707 + f"sampled_token_ids={sampled_token_ids_repr}, "
708 + f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
vllm/spec_decode/batch_expansion.py
+18 -5
@@ -6,10 +6,10 @@ import torch
6 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeScorer, SpeculativeScores)
9 -from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
10 - sampler_output_to_torch,
9 +from vllm.spec_decode.util import (get_all_seq_ids, maybe_mock_device_tensors,
10 + nvtx_range, sampler_output_to_torch,
11 split_batch_by_proposal_len)
12 -from vllm.worker.worker import Worker
12 +from vllm.worker.worker_base import WorkerBase
13
14 SeqId = int
15 TargetSeqId = int
@@ -31,7 +31,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
31 of topk/tree.
32 """
33
34 - def __init__(self, scorer_worker: Worker, device: str, vocab_size: int):
34 + def __init__(self, scorer_worker: WorkerBase, device: str,
35 + vocab_size: int):
36 self._scorer_worker = scorer_worker
37 self._device = device
38 self._vocab_size = vocab_size
@@ -83,7 +84,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
84 blocks_to_swap_in=blocks_to_swap_in,
85 blocks_to_swap_out=blocks_to_swap_out,
86 blocks_to_copy=blocks_to_copy,
86 - return_python_output=False)
87 + )
88 + assert len(target_sampler_output) == 1, "expected single-step output"
89 + target_sampler_output = target_sampler_output[0]
90
91 all_tokens, all_probs = self._contract_batch(
92 original_bs=len(seq_group_metadata_list),
@@ -142,6 +145,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
145 This maps the scores of speculative tokens back to their original
146 sequences.
147 """
148 +
149 + # We mock the device tensors until PR 7/9 is merged (e2e correctness).
150 + # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
151 + maybe_mock_device_tensors(
152 + sampler_output=target_sampler_output,
153 + batch_size=len(non_spec_indices) + num_scoring_tokens,
154 + vocab_size=self._vocab_size,
155 + device=self._device,
156 + )
157 +
158 (target_token_ids, target_probs, non_spec_target_token_ids,
159 non_spec_target_probs) = self._split_scoring_output(
160 target_sampler_output, num_scoring_tokens)
vllm/spec_decode/multi_step_worker.py
+15 -1
@@ -6,7 +6,8 @@ import torch
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeProposer)
9 -from vllm.spec_decode.util import sampler_output_to_torch
9 +from vllm.spec_decode.util import (maybe_mock_device_tensors,
10 + sampler_output_to_torch)
11 from vllm.worker.worker import Worker
12
13
@@ -69,6 +70,9 @@ class MultiStepWorker(Worker):
70 blocks_to_swap_out=blocks_to_swap_out,
71 blocks_to_copy=blocks_to_copy,
72 )
73 + assert (len(model_output) == 1
74 + ), "composing multistep workers not supported"
75 + model_output = model_output[0]
76
77 self._append_new_tokens(model_output,
78 copied_seq_group_metadata_list)
@@ -341,6 +345,16 @@ class DraftModelTop1Proposer(SpeculativeProposer):
345
346 sampler_output = maybe_sampler_output
347
348 + # We mock the device tensors until PR 7/9 is merged (e2e correctness).
349 + # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
350 + for step_output in sampler_output:
351 + maybe_mock_device_tensors(
352 + sampler_output=step_output,
353 + batch_size=len(proposal_lens),
354 + vocab_size=self._vocab_size,
355 + device=self._device,
356 + )
357 +
358 proposal_tokens, proposal_probs = sampler_output_to_torch(
359 sampler_output)
360
vllm/spec_decode/spec_decode_worker.py
+35 -9
@@ -3,8 +3,9 @@ from typing import Dict, List, Optional, Tuple
3
4 import torch
5
6 +from vllm.logger import init_logger
7 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7 -from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
8 +from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
9 SequenceGroupOutput, SequenceOutput)
10 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
11 from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -13,8 +14,9 @@ from vllm.spec_decode.metrics import AsyncMetricsCollector
14 from vllm.spec_decode.multi_step_worker import MultiStepWorker
15 from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
16 split_batch_by_proposal_len)
16 -from vllm.worker.worker import Worker
17 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
17 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
18 +
19 +logger = init_logger(__name__)
20
21
22 class SpecDecodeWorker(LoraNotSupportedWorkerBase):
@@ -45,10 +47,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
47 More info here https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit.
48 """
49
50 + @classmethod
51 + def from_workers(cls, proposer_worker: MultiStepWorker,
52 + scorer_worker: WorkerBase) -> "SpecDecodeWorker":
53 + return SpecDecodeWorker(
54 + proposer_worker,
55 + scorer_worker,
56 + # TODO(cade) disable strict mode for speedup.
57 + rejection_sampler=RejectionSampler(strict_mode=True),
58 + )
59 +
60 def __init__(
61 self,
62 proposer_worker: MultiStepWorker,
51 - scorer_worker: Worker,
63 + scorer_worker: WorkerBase,
64 rejection_sampler: RejectionSampler,
65 metrics_collector: Optional[AsyncMetricsCollector] = None,
66 ):
@@ -87,6 +99,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
99 self.scorer_worker.init_device()
100 self.proposer_worker.init_device()
101
102 + # NOTE(cade): load_model is not part of the WorkerBase interface.
103 + self.scorer_worker.load_model()
104 + self.proposer_worker.load_model()
105 +
106 self._metrics.init_gpu_tensors(self.rank)
107 self.rejection_sampler.init_gpu_tensors(self.rank)
108 self.scorer = BatchExpansionTop1Scorer(
@@ -131,7 +147,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
147 blocks_to_swap_in: Optional[Dict[int, int]],
148 blocks_to_swap_out: Optional[Dict[int, int]],
149 blocks_to_copy: Optional[Dict[int, List[int]]],
134 - num_spec_tokens: int,
150 + num_lookahead_slots: int,
151 ) -> List[SamplerOutput]:
152 """Perform speculative decoding on the input batch.
153 """
@@ -140,9 +156,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
156 "speculative decoding "
157 "requires non-None seq_group_metadata_list")
158
159 + logger.info(f"spec_decode_worker.execute_model {num_lookahead_slots=}")
160 +
161 # If no spec tokens, call the proposer and scorer workers normally.
162 # Used for prefill.
145 - if num_spec_tokens == 0 or len(seq_group_metadata_list) == 0:
163 + if num_lookahead_slots == 0 or len(seq_group_metadata_list) == 0:
164 return self._run_no_spec(
165 seq_group_metadata_list=seq_group_metadata_list,
166 blocks_to_swap_in=blocks_to_swap_in,
@@ -155,7 +173,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
173 blocks_to_swap_in=blocks_to_swap_in,
174 blocks_to_swap_out=blocks_to_swap_out,
175 blocks_to_copy=blocks_to_copy,
158 - k=num_spec_tokens,
176 + k=num_lookahead_slots,
177 )
178
179 @nvtx_range("spec_decode_worker._run_no_spec")
@@ -170,20 +188,24 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
188 proposer and scorer model so that the KV cache is consistent between the
189 two.
190 """
191 + logger.info("run proposer worker no spec")
192
193 self.proposer_worker.execute_model(
194 seq_group_metadata_list=seq_group_metadata_list,
195 blocks_to_swap_in=blocks_to_swap_in,
196 blocks_to_swap_out=blocks_to_swap_out,
197 blocks_to_copy=blocks_to_copy,
179 - return_python_output=False)
198 + )
199
200 + logger.info("run target worker no spec")
201 sampler_output = self.scorer_worker.execute_model(
202 seq_group_metadata_list=seq_group_metadata_list,
203 blocks_to_swap_in=blocks_to_swap_in,
204 blocks_to_swap_out=blocks_to_swap_out,
205 blocks_to_copy=blocks_to_copy,
206 )
207 + assert len(sampler_output) == 1
208 + sampler_output = sampler_output[0]
209
210 # Clear device tensors from sampler output. This reduces communication
211 # overhead when the engine runs in a different process than the workers.
@@ -209,11 +231,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
231 sequence.
232 """
233
234 + logger.info("get spec proposals")
235 # Generate proposals using draft worker.
236 proposals = self.proposer_worker.get_spec_proposals(
237 seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
238 blocks_to_copy, k)
239
240 + logger.info("score proposals")
241 proposal_scores = self.scorer.score_proposals(
242 seq_group_metadata_list,
243 blocks_to_swap_in,
@@ -223,9 +247,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
247 proposals,
248 )
249
250 + logger.info("verify proposals")
251 accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
252 proposal_scores, proposals, k)
253
254 + logger.info("create output list")
255 return self._create_output_sampler_list(seq_group_metadata_list,
256 accepted_token_ids, k)
257
@@ -311,7 +337,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
337 parent_seq_id=seq_id,
338 output_token=token_id,
339 # TODO Add verifier logprobs.
314 - logprobs={token_id: 0.0},
340 + logprobs={token_id: Logprob(0.0)},
341 )
342 ],
343 prompt_logprobs=None,
vllm/spec_decode/util.py
+26
@@ -82,6 +82,32 @@ def sampler_output_to_torch(
82 return sampled_token_ids, sampled_token_probs
83
84
85 +def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int,
86 + vocab_size: int, device: str) -> None:
87 + """Helper method which mocks out the GPU tensors in SamplerOutput with dummy
88 + values. This will be removed in PR 7/9.
89 + https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
90 + """
91 + values = [
92 + sampler_output.sampled_token_probs, sampler_output.sampled_token_ids
93 + ]
94 + assert all(v is None for v in values) or not any(v is None for v in values)
95 + if not any(v is None for v in values):
96 + # Do nothing if the tensors are already created (usually in unit tests).
97 + return
98 +
99 + # Softmax to ensure valid probs.
100 + sampler_output.sampled_token_probs = torch.nn.functional.softmax(
101 + torch.rand(batch_size, vocab_size, dtype=torch.float32, device=device),
102 + dim=-1)
103 +
104 + sampler_output.sampled_token_ids = torch.randint(low=10,
105 + high=100,
106 + size=(batch_size, ),
107 + dtype=torch.long,
108 + device=device)
109 +
110 +
111 @contextmanager
112 def nvtx_range(msg, *args, **kwargs):
113 """
vllm/worker/cpu_worker.py
+5 -3
@@ -251,7 +251,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
251 blocks_to_swap_in: Optional[Dict[int, int]] = None,
252 blocks_to_swap_out: Optional[Dict[int, int]] = None,
253 blocks_to_copy: Optional[Dict[int, List[int]]] = None,
254 - ) -> Optional[SamplerOutput]:
254 + ) -> List[SamplerOutput]:
255 if self.is_driver_worker:
256 assert seq_group_metadata_list is not None
257 num_seq_groups = len(seq_group_metadata_list)
@@ -274,11 +274,13 @@ class CPUWorker(LoraNotSupportedWorkerBase):
274
275 # If there is no input, we don't need to execute the model.
276 if num_seq_groups == 0:
277 - return {}
277 + return []
278
279 output = self.model_runner.execute_model(seq_group_metadata_list,
280 self.cpu_cache)
281 - return output
281 +
282 + # CPU worker only supports single-step execution.
283 + return [output]
284
285 def init_distributed_environment(self) -> None:
286 """Initialize the distributed environment."""
vllm/worker/neuron_worker.py
+7 -4
@@ -1,5 +1,5 @@
1 """A Neuron worker class."""
2 -from typing import List, Optional, Tuple
2 +from typing import List, Tuple
3
4 import torch
5 import torch.distributed
@@ -73,15 +73,18 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
73 def execute_model(
74 self,
75 seq_group_metadata_list: List[SequenceGroupMetadata],
76 - ) -> Optional[SamplerOutput]:
76 + ) -> List[SamplerOutput]:
77 num_seq_groups = len(seq_group_metadata_list)
78
79 # If there is no input, we don't need to execute the model.
80 if num_seq_groups == 0:
81 - return {}
81 + return []
82
83 output = self.model_runner.execute_model(seq_group_metadata_list)
84 - return output
84 +
85 + # Neuron worker only supports single-step output. Wrap the output in a
86 + # list to conform to interface.
87 + return [output]
88
89 def get_cache_block_size_bytes(self) -> int:
90 """Determine the size in bytes of a cache block.
vllm/worker/worker.py
+8 -3
@@ -210,7 +210,9 @@ class Worker(WorkerBase):
210 blocks_to_swap_in: Optional[Dict[int, int]] = None,
211 blocks_to_swap_out: Optional[Dict[int, int]] = None,
212 blocks_to_copy: Optional[Dict[int, List[int]]] = None,
213 - ) -> Optional[SamplerOutput]:
213 + num_lookahead_slots: int = 0,
214 + ) -> List[SamplerOutput]:
215 +
216 if self.is_driver_worker:
217 assert seq_group_metadata_list is not None
218 num_seq_groups = len(seq_group_metadata_list)
@@ -235,11 +237,14 @@ class Worker(WorkerBase):
237
238 # If there is no input, we don't need to execute the model.
239 if num_seq_groups == 0:
238 - return {}
240 + return []
241
242 output = self.model_runner.execute_model(seq_group_metadata_list,
243 self.gpu_cache)
242 - return output
244 +
245 + # Worker only supports single-step execution. Wrap the output in a list
246 + # to conform to interface.
247 + return [output]
248
249 def add_lora(self, lora_request: LoRARequest) -> bool:
250 return self.model_runner.add_lora(lora_request)
vllm/worker/worker_base.py
+7 -6
@@ -40,12 +40,13 @@ class WorkerBase(ABC):
40 raise NotImplementedError
41
42 @abstractmethod
43 - def execute_model(self,
44 - seq_group_metadata_list: List[SequenceGroupMetadata],
45 - blocks_to_swap_in: Dict[int, int],
46 - blocks_to_swap_out: Dict[int, int],
47 - blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
48 - """Executes one model step on the given sequences."""
43 + def execute_model(
44 + self, seq_group_metadata_list: List[SequenceGroupMetadata],
45 + blocks_to_swap_in: Dict[int, int], blocks_to_swap_out: Dict[int,
46 + int],
47 + blocks_to_copy: Dict[int, List[int]]) -> List[SamplerOutput]:
48 + """Executes at least one model step on the given sequences, unless no
49 + sequences are provided."""
50 raise NotImplementedError
51
52 @abstractmethod