@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc8ad68455
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc][Refactor] Introduce ExecuteModelData (#4540)

Cody Yu committed May 3, 2024 at 17:47 UTC bc8ad68455ce41ba672764f4a53df5a87d1dbe99
23 files changed +359 -515
tests/spec_decode/test_multi_step_worker.py
+48 -50
@@ -5,13 +5,12 @@ import pytest
5 import torch
6
7 from vllm.model_executor.utils import set_random_seed
8 -from vllm.sequence import SamplerOutput
8 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 from vllm.spec_decode.multi_step_worker import MultiStepWorker
10 from vllm.spec_decode.top1_proposer import Top1Proposer
11 from vllm.worker.worker import Worker
12
13 from .utils import (assert_logprobs_dict_allclose, create_batch,
14 - create_execute_model_data,
14 create_seq_group_metadata_from_prompts, create_worker,
15 patch_execute_model_with_seeds, zero_kv_cache)
16
@@ -105,31 +104,32 @@ def test_same_output_for_single_step():
104
105 final_prompt_lens = [len(prompt) + num_steps for prompt in prompts]
106
108 - multi_step_execute_model_data = create_execute_model_data(
109 - seq_group_metadata_list=create_seq_group_metadata_from_prompts(
110 - prompts,
111 - num_gpu_blocks,
112 - block_size,
113 - final_prompt_lens=final_prompt_lens))
114 -
115 - single_step_execute_model_data = create_execute_model_data(
116 - seq_group_metadata_list=create_seq_group_metadata_from_prompts(
117 - prompts,
118 - num_gpu_blocks,
119 - block_size,
120 - final_prompt_lens=final_prompt_lens))
107 + multi_step_seq_group = create_seq_group_metadata_from_prompts(
108 + prompts,
109 + num_gpu_blocks,
110 + block_size,
111 + final_prompt_lens=final_prompt_lens)
112
113 zero_kv_cache(multi_step_worker.cache_engine)
114 set_random_seed(seed)
115 actual_output, _ = multi_step_worker.sampler_output(
125 - **multi_step_execute_model_data.to_dict(), sample_len=num_steps)
116 + execute_model_req=ExecuteModelRequest(
117 + seq_group_metadata_list=multi_step_seq_group),
118 + sample_len=num_steps)
119 assert len(actual_output) == num_steps
120 actual_output = actual_output[0]
121
122 + single_step_seq_group = create_seq_group_metadata_from_prompts(
123 + prompts,
124 + num_gpu_blocks,
125 + block_size,
126 + final_prompt_lens=final_prompt_lens)
127 +
128 zero_kv_cache(worker.cache_engine)
129 set_random_seed(seed)
130 expected_output = worker.execute_model(
132 - **single_step_execute_model_data.to_dict(), )[0]
131 + execute_model_req=ExecuteModelRequest(
132 + seq_group_metadata_list=single_step_seq_group))[0]
133
134 actual_token_ids = [
135 output.samples[0].output_token for output in actual_output
@@ -193,19 +193,20 @@ def test_same_output_for_multi_step():
193 worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
194
195 continuations = [[1] for _ in prompts]
196 - execute_model_data = create_execute_model_data(
197 - create_seq_group_metadata_from_prompts(
198 - prompts,
199 - num_gpu_blocks,
200 - block_size,
201 - continuations=continuations,
202 - final_prompt_lens=final_prompt_lens), )
196 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
197 + prompts,
198 + num_gpu_blocks,
199 + block_size,
200 + continuations=continuations,
201 + final_prompt_lens=final_prompt_lens)
202
203 # Run multi-step.
204 zero_kv_cache(multi_step_worker.cache_engine)
205 set_random_seed(seed)
206 multi_step_output, _ = multi_step_worker.sampler_output(
208 - **execute_model_data.to_dict(), sample_len=num_steps)
207 + execute_model_req=ExecuteModelRequest(
208 + seq_group_metadata_list=seq_group_metadata_list),
209 + sample_len=num_steps)
210
211 # Run single-step repeatedly.
212 zero_kv_cache(worker.cache_engine)
@@ -215,16 +216,16 @@ def test_same_output_for_multi_step():
216
217 for _ in multi_step_output:
218
218 - execute_model_data = create_execute_model_data(
219 - create_seq_group_metadata_from_prompts(
220 - prompts,
221 - num_gpu_blocks,
222 - block_size,
223 - continuations=continuations,
224 - final_prompt_lens=final_prompt_lens))
219 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
220 + prompts,
221 + num_gpu_blocks,
222 + block_size,
223 + continuations=continuations,
224 + final_prompt_lens=final_prompt_lens)
225
226 single_step_output.extend(
227 - worker.execute_model(**execute_model_data.to_dict(), ))
227 + worker.execute_model(execute_model_req=ExecuteModelRequest(
228 + seq_group_metadata_list=seq_group_metadata_list)))
229
230 # Append output tokens to new sequence data.
231 for i, seq_group_output in enumerate(single_step_output[-1]):
@@ -304,12 +305,11 @@ def test_draft_proposals_full_speculation_len():
305 ) for _ in range(k)
306 ], True
307
307 - execute_model_data, _, _ = create_batch(batch_size, k)
308 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
309
309 - proposals = proposer.get_proposals(
310 - **execute_model_data.to_dict(),
311 - proposal_len=k,
312 - )
310 + proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
311 + seq_group_metadata_list=seq_group_metadata_list,
312 + num_lookahead_slots=k), )
313
314 assert torch.is_tensor(proposals.proposal_token_ids)
315 assert torch.is_tensor(proposals.proposal_probs)
@@ -340,14 +340,13 @@ def test_draft_proposals_no_speculations():
340 max_proposal_len=prompt_len + k - 1,
341 )
342
343 - execute_model_data, _, _ = create_batch(batch_size,
344 - k,
345 - prompt_len=prompt_len)
343 + seq_group_metadata_list, _, _ = create_batch(batch_size,
344 + k,
345 + prompt_len=prompt_len)
346
347 - proposals = proposer.get_proposals(
348 - **execute_model_data.to_dict(),
349 - proposal_len=k,
350 - )
347 + proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
348 + seq_group_metadata_list=seq_group_metadata_list,
349 + num_lookahead_slots=k), )
350
351 assert torch.is_tensor(proposals.proposal_token_ids)
352 assert torch.is_tensor(proposals.proposal_probs)
@@ -409,17 +408,16 @@ def test_draft_proposals_mixed_k():
408 ) for _ in range(k)
409 ], True
410
412 - execute_model_data, _, _ = create_batch(
411 + seq_group_metadata_list, _, _ = create_batch(
412 batch_size,
413 k,
414 prompt_len=prompt_len,
415 prev_output_token_len=prev_output_token_len,
416 )
417
419 - proposals = proposer.get_proposals(
420 - **execute_model_data.to_dict(),
421 - proposal_len=k,
422 - )
418 + proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
419 + seq_group_metadata_list=seq_group_metadata_list,
420 + num_lookahead_slots=k), )
421
422 assert torch.is_tensor(proposals.proposal_token_ids)
423 assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_ngram_worker.py
+29 -35
@@ -1,10 +1,10 @@
1 import torch
2
3 +from vllm.sequence import ExecuteModelRequest
4 from vllm.spec_decode.ngram_worker import NGramWorker
5 from vllm.spec_decode.top1_proposer import Top1Proposer
6
6 -from .utils import (create_execute_model_data,
7 - create_seq_group_metadata_from_prompts, create_worker)
7 +from .utils import create_seq_group_metadata_from_prompts, create_worker
8
9
10 def test_ngram_algo_correctness_for_single_no_match():
@@ -44,17 +44,15 @@ def test_ngram_algo_correctness_for_single_no_match():
44
45 proposal_len = 5
46 final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
47 - ngram_sampler_output_data = create_execute_model_data(
48 - seq_group_metadata_list=create_seq_group_metadata_from_prompts(
49 - prompts,
50 - num_gpu_blocks,
51 - block_size,
52 - final_prompt_lens=final_prompt_lens))
53 -
54 - proposals = proposer.get_proposals(
55 - **ngram_sampler_output_data.to_dict(),
56 - proposal_len=proposal_len,
57 - )
47 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
48 + prompts,
49 + num_gpu_blocks,
50 + block_size,
51 + final_prompt_lens=final_prompt_lens)
52 +
53 + proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
54 + seq_group_metadata_list=seq_group_metadata_list,
55 + num_lookahead_slots=proposal_len), )
56
57 assert torch.is_tensor(proposals.proposal_token_ids)
58 assert torch.is_tensor(proposals.proposal_probs)
@@ -113,17 +111,15 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
111
112 proposal_len = 5
113 final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
116 - ngram_sampler_output_data = create_execute_model_data(
117 - seq_group_metadata_list=create_seq_group_metadata_from_prompts(
118 - prompts,
119 - num_gpu_blocks,
120 - block_size,
121 - final_prompt_lens=final_prompt_lens))
122 -
123 - proposals = proposer.get_proposals(
124 - **ngram_sampler_output_data.to_dict(),
125 - proposal_len=proposal_len,
126 - )
114 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
115 + prompts,
116 + num_gpu_blocks,
117 + block_size,
118 + final_prompt_lens=final_prompt_lens)
119 +
120 + proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
121 + seq_group_metadata_list=seq_group_metadata_list,
122 + num_lookahead_slots=proposal_len), )
123
124 assert torch.is_tensor(proposals.proposal_token_ids)
125 assert torch.is_tensor(proposals.proposal_probs)
@@ -185,17 +181,15 @@ def test_ngram_algo_correctness_for_batches_match_all():
181
182 proposal_len = 5
183 final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
188 - ngram_sampler_output_data = create_execute_model_data(
189 - seq_group_metadata_list=create_seq_group_metadata_from_prompts(
190 - prompts,
191 - num_gpu_blocks,
192 - block_size,
193 - final_prompt_lens=final_prompt_lens))
194 -
195 - proposals = proposer.get_proposals(
196 - **ngram_sampler_output_data.to_dict(),
197 - proposal_len=proposal_len,
198 - )
184 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
185 + prompts,
186 + num_gpu_blocks,
187 + block_size,
188 + final_prompt_lens=final_prompt_lens)
189 +
190 + proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
191 + seq_group_metadata_list=seq_group_metadata_list,
192 + num_lookahead_slots=proposal_len), )
193
194 assert torch.is_tensor(proposals.proposal_token_ids)
195 assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_spec_decode_worker.py
+46 -49
@@ -7,7 +7,7 @@ import torch
7
8 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9 from vllm.model_executor.utils import set_random_seed
10 -from vllm.sequence import SamplerOutput
10 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
11 from vllm.spec_decode.interfaces import SpeculativeProposals
12 from vllm.spec_decode.metrics import (AsyncMetricsCollector,
13 SpecDecodeWorkerMetrics)
@@ -15,8 +15,7 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
15 from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
16 split_num_cache_blocks_evenly)
17
18 -from .utils import (ExecuteModelData, create_batch, create_sampler_output_list,
19 - mock_worker)
18 +from .utils import create_batch, create_sampler_output_list, mock_worker
19
20
21 @pytest.mark.parametrize('k', [1, 2, 6])
@@ -36,24 +35,19 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
35 exception_secret = 'artificial stop'
36 draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
37
39 - execute_model_data, _, _ = create_batch(batch_size, k)
38 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
39 + execute_model_req = ExecuteModelRequest(
40 + seq_group_metadata_list=seq_group_metadata_list, num_lookahead_slots=k)
41
42 with pytest.raises(ValueError, match=exception_secret):
42 - worker.execute_model(**execute_model_data.to_dict(),
43 - num_lookahead_slots=k)
43 + worker.execute_model(execute_model_req=execute_model_req)
44
45 call_args_list = draft_worker.get_spec_proposals.call_args_list
46 assert len(call_args_list) == 1
47
48 for args, _ in call_args_list:
49 - (seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
50 - blocks_to_copy, actual_k) = args
51 - actual_execute_model_data = ExecuteModelData(seq_group_metadata_list,
52 - blocks_to_swap_in,
53 - blocks_to_swap_out,
54 - blocks_to_copy)
55 - assert actual_execute_model_data == execute_model_data
56 - assert actual_k == k
49 + actual_execute_model_data = args[0]
50 + assert actual_execute_model_data == execute_model_req
51
52
53 @pytest.mark.parametrize('k', [1, 2, 6])
@@ -93,7 +87,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
87 proposal_lens = torch.ones(batch_size, dtype=torch.int64,
88 device='cuda') * k
89
96 - execute_model_data, prompts, prev_output_tokens = create_batch(
90 + seq_group_metadata_list, prompts, prev_output_tokens = create_batch(
91 batch_size, k)
92
93 draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
@@ -105,20 +99,20 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
99 target_worker.execute_model.side_effect = ValueError(exception_secret)
100
101 with pytest.raises(ValueError, match=exception_secret):
108 - worker.execute_model(**execute_model_data.to_dict(),
109 - num_lookahead_slots=k)
102 + worker.execute_model(execute_model_req=ExecuteModelRequest(
103 + seq_group_metadata_list=seq_group_metadata_list,
104 + num_lookahead_slots=k))
105
106 seen_contexts = []
107
108 call_args_list = target_worker.execute_model.call_args_list
109 assert len(call_args_list) == 1
115 - for args, kwargs in call_args_list:
116 - target_execute_model_data = ExecuteModelData.from_dict(kwargs)
110 + for _, kwargs in call_args_list:
111 + seq_group_metadata_list = kwargs[
112 + "execute_model_req"].seq_group_metadata_list
113
118 - assert len(target_execute_model_data.seq_group_metadata_list) == (
119 - k + 1) * batch_size
120 - for seq_group_metadata in (
121 - target_execute_model_data.seq_group_metadata_list):
114 + assert len(seq_group_metadata_list) == (k + 1) * batch_size
115 + for seq_group_metadata in seq_group_metadata_list:
116 for seq_data in seq_group_metadata.seq_data.values():
117 seen_contexts.append(seq_data.get_token_ids())
118
@@ -175,7 +169,7 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
169 proposal_lens = torch.ones(batch_size, dtype=torch.int64,
170 device='cuda') * k
171
178 - execute_model_data, _, _ = create_batch(batch_size, k)
172 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
173
174 draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
175 proposal_token_ids=proposal_token_ids,
@@ -207,8 +201,9 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
201 rejection_sampler.side_effect = ValueError(exception_secret)
202
203 with pytest.raises(ValueError, match=exception_secret):
210 - worker.execute_model(**execute_model_data.to_dict(),
211 - num_lookahead_slots=k)
204 + worker.execute_model(execute_model_req=ExecuteModelRequest(
205 + seq_group_metadata_list=seq_group_metadata_list,
206 + num_lookahead_slots=k))
207
208 assert len(rejection_sampler.call_args_list) == 1
209 _, kwargs = rejection_sampler.call_args_list[0]
@@ -262,7 +257,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
257 proposal_lens = torch.ones(batch_size, dtype=torch.int64,
258 device='cuda') * k
259
265 - execute_model_data, _, _ = create_batch(batch_size, k)
260 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
261
262 draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
263 proposal_token_ids=proposal_token_ids,
@@ -302,8 +297,9 @@ def test_correctly_formats_output(k: int, batch_size: int):
297
298 rejection_sampler.return_value = rejection_sampler_output
299
305 - output = worker.execute_model(**execute_model_data.to_dict(),
306 - num_lookahead_slots=k)
300 + output = worker.execute_model(execute_model_req=ExecuteModelRequest(
301 + seq_group_metadata_list=seq_group_metadata_list,
302 + num_lookahead_slots=k))
303
304 expected_output = create_sampler_output_list(
305 token_ids=rejection_sampler_output.transpose(0, 1),
@@ -312,7 +308,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
308
309 seq_ids = [
310 next(iter(seq_group_metadata.seq_data.keys()))
315 - for seq_group_metadata in execute_model_data.seq_group_metadata_list
311 + for seq_group_metadata in seq_group_metadata_list
312 ]
313 actual_output_by_seq = {seq_id: [] for seq_id in seq_ids}
314 expected_output_by_seq = {seq_id: [] for seq_id in seq_ids}
@@ -383,7 +379,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
379 proposal_lens = torch.ones(batch_size, dtype=torch.int64,
380 device='cuda') * k
381
386 - execute_model_data, _, _ = create_batch(batch_size, k)
382 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
383
384 draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
385 proposal_token_ids=proposal_token_ids,
@@ -428,8 +424,9 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
424 metrics_collector.maybe_collect_rejsample_metrics.return_value = (
425 mock_rejsample_metrics)
426
431 - output = worker.execute_model(**execute_model_data.to_dict(),
432 - num_lookahead_slots=k)
427 + output = worker.execute_model(execute_model_req=ExecuteModelRequest(
428 + seq_group_metadata_list=seq_group_metadata_list,
429 + num_lookahead_slots=k))
430 assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
431
432 call_args_list = (
@@ -462,21 +459,21 @@ def test_k_equals_zero(k: int, batch_size: int):
459 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
460 metrics_collector)
461
465 - execute_model_data, prompts, prev_output_tokens = create_batch(
466 - batch_size, k, prev_output_token_len=0)
462 + seq_group_metadata_list, _, _ = create_batch(batch_size,
463 + k,
464 + prev_output_token_len=0)
465 + execute_model_req = ExecuteModelRequest(
466 + seq_group_metadata_list=seq_group_metadata_list, num_lookahead_slots=k)
467
468 - out = worker.execute_model(**execute_model_data.to_dict(),
469 - num_lookahead_slots=k)
468 + out = worker.execute_model(execute_model_req=execute_model_req)
469
470 assert len(out) == 1, f"expected only one token output when {k=}"
471 assert out[0].probs is None, "expect gpu tensor references to be None"
472 assert out[
473 0].sampled_tokens is None, "expect gpu tensor references to be None"
474
476 - draft_worker.execute_model.assert_called_once_with(
477 - **execute_model_data.to_dict())
478 - target_worker.execute_model.assert_called_once_with(
479 - **execute_model_data.to_dict())
475 + draft_worker.execute_model.assert_called_once_with(execute_model_req)
476 + target_worker.execute_model.assert_called_once_with(execute_model_req)
477
478
479 @pytest.mark.parametrize('k', [0, 5])
@@ -503,21 +500,21 @@ def test_empty_input_batch(k: int, batch_size: int):
500 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
501 metrics_collector)
502
506 - execute_model_data, prompts, prev_output_tokens = create_batch(
507 - batch_size, k, prev_output_token_len=0)
503 + seq_group_metadata_list, _, _ = create_batch(batch_size,
504 + k,
505 + prev_output_token_len=0)
506 + execute_model_req = ExecuteModelRequest(
507 + seq_group_metadata_list=seq_group_metadata_list, num_lookahead_slots=k)
508
509 - out = worker.execute_model(**execute_model_data.to_dict(),
510 - num_lookahead_slots=k)
509 + out = worker.execute_model(execute_model_req=execute_model_req)
510
511 assert len(out) == 1, f"expected only one token output when {k=}"
512 assert out[0].probs is None, "expect gpu tensor references to be None"
513 assert out[
514 0].sampled_tokens is None, "expect gpu tensor references to be None"
515
517 - draft_worker.execute_model.assert_called_once_with(
518 - **execute_model_data.to_dict())
519 - target_worker.execute_model.assert_called_once_with(
520 - **execute_model_data.to_dict())
516 + draft_worker.execute_model.assert_called_once_with(execute_model_req)
517 + target_worker.execute_model.assert_called_once_with(execute_model_req)
518
519
520 @pytest.mark.skip_global_cleanup
tests/spec_decode/utils.py
+4 -46
@@ -1,4 +1,3 @@
1 -from dataclasses import dataclass, fields
1 from itertools import count
2 from typing import Dict, Iterable, List, Optional, Union
3 from unittest.mock import MagicMock
@@ -16,50 +15,10 @@ from vllm.worker.cache_engine import CacheEngine
15 from vllm.worker.worker import Worker
16
17
19 -@dataclass
20 -class ExecuteModelData:
21 - """Helper data structure which facilitates cleaner tests.
22 - """
23 - seq_group_metadata_list: List[SequenceGroupMetadata]
24 - blocks_to_swap_in: Dict[int, int]
25 - blocks_to_swap_out: Dict[int, int]
26 - blocks_to_copy: Dict[int, List[int]]
27 -
28 - def to_dict(self):
29 - return dict(
30 - (field.name, getattr(self, field.name)) for field in fields(self))
31 -
32 - @classmethod
33 - def from_dict(cls, d):
34 - cleaned = dict((field.name, d[field.name]) for field in fields(cls))
35 - return cls(**cleaned)
36 -
37 -
18 def round_up_to_next_block(seq_len: int, block_size: int) -> int:
19 return (seq_len + block_size - 1) // block_size
20
21
42 -def create_execute_model_data(
43 - seq_group_metadata_list: List[SequenceGroupMetadata],
44 - blocks_to_swap_in: Optional[Dict[int, int]] = None,
45 - blocks_to_swap_out: Optional[Dict[int, int]] = None,
46 - blocks_to_copy: Optional[Dict[int, int]] = None,
47 -) -> ExecuteModelData:
48 - if blocks_to_swap_in is None:
49 - blocks_to_swap_in = {}
50 - if blocks_to_swap_out is None:
51 - blocks_to_swap_out = {}
52 - if blocks_to_copy is None:
53 - blocks_to_copy = {}
54 -
55 - return ExecuteModelData(
56 - seq_group_metadata_list=seq_group_metadata_list,
57 - blocks_to_swap_in=blocks_to_swap_in,
58 - blocks_to_swap_out=blocks_to_swap_out,
59 - blocks_to_copy=blocks_to_copy,
60 - )
61 -
62 -
22 def mock_worker(cls=None,
23 vocab_size: int = 30_000,
24 max_model_len: int = 2048,
@@ -258,8 +217,7 @@ def create_batch(batch_size,
217 for prompt, prev_output_token in zip(prompts, prev_output_tokens)
218 ]
219
261 - execute_model_data = create_execute_model_data(
262 - create_seq_group_metadata_from_prompts(prompts, num_gpu_blocks,
263 - block_size, final_prompt_lens,
264 - prev_output_tokens, seq_ids), )
265 - return execute_model_data, prompts, prev_output_tokens
220 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
221 + prompts, num_gpu_blocks, block_size, final_prompt_lens,
222 + prev_output_tokens, seq_ids)
223 + return seq_group_metadata_list, prompts, prev_output_tokens
tests/worker/test_swap.py
+20 -10
@@ -1,6 +1,7 @@
1 import torch
2
3 from vllm.engine.arg_utils import EngineArgs
4 +from vllm.sequence import ExecuteModelRequest
5 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6 from vllm.worker.worker import Worker
7
@@ -54,10 +55,14 @@ def test_swap() -> None:
55
56 # Test swap out.
57 blocks_to_swap_out = {3: 72, 56: 35, 84: 34}
57 - worker.execute_model(seq_group_metadata_list=[],
58 - blocks_to_swap_in={},
59 - blocks_to_swap_out=blocks_to_swap_out,
60 - blocks_to_copy={})
58 + execute_model_req = ExecuteModelRequest(
59 + seq_group_metadata_list=[],
60 + blocks_to_swap_in={},
61 + blocks_to_swap_out=blocks_to_swap_out,
62 + blocks_to_copy={},
63 + )
64 + worker.execute_model(execute_model_req=execute_model_req)
65 +
66 for i in range(num_layers):
67 gpu_key_cache, gpu_value_cache = gpu_cache[i]
68 cpu_key_cache, cpu_value_cache = cpu_cache[i]
@@ -66,14 +71,19 @@ def test_swap() -> None:
71 assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
72
73 # Test swap in.
69 - blocks_to_swap_in = {19: 45, 67: 23, 12: 78, 40: 99, 1: 71}
70 - worker.execute_model(seq_group_metadata_list=[],
71 - blocks_to_swap_in=blocks_to_swap_in,
72 - blocks_to_swap_out={},
73 - blocks_to_copy={})
74 + execute_model_req.blocks_to_swap_out = {}
75 + execute_model_req.blocks_to_swap_in = {
76 + 19: 45,
77 + 67: 23,
78 + 12: 78,
79 + 40: 99,
80 + 1: 71
81 + }
82 + worker.execute_model(execute_model_req=execute_model_req)
83 +
84 for i in range(num_layers):
85 gpu_key_cache, gpu_value_cache = gpu_cache[i]
86 cpu_key_cache, cpu_value_cache = cpu_cache[i]
77 - for src, dst in blocks_to_swap_in.items():
87 + for src, dst in execute_model_req.blocks_to_swap_in.items():
88 assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
89 assert allclose(gpu_value_cache[dst], cpu_value_cache[src])
vllm/core/scheduler.py
+4
@@ -128,6 +128,8 @@ class SchedulerOutputs:
128 ignored_seq_groups: List[SequenceGroup]
129 # The number of slots for lookahead decoding.
130 num_lookahead_slots: int
131 + # The number of requests in the running queue
132 + running_queue_size: int
133
134 def __post_init__(self):
135 # Swap in and swap out should never happen at the same time.
@@ -797,6 +799,7 @@ class Scheduler:
799 ignored_seq_groups=prefills.ignored_seq_groups +
800 swapped_in.infeasible_seq_groups,
801 num_lookahead_slots=running_scheduled.num_lookahead_slots,
802 + running_queue_size=len(self.running),
803 )
804
805 def _schedule_chunked_prefill(self):
@@ -883,6 +886,7 @@ class Scheduler:
886 swapped_in.blocks_to_copy),
887 ignored_seq_groups=prefills.ignored_seq_groups,
888 num_lookahead_slots=running_scheduled.num_lookahead_slots,
889 + running_queue_size=len(self.running),
890 )
891
892 def _schedule(self) -> SchedulerOutputs:
vllm/engine/async_llm_engine.py
+10 -6
@@ -16,7 +16,7 @@ from vllm.logger import init_logger
16 from vllm.lora.request import LoRARequest
17 from vllm.outputs import RequestOutput
18 from vllm.sampling_params import SamplingParams
19 -from vllm.sequence import MultiModalData, SamplerOutput
19 +from vllm.sequence import ExecuteModelRequest, MultiModalData, SamplerOutput
20 from vllm.usage.usage_lib import UsageContext
21
22 logger = init_logger(__name__)
@@ -210,12 +210,16 @@ class _AsyncLLMEngine(LLMEngine):
210
211 if not scheduler_outputs.is_empty():
212 # Execute the model.
213 + execute_model_req = ExecuteModelRequest(
214 + seq_group_metadata_list=seq_group_metadata_list,
215 + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
216 + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
217 + blocks_to_copy=scheduler_outputs.blocks_to_copy,
218 + num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
219 + running_queue_size=scheduler_outputs.running_queue_size,
220 + )
221 output = await self.model_executor.execute_model_async(
214 - seq_group_metadata_list,
215 - scheduler_outputs.blocks_to_swap_in,
216 - scheduler_outputs.blocks_to_swap_out,
217 - scheduler_outputs.blocks_to_copy,
218 - num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
222 + execute_model_req)
223 else:
224 output = []
225
vllm/engine/llm_engine.py
+8 -4
@@ -22,8 +22,8 @@ from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 from vllm.outputs import RequestOutput
24 from vllm.sampling_params import SamplingParams
25 -from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
26 - SequenceGroup, SequenceGroupMetadata,
25 +from vllm.sequence import (ExecuteModelRequest, MultiModalData, SamplerOutput,
26 + Sequence, SequenceGroup, SequenceGroupMetadata,
27 SequenceStatus)
28 from vllm.transformers_utils.detokenizer import Detokenizer
29 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
@@ -583,12 +583,16 @@ class LLMEngine:
583 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
584
585 if not scheduler_outputs.is_empty():
586 - output = self.model_executor.execute_model(
586 + execute_model_req = ExecuteModelRequest(
587 seq_group_metadata_list=seq_group_metadata_list,
588 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
589 blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
590 blocks_to_copy=scheduler_outputs.blocks_to_copy,
591 - num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
591 + num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
592 + running_queue_size=scheduler_outputs.running_queue_size,
593 + )
594 + output = self.model_executor.execute_model(
595 + execute_model_req=execute_model_req)
596 else:
597 output = []
598
vllm/executor/cpu_executor.py
+10 -27
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Set, Tuple
1 +from typing import List, Set, Tuple
2
3 import torch
4
@@ -7,7 +7,7 @@ from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8 from vllm.logger import init_logger
9 from vllm.lora.request import LoRARequest
10 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
11 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12 make_async)
13
@@ -72,18 +72,10 @@ class CPUExecutor(ExecutorBase):
72 logger.info("# CPU blocks: %d", num_gpu_blocks)
73 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
74
75 - def execute_model(self,
76 - seq_group_metadata_list: List[SequenceGroupMetadata],
77 - blocks_to_swap_in: Dict[int, int],
78 - blocks_to_swap_out: Dict[int, int],
79 - blocks_to_copy: Dict[int, List[int]],
80 - num_lookahead_slots: int) -> List[SamplerOutput]:
81 - output = self.driver_worker.execute_model(
82 - seq_group_metadata_list=seq_group_metadata_list,
83 - blocks_to_swap_in=blocks_to_swap_in,
84 - blocks_to_swap_out=blocks_to_swap_out,
85 - blocks_to_copy=blocks_to_copy,
86 - )
75 + def execute_model(
76 + self,
77 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
78 + output = self.driver_worker.execute_model(execute_model_req)
79 return output
80
81 def add_lora(self, lora_request: LoRARequest) -> bool:
@@ -104,19 +96,10 @@ class CPUExecutor(ExecutorBase):
96 class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
97
98 async def execute_model_async(
107 - self,
108 - seq_group_metadata_list: List[SequenceGroupMetadata],
109 - blocks_to_swap_in: Dict[int, int],
110 - blocks_to_swap_out: Dict[int, int],
111 - blocks_to_copy: Dict[int, List[int]],
112 - num_lookahead_slots: int,
113 - ) -> List[SamplerOutput]:
114 - output = await make_async(self.driver_worker.execute_model)(
115 - seq_group_metadata_list=seq_group_metadata_list,
116 - blocks_to_swap_in=blocks_to_swap_in,
117 - blocks_to_swap_out=blocks_to_swap_out,
118 - blocks_to_copy=blocks_to_copy,
119 - num_lookahead_slots=num_lookahead_slots)
99 + self,
100 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
101 + output = await make_async(self.driver_worker.execute_model
102 + )(execute_model_req=execute_model_req, )
103 return output
104
105 async def check_health_async(self) -> None:
vllm/executor/executor_base.py
+7 -15
@@ -1,11 +1,11 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, List, Optional, Set, Tuple
2 +from typing import List, Optional, Set, Tuple
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5 ModelConfig, ParallelConfig, SchedulerConfig,
6 SpeculativeConfig, VisionLanguageConfig)
7 from vllm.lora.request import LoRARequest
8 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
10
11 class ExecutorBase(ABC):
@@ -68,12 +68,9 @@ class ExecutorBase(ABC):
68 raise NotImplementedError
69
70 @abstractmethod
71 - def execute_model(self,
72 - seq_group_metadata_list: List[SequenceGroupMetadata],
73 - blocks_to_swap_in: Dict[int, int],
74 - blocks_to_swap_out: Dict[int, int],
75 - blocks_to_copy: Dict[int, List[int]],
76 - num_lookahead_slots: int) -> List[SamplerOutput]:
71 + def execute_model(
72 + self,
73 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
74 """Executes at least one model step on the given sequences."""
75 raise NotImplementedError
76
@@ -107,13 +104,8 @@ class ExecutorAsyncBase(ExecutorBase):
104
105 @abstractmethod
106 async def execute_model_async(
110 - self,
111 - seq_group_metadata_list: List[SequenceGroupMetadata],
112 - blocks_to_swap_in: Dict[int, int],
113 - blocks_to_swap_out: Dict[int, int],
114 - blocks_to_copy: Dict[int, List[int]],
115 - num_lookahead_slots: int,
116 - ) -> List[SamplerOutput]:
107 + self,
108 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
109 """Executes one model step on the given sequences."""
110 raise NotImplementedError
111
vllm/executor/gpu_executor.py
+7 -26
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
7 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8 make_async)
9 from vllm.worker.worker_base import WorkerWrapperBase
@@ -117,20 +117,9 @@ class GPUExecutor(ExecutorBase):
117 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
118
119 def execute_model(
120 - self,
121 - seq_group_metadata_list: List[SequenceGroupMetadata],
122 - blocks_to_swap_in: Dict[int, int],
123 - blocks_to_swap_out: Dict[int, int],
124 - blocks_to_copy: Dict[int, List[int]],
125 - num_lookahead_slots: int,
126 - ) -> List[SamplerOutput]:
127 - output = self.driver_worker.execute_model(
128 - seq_group_metadata_list=seq_group_metadata_list,
129 - blocks_to_swap_in=blocks_to_swap_in,
130 - blocks_to_swap_out=blocks_to_swap_out,
131 - blocks_to_copy=blocks_to_copy,
132 - num_lookahead_slots=num_lookahead_slots,
133 - )
120 + self,
121 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
122 + output = self.driver_worker.execute_model(execute_model_req)
123 return output
124
125 def add_lora(self, lora_request: LoRARequest) -> bool:
@@ -154,16 +143,8 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
143
144 async def execute_model_async(
145 self,
157 - seq_group_metadata_list: List[SequenceGroupMetadata],
158 - blocks_to_swap_in: Dict[int, int],
159 - blocks_to_swap_out: Dict[int, int],
160 - blocks_to_copy: Dict[int, List[int]],
161 - num_lookahead_slots: int,
146 + execute_model_req: ExecuteModelRequest,
147 ) -> List[SamplerOutput]:
163 - output = await make_async(self.driver_worker.execute_model)(
164 - seq_group_metadata_list=seq_group_metadata_list,
165 - blocks_to_swap_in=blocks_to_swap_in,
166 - blocks_to_swap_out=blocks_to_swap_out,
167 - blocks_to_copy=blocks_to_copy,
168 - num_lookahead_slots=num_lookahead_slots)
148 + output = await make_async(self.driver_worker.execute_model
149 + )(execute_model_req=execute_model_req, )
150 return output
vllm/executor/neuron_executor.py
+14 -19
@@ -1,9 +1,9 @@
1 -from typing import Dict, List, Set, Tuple
1 +from typing import List, Set, Tuple
2
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
7 from vllm.utils import make_async
8
9 logger = init_logger(__name__)
@@ -45,20 +45,18 @@ class NeuronExecutor(ExecutorBase):
45 """
46 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
47
48 - def execute_model(self,
49 - seq_group_metadata_list: List[SequenceGroupMetadata],
50 - blocks_to_swap_in: Dict[int, int],
51 - blocks_to_swap_out: Dict[int, int],
52 - blocks_to_copy: Dict[int, List[int]],
53 - num_lookahead_slots: int) -> List[SamplerOutput]:
54 - assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
55 - and blocks_to_copy == {}), (
48 + def execute_model(
49 + self,
50 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
51 + assert (execute_model_req.blocks_to_swap_in == {}
52 + and execute_model_req.blocks_to_swap_out == {}
53 + and execute_model_req.blocks_to_copy == {}), (
54 "Cache operations are not supported for Neuron backend.")
57 - assert num_lookahead_slots == 0, (
55 + assert execute_model_req.num_lookahead_slots == 0, (
56 "lookahead not supported for Neuron backend.")
57
58 output = self.driver_worker.execute_model(
61 - seq_group_metadata_list=seq_group_metadata_list)
59 + execute_model_req.seq_group_metadata_list)
60 return output
61
62 def add_lora(self, lora_request: LoRARequest) -> bool:
@@ -80,14 +78,11 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
78
79 async def execute_model_async(
80 self,
83 - seq_group_metadata_list: List[SequenceGroupMetadata],
84 - blocks_to_swap_in: Dict[int, int],
85 - blocks_to_swap_out: Dict[int, int],
86 - blocks_to_copy: Dict[int, List[int]],
87 - num_lookahead_slots: int,
81 + execute_model_req: ExecuteModelRequest,
82 ) -> List[SamplerOutput]:
89 - output = await make_async(self.driver_worker.execute_model)(
90 - seq_group_metadata_list=seq_group_metadata_list, )
83 + output = await make_async(
84 + self.driver_worker.execute_model
85 + )(seq_group_metadata_list=execute_model_req.seq_group_metadata_list, )
86 return output
87
88 async def check_health_async(self) -> None:
vllm/executor/ray_gpu_executor.py
+5 -14
@@ -10,7 +10,7 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10 DistributedGPUExecutor, DistributedGPUExecutorAsync)
11 from vllm.executor.ray_utils import RayWorkerWrapper, ray
12 from vllm.logger import init_logger
13 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
13 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
14 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
15 get_vllm_instance_id, make_async)
16
@@ -166,21 +166,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
166 max_concurrent_workers=self.parallel_config.
167 max_parallel_loading_workers)
168
169 - def execute_model(self,
170 - seq_group_metadata_list: List[SequenceGroupMetadata],
171 - blocks_to_swap_in: Dict[int, int],
172 - blocks_to_swap_out: Dict[int, int],
173 - blocks_to_copy: Dict[int, List[int]],
174 - num_lookahead_slots: int = 0) -> List[SamplerOutput]:
169 + def execute_model(
170 + self,
171 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
172 all_outputs = self._run_workers(
173 "execute_model",
177 - driver_kwargs={
178 - "seq_group_metadata_list": seq_group_metadata_list,
179 - "blocks_to_swap_in": blocks_to_swap_in,
180 - "blocks_to_swap_out": blocks_to_swap_out,
181 - "blocks_to_copy": blocks_to_copy,
182 - "num_lookahead_slots": num_lookahead_slots,
183 - },
174 + driver_kwargs={"execute_model_req": execute_model_req},
175 use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
176
177 # Only the driver worker returns the sampling results.
vllm/sequence.py
+31 -1
@@ -1,7 +1,7 @@
1 """Sequence and its related classes."""
2 import copy
3 import enum
4 -from dataclasses import dataclass
4 +from dataclasses import dataclass, field
5 from typing import TYPE_CHECKING, Dict, List, Optional, Union
6
7 from vllm.block import LogicalTokenBlock
@@ -734,3 +734,33 @@ class SamplerOutput:
734 f"sampled_token_probs={sampled_token_probs_repr}, "
735 f"sampled_token_ids={sampled_token_ids_repr}, "
736 f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
737 +
738 +
739 +@dataclass
740 +class ExecuteModelRequest:
741 + """The model execution request."""
742 + # The sequence group metadata list.
743 + seq_group_metadata_list: List[SequenceGroupMetadata]
744 + # Blocks to swap in. Dict of CPU -> GPU block number.
745 + blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
746 + # Blocks to swap out. Dict of GPU -> CPU block number.
747 + blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
748 + # Blocks to copy. Source to a list of dest blocks.
749 + blocks_to_copy: Dict[int, List[int]] = field(default_factory=dict)
750 + # The number of slots for lookahead decoding.
751 + num_lookahead_slots: int = 0
752 + # The number of requests in the running queue.
753 + running_queue_size: int = 0
754 +
755 + def clone(
756 + self, seq_group_metadata_list: List[SequenceGroupMetadata]
757 + ) -> "ExecuteModelRequest":
758 + """Clone the request with a new sequence group metadata list."""
759 + return ExecuteModelRequest(
760 + seq_group_metadata_list=seq_group_metadata_list,
761 + blocks_to_swap_in=self.blocks_to_swap_in.copy(),
762 + blocks_to_swap_out=self.blocks_to_swap_out.copy(),
763 + blocks_to_copy=self.blocks_to_copy.copy(),
764 + num_lookahead_slots=self.num_lookahead_slots,
765 + running_queue_size=self.running_queue_size,
766 + )
vllm/spec_decode/batch_expansion.py
+10 -20
@@ -1,9 +1,10 @@
1 from itertools import chain, count
2 -from typing import Dict, Iterator, List, Optional, Tuple
2 +from typing import Iterator, List, Tuple
3
4 import torch
5
6 -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
6 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7 + SequenceGroupMetadata)
8 from vllm.spec_decode.interfaces import (SpeculativeProposals,
9 SpeculativeScorer, SpeculativeScores)
10 from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
@@ -40,11 +41,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
41 @nvtx_range("BatchExpansionTop1Scorer.score_proposals")
42 def score_proposals(
43 self,
43 - seq_group_metadata_list: List[SequenceGroupMetadata],
44 - blocks_to_swap_in: Optional[Dict[int, int]],
45 - blocks_to_swap_out: Optional[Dict[int, int]],
46 - blocks_to_copy: Optional[Dict[int, List[int]]],
47 - k: int,
44 + execute_model_req: ExecuteModelRequest,
45 proposals: SpeculativeProposals,
46 ) -> SpeculativeScores:
47 """Score the proposed tokens via the scorer model.
@@ -57,11 +54,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
54 no speculation is produced for that sequence.
55
56 Args:
60 - seq_group_metadata_list: The input sequence group metadata.
61 - blocks_to_swap_in: This is passed to the worker during scoring.
62 - blocks_to_swap_out: This is passed to the worker during scoring.
63 - blocks_to_copy: This is passed to the worker during scoring.
64 - k: The fixed proposal length.
57 + execute_model_req: The execution request.
58 proposals: The speculative proposals to score.
59 Returns:
60 SpeculativeScores: The scores of each speculative token, along with
@@ -80,28 +73,25 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
73
74 (spec_indices, non_spec_indices, target_seq_group_metadata_list,
75 num_scoring_tokens) = self._expand_batch(
83 - seq_group_metadata_list=seq_group_metadata_list,
76 + seq_group_metadata_list=execute_model_req.seq_group_metadata_list,
77 proposal_token_ids_list=proposal_token_ids_list_without_skips,
78 proposal_lens_list=proposal_lens_list,
79 )
80
81 target_sampler_output = self._scorer_worker.execute_model(
89 - seq_group_metadata_list=target_seq_group_metadata_list,
90 - blocks_to_swap_in=blocks_to_swap_in,
91 - blocks_to_swap_out=blocks_to_swap_out,
92 - blocks_to_copy=blocks_to_copy,
93 - )
82 + execute_model_req=execute_model_req.clone(
83 + seq_group_metadata_list=target_seq_group_metadata_list, ))
84 assert len(target_sampler_output) == 1, "expected single-step output"
85 target_sampler_output = target_sampler_output[0]
86
87 all_tokens, all_probs, spec_logprobs = self._contract_batch(
98 - contracted_bs=len(seq_group_metadata_list),
88 + contracted_bs=len(execute_model_req.seq_group_metadata_list),
89 target_sampler_output=target_sampler_output,
90 proposals=proposals,
91 num_scoring_tokens=num_scoring_tokens,
92 non_spec_indices=non_spec_indices,
93 spec_indices=spec_indices,
104 - k=k,
94 + k=execute_model_req.num_lookahead_slots,
95 )
96
97 return SpeculativeScores(
vllm/spec_decode/interfaces.py
+3 -12
@@ -1,10 +1,9 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3 -from typing import Dict, List, Optional
3
4 import torch
5
7 -from vllm.sequence import SequenceGroupMetadata
6 +from vllm.sequence import ExecuteModelRequest
7
8
9 @dataclass
@@ -58,11 +57,7 @@ class SpeculativeProposer(ABC):
57 @abstractmethod
58 def get_proposals(
59 self,
61 - seq_group_metadata_list: List[SequenceGroupMetadata],
62 - blocks_to_swap_in: Dict[int, int],
63 - blocks_to_swap_out: Dict[int, int],
64 - blocks_to_copy: Dict[int, List[int]],
65 - max_proposal_len: int,
60 + execute_model_req: ExecuteModelRequest,
61 ) -> SpeculativeProposals:
62 raise NotImplementedError
63
@@ -72,11 +67,7 @@ class SpeculativeScorer(ABC):
67 @abstractmethod
68 def score_proposals(
69 self,
75 - seq_group_metadata_list: List[SequenceGroupMetadata],
76 - blocks_to_swap_in: Optional[Dict[int, int]],
77 - blocks_to_swap_out: Optional[Dict[int, int]],
78 - blocks_to_copy: Optional[Dict[int, List[int]]],
79 - k: int,
70 + execute_model_req: ExecuteModelRequest,
71 proposals: SpeculativeProposals,
72 ) -> SpeculativeScores:
73 raise NotImplementedError
vllm/spec_decode/multi_step_worker.py
+21 -33
@@ -1,9 +1,10 @@
1 import copy
2 -from typing import Dict, List, Tuple
2 +from typing import List, Tuple
3
4 import torch
5
6 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7 + SequenceGroupMetadata)
8 from vllm.spec_decode.interfaces import SpeculativeProposals
9 from vllm.spec_decode.top1_proposer import Top1Proposer
10 from vllm.worker.worker import Worker
@@ -44,10 +45,7 @@ class MultiStepWorker(Worker):
45 @torch.inference_mode()
46 def sampler_output(
47 self,
47 - seq_group_metadata_list: List[SequenceGroupMetadata],
48 - blocks_to_swap_in: Dict[int, int],
49 - blocks_to_swap_out: Dict[int, int],
50 - blocks_to_copy: Dict[int, List[int]],
48 + execute_model_req: ExecuteModelRequest,
49 sample_len: int,
50 ) -> Tuple[List[SamplerOutput], bool]:
51 """Run the model forward pass sample_len times. Returns the list of
@@ -57,26 +55,24 @@ class MultiStepWorker(Worker):
55
56 For multi step worker, this indicator shall be True.
57 """
60 - self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
61 - blocks_to_swap_out, blocks_to_copy)
58 + self._raise_if_unsupported(execute_model_req)
59
60 # Shallow copy input data so modifications (such as appending tokens)
61 # do not cause side-effects.
62 copied_seq_group_metadata_list = self._shallow_copy_inputs(
66 - seq_group_metadata_list)
63 + execute_model_req.seq_group_metadata_list)
64 + copied_execute_model_req = execute_model_req.clone(
65 + copied_seq_group_metadata_list)
66
67 # Assert enough KV space for sample_len tokens per sequence.
69 - self._assert_enough_kv_space(seq_group_metadata_list, sample_len)
68 + self._assert_enough_kv_space(execute_model_req.seq_group_metadata_list,
69 + sample_len)
70
71 # Run model sample_len times.
72 model_outputs = []
73 for _ in range(sample_len):
74 model_output = super().execute_model(
75 - seq_group_metadata_list=copied_seq_group_metadata_list,
76 - blocks_to_swap_in=blocks_to_swap_in,
77 - blocks_to_swap_out=blocks_to_swap_out,
78 - blocks_to_copy=blocks_to_copy,
79 - )
75 + execute_model_req=copied_execute_model_req)
76 assert (len(model_output) == 1
77 ), "composing multistep workers not supported"
78 model_output = model_output[0]
@@ -89,23 +85,13 @@ class MultiStepWorker(Worker):
85
86 def get_spec_proposals(
87 self,
92 - seq_group_metadata_list: List[SequenceGroupMetadata],
93 - blocks_to_swap_in: Dict[int, int],
94 - blocks_to_swap_out: Dict[int, int],
95 - blocks_to_copy: Dict[int, List[int]],
96 - max_proposal_len: int,
88 + execute_model_req: ExecuteModelRequest,
89 ) -> SpeculativeProposals:
90 """Produce speculations given an input batch of sequences. The number of
91 speculative tokens per sequence is determined by max_proposal_len.
92 """
93
102 - return self._proposer.get_proposals(
103 - seq_group_metadata_list,
104 - blocks_to_swap_in,
105 - blocks_to_swap_out,
106 - blocks_to_copy,
107 - max_proposal_len,
108 - )
94 + return self._proposer.get_proposals(execute_model_req)
95
96 def _append_new_tokens(
97 self, model_output: SamplerOutput,
@@ -196,20 +182,22 @@ class MultiStepWorker(Worker):
182
183 def _raise_if_unsupported(
184 self,
199 - seq_group_metadata_list: List[SequenceGroupMetadata],
200 - blocks_to_swap_in: Dict[int, int],
201 - blocks_to_swap_out: Dict[int, int],
202 - blocks_to_copy: Dict[int, List[int]],
185 + execute_model_req: ExecuteModelRequest,
186 ) -> None:
187 """MultiStepWorker does not yet implement support for cache swap
188 operations or beam search.
189 """
207 - if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
190 + if any([
191 + execute_model_req.blocks_to_swap_in,
192 + execute_model_req.blocks_to_swap_out,
193 + execute_model_req.blocks_to_copy
194 + ]):
195 raise NotImplementedError(
196 "MultiStepWorker does not support cache operations")
197
198 if any(
199 len(seq_group_metadata.seq_data.keys()) != 1
213 - for seq_group_metadata in seq_group_metadata_list):
200 + for seq_group_metadata in
201 + execute_model_req.seq_group_metadata_list):
202 raise NotImplementedError(
203 "MultiStepWorker does not support beam search.")
vllm/spec_decode/ngram_worker.py
+21 -41
@@ -1,8 +1,8 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import List, Optional, Tuple
2
3 import torch
4
5 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
5 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
6 from vllm.spec_decode.interfaces import SpeculativeProposals
7 from vllm.spec_decode.top1_proposer import Top1Proposer
8 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
@@ -46,13 +46,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
46 # NGram don't need gpu sampler
47 pass
48
49 - def execute_model(
50 - self,
51 - seq_group_metadata_list: List[SequenceGroupMetadata],
52 - blocks_to_swap_in: Optional[Dict[int, int]],
53 - blocks_to_swap_out: Optional[Dict[int, int]],
54 - blocks_to_copy: Optional[Dict[int, List[int]]],
55 - ) -> None:
49 + def execute_model(self, execute_model_req: ExecuteModelRequest) -> None:
50 """NGram doesn't depend on model execution, just pass this function"""
51 pass
52
@@ -71,10 +65,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
65
66 def sampler_output(
67 self,
74 - seq_group_metadata_list: List[SequenceGroupMetadata],
75 - blocks_to_swap_in: Dict[int, int],
76 - blocks_to_swap_out: Dict[int, int],
77 - blocks_to_copy: Dict[int, List[int]],
68 + execute_model_req: ExecuteModelRequest,
69 sample_len: int,
70 ) -> Tuple[Optional[List[SamplerOutput]], bool]:
71 """NGram match algo to pick proposal candidate. Returns the list of
@@ -83,16 +74,11 @@ class NGramWorker(LoraNotSupportedWorkerBase):
74 For ngram worker, we already done needed transposed internal, so the
75 indicator pass to sampler_output_to_torch shall be False.
76 """
86 - self._raise_if_unsupported(
87 - seq_group_metadata_list,
88 - blocks_to_swap_in,
89 - blocks_to_swap_out,
90 - blocks_to_copy,
91 - )
77 + self._raise_if_unsupported(execute_model_req)
78
79 arr = []
80 has_spec_out = False
95 - for seq_group_metadata in seq_group_metadata_list:
81 + for seq_group_metadata in execute_model_req.seq_group_metadata_list:
82 seq_data = next(iter(seq_group_metadata.seq_data.values()))
83
84 input_ids = torch.as_tensor(seq_data.get_token_ids(),
@@ -135,17 +121,19 @@ class NGramWorker(LoraNotSupportedWorkerBase):
121 indices = token_ids.unsqueeze(2)
122
123 token_probs = torch.zeros(
138 - (len(seq_group_metadata_list), sample_len, self.vocab_size),
124 + (len(execute_model_req.seq_group_metadata_list), sample_len,
125 + self.vocab_size),
126 dtype=torch.float32,
127 device=self.device,
128 )
129 token_probs.scatter_(2, indices, 1)
130 token_logprobs = torch.zeros(
144 - (len(seq_group_metadata_list), sample_len, self.vocab_size),
131 + (len(execute_model_req.seq_group_metadata_list), sample_len,
132 + self.vocab_size),
133 dtype=torch.float32,
134 device=self.device,
135 )
148 - for i in range(len(seq_group_metadata_list)):
136 + for i in range(len(execute_model_req.seq_group_metadata_list)):
137 outputs.append(
138 SamplerOutput(
139 outputs=None,
@@ -157,40 +145,32 @@ class NGramWorker(LoraNotSupportedWorkerBase):
145
146 def get_spec_proposals(
147 self,
160 - seq_group_metadata_list: List[SequenceGroupMetadata],
161 - blocks_to_swap_in: Dict[int, int],
162 - blocks_to_swap_out: Dict[int, int],
163 - blocks_to_copy: Dict[int, List[int]],
164 - max_proposal_len: int,
148 + execute_model_req: ExecuteModelRequest,
149 ) -> SpeculativeProposals:
150 """Produce speculations given an input batch of sequences. The number of
151 speculative tokens per sequence is determined by max_proposal_len.
152 """
153
170 - return self._proposer.get_proposals(
171 - seq_group_metadata_list,
172 - blocks_to_swap_in,
173 - blocks_to_swap_out,
174 - blocks_to_copy,
175 - max_proposal_len,
176 - )
154 + return self._proposer.get_proposals(execute_model_req)
155
156 def _raise_if_unsupported(
157 self,
180 - seq_group_metadata_list: List[SequenceGroupMetadata],
181 - blocks_to_swap_in: Dict[int, int],
182 - blocks_to_swap_out: Dict[int, int],
183 - blocks_to_copy: Dict[int, List[int]],
158 + execute_model_req: ExecuteModelRequest,
159 ) -> None:
160 """NGramWorker does not yet implement support for cache swap
161 operations or beam search.
162 """
188 - if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
163 + if any([
164 + execute_model_req.blocks_to_swap_in,
165 + execute_model_req.blocks_to_swap_out,
166 + execute_model_req.blocks_to_copy
167 + ]):
168 raise NotImplementedError(
169 "NGramWorker does not support cache operations")
170
171 if any(
172 len(seq_group_metadata.seq_data.keys()) != 1
194 - for seq_group_metadata in seq_group_metadata_list):
173 + for seq_group_metadata in
174 + execute_model_req.seq_group_metadata_list):
175 raise NotImplementedError(
176 "NGramWorker does not support beam search.")
vllm/spec_decode/spec_decode_worker.py
+23 -67
@@ -1,11 +1,12 @@
1 from functools import cached_property
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import List, Optional, Tuple
3
4 import torch
5
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
9 + SequenceGroupMetadata)
10 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
11 from vllm.spec_decode.interfaces import (SpeculativeProposals,
12 SpeculativeScorer, SpeculativeScores)
@@ -189,69 +190,37 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
190
191 @torch.inference_mode()
192 def execute_model(
192 - self,
193 - seq_group_metadata_list: List[SequenceGroupMetadata],
194 - blocks_to_swap_in: Optional[Dict[int, int]],
195 - blocks_to_swap_out: Optional[Dict[int, int]],
196 - blocks_to_copy: Optional[Dict[int, List[int]]],
197 - num_lookahead_slots: int,
198 - ) -> List[SamplerOutput]:
193 + self,
194 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
195 """Perform speculative decoding on the input batch.
196 """
197
202 - assert seq_group_metadata_list is not None, (
198 + assert execute_model_req.seq_group_metadata_list is not None, (
199 "speculative decoding "
200 "requires non-None seq_group_metadata_list")
201
206 - #logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
207 - # num_lookahead_slots)
208 -
202 # If no spec tokens, call the proposer and scorer workers normally.
203 # Used for prefill.
211 - if num_lookahead_slots == 0 or len(seq_group_metadata_list) == 0:
212 - return self._run_no_spec(
213 - seq_group_metadata_list=seq_group_metadata_list,
214 - blocks_to_swap_in=blocks_to_swap_in,
215 - blocks_to_swap_out=blocks_to_swap_out,
216 - blocks_to_copy=blocks_to_copy,
217 - )
218 -
219 - return self._run_speculative_decoding_step(
220 - seq_group_metadata_list=seq_group_metadata_list,
221 - blocks_to_swap_in=blocks_to_swap_in,
222 - blocks_to_swap_out=blocks_to_swap_out,
223 - blocks_to_copy=blocks_to_copy,
224 - k=num_lookahead_slots,
225 - )
204 + if execute_model_req.num_lookahead_slots == 0 or len(
205 + execute_model_req.seq_group_metadata_list) == 0:
206 + return self._run_no_spec(execute_model_req)
207 +
208 + return self._run_speculative_decoding_step(execute_model_req)
209
210 @nvtx_range("spec_decode_worker._run_no_spec")
211 def _run_no_spec(
229 - self,
230 - seq_group_metadata_list: List[SequenceGroupMetadata],
231 - blocks_to_swap_in: Optional[Dict[int, int]],
232 - blocks_to_swap_out: Optional[Dict[int, int]],
233 - blocks_to_copy: Optional[Dict[int, List[int]]],
234 - ) -> List[SamplerOutput]:
212 + self,
213 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
214 """Run a prefill step, without any speculation. The input is sent to the
215 proposer and scorer model so that the KV cache is consistent between the
216 two.
217 """
218 #logger.info("run proposer worker no spec")
219
241 - self.proposer_worker.execute_model(
242 - seq_group_metadata_list=seq_group_metadata_list,
243 - blocks_to_swap_in=blocks_to_swap_in,
244 - blocks_to_swap_out=blocks_to_swap_out,
245 - blocks_to_copy=blocks_to_copy,
246 - )
220 + self.proposer_worker.execute_model(execute_model_req)
221
222 #logger.info("run target worker no spec")
249 - sampler_output = self.scorer_worker.execute_model(
250 - seq_group_metadata_list=seq_group_metadata_list,
251 - blocks_to_swap_in=blocks_to_swap_in,
252 - blocks_to_swap_out=blocks_to_swap_out,
253 - blocks_to_copy=blocks_to_copy,
254 - )
223 + sampler_output = self.scorer_worker.execute_model(execute_model_req)
224 assert len(sampler_output) == 1
225 sampler_output = sampler_output[0]
226
@@ -264,13 +233,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
233
234 @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
235 def _run_speculative_decoding_step(
267 - self,
268 - seq_group_metadata_list: List[SequenceGroupMetadata],
269 - blocks_to_swap_in: Optional[Dict[int, int]],
270 - blocks_to_swap_out: Optional[Dict[int, int]],
271 - blocks_to_copy: Optional[Dict[int, List[int]]],
272 - k: int,
273 - ) -> List[SamplerOutput]:
236 + self,
237 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
238 """Execute a single step of speculative decoding.
239
240 This invokes the proposer worker to get k speculative tokens for each
@@ -282,33 +246,25 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
246
247 #logger.info("get spec proposals")
248 # Generate proposals using draft worker.
285 - assert blocks_to_swap_in is not None
286 - assert blocks_to_swap_out is not None
287 - assert blocks_to_copy is not None
288 - proposals = self.proposer_worker.get_spec_proposals(
289 - seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
290 - blocks_to_copy, k)
249 + proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
250
251 #logger.info("score proposals")
252 proposal_scores = self.scorer.score_proposals(
294 - seq_group_metadata_list,
295 - blocks_to_swap_in,
296 - blocks_to_swap_out,
297 - blocks_to_copy,
298 - k,
253 + execute_model_req,
254 proposals,
255 )
256
257 #logger.info("verify proposals")
258 accepted_token_ids, target_logprobs = self._verify_tokens(
304 - seq_group_metadata_list, proposal_scores, proposals, k)
259 + execute_model_req.seq_group_metadata_list, proposal_scores,
260 + proposals, execute_model_req.num_lookahead_slots)
261
262 #logger.info("create output list")
263 return self._create_output_sampler_list(
308 - seq_group_metadata_list,
264 + execute_model_req.seq_group_metadata_list,
265 accepted_token_ids,
266 target_logprobs=target_logprobs,
311 - k=k)
267 + k=execute_model_req.num_lookahead_slots)
268
269 @nvtx_range("spec_decode_worker._verify_tokens")
270 def _verify_tokens(
vllm/spec_decode/top1_proposer.py
+11 -11
@@ -1,8 +1,9 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import List, Optional, Tuple
2
3 import torch
4
5 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
5 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
6 + SequenceGroupMetadata)
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeProposer)
9 from vllm.spec_decode.util import sampler_output_to_torch
@@ -40,17 +41,15 @@ class Top1Proposer(SpeculativeProposer):
41
42 def get_proposals(
43 self,
43 - seq_group_metadata_list: List[SequenceGroupMetadata],
44 - blocks_to_swap_in: Dict[int, int],
45 - blocks_to_swap_out: Dict[int, int],
46 - blocks_to_copy: Dict[int, List[int]],
47 - proposal_len: int,
44 + execute_model_req: ExecuteModelRequest,
45 ) -> SpeculativeProposals:
46 """Get speculative proposals given the input batch.
47
48 Sequences which would exceed the max model length are skipped during
49 speculation.
50 """
51 + proposal_len = execute_model_req.num_lookahead_slots
52 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
53
54 # Split speculative- and non-speculative- sequences.
55 (
@@ -66,11 +65,12 @@ class Top1Proposer(SpeculativeProposer):
65 # token_ids is like [batch] format in proposal_len size list,
66 # while if it is false, the format would be [proposal_len]
67 # in batch size list
69 - maybe_sampler_output, transposed = self._worker.sampler_output(
68 + nonzero_execute_model_req = ExecuteModelRequest(
69 seq_group_metadata_list=nonzero_proposal_len_seqs,
71 - blocks_to_swap_in=blocks_to_swap_in,
72 - blocks_to_swap_out=blocks_to_swap_out,
73 - blocks_to_copy=blocks_to_copy,
70 + num_lookahead_slots=proposal_len,
71 + )
72 + maybe_sampler_output, transposed = self._worker.sampler_output(
73 + execute_model_req=nonzero_execute_model_req,
74 sample_len=proposal_len,
75 )
76 else:
vllm/worker/cpu_worker.py
+13 -12
@@ -13,7 +13,7 @@ from vllm.distributed import (broadcast_tensor_dict,
13 init_distributed_environment)
14 from vllm.logger import init_logger
15 from vllm.model_executor import set_random_seed
16 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18 from vllm.worker.cpu_model_runner import CPUModelRunner
19 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
@@ -256,22 +256,24 @@ class CPUWorker(LoraNotSupportedWorkerBase):
256 @torch.inference_mode()
257 def execute_model(
258 self,
259 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
260 - blocks_to_swap_in: Optional[Dict[int, int]] = None,
261 - blocks_to_swap_out: Optional[Dict[int, int]] = None,
262 - blocks_to_copy: Optional[Dict[int, List[int]]] = None,
259 + execute_model_req: Optional[ExecuteModelRequest] = None,
260 ) -> List[SamplerOutput]:
261 +
262 + if execute_model_req is None:
263 + seq_group_metadata_list = None
264 + else:
265 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
266 +
267 if self.is_driver_worker:
268 assert seq_group_metadata_list is not None
269 num_seq_groups: int = len(seq_group_metadata_list)
267 - assert blocks_to_swap_in is not None
268 - assert blocks_to_swap_out is not None
269 - assert blocks_to_copy is not None
270 - assert len(blocks_to_swap_in) == 0
271 - assert len(blocks_to_swap_out) == 0
270 + assert execute_model_req is not None
271 + blocks_to_copy = execute_model_req.blocks_to_copy
272 + assert len(execute_model_req.blocks_to_swap_in) == 0
273 + assert len(execute_model_req.blocks_to_swap_out) == 0
274 data: Dict[str, Any] = {
275 "num_seq_groups": num_seq_groups,
274 - "blocks_to_copy": blocks_to_copy,
276 + "blocks_to_copy": execute_model_req.blocks_to_copy,
277 }
278 broadcast_tensor_dict(data, src=0)
279 else:
@@ -279,7 +281,6 @@ class CPUWorker(LoraNotSupportedWorkerBase):
281 num_seq_groups = data["num_seq_groups"]
282 blocks_to_copy = data["blocks_to_copy"]
283
282 - assert blocks_to_copy is not None
284 self.cache_copy(blocks_to_copy)
285
286 # If there is no input, we don't need to execute the model.
vllm/worker/worker.py
+11 -12
@@ -18,7 +18,7 @@ from vllm.distributed.device_communicators.custom_all_reduce import (
18 init_custom_ar)
19 from vllm.lora.request import LoRARequest
20 from vllm.model_executor import set_random_seed
21 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
21 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
22 from vllm.worker.cache_engine import CacheEngine
23 from vllm.worker.model_runner import ModelRunner
24 from vllm.worker.worker_base import WorkerBase
@@ -211,19 +211,21 @@ class Worker(WorkerBase):
211 @torch.inference_mode()
212 def execute_model(
213 self,
214 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
215 - blocks_to_swap_in: Optional[Dict[int, int]] = None,
216 - blocks_to_swap_out: Optional[Dict[int, int]] = None,
217 - blocks_to_copy: Optional[Dict[int, List[int]]] = None,
218 - num_lookahead_slots: int = 0,
214 + execute_model_req: Optional[ExecuteModelRequest] = None
215 ) -> List[SamplerOutput]:
216
217 + if execute_model_req is None:
218 + seq_group_metadata_list = None
219 + else:
220 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
221 +
222 if self.is_driver_worker:
223 assert seq_group_metadata_list is not None
224 + assert execute_model_req is not None
225 num_seq_groups = len(seq_group_metadata_list)
224 - assert blocks_to_swap_in is not None
225 - assert blocks_to_swap_out is not None
226 - assert blocks_to_copy is not None
226 + blocks_to_swap_in = execute_model_req.blocks_to_swap_in
227 + blocks_to_swap_out = execute_model_req.blocks_to_swap_out
228 + blocks_to_copy = execute_model_req.blocks_to_copy
229 data: Dict[str, Any] = {
230 "num_seq_groups": num_seq_groups,
231 "blocks_to_swap_in": blocks_to_swap_in,
@@ -238,9 +240,6 @@ class Worker(WorkerBase):
240 blocks_to_swap_out = data["blocks_to_swap_out"]
241 blocks_to_copy = data["blocks_to_copy"]
242
241 - assert blocks_to_swap_in is not None
242 - assert blocks_to_swap_out is not None
243 - assert blocks_to_copy is not None
243 self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
244
245 # If there is no input, we don't need to execute the model.
vllm/worker/worker_base.py
+3 -5
@@ -5,7 +5,7 @@ from typing import Dict, List, Set, Tuple
5
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 from vllm.utils import (enable_trace_function_call_for_thread,
10 update_environment_variables)
11
@@ -48,10 +48,8 @@ class WorkerBase(ABC):
48
49 @abstractmethod
50 def execute_model(
51 - self, seq_group_metadata_list: List[SequenceGroupMetadata],
52 - blocks_to_swap_in: Dict[int, int], blocks_to_swap_out: Dict[int,
53 - int],
54 - blocks_to_copy: Dict[int, List[int]]) -> List[SamplerOutput]:
51 + self,
52 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
53 """Executes at least one model step on the given sequences, unless no
54 sequences are provided."""
55 raise NotImplementedError