A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc][Refactor] Introduce ExecuteModelData (#4540)
Cody Yu committed
May 3, 2024 at 17:47 UTC
bc8ad68455ce41ba672764f4a53df5a87d1dbe99
23 files changed
+359
-515
tests/spec_decode/test_multi_step_worker.py
+48
-50
@@ -5,13 +5,12 @@ import pytest
5
import torch
6
7
from vllm.model_executor.utils import set_random_seed
8
-from vllm.sequence import SamplerOutput
8
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
from vllm.spec_decode.multi_step_worker import MultiStepWorker
10
from vllm.spec_decode.top1_proposer import Top1Proposer
11
from vllm.worker.worker import Worker
12
13
from .utils import (assert_logprobs_dict_allclose, create_batch,
14
- create_execute_model_data,
14
create_seq_group_metadata_from_prompts, create_worker,
15
patch_execute_model_with_seeds, zero_kv_cache)
16
@@ -105,31 +104,32 @@ def test_same_output_for_single_step():
104
105
final_prompt_lens = [len(prompt) + num_steps for prompt in prompts]
106
108
- multi_step_execute_model_data = create_execute_model_data(
109
- seq_group_metadata_list=create_seq_group_metadata_from_prompts(
110
- prompts,
111
- num_gpu_blocks,
112
- block_size,
113
- final_prompt_lens=final_prompt_lens))
114
-
115
- single_step_execute_model_data = create_execute_model_data(
116
- seq_group_metadata_list=create_seq_group_metadata_from_prompts(
117
- prompts,
118
- num_gpu_blocks,
119
- block_size,
120
- final_prompt_lens=final_prompt_lens))
107
+ multi_step_seq_group = create_seq_group_metadata_from_prompts(
108
+ prompts,
109
+ num_gpu_blocks,
110
+ block_size,
111
+ final_prompt_lens=final_prompt_lens)
112
113
zero_kv_cache(multi_step_worker.cache_engine)
114
set_random_seed(seed)
115
actual_output, _ = multi_step_worker.sampler_output(
125
- **multi_step_execute_model_data.to_dict(), sample_len=num_steps)
116
+ execute_model_req=ExecuteModelRequest(
117
+ seq_group_metadata_list=multi_step_seq_group),
118
+ sample_len=num_steps)
119
assert len(actual_output) == num_steps
120
actual_output = actual_output[0]
121
122
+ single_step_seq_group = create_seq_group_metadata_from_prompts(
123
+ prompts,
124
+ num_gpu_blocks,
125
+ block_size,
126
+ final_prompt_lens=final_prompt_lens)
127
+
128
zero_kv_cache(worker.cache_engine)
129
set_random_seed(seed)
130
expected_output = worker.execute_model(
132
- **single_step_execute_model_data.to_dict(), )[0]
131
+ execute_model_req=ExecuteModelRequest(
132
+ seq_group_metadata_list=single_step_seq_group))[0]
133
134
actual_token_ids = [
135
output.samples[0].output_token for output in actual_output
@@ -193,19 +193,20 @@ def test_same_output_for_multi_step():
193
worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
194
195
continuations = [[1] for _ in prompts]
196
- execute_model_data = create_execute_model_data(
197
- create_seq_group_metadata_from_prompts(
198
- prompts,
199
- num_gpu_blocks,
200
- block_size,
201
- continuations=continuations,
202
- final_prompt_lens=final_prompt_lens), )
196
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
197
+ prompts,
198
+ num_gpu_blocks,
199
+ block_size,
200
+ continuations=continuations,
201
+ final_prompt_lens=final_prompt_lens)
202
203
# Run multi-step.
204
zero_kv_cache(multi_step_worker.cache_engine)
205
set_random_seed(seed)
206
multi_step_output, _ = multi_step_worker.sampler_output(
208
- **execute_model_data.to_dict(), sample_len=num_steps)
207
+ execute_model_req=ExecuteModelRequest(
208
+ seq_group_metadata_list=seq_group_metadata_list),
209
+ sample_len=num_steps)
210
211
# Run single-step repeatedly.
212
zero_kv_cache(worker.cache_engine)
@@ -215,16 +216,16 @@ def test_same_output_for_multi_step():
216
217
for _ in multi_step_output:
218
218
- execute_model_data = create_execute_model_data(
219
- create_seq_group_metadata_from_prompts(
220
- prompts,
221
- num_gpu_blocks,
222
- block_size,
223
- continuations=continuations,
224
- final_prompt_lens=final_prompt_lens))
219
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
220
+ prompts,
221
+ num_gpu_blocks,
222
+ block_size,
223
+ continuations=continuations,
224
+ final_prompt_lens=final_prompt_lens)
225
226
single_step_output.extend(
227
- worker.execute_model(**execute_model_data.to_dict(), ))
227
+ worker.execute_model(execute_model_req=ExecuteModelRequest(
228
+ seq_group_metadata_list=seq_group_metadata_list)))
229
230
# Append output tokens to new sequence data.
231
for i, seq_group_output in enumerate(single_step_output[-1]):
@@ -304,12 +305,11 @@ def test_draft_proposals_full_speculation_len():
305
) for _ in range(k)
306
], True
307
307
- execute_model_data, _, _ = create_batch(batch_size, k)
308
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
309
309
- proposals = proposer.get_proposals(
310
- **execute_model_data.to_dict(),
311
- proposal_len=k,
312
- )
310
+ proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
311
+ seq_group_metadata_list=seq_group_metadata_list,
312
+ num_lookahead_slots=k), )
313
314
assert torch.is_tensor(proposals.proposal_token_ids)
315
assert torch.is_tensor(proposals.proposal_probs)
@@ -340,14 +340,13 @@ def test_draft_proposals_no_speculations():
340
max_proposal_len=prompt_len + k - 1,
341
)
342
343
- execute_model_data, _, _ = create_batch(batch_size,
344
- k,
345
- prompt_len=prompt_len)
343
+ seq_group_metadata_list, _, _ = create_batch(batch_size,
344
+ k,
345
+ prompt_len=prompt_len)
346
347
- proposals = proposer.get_proposals(
348
- **execute_model_data.to_dict(),
349
- proposal_len=k,
350
- )
347
+ proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
348
+ seq_group_metadata_list=seq_group_metadata_list,
349
+ num_lookahead_slots=k), )
350
351
assert torch.is_tensor(proposals.proposal_token_ids)
352
assert torch.is_tensor(proposals.proposal_probs)
@@ -409,17 +408,16 @@ def test_draft_proposals_mixed_k():
408
) for _ in range(k)
409
], True
410
412
- execute_model_data, _, _ = create_batch(
411
+ seq_group_metadata_list, _, _ = create_batch(
412
batch_size,
413
k,
414
prompt_len=prompt_len,
415
prev_output_token_len=prev_output_token_len,
416
)
417
419
- proposals = proposer.get_proposals(
420
- **execute_model_data.to_dict(),
421
- proposal_len=k,
422
- )
418
+ proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
419
+ seq_group_metadata_list=seq_group_metadata_list,
420
+ num_lookahead_slots=k), )
421
422
assert torch.is_tensor(proposals.proposal_token_ids)
423
assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_ngram_worker.py
+29
-35
@@ -1,10 +1,10 @@
1
import torch
2
3
+from vllm.sequence import ExecuteModelRequest
4
from vllm.spec_decode.ngram_worker import NGramWorker
5
from vllm.spec_decode.top1_proposer import Top1Proposer
6
6
-from .utils import (create_execute_model_data,
7
- create_seq_group_metadata_from_prompts, create_worker)
7
+from .utils import create_seq_group_metadata_from_prompts, create_worker
8
9
10
def test_ngram_algo_correctness_for_single_no_match():
@@ -44,17 +44,15 @@ def test_ngram_algo_correctness_for_single_no_match():
44
45
proposal_len = 5
46
final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
47
- ngram_sampler_output_data = create_execute_model_data(
48
- seq_group_metadata_list=create_seq_group_metadata_from_prompts(
49
- prompts,
50
- num_gpu_blocks,
51
- block_size,
52
- final_prompt_lens=final_prompt_lens))
53
-
54
- proposals = proposer.get_proposals(
55
- **ngram_sampler_output_data.to_dict(),
56
- proposal_len=proposal_len,
57
- )
47
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
48
+ prompts,
49
+ num_gpu_blocks,
50
+ block_size,
51
+ final_prompt_lens=final_prompt_lens)
52
+
53
+ proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
54
+ seq_group_metadata_list=seq_group_metadata_list,
55
+ num_lookahead_slots=proposal_len), )
56
57
assert torch.is_tensor(proposals.proposal_token_ids)
58
assert torch.is_tensor(proposals.proposal_probs)
@@ -113,17 +111,15 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
111
112
proposal_len = 5
113
final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
116
- ngram_sampler_output_data = create_execute_model_data(
117
- seq_group_metadata_list=create_seq_group_metadata_from_prompts(
118
- prompts,
119
- num_gpu_blocks,
120
- block_size,
121
- final_prompt_lens=final_prompt_lens))
122
-
123
- proposals = proposer.get_proposals(
124
- **ngram_sampler_output_data.to_dict(),
125
- proposal_len=proposal_len,
126
- )
114
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
115
+ prompts,
116
+ num_gpu_blocks,
117
+ block_size,
118
+ final_prompt_lens=final_prompt_lens)
119
+
120
+ proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
121
+ seq_group_metadata_list=seq_group_metadata_list,
122
+ num_lookahead_slots=proposal_len), )
123
124
assert torch.is_tensor(proposals.proposal_token_ids)
125
assert torch.is_tensor(proposals.proposal_probs)
@@ -185,17 +181,15 @@ def test_ngram_algo_correctness_for_batches_match_all():
181
182
proposal_len = 5
183
final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
188
- ngram_sampler_output_data = create_execute_model_data(
189
- seq_group_metadata_list=create_seq_group_metadata_from_prompts(
190
- prompts,
191
- num_gpu_blocks,
192
- block_size,
193
- final_prompt_lens=final_prompt_lens))
194
-
195
- proposals = proposer.get_proposals(
196
- **ngram_sampler_output_data.to_dict(),
197
- proposal_len=proposal_len,
198
- )
184
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
185
+ prompts,
186
+ num_gpu_blocks,
187
+ block_size,
188
+ final_prompt_lens=final_prompt_lens)
189
+
190
+ proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
191
+ seq_group_metadata_list=seq_group_metadata_list,
192
+ num_lookahead_slots=proposal_len), )
193
194
assert torch.is_tensor(proposals.proposal_token_ids)
195
assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_spec_decode_worker.py
+46
-49
@@ -7,7 +7,7 @@ import torch
7
8
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9
from vllm.model_executor.utils import set_random_seed
10
-from vllm.sequence import SamplerOutput
10
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
11
from vllm.spec_decode.interfaces import SpeculativeProposals
12
from vllm.spec_decode.metrics import (AsyncMetricsCollector,
13
SpecDecodeWorkerMetrics)
@@ -15,8 +15,7 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
15
from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
16
split_num_cache_blocks_evenly)
17
18
-from .utils import (ExecuteModelData, create_batch, create_sampler_output_list,
19
- mock_worker)
18
+from .utils import create_batch, create_sampler_output_list, mock_worker
19
20
21
@pytest.mark.parametrize('k', [1, 2, 6])
@@ -36,24 +35,19 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
35
exception_secret = 'artificial stop'
36
draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
37
39
- execute_model_data, _, _ = create_batch(batch_size, k)
38
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
39
+ execute_model_req = ExecuteModelRequest(
40
+ seq_group_metadata_list=seq_group_metadata_list, num_lookahead_slots=k)
41
42
with pytest.raises(ValueError, match=exception_secret):
42
- worker.execute_model(**execute_model_data.to_dict(),
43
- num_lookahead_slots=k)
43
+ worker.execute_model(execute_model_req=execute_model_req)
44
45
call_args_list = draft_worker.get_spec_proposals.call_args_list
46
assert len(call_args_list) == 1
47
48
for args, _ in call_args_list:
49
- (seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
50
- blocks_to_copy, actual_k) = args
51
- actual_execute_model_data = ExecuteModelData(seq_group_metadata_list,
52
- blocks_to_swap_in,
53
- blocks_to_swap_out,
54
- blocks_to_copy)
55
- assert actual_execute_model_data == execute_model_data
56
- assert actual_k == k
49
+ actual_execute_model_data = args[0]
50
+ assert actual_execute_model_data == execute_model_req
51
52
53
@pytest.mark.parametrize('k', [1, 2, 6])
@@ -93,7 +87,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
87
proposal_lens = torch.ones(batch_size, dtype=torch.int64,
88
device='cuda') * k
89
96
- execute_model_data, prompts, prev_output_tokens = create_batch(
90
+ seq_group_metadata_list, prompts, prev_output_tokens = create_batch(
91
batch_size, k)
92
93
draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
@@ -105,20 +99,20 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
99
target_worker.execute_model.side_effect = ValueError(exception_secret)
100
101
with pytest.raises(ValueError, match=exception_secret):
108
- worker.execute_model(**execute_model_data.to_dict(),
109
- num_lookahead_slots=k)
102
+ worker.execute_model(execute_model_req=ExecuteModelRequest(
103
+ seq_group_metadata_list=seq_group_metadata_list,
104
+ num_lookahead_slots=k))
105
106
seen_contexts = []
107
108
call_args_list = target_worker.execute_model.call_args_list
109
assert len(call_args_list) == 1
115
- for args, kwargs in call_args_list:
116
- target_execute_model_data = ExecuteModelData.from_dict(kwargs)
110
+ for _, kwargs in call_args_list:
111
+ seq_group_metadata_list = kwargs[
112
+ "execute_model_req"].seq_group_metadata_list
113
118
- assert len(target_execute_model_data.seq_group_metadata_list) == (
119
- k + 1) * batch_size
120
- for seq_group_metadata in (
121
- target_execute_model_data.seq_group_metadata_list):
114
+ assert len(seq_group_metadata_list) == (k + 1) * batch_size
115
+ for seq_group_metadata in seq_group_metadata_list:
116
for seq_data in seq_group_metadata.seq_data.values():
117
seen_contexts.append(seq_data.get_token_ids())
118
@@ -175,7 +169,7 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
169
proposal_lens = torch.ones(batch_size, dtype=torch.int64,
170
device='cuda') * k
171
178
- execute_model_data, _, _ = create_batch(batch_size, k)
172
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
173
174
draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
175
proposal_token_ids=proposal_token_ids,
@@ -207,8 +201,9 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
201
rejection_sampler.side_effect = ValueError(exception_secret)
202
203
with pytest.raises(ValueError, match=exception_secret):
210
- worker.execute_model(**execute_model_data.to_dict(),
211
- num_lookahead_slots=k)
204
+ worker.execute_model(execute_model_req=ExecuteModelRequest(
205
+ seq_group_metadata_list=seq_group_metadata_list,
206
+ num_lookahead_slots=k))
207
208
assert len(rejection_sampler.call_args_list) == 1
209
_, kwargs = rejection_sampler.call_args_list[0]
@@ -262,7 +257,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
257
proposal_lens = torch.ones(batch_size, dtype=torch.int64,
258
device='cuda') * k
259
265
- execute_model_data, _, _ = create_batch(batch_size, k)
260
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
261
262
draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
263
proposal_token_ids=proposal_token_ids,
@@ -302,8 +297,9 @@ def test_correctly_formats_output(k: int, batch_size: int):
297
298
rejection_sampler.return_value = rejection_sampler_output
299
305
- output = worker.execute_model(**execute_model_data.to_dict(),
306
- num_lookahead_slots=k)
300
+ output = worker.execute_model(execute_model_req=ExecuteModelRequest(
301
+ seq_group_metadata_list=seq_group_metadata_list,
302
+ num_lookahead_slots=k))
303
304
expected_output = create_sampler_output_list(
305
token_ids=rejection_sampler_output.transpose(0, 1),
@@ -312,7 +308,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
308
309
seq_ids = [
310
next(iter(seq_group_metadata.seq_data.keys()))
315
- for seq_group_metadata in execute_model_data.seq_group_metadata_list
311
+ for seq_group_metadata in seq_group_metadata_list
312
]
313
actual_output_by_seq = {seq_id: [] for seq_id in seq_ids}
314
expected_output_by_seq = {seq_id: [] for seq_id in seq_ids}
@@ -383,7 +379,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
379
proposal_lens = torch.ones(batch_size, dtype=torch.int64,
380
device='cuda') * k
381
386
- execute_model_data, _, _ = create_batch(batch_size, k)
382
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
383
384
draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
385
proposal_token_ids=proposal_token_ids,
@@ -428,8 +424,9 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
424
metrics_collector.maybe_collect_rejsample_metrics.return_value = (
425
mock_rejsample_metrics)
426
431
- output = worker.execute_model(**execute_model_data.to_dict(),
432
- num_lookahead_slots=k)
427
+ output = worker.execute_model(execute_model_req=ExecuteModelRequest(
428
+ seq_group_metadata_list=seq_group_metadata_list,
429
+ num_lookahead_slots=k))
430
assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
431
432
call_args_list = (
@@ -462,21 +459,21 @@ def test_k_equals_zero(k: int, batch_size: int):
459
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
460
metrics_collector)
461
465
- execute_model_data, prompts, prev_output_tokens = create_batch(
466
- batch_size, k, prev_output_token_len=0)
462
+ seq_group_metadata_list, _, _ = create_batch(batch_size,
463
+ k,
464
+ prev_output_token_len=0)
465
+ execute_model_req = ExecuteModelRequest(
466
+ seq_group_metadata_list=seq_group_metadata_list, num_lookahead_slots=k)
467
468
- out = worker.execute_model(**execute_model_data.to_dict(),
469
- num_lookahead_slots=k)
468
+ out = worker.execute_model(execute_model_req=execute_model_req)
469
470
assert len(out) == 1, f"expected only one token output when {k=}"
471
assert out[0].probs is None, "expect gpu tensor references to be None"
472
assert out[
473
0].sampled_tokens is None, "expect gpu tensor references to be None"
474
476
- draft_worker.execute_model.assert_called_once_with(
477
- **execute_model_data.to_dict())
478
- target_worker.execute_model.assert_called_once_with(
479
- **execute_model_data.to_dict())
475
+ draft_worker.execute_model.assert_called_once_with(execute_model_req)
476
+ target_worker.execute_model.assert_called_once_with(execute_model_req)
477
478
479
@pytest.mark.parametrize('k', [0, 5])
@@ -503,21 +500,21 @@ def test_empty_input_batch(k: int, batch_size: int):
500
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
501
metrics_collector)
502
506
- execute_model_data, prompts, prev_output_tokens = create_batch(
507
- batch_size, k, prev_output_token_len=0)
503
+ seq_group_metadata_list, _, _ = create_batch(batch_size,
504
+ k,
505
+ prev_output_token_len=0)
506
+ execute_model_req = ExecuteModelRequest(
507
+ seq_group_metadata_list=seq_group_metadata_list, num_lookahead_slots=k)
508
509
- out = worker.execute_model(**execute_model_data.to_dict(),
510
- num_lookahead_slots=k)
509
+ out = worker.execute_model(execute_model_req=execute_model_req)
510
511
assert len(out) == 1, f"expected only one token output when {k=}"
512
assert out[0].probs is None, "expect gpu tensor references to be None"
513
assert out[
514
0].sampled_tokens is None, "expect gpu tensor references to be None"
515
517
- draft_worker.execute_model.assert_called_once_with(
518
- **execute_model_data.to_dict())
519
- target_worker.execute_model.assert_called_once_with(
520
- **execute_model_data.to_dict())
516
+ draft_worker.execute_model.assert_called_once_with(execute_model_req)
517
+ target_worker.execute_model.assert_called_once_with(execute_model_req)
518
519
520
@pytest.mark.skip_global_cleanup
tests/spec_decode/utils.py
+4
-46
@@ -1,4 +1,3 @@
1
-from dataclasses import dataclass, fields
1
from itertools import count
2
from typing import Dict, Iterable, List, Optional, Union
3
from unittest.mock import MagicMock
@@ -16,50 +15,10 @@ from vllm.worker.cache_engine import CacheEngine
15
from vllm.worker.worker import Worker
16
17
19
-@dataclass
20
-class ExecuteModelData:
21
- """Helper data structure which facilitates cleaner tests.
22
- """
23
- seq_group_metadata_list: List[SequenceGroupMetadata]
24
- blocks_to_swap_in: Dict[int, int]
25
- blocks_to_swap_out: Dict[int, int]
26
- blocks_to_copy: Dict[int, List[int]]
27
-
28
- def to_dict(self):
29
- return dict(
30
- (field.name, getattr(self, field.name)) for field in fields(self))
31
-
32
- @classmethod
33
- def from_dict(cls, d):
34
- cleaned = dict((field.name, d[field.name]) for field in fields(cls))
35
- return cls(**cleaned)
36
-
37
-
18
def round_up_to_next_block(seq_len: int, block_size: int) -> int:
19
return (seq_len + block_size - 1) // block_size
20
21
42
-def create_execute_model_data(
43
- seq_group_metadata_list: List[SequenceGroupMetadata],
44
- blocks_to_swap_in: Optional[Dict[int, int]] = None,
45
- blocks_to_swap_out: Optional[Dict[int, int]] = None,
46
- blocks_to_copy: Optional[Dict[int, int]] = None,
47
-) -> ExecuteModelData:
48
- if blocks_to_swap_in is None:
49
- blocks_to_swap_in = {}
50
- if blocks_to_swap_out is None:
51
- blocks_to_swap_out = {}
52
- if blocks_to_copy is None:
53
- blocks_to_copy = {}
54
-
55
- return ExecuteModelData(
56
- seq_group_metadata_list=seq_group_metadata_list,
57
- blocks_to_swap_in=blocks_to_swap_in,
58
- blocks_to_swap_out=blocks_to_swap_out,
59
- blocks_to_copy=blocks_to_copy,
60
- )
61
-
62
-
22
def mock_worker(cls=None,
23
vocab_size: int = 30_000,
24
max_model_len: int = 2048,
@@ -258,8 +217,7 @@ def create_batch(batch_size,
217
for prompt, prev_output_token in zip(prompts, prev_output_tokens)
218
]
219
261
- execute_model_data = create_execute_model_data(
262
- create_seq_group_metadata_from_prompts(prompts, num_gpu_blocks,
263
- block_size, final_prompt_lens,
264
- prev_output_tokens, seq_ids), )
265
- return execute_model_data, prompts, prev_output_tokens
220
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
221
+ prompts, num_gpu_blocks, block_size, final_prompt_lens,
222
+ prev_output_tokens, seq_ids)
223
+ return seq_group_metadata_list, prompts, prev_output_tokens
tests/worker/test_swap.py
+20
-10
@@ -1,6 +1,7 @@
1
import torch
2
3
from vllm.engine.arg_utils import EngineArgs
4
+from vllm.sequence import ExecuteModelRequest
5
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6
from vllm.worker.worker import Worker
7
@@ -54,10 +55,14 @@ def test_swap() -> None:
55
56
# Test swap out.
57
blocks_to_swap_out = {3: 72, 56: 35, 84: 34}
57
- worker.execute_model(seq_group_metadata_list=[],
58
- blocks_to_swap_in={},
59
- blocks_to_swap_out=blocks_to_swap_out,
60
- blocks_to_copy={})
58
+ execute_model_req = ExecuteModelRequest(
59
+ seq_group_metadata_list=[],
60
+ blocks_to_swap_in={},
61
+ blocks_to_swap_out=blocks_to_swap_out,
62
+ blocks_to_copy={},
63
+ )
64
+ worker.execute_model(execute_model_req=execute_model_req)
65
+
66
for i in range(num_layers):
67
gpu_key_cache, gpu_value_cache = gpu_cache[i]
68
cpu_key_cache, cpu_value_cache = cpu_cache[i]
@@ -66,14 +71,19 @@ def test_swap() -> None:
71
assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
72
73
# Test swap in.
69
- blocks_to_swap_in = {19: 45, 67: 23, 12: 78, 40: 99, 1: 71}
70
- worker.execute_model(seq_group_metadata_list=[],
71
- blocks_to_swap_in=blocks_to_swap_in,
72
- blocks_to_swap_out={},
73
- blocks_to_copy={})
74
+ execute_model_req.blocks_to_swap_out = {}
75
+ execute_model_req.blocks_to_swap_in = {
76
+ 19: 45,
77
+ 67: 23,
78
+ 12: 78,
79
+ 40: 99,
80
+ 1: 71
81
+ }
82
+ worker.execute_model(execute_model_req=execute_model_req)
83
+
84
for i in range(num_layers):
85
gpu_key_cache, gpu_value_cache = gpu_cache[i]
86
cpu_key_cache, cpu_value_cache = cpu_cache[i]
77
- for src, dst in blocks_to_swap_in.items():
87
+ for src, dst in execute_model_req.blocks_to_swap_in.items():
88
assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
89
assert allclose(gpu_value_cache[dst], cpu_value_cache[src])
vllm/core/scheduler.py
+4
@@ -128,6 +128,8 @@ class SchedulerOutputs:
128
ignored_seq_groups: List[SequenceGroup]
129
# The number of slots for lookahead decoding.
130
num_lookahead_slots: int
131
+ # The number of requests in the running queue
132
+ running_queue_size: int
133
134
def __post_init__(self):
135
# Swap in and swap out should never happen at the same time.
@@ -797,6 +799,7 @@ class Scheduler:
799
ignored_seq_groups=prefills.ignored_seq_groups +
800
swapped_in.infeasible_seq_groups,
801
num_lookahead_slots=running_scheduled.num_lookahead_slots,
802
+ running_queue_size=len(self.running),
803
)
804
805
def _schedule_chunked_prefill(self):
@@ -883,6 +886,7 @@ class Scheduler:
886
swapped_in.blocks_to_copy),
887
ignored_seq_groups=prefills.ignored_seq_groups,
888
num_lookahead_slots=running_scheduled.num_lookahead_slots,
889
+ running_queue_size=len(self.running),
890
)
891
892
def _schedule(self) -> SchedulerOutputs:
vllm/engine/async_llm_engine.py
+10
-6
@@ -16,7 +16,7 @@ from vllm.logger import init_logger
16
from vllm.lora.request import LoRARequest
17
from vllm.outputs import RequestOutput
18
from vllm.sampling_params import SamplingParams
19
-from vllm.sequence import MultiModalData, SamplerOutput
19
+from vllm.sequence import ExecuteModelRequest, MultiModalData, SamplerOutput
20
from vllm.usage.usage_lib import UsageContext
21
22
logger = init_logger(__name__)
@@ -210,12 +210,16 @@ class _AsyncLLMEngine(LLMEngine):
210
211
if not scheduler_outputs.is_empty():
212
# Execute the model.
213
+ execute_model_req = ExecuteModelRequest(
214
+ seq_group_metadata_list=seq_group_metadata_list,
215
+ blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
216
+ blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
217
+ blocks_to_copy=scheduler_outputs.blocks_to_copy,
218
+ num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
219
+ running_queue_size=scheduler_outputs.running_queue_size,
220
+ )
221
output = await self.model_executor.execute_model_async(
214
- seq_group_metadata_list,
215
- scheduler_outputs.blocks_to_swap_in,
216
- scheduler_outputs.blocks_to_swap_out,
217
- scheduler_outputs.blocks_to_copy,
218
- num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
222
+ execute_model_req)
223
else:
224
output = []
225
vllm/engine/llm_engine.py
+8
-4
@@ -22,8 +22,8 @@ from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
23
from vllm.outputs import RequestOutput
24
from vllm.sampling_params import SamplingParams
25
-from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
26
- SequenceGroup, SequenceGroupMetadata,
25
+from vllm.sequence import (ExecuteModelRequest, MultiModalData, SamplerOutput,
26
+ Sequence, SequenceGroup, SequenceGroupMetadata,
27
SequenceStatus)
28
from vllm.transformers_utils.detokenizer import Detokenizer
29
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
@@ -583,12 +583,16 @@ class LLMEngine:
583
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
584
585
if not scheduler_outputs.is_empty():
586
- output = self.model_executor.execute_model(
586
+ execute_model_req = ExecuteModelRequest(
587
seq_group_metadata_list=seq_group_metadata_list,
588
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
589
blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
590
blocks_to_copy=scheduler_outputs.blocks_to_copy,
591
- num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
591
+ num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
592
+ running_queue_size=scheduler_outputs.running_queue_size,
593
+ )
594
+ output = self.model_executor.execute_model(
595
+ execute_model_req=execute_model_req)
596
else:
597
output = []
598
vllm/executor/cpu_executor.py
+10
-27
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Set, Tuple
1
+from typing import List, Set, Tuple
2
3
import torch
4
@@ -7,7 +7,7 @@ from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
10
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
11
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12
make_async)
13
@@ -72,18 +72,10 @@ class CPUExecutor(ExecutorBase):
72
logger.info("# CPU blocks: %d", num_gpu_blocks)
73
self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
74
75
- def execute_model(self,
76
- seq_group_metadata_list: List[SequenceGroupMetadata],
77
- blocks_to_swap_in: Dict[int, int],
78
- blocks_to_swap_out: Dict[int, int],
79
- blocks_to_copy: Dict[int, List[int]],
80
- num_lookahead_slots: int) -> List[SamplerOutput]:
81
- output = self.driver_worker.execute_model(
82
- seq_group_metadata_list=seq_group_metadata_list,
83
- blocks_to_swap_in=blocks_to_swap_in,
84
- blocks_to_swap_out=blocks_to_swap_out,
85
- blocks_to_copy=blocks_to_copy,
86
- )
75
+ def execute_model(
76
+ self,
77
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
78
+ output = self.driver_worker.execute_model(execute_model_req)
79
return output
80
81
def add_lora(self, lora_request: LoRARequest) -> bool:
@@ -104,19 +96,10 @@ class CPUExecutor(ExecutorBase):
96
class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
97
98
async def execute_model_async(
107
- self,
108
- seq_group_metadata_list: List[SequenceGroupMetadata],
109
- blocks_to_swap_in: Dict[int, int],
110
- blocks_to_swap_out: Dict[int, int],
111
- blocks_to_copy: Dict[int, List[int]],
112
- num_lookahead_slots: int,
113
- ) -> List[SamplerOutput]:
114
- output = await make_async(self.driver_worker.execute_model)(
115
- seq_group_metadata_list=seq_group_metadata_list,
116
- blocks_to_swap_in=blocks_to_swap_in,
117
- blocks_to_swap_out=blocks_to_swap_out,
118
- blocks_to_copy=blocks_to_copy,
119
- num_lookahead_slots=num_lookahead_slots)
99
+ self,
100
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
101
+ output = await make_async(self.driver_worker.execute_model
102
+ )(execute_model_req=execute_model_req, )
103
return output
104
105
async def check_health_async(self) -> None:
vllm/executor/executor_base.py
+7
-15
@@ -1,11 +1,11 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, List, Optional, Set, Tuple
2
+from typing import List, Optional, Set, Tuple
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5
ModelConfig, ParallelConfig, SchedulerConfig,
6
SpeculativeConfig, VisionLanguageConfig)
7
from vllm.lora.request import LoRARequest
8
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
10
11
class ExecutorBase(ABC):
@@ -68,12 +68,9 @@ class ExecutorBase(ABC):
68
raise NotImplementedError
69
70
@abstractmethod
71
- def execute_model(self,
72
- seq_group_metadata_list: List[SequenceGroupMetadata],
73
- blocks_to_swap_in: Dict[int, int],
74
- blocks_to_swap_out: Dict[int, int],
75
- blocks_to_copy: Dict[int, List[int]],
76
- num_lookahead_slots: int) -> List[SamplerOutput]:
71
+ def execute_model(
72
+ self,
73
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
74
"""Executes at least one model step on the given sequences."""
75
raise NotImplementedError
76
@@ -107,13 +104,8 @@ class ExecutorAsyncBase(ExecutorBase):
104
105
@abstractmethod
106
async def execute_model_async(
110
- self,
111
- seq_group_metadata_list: List[SequenceGroupMetadata],
112
- blocks_to_swap_in: Dict[int, int],
113
- blocks_to_swap_out: Dict[int, int],
114
- blocks_to_copy: Dict[int, List[int]],
115
- num_lookahead_slots: int,
116
- ) -> List[SamplerOutput]:
107
+ self,
108
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
109
"""Executes one model step on the given sequences."""
110
raise NotImplementedError
111
vllm/executor/gpu_executor.py
+7
-26
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
7
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8
make_async)
9
from vllm.worker.worker_base import WorkerWrapperBase
@@ -117,20 +117,9 @@ class GPUExecutor(ExecutorBase):
117
self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
118
119
def execute_model(
120
- self,
121
- seq_group_metadata_list: List[SequenceGroupMetadata],
122
- blocks_to_swap_in: Dict[int, int],
123
- blocks_to_swap_out: Dict[int, int],
124
- blocks_to_copy: Dict[int, List[int]],
125
- num_lookahead_slots: int,
126
- ) -> List[SamplerOutput]:
127
- output = self.driver_worker.execute_model(
128
- seq_group_metadata_list=seq_group_metadata_list,
129
- blocks_to_swap_in=blocks_to_swap_in,
130
- blocks_to_swap_out=blocks_to_swap_out,
131
- blocks_to_copy=blocks_to_copy,
132
- num_lookahead_slots=num_lookahead_slots,
133
- )
120
+ self,
121
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
122
+ output = self.driver_worker.execute_model(execute_model_req)
123
return output
124
125
def add_lora(self, lora_request: LoRARequest) -> bool:
@@ -154,16 +143,8 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
143
144
async def execute_model_async(
145
self,
157
- seq_group_metadata_list: List[SequenceGroupMetadata],
158
- blocks_to_swap_in: Dict[int, int],
159
- blocks_to_swap_out: Dict[int, int],
160
- blocks_to_copy: Dict[int, List[int]],
161
- num_lookahead_slots: int,
146
+ execute_model_req: ExecuteModelRequest,
147
) -> List[SamplerOutput]:
163
- output = await make_async(self.driver_worker.execute_model)(
164
- seq_group_metadata_list=seq_group_metadata_list,
165
- blocks_to_swap_in=blocks_to_swap_in,
166
- blocks_to_swap_out=blocks_to_swap_out,
167
- blocks_to_copy=blocks_to_copy,
168
- num_lookahead_slots=num_lookahead_slots)
148
+ output = await make_async(self.driver_worker.execute_model
149
+ )(execute_model_req=execute_model_req, )
150
return output
vllm/executor/neuron_executor.py
+14
-19
@@ -1,9 +1,9 @@
1
-from typing import Dict, List, Set, Tuple
1
+from typing import List, Set, Tuple
2
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
7
from vllm.utils import make_async
8
9
logger = init_logger(__name__)
@@ -45,20 +45,18 @@ class NeuronExecutor(ExecutorBase):
45
"""
46
self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
47
48
- def execute_model(self,
49
- seq_group_metadata_list: List[SequenceGroupMetadata],
50
- blocks_to_swap_in: Dict[int, int],
51
- blocks_to_swap_out: Dict[int, int],
52
- blocks_to_copy: Dict[int, List[int]],
53
- num_lookahead_slots: int) -> List[SamplerOutput]:
54
- assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
55
- and blocks_to_copy == {}), (
48
+ def execute_model(
49
+ self,
50
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
51
+ assert (execute_model_req.blocks_to_swap_in == {}
52
+ and execute_model_req.blocks_to_swap_out == {}
53
+ and execute_model_req.blocks_to_copy == {}), (
54
"Cache operations are not supported for Neuron backend.")
57
- assert num_lookahead_slots == 0, (
55
+ assert execute_model_req.num_lookahead_slots == 0, (
56
"lookahead not supported for Neuron backend.")
57
58
output = self.driver_worker.execute_model(
61
- seq_group_metadata_list=seq_group_metadata_list)
59
+ execute_model_req.seq_group_metadata_list)
60
return output
61
62
def add_lora(self, lora_request: LoRARequest) -> bool:
@@ -80,14 +78,11 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
78
79
async def execute_model_async(
80
self,
83
- seq_group_metadata_list: List[SequenceGroupMetadata],
84
- blocks_to_swap_in: Dict[int, int],
85
- blocks_to_swap_out: Dict[int, int],
86
- blocks_to_copy: Dict[int, List[int]],
87
- num_lookahead_slots: int,
81
+ execute_model_req: ExecuteModelRequest,
82
) -> List[SamplerOutput]:
89
- output = await make_async(self.driver_worker.execute_model)(
90
- seq_group_metadata_list=seq_group_metadata_list, )
83
+ output = await make_async(
84
+ self.driver_worker.execute_model
85
+ )(seq_group_metadata_list=execute_model_req.seq_group_metadata_list, )
86
return output
87
88
async def check_health_async(self) -> None:
vllm/executor/ray_gpu_executor.py
+5
-14
@@ -10,7 +10,7 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10
DistributedGPUExecutor, DistributedGPUExecutorAsync)
11
from vllm.executor.ray_utils import RayWorkerWrapper, ray
12
from vllm.logger import init_logger
13
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
13
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
14
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
15
get_vllm_instance_id, make_async)
16
@@ -166,21 +166,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
166
max_concurrent_workers=self.parallel_config.
167
max_parallel_loading_workers)
168
169
- def execute_model(self,
170
- seq_group_metadata_list: List[SequenceGroupMetadata],
171
- blocks_to_swap_in: Dict[int, int],
172
- blocks_to_swap_out: Dict[int, int],
173
- blocks_to_copy: Dict[int, List[int]],
174
- num_lookahead_slots: int = 0) -> List[SamplerOutput]:
169
+ def execute_model(
170
+ self,
171
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
172
all_outputs = self._run_workers(
173
"execute_model",
177
- driver_kwargs={
178
- "seq_group_metadata_list": seq_group_metadata_list,
179
- "blocks_to_swap_in": blocks_to_swap_in,
180
- "blocks_to_swap_out": blocks_to_swap_out,
181
- "blocks_to_copy": blocks_to_copy,
182
- "num_lookahead_slots": num_lookahead_slots,
183
- },
174
+ driver_kwargs={"execute_model_req": execute_model_req},
175
use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
176
177
# Only the driver worker returns the sampling results.
vllm/sequence.py
+31
-1
@@ -1,7 +1,7 @@
1
"""Sequence and its related classes."""
2
import copy
3
import enum
4
-from dataclasses import dataclass
4
+from dataclasses import dataclass, field
5
from typing import TYPE_CHECKING, Dict, List, Optional, Union
6
7
from vllm.block import LogicalTokenBlock
@@ -734,3 +734,33 @@ class SamplerOutput:
734
f"sampled_token_probs={sampled_token_probs_repr}, "
735
f"sampled_token_ids={sampled_token_ids_repr}, "
736
f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
737
+
738
+
739
+@dataclass
740
+class ExecuteModelRequest:
741
+ """The model execution request."""
742
+ # The sequence group metadata list.
743
+ seq_group_metadata_list: List[SequenceGroupMetadata]
744
+ # Blocks to swap in. Dict of CPU -> GPU block number.
745
+ blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
746
+ # Blocks to swap out. Dict of GPU -> CPU block number.
747
+ blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
748
+ # Blocks to copy. Source to a list of dest blocks.
749
+ blocks_to_copy: Dict[int, List[int]] = field(default_factory=dict)
750
+ # The number of slots for lookahead decoding.
751
+ num_lookahead_slots: int = 0
752
+ # The number of requests in the running queue.
753
+ running_queue_size: int = 0
754
+
755
+ def clone(
756
+ self, seq_group_metadata_list: List[SequenceGroupMetadata]
757
+ ) -> "ExecuteModelRequest":
758
+ """Clone the request with a new sequence group metadata list."""
759
+ return ExecuteModelRequest(
760
+ seq_group_metadata_list=seq_group_metadata_list,
761
+ blocks_to_swap_in=self.blocks_to_swap_in.copy(),
762
+ blocks_to_swap_out=self.blocks_to_swap_out.copy(),
763
+ blocks_to_copy=self.blocks_to_copy.copy(),
764
+ num_lookahead_slots=self.num_lookahead_slots,
765
+ running_queue_size=self.running_queue_size,
766
+ )
vllm/spec_decode/batch_expansion.py
+10
-20
@@ -1,9 +1,10 @@
1
from itertools import chain, count
2
-from typing import Dict, Iterator, List, Optional, Tuple
2
+from typing import Iterator, List, Tuple
3
4
import torch
5
6
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
6
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7
+ SequenceGroupMetadata)
8
from vllm.spec_decode.interfaces import (SpeculativeProposals,
9
SpeculativeScorer, SpeculativeScores)
10
from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
@@ -40,11 +41,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
41
@nvtx_range("BatchExpansionTop1Scorer.score_proposals")
42
def score_proposals(
43
self,
43
- seq_group_metadata_list: List[SequenceGroupMetadata],
44
- blocks_to_swap_in: Optional[Dict[int, int]],
45
- blocks_to_swap_out: Optional[Dict[int, int]],
46
- blocks_to_copy: Optional[Dict[int, List[int]]],
47
- k: int,
44
+ execute_model_req: ExecuteModelRequest,
45
proposals: SpeculativeProposals,
46
) -> SpeculativeScores:
47
"""Score the proposed tokens via the scorer model.
@@ -57,11 +54,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
54
no speculation is produced for that sequence.
55
56
Args:
60
- seq_group_metadata_list: The input sequence group metadata.
61
- blocks_to_swap_in: This is passed to the worker during scoring.
62
- blocks_to_swap_out: This is passed to the worker during scoring.
63
- blocks_to_copy: This is passed to the worker during scoring.
64
- k: The fixed proposal length.
57
+ execute_model_req: The execution request.
58
proposals: The speculative proposals to score.
59
Returns:
60
SpeculativeScores: The scores of each speculative token, along with
@@ -80,28 +73,25 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
73
74
(spec_indices, non_spec_indices, target_seq_group_metadata_list,
75
num_scoring_tokens) = self._expand_batch(
83
- seq_group_metadata_list=seq_group_metadata_list,
76
+ seq_group_metadata_list=execute_model_req.seq_group_metadata_list,
77
proposal_token_ids_list=proposal_token_ids_list_without_skips,
78
proposal_lens_list=proposal_lens_list,
79
)
80
81
target_sampler_output = self._scorer_worker.execute_model(
89
- seq_group_metadata_list=target_seq_group_metadata_list,
90
- blocks_to_swap_in=blocks_to_swap_in,
91
- blocks_to_swap_out=blocks_to_swap_out,
92
- blocks_to_copy=blocks_to_copy,
93
- )
82
+ execute_model_req=execute_model_req.clone(
83
+ seq_group_metadata_list=target_seq_group_metadata_list, ))
84
assert len(target_sampler_output) == 1, "expected single-step output"
85
target_sampler_output = target_sampler_output[0]
86
87
all_tokens, all_probs, spec_logprobs = self._contract_batch(
98
- contracted_bs=len(seq_group_metadata_list),
88
+ contracted_bs=len(execute_model_req.seq_group_metadata_list),
89
target_sampler_output=target_sampler_output,
90
proposals=proposals,
91
num_scoring_tokens=num_scoring_tokens,
92
non_spec_indices=non_spec_indices,
93
spec_indices=spec_indices,
104
- k=k,
94
+ k=execute_model_req.num_lookahead_slots,
95
)
96
97
return SpeculativeScores(
vllm/spec_decode/interfaces.py
+3
-12
@@ -1,10 +1,9 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass
3
-from typing import Dict, List, Optional
3
4
import torch
5
7
-from vllm.sequence import SequenceGroupMetadata
6
+from vllm.sequence import ExecuteModelRequest
7
8
9
@dataclass
@@ -58,11 +57,7 @@ class SpeculativeProposer(ABC):
57
@abstractmethod
58
def get_proposals(
59
self,
61
- seq_group_metadata_list: List[SequenceGroupMetadata],
62
- blocks_to_swap_in: Dict[int, int],
63
- blocks_to_swap_out: Dict[int, int],
64
- blocks_to_copy: Dict[int, List[int]],
65
- max_proposal_len: int,
60
+ execute_model_req: ExecuteModelRequest,
61
) -> SpeculativeProposals:
62
raise NotImplementedError
63
@@ -72,11 +67,7 @@ class SpeculativeScorer(ABC):
67
@abstractmethod
68
def score_proposals(
69
self,
75
- seq_group_metadata_list: List[SequenceGroupMetadata],
76
- blocks_to_swap_in: Optional[Dict[int, int]],
77
- blocks_to_swap_out: Optional[Dict[int, int]],
78
- blocks_to_copy: Optional[Dict[int, List[int]]],
79
- k: int,
70
+ execute_model_req: ExecuteModelRequest,
71
proposals: SpeculativeProposals,
72
) -> SpeculativeScores:
73
raise NotImplementedError
vllm/spec_decode/multi_step_worker.py
+21
-33
@@ -1,9 +1,10 @@
1
import copy
2
-from typing import Dict, List, Tuple
2
+from typing import List, Tuple
3
4
import torch
5
6
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7
+ SequenceGroupMetadata)
8
from vllm.spec_decode.interfaces import SpeculativeProposals
9
from vllm.spec_decode.top1_proposer import Top1Proposer
10
from vllm.worker.worker import Worker
@@ -44,10 +45,7 @@ class MultiStepWorker(Worker):
45
@torch.inference_mode()
46
def sampler_output(
47
self,
47
- seq_group_metadata_list: List[SequenceGroupMetadata],
48
- blocks_to_swap_in: Dict[int, int],
49
- blocks_to_swap_out: Dict[int, int],
50
- blocks_to_copy: Dict[int, List[int]],
48
+ execute_model_req: ExecuteModelRequest,
49
sample_len: int,
50
) -> Tuple[List[SamplerOutput], bool]:
51
"""Run the model forward pass sample_len times. Returns the list of
@@ -57,26 +55,24 @@ class MultiStepWorker(Worker):
55
56
For multi step worker, this indicator shall be True.
57
"""
60
- self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
61
- blocks_to_swap_out, blocks_to_copy)
58
+ self._raise_if_unsupported(execute_model_req)
59
60
# Shallow copy input data so modifications (such as appending tokens)
61
# do not cause side-effects.
62
copied_seq_group_metadata_list = self._shallow_copy_inputs(
66
- seq_group_metadata_list)
63
+ execute_model_req.seq_group_metadata_list)
64
+ copied_execute_model_req = execute_model_req.clone(
65
+ copied_seq_group_metadata_list)
66
67
# Assert enough KV space for sample_len tokens per sequence.
69
- self._assert_enough_kv_space(seq_group_metadata_list, sample_len)
68
+ self._assert_enough_kv_space(execute_model_req.seq_group_metadata_list,
69
+ sample_len)
70
71
# Run model sample_len times.
72
model_outputs = []
73
for _ in range(sample_len):
74
model_output = super().execute_model(
75
- seq_group_metadata_list=copied_seq_group_metadata_list,
76
- blocks_to_swap_in=blocks_to_swap_in,
77
- blocks_to_swap_out=blocks_to_swap_out,
78
- blocks_to_copy=blocks_to_copy,
79
- )
75
+ execute_model_req=copied_execute_model_req)
76
assert (len(model_output) == 1
77
), "composing multistep workers not supported"
78
model_output = model_output[0]
@@ -89,23 +85,13 @@ class MultiStepWorker(Worker):
85
86
def get_spec_proposals(
87
self,
92
- seq_group_metadata_list: List[SequenceGroupMetadata],
93
- blocks_to_swap_in: Dict[int, int],
94
- blocks_to_swap_out: Dict[int, int],
95
- blocks_to_copy: Dict[int, List[int]],
96
- max_proposal_len: int,
88
+ execute_model_req: ExecuteModelRequest,
89
) -> SpeculativeProposals:
90
"""Produce speculations given an input batch of sequences. The number of
91
speculative tokens per sequence is determined by max_proposal_len.
92
"""
93
102
- return self._proposer.get_proposals(
103
- seq_group_metadata_list,
104
- blocks_to_swap_in,
105
- blocks_to_swap_out,
106
- blocks_to_copy,
107
- max_proposal_len,
108
- )
94
+ return self._proposer.get_proposals(execute_model_req)
95
96
def _append_new_tokens(
97
self, model_output: SamplerOutput,
@@ -196,20 +182,22 @@ class MultiStepWorker(Worker):
182
183
def _raise_if_unsupported(
184
self,
199
- seq_group_metadata_list: List[SequenceGroupMetadata],
200
- blocks_to_swap_in: Dict[int, int],
201
- blocks_to_swap_out: Dict[int, int],
202
- blocks_to_copy: Dict[int, List[int]],
185
+ execute_model_req: ExecuteModelRequest,
186
) -> None:
187
"""MultiStepWorker does not yet implement support for cache swap
188
operations or beam search.
189
"""
207
- if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
190
+ if any([
191
+ execute_model_req.blocks_to_swap_in,
192
+ execute_model_req.blocks_to_swap_out,
193
+ execute_model_req.blocks_to_copy
194
+ ]):
195
raise NotImplementedError(
196
"MultiStepWorker does not support cache operations")
197
198
if any(
199
len(seq_group_metadata.seq_data.keys()) != 1
213
- for seq_group_metadata in seq_group_metadata_list):
200
+ for seq_group_metadata in
201
+ execute_model_req.seq_group_metadata_list):
202
raise NotImplementedError(
203
"MultiStepWorker does not support beam search.")
vllm/spec_decode/ngram_worker.py
+21
-41
@@ -1,8 +1,8 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import List, Optional, Tuple
2
3
import torch
4
5
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
5
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
6
from vllm.spec_decode.interfaces import SpeculativeProposals
7
from vllm.spec_decode.top1_proposer import Top1Proposer
8
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
@@ -46,13 +46,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
46
# NGram don't need gpu sampler
47
pass
48
49
- def execute_model(
50
- self,
51
- seq_group_metadata_list: List[SequenceGroupMetadata],
52
- blocks_to_swap_in: Optional[Dict[int, int]],
53
- blocks_to_swap_out: Optional[Dict[int, int]],
54
- blocks_to_copy: Optional[Dict[int, List[int]]],
55
- ) -> None:
49
+ def execute_model(self, execute_model_req: ExecuteModelRequest) -> None:
50
"""NGram doesn't depend on model execution, just pass this function"""
51
pass
52
@@ -71,10 +65,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
65
66
def sampler_output(
67
self,
74
- seq_group_metadata_list: List[SequenceGroupMetadata],
75
- blocks_to_swap_in: Dict[int, int],
76
- blocks_to_swap_out: Dict[int, int],
77
- blocks_to_copy: Dict[int, List[int]],
68
+ execute_model_req: ExecuteModelRequest,
69
sample_len: int,
70
) -> Tuple[Optional[List[SamplerOutput]], bool]:
71
"""NGram match algo to pick proposal candidate. Returns the list of
@@ -83,16 +74,11 @@ class NGramWorker(LoraNotSupportedWorkerBase):
74
For ngram worker, we already done needed transposed internal, so the
75
indicator pass to sampler_output_to_torch shall be False.
76
"""
86
- self._raise_if_unsupported(
87
- seq_group_metadata_list,
88
- blocks_to_swap_in,
89
- blocks_to_swap_out,
90
- blocks_to_copy,
91
- )
77
+ self._raise_if_unsupported(execute_model_req)
78
79
arr = []
80
has_spec_out = False
95
- for seq_group_metadata in seq_group_metadata_list:
81
+ for seq_group_metadata in execute_model_req.seq_group_metadata_list:
82
seq_data = next(iter(seq_group_metadata.seq_data.values()))
83
84
input_ids = torch.as_tensor(seq_data.get_token_ids(),
@@ -135,17 +121,19 @@ class NGramWorker(LoraNotSupportedWorkerBase):
121
indices = token_ids.unsqueeze(2)
122
123
token_probs = torch.zeros(
138
- (len(seq_group_metadata_list), sample_len, self.vocab_size),
124
+ (len(execute_model_req.seq_group_metadata_list), sample_len,
125
+ self.vocab_size),
126
dtype=torch.float32,
127
device=self.device,
128
)
129
token_probs.scatter_(2, indices, 1)
130
token_logprobs = torch.zeros(
144
- (len(seq_group_metadata_list), sample_len, self.vocab_size),
131
+ (len(execute_model_req.seq_group_metadata_list), sample_len,
132
+ self.vocab_size),
133
dtype=torch.float32,
134
device=self.device,
135
)
148
- for i in range(len(seq_group_metadata_list)):
136
+ for i in range(len(execute_model_req.seq_group_metadata_list)):
137
outputs.append(
138
SamplerOutput(
139
outputs=None,
@@ -157,40 +145,32 @@ class NGramWorker(LoraNotSupportedWorkerBase):
145
146
def get_spec_proposals(
147
self,
160
- seq_group_metadata_list: List[SequenceGroupMetadata],
161
- blocks_to_swap_in: Dict[int, int],
162
- blocks_to_swap_out: Dict[int, int],
163
- blocks_to_copy: Dict[int, List[int]],
164
- max_proposal_len: int,
148
+ execute_model_req: ExecuteModelRequest,
149
) -> SpeculativeProposals:
150
"""Produce speculations given an input batch of sequences. The number of
151
speculative tokens per sequence is determined by max_proposal_len.
152
"""
153
170
- return self._proposer.get_proposals(
171
- seq_group_metadata_list,
172
- blocks_to_swap_in,
173
- blocks_to_swap_out,
174
- blocks_to_copy,
175
- max_proposal_len,
176
- )
154
+ return self._proposer.get_proposals(execute_model_req)
155
156
def _raise_if_unsupported(
157
self,
180
- seq_group_metadata_list: List[SequenceGroupMetadata],
181
- blocks_to_swap_in: Dict[int, int],
182
- blocks_to_swap_out: Dict[int, int],
183
- blocks_to_copy: Dict[int, List[int]],
158
+ execute_model_req: ExecuteModelRequest,
159
) -> None:
160
"""NGramWorker does not yet implement support for cache swap
161
operations or beam search.
162
"""
188
- if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
163
+ if any([
164
+ execute_model_req.blocks_to_swap_in,
165
+ execute_model_req.blocks_to_swap_out,
166
+ execute_model_req.blocks_to_copy
167
+ ]):
168
raise NotImplementedError(
169
"NGramWorker does not support cache operations")
170
171
if any(
172
len(seq_group_metadata.seq_data.keys()) != 1
194
- for seq_group_metadata in seq_group_metadata_list):
173
+ for seq_group_metadata in
174
+ execute_model_req.seq_group_metadata_list):
175
raise NotImplementedError(
176
"NGramWorker does not support beam search.")
vllm/spec_decode/spec_decode_worker.py
+23
-67
@@ -1,11 +1,12 @@
1
from functools import cached_property
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import List, Optional, Tuple
3
4
import torch
5
6
from vllm.logger import init_logger
7
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
9
+ SequenceGroupMetadata)
10
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
11
from vllm.spec_decode.interfaces import (SpeculativeProposals,
12
SpeculativeScorer, SpeculativeScores)
@@ -189,69 +190,37 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
190
191
@torch.inference_mode()
192
def execute_model(
192
- self,
193
- seq_group_metadata_list: List[SequenceGroupMetadata],
194
- blocks_to_swap_in: Optional[Dict[int, int]],
195
- blocks_to_swap_out: Optional[Dict[int, int]],
196
- blocks_to_copy: Optional[Dict[int, List[int]]],
197
- num_lookahead_slots: int,
198
- ) -> List[SamplerOutput]:
193
+ self,
194
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
195
"""Perform speculative decoding on the input batch.
196
"""
197
202
- assert seq_group_metadata_list is not None, (
198
+ assert execute_model_req.seq_group_metadata_list is not None, (
199
"speculative decoding "
200
"requires non-None seq_group_metadata_list")
201
206
- #logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
207
- # num_lookahead_slots)
208
-
202
# If no spec tokens, call the proposer and scorer workers normally.
203
# Used for prefill.
211
- if num_lookahead_slots == 0 or len(seq_group_metadata_list) == 0:
212
- return self._run_no_spec(
213
- seq_group_metadata_list=seq_group_metadata_list,
214
- blocks_to_swap_in=blocks_to_swap_in,
215
- blocks_to_swap_out=blocks_to_swap_out,
216
- blocks_to_copy=blocks_to_copy,
217
- )
218
-
219
- return self._run_speculative_decoding_step(
220
- seq_group_metadata_list=seq_group_metadata_list,
221
- blocks_to_swap_in=blocks_to_swap_in,
222
- blocks_to_swap_out=blocks_to_swap_out,
223
- blocks_to_copy=blocks_to_copy,
224
- k=num_lookahead_slots,
225
- )
204
+ if execute_model_req.num_lookahead_slots == 0 or len(
205
+ execute_model_req.seq_group_metadata_list) == 0:
206
+ return self._run_no_spec(execute_model_req)
207
+
208
+ return self._run_speculative_decoding_step(execute_model_req)
209
210
@nvtx_range("spec_decode_worker._run_no_spec")
211
def _run_no_spec(
229
- self,
230
- seq_group_metadata_list: List[SequenceGroupMetadata],
231
- blocks_to_swap_in: Optional[Dict[int, int]],
232
- blocks_to_swap_out: Optional[Dict[int, int]],
233
- blocks_to_copy: Optional[Dict[int, List[int]]],
234
- ) -> List[SamplerOutput]:
212
+ self,
213
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
214
"""Run a prefill step, without any speculation. The input is sent to the
215
proposer and scorer model so that the KV cache is consistent between the
216
two.
217
"""
218
#logger.info("run proposer worker no spec")
219
241
- self.proposer_worker.execute_model(
242
- seq_group_metadata_list=seq_group_metadata_list,
243
- blocks_to_swap_in=blocks_to_swap_in,
244
- blocks_to_swap_out=blocks_to_swap_out,
245
- blocks_to_copy=blocks_to_copy,
246
- )
220
+ self.proposer_worker.execute_model(execute_model_req)
221
222
#logger.info("run target worker no spec")
249
- sampler_output = self.scorer_worker.execute_model(
250
- seq_group_metadata_list=seq_group_metadata_list,
251
- blocks_to_swap_in=blocks_to_swap_in,
252
- blocks_to_swap_out=blocks_to_swap_out,
253
- blocks_to_copy=blocks_to_copy,
254
- )
223
+ sampler_output = self.scorer_worker.execute_model(execute_model_req)
224
assert len(sampler_output) == 1
225
sampler_output = sampler_output[0]
226
@@ -264,13 +233,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
233
234
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
235
def _run_speculative_decoding_step(
267
- self,
268
- seq_group_metadata_list: List[SequenceGroupMetadata],
269
- blocks_to_swap_in: Optional[Dict[int, int]],
270
- blocks_to_swap_out: Optional[Dict[int, int]],
271
- blocks_to_copy: Optional[Dict[int, List[int]]],
272
- k: int,
273
- ) -> List[SamplerOutput]:
236
+ self,
237
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
238
"""Execute a single step of speculative decoding.
239
240
This invokes the proposer worker to get k speculative tokens for each
@@ -282,33 +246,25 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
246
247
#logger.info("get spec proposals")
248
# Generate proposals using draft worker.
285
- assert blocks_to_swap_in is not None
286
- assert blocks_to_swap_out is not None
287
- assert blocks_to_copy is not None
288
- proposals = self.proposer_worker.get_spec_proposals(
289
- seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
290
- blocks_to_copy, k)
249
+ proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
250
251
#logger.info("score proposals")
252
proposal_scores = self.scorer.score_proposals(
294
- seq_group_metadata_list,
295
- blocks_to_swap_in,
296
- blocks_to_swap_out,
297
- blocks_to_copy,
298
- k,
253
+ execute_model_req,
254
proposals,
255
)
256
257
#logger.info("verify proposals")
258
accepted_token_ids, target_logprobs = self._verify_tokens(
304
- seq_group_metadata_list, proposal_scores, proposals, k)
259
+ execute_model_req.seq_group_metadata_list, proposal_scores,
260
+ proposals, execute_model_req.num_lookahead_slots)
261
262
#logger.info("create output list")
263
return self._create_output_sampler_list(
308
- seq_group_metadata_list,
264
+ execute_model_req.seq_group_metadata_list,
265
accepted_token_ids,
266
target_logprobs=target_logprobs,
311
- k=k)
267
+ k=execute_model_req.num_lookahead_slots)
268
269
@nvtx_range("spec_decode_worker._verify_tokens")
270
def _verify_tokens(
vllm/spec_decode/top1_proposer.py
+11
-11
@@ -1,8 +1,9 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import List, Optional, Tuple
2
3
import torch
4
5
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
5
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
6
+ SequenceGroupMetadata)
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeProposer)
9
from vllm.spec_decode.util import sampler_output_to_torch
@@ -40,17 +41,15 @@ class Top1Proposer(SpeculativeProposer):
41
42
def get_proposals(
43
self,
43
- seq_group_metadata_list: List[SequenceGroupMetadata],
44
- blocks_to_swap_in: Dict[int, int],
45
- blocks_to_swap_out: Dict[int, int],
46
- blocks_to_copy: Dict[int, List[int]],
47
- proposal_len: int,
44
+ execute_model_req: ExecuteModelRequest,
45
) -> SpeculativeProposals:
46
"""Get speculative proposals given the input batch.
47
48
Sequences which would exceed the max model length are skipped during
49
speculation.
50
"""
51
+ proposal_len = execute_model_req.num_lookahead_slots
52
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
53
54
# Split speculative- and non-speculative- sequences.
55
(
@@ -66,11 +65,12 @@ class Top1Proposer(SpeculativeProposer):
65
# token_ids is like [batch] format in proposal_len size list,
66
# while if it is false, the format would be [proposal_len]
67
# in batch size list
69
- maybe_sampler_output, transposed = self._worker.sampler_output(
68
+ nonzero_execute_model_req = ExecuteModelRequest(
69
seq_group_metadata_list=nonzero_proposal_len_seqs,
71
- blocks_to_swap_in=blocks_to_swap_in,
72
- blocks_to_swap_out=blocks_to_swap_out,
73
- blocks_to_copy=blocks_to_copy,
70
+ num_lookahead_slots=proposal_len,
71
+ )
72
+ maybe_sampler_output, transposed = self._worker.sampler_output(
73
+ execute_model_req=nonzero_execute_model_req,
74
sample_len=proposal_len,
75
)
76
else:
vllm/worker/cpu_worker.py
+13
-12
@@ -13,7 +13,7 @@ from vllm.distributed import (broadcast_tensor_dict,
13
init_distributed_environment)
14
from vllm.logger import init_logger
15
from vllm.model_executor import set_random_seed
16
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
from vllm.worker.cpu_model_runner import CPUModelRunner
19
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
@@ -256,22 +256,24 @@ class CPUWorker(LoraNotSupportedWorkerBase):
256
@torch.inference_mode()
257
def execute_model(
258
self,
259
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
260
- blocks_to_swap_in: Optional[Dict[int, int]] = None,
261
- blocks_to_swap_out: Optional[Dict[int, int]] = None,
262
- blocks_to_copy: Optional[Dict[int, List[int]]] = None,
259
+ execute_model_req: Optional[ExecuteModelRequest] = None,
260
) -> List[SamplerOutput]:
261
+
262
+ if execute_model_req is None:
263
+ seq_group_metadata_list = None
264
+ else:
265
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
266
+
267
if self.is_driver_worker:
268
assert seq_group_metadata_list is not None
269
num_seq_groups: int = len(seq_group_metadata_list)
267
- assert blocks_to_swap_in is not None
268
- assert blocks_to_swap_out is not None
269
- assert blocks_to_copy is not None
270
- assert len(blocks_to_swap_in) == 0
271
- assert len(blocks_to_swap_out) == 0
270
+ assert execute_model_req is not None
271
+ blocks_to_copy = execute_model_req.blocks_to_copy
272
+ assert len(execute_model_req.blocks_to_swap_in) == 0
273
+ assert len(execute_model_req.blocks_to_swap_out) == 0
274
data: Dict[str, Any] = {
275
"num_seq_groups": num_seq_groups,
274
- "blocks_to_copy": blocks_to_copy,
276
+ "blocks_to_copy": execute_model_req.blocks_to_copy,
277
}
278
broadcast_tensor_dict(data, src=0)
279
else:
@@ -279,7 +281,6 @@ class CPUWorker(LoraNotSupportedWorkerBase):
281
num_seq_groups = data["num_seq_groups"]
282
blocks_to_copy = data["blocks_to_copy"]
283
282
- assert blocks_to_copy is not None
284
self.cache_copy(blocks_to_copy)
285
286
# If there is no input, we don't need to execute the model.
vllm/worker/worker.py
+11
-12
@@ -18,7 +18,7 @@ from vllm.distributed.device_communicators.custom_all_reduce import (
18
init_custom_ar)
19
from vllm.lora.request import LoRARequest
20
from vllm.model_executor import set_random_seed
21
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
21
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
22
from vllm.worker.cache_engine import CacheEngine
23
from vllm.worker.model_runner import ModelRunner
24
from vllm.worker.worker_base import WorkerBase
@@ -211,19 +211,21 @@ class Worker(WorkerBase):
211
@torch.inference_mode()
212
def execute_model(
213
self,
214
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
215
- blocks_to_swap_in: Optional[Dict[int, int]] = None,
216
- blocks_to_swap_out: Optional[Dict[int, int]] = None,
217
- blocks_to_copy: Optional[Dict[int, List[int]]] = None,
218
- num_lookahead_slots: int = 0,
214
+ execute_model_req: Optional[ExecuteModelRequest] = None
215
) -> List[SamplerOutput]:
216
217
+ if execute_model_req is None:
218
+ seq_group_metadata_list = None
219
+ else:
220
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
221
+
222
if self.is_driver_worker:
223
assert seq_group_metadata_list is not None
224
+ assert execute_model_req is not None
225
num_seq_groups = len(seq_group_metadata_list)
224
- assert blocks_to_swap_in is not None
225
- assert blocks_to_swap_out is not None
226
- assert blocks_to_copy is not None
226
+ blocks_to_swap_in = execute_model_req.blocks_to_swap_in
227
+ blocks_to_swap_out = execute_model_req.blocks_to_swap_out
228
+ blocks_to_copy = execute_model_req.blocks_to_copy
229
data: Dict[str, Any] = {
230
"num_seq_groups": num_seq_groups,
231
"blocks_to_swap_in": blocks_to_swap_in,
@@ -238,9 +240,6 @@ class Worker(WorkerBase):
240
blocks_to_swap_out = data["blocks_to_swap_out"]
241
blocks_to_copy = data["blocks_to_copy"]
242
241
- assert blocks_to_swap_in is not None
242
- assert blocks_to_swap_out is not None
243
- assert blocks_to_copy is not None
243
self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
244
245
# If there is no input, we don't need to execute the model.
vllm/worker/worker_base.py
+3
-5
@@ -5,7 +5,7 @@ from typing import Dict, List, Set, Tuple
5
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
from vllm.utils import (enable_trace_function_call_for_thread,
10
update_environment_variables)
11
@@ -48,10 +48,8 @@ class WorkerBase(ABC):
48
49
@abstractmethod
50
def execute_model(
51
- self, seq_group_metadata_list: List[SequenceGroupMetadata],
52
- blocks_to_swap_in: Dict[int, int], blocks_to_swap_out: Dict[int,
53
- int],
54
- blocks_to_copy: Dict[int, List[int]]) -> List[SamplerOutput]:
51
+ self,
52
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
53
"""Executes at least one model step on the given sequences, unless no
54
sequences are provided."""
55
raise NotImplementedError