A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Speculative Decoding] Add `ProposerWorkerBase` abstract class (#5252)
Nick Hill committed
Jun 5, 2024 at 14:53 UTC
faf71bcd4b11b6d350431f432af08ccd9f30016f
9 files changed
+91
-60
tests/spec_decode/test_dynamic_spec_decode.py
+2
-2
@@ -68,13 +68,13 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
68
if queue_size < disable_by_batch_size:
69
# Should raise exception when executing the mocked draft model.
70
with pytest.raises(ValueError, match=exception_secret):
71
- proposer.get_proposals(execute_model_req=ExecuteModelRequest(
71
+ proposer.get_spec_proposals(execute_model_req=ExecuteModelRequest(
72
seq_group_metadata_list=seq_group_metadata_list,
73
num_lookahead_slots=k), )
74
else:
75
# Should not execute the draft model because spec decode is disabled
76
# for all requests. Accordingly, the proposal length should be 0.
77
- proposals = proposer.get_proposals(
77
+ proposals = proposer.get_spec_proposals(
78
execute_model_req=ExecuteModelRequest(
79
seq_group_metadata_list=seq_group_metadata_list,
80
num_lookahead_slots=k), )
tests/spec_decode/test_multi_step_worker.py
+12
-9
@@ -307,9 +307,10 @@ def test_draft_proposals_full_speculation_len():
307
308
seq_group_metadata_list, _, _ = create_batch(batch_size, k)
309
310
- proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
311
- seq_group_metadata_list=seq_group_metadata_list,
312
- num_lookahead_slots=k), )
310
+ proposals = proposer.get_spec_proposals(
311
+ execute_model_req=ExecuteModelRequest(
312
+ seq_group_metadata_list=seq_group_metadata_list,
313
+ num_lookahead_slots=k), )
314
315
assert torch.is_tensor(proposals.proposal_token_ids)
316
assert torch.is_tensor(proposals.proposal_probs)
@@ -344,9 +345,10 @@ def test_draft_proposals_no_speculations():
345
k,
346
prompt_len=prompt_len)
347
347
- proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
348
- seq_group_metadata_list=seq_group_metadata_list,
349
- num_lookahead_slots=k), )
348
+ proposals = proposer.get_spec_proposals(
349
+ execute_model_req=ExecuteModelRequest(
350
+ seq_group_metadata_list=seq_group_metadata_list,
351
+ num_lookahead_slots=k), )
352
353
assert torch.is_tensor(proposals.proposal_token_ids)
354
assert torch.is_tensor(proposals.proposal_probs)
@@ -415,9 +417,10 @@ def test_draft_proposals_mixed_k():
417
prev_output_token_len=prev_output_token_len,
418
)
419
418
- proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
419
- seq_group_metadata_list=seq_group_metadata_list,
420
- num_lookahead_slots=k), )
420
+ proposals = proposer.get_spec_proposals(
421
+ execute_model_req=ExecuteModelRequest(
422
+ seq_group_metadata_list=seq_group_metadata_list,
423
+ num_lookahead_slots=k), )
424
425
assert torch.is_tensor(proposals.proposal_token_ids)
426
assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_ngram_worker.py
+12
-9
@@ -50,9 +50,10 @@ def test_ngram_algo_correctness_for_single_no_match():
50
block_size,
51
final_prompt_lens=final_prompt_lens)
52
53
- proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
54
- seq_group_metadata_list=seq_group_metadata_list,
55
- num_lookahead_slots=proposal_len), )
53
+ proposals = proposer.get_spec_proposals(
54
+ execute_model_req=ExecuteModelRequest(
55
+ seq_group_metadata_list=seq_group_metadata_list,
56
+ num_lookahead_slots=proposal_len), )
57
58
assert torch.is_tensor(proposals.proposal_token_ids)
59
assert torch.is_tensor(proposals.proposal_probs)
@@ -117,9 +118,10 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
118
block_size,
119
final_prompt_lens=final_prompt_lens)
120
120
- proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
121
- seq_group_metadata_list=seq_group_metadata_list,
122
- num_lookahead_slots=proposal_len), )
121
+ proposals = proposer.get_spec_proposals(
122
+ execute_model_req=ExecuteModelRequest(
123
+ seq_group_metadata_list=seq_group_metadata_list,
124
+ num_lookahead_slots=proposal_len), )
125
126
assert torch.is_tensor(proposals.proposal_token_ids)
127
assert torch.is_tensor(proposals.proposal_probs)
@@ -188,9 +190,10 @@ def test_ngram_algo_correctness_for_batches_match_all():
190
block_size,
191
final_prompt_lens=final_prompt_lens)
192
191
- proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
192
- seq_group_metadata_list=seq_group_metadata_list,
193
- num_lookahead_slots=proposal_len), )
193
+ proposals = proposer.get_spec_proposals(
194
+ execute_model_req=ExecuteModelRequest(
195
+ seq_group_metadata_list=seq_group_metadata_list,
196
+ num_lookahead_slots=proposal_len), )
197
198
assert torch.is_tensor(proposals.proposal_token_ids)
199
assert torch.is_tensor(proposals.proposal_probs)
vllm/spec_decode/interfaces.py
+1
-1
@@ -55,7 +55,7 @@ class SpeculativeScores:
55
class SpeculativeProposer(ABC):
56
57
@abstractmethod
58
- def get_proposals(
58
+ def get_spec_proposals(
59
self,
60
execute_model_req: ExecuteModelRequest,
61
) -> SpeculativeProposals:
vllm/spec_decode/multi_step_worker.py
+9
-6
@@ -7,11 +7,12 @@ import torch
7
from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
8
SequenceGroupMetadata)
9
from vllm.spec_decode.interfaces import SpeculativeProposals
10
+from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
11
from vllm.spec_decode.top1_proposer import Top1Proposer
12
from vllm.worker.worker import Worker
13
14
14
-class MultiStepWorker(Worker):
15
+class MultiStepWorker(Worker, ProposerWorkerBase):
16
"""The MultiStepWorker is equivalent to a Worker except that it allows
17
multiple forward passes in a single call, assuming the scheduler has
18
allocated enough space to store the additional KV. This reduces overhead
@@ -33,7 +34,7 @@ class MultiStepWorker(Worker):
34
super().init_device()
35
36
self._proposer = Top1Proposer(
36
- weakref.proxy(self),
37
+ weakref.proxy(self), # type: ignore[arg-type]
38
self.device,
39
self.vocab_size,
40
max_proposal_len=self.max_model_len,
@@ -92,11 +93,12 @@ class MultiStepWorker(Worker):
93
speculative tokens per sequence is determined by max_proposal_len.
94
"""
95
95
- return self._proposer.get_proposals(execute_model_req)
96
+ return self._proposer.get_spec_proposals(execute_model_req)
97
98
+ @staticmethod
99
def _append_new_tokens(
98
- self, model_output: SamplerOutput,
99
- seq_group_metadata_list: SequenceGroupMetadata) -> None:
100
+ model_output: List[SamplerOutput],
101
+ seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
102
"""Given model output from a single run, append the tokens to the
103
sequences. This is normally done outside of the worker, but it is
104
required if the worker is to perform multiple forward passes.
@@ -116,8 +118,9 @@ class MultiStepWorker(Worker):
118
seq.append_token_id(token_id, token_logprob.logprob)
119
seq.update_num_computed_tokens(1)
120
121
+ @staticmethod
122
def _shallow_copy_inputs(
120
- self, seq_group_metadata_list: List[SequenceGroupMetadata]
123
+ seq_group_metadata_list: List[SequenceGroupMetadata]
124
) -> List[SequenceGroupMetadata]:
125
"""Copy input data structures to remove side-effects when input data
126
structures are shared with other modules.
vllm/spec_decode/ngram_worker.py
+5
-28
@@ -5,15 +5,16 @@ import torch
5
6
from vllm.sequence import ExecuteModelRequest, SamplerOutput
7
from vllm.spec_decode.interfaces import SpeculativeProposals
8
+from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
9
from vllm.spec_decode.top1_proposer import Top1Proposer
10
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
11
12
12
-class NGramWorker(LoraNotSupportedWorkerBase):
13
+class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
14
"""NGramWorker provides a light drafter without need for model.
15
16
Current NGramWorker only implement prompt lookup decoding,
16
- and in future we may also do RAG type drafter and other scenerios
17
+ and in future we may also do RAG type drafter and other scenarios
18
which don't rely on LLM model to give proposals.
19
"""
20
@@ -38,34 +39,11 @@ class NGramWorker(LoraNotSupportedWorkerBase):
39
40
# Current only support Top1Proposer
41
self._proposer = Top1Proposer(
41
- weakref.proxy(self),
42
+ weakref.proxy(self), # type: ignore[arg-type]
43
device=self.device,
44
vocab_size=self.vocab_size,
45
)
46
46
- def set_include_gpu_probs_tensor(self):
47
- # NGram don't need gpu sampler
48
- pass
49
-
50
- def execute_model(
51
- self,
52
- execute_model_req: Optional[ExecuteModelRequest] = None) -> None:
53
- """NGram doesn't depend on model execution, just pass this function"""
54
- pass
55
-
56
- def determine_num_available_blocks(self) -> None:
57
- """NGram doesn't depend on model execution, no need to check blocks"""
58
- pass
59
-
60
- def initialize_cache(self, num_gpu_blocks: int,
61
- num_cpu_blocks: int) -> None:
62
- """As there is no cache need to handle, just pass this function"""
63
- pass
64
-
65
- def get_cache_block_size_bytes(self):
66
- """Return the size of a cache block in bytes."""
67
- return 0
68
-
47
def sampler_output(
48
self,
49
execute_model_req: ExecuteModelRequest,
@@ -97,7 +75,6 @@ class NGramWorker(LoraNotSupportedWorkerBase):
75
-1,
76
):
77
ngram_tensor = input_ids[-ngram_size:]
100
- proposal_start_idx = None
78
if ngram_size == 1:
79
# Do not match itself and do not use unfold and all
80
matches = (input_ids[:-1] == ngram_tensor)
@@ -161,7 +138,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
138
speculative tokens per sequence is determined by max_proposal_len.
139
"""
140
164
- return self._proposer.get_proposals(execute_model_req)
141
+ return self._proposer.get_spec_proposals(execute_model_req)
142
143
def _raise_if_unsupported(
144
self,
vllm/spec_decode/proposer_worker_base.py
new
+44
@@ -0,0 +1,44 @@
1
+from abc import ABC, abstractmethod
2
+from typing import List, Optional, Tuple
3
+
4
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
5
+from vllm.spec_decode.interfaces import SpeculativeProposer
6
+from vllm.worker.worker_base import WorkerBase
7
+
8
+
9
+class ProposerWorkerBase(WorkerBase, SpeculativeProposer):
10
+ """Interface for proposer workers"""
11
+
12
+ @abstractmethod
13
+ def sampler_output(
14
+ self,
15
+ execute_model_req: ExecuteModelRequest,
16
+ sample_len: int,
17
+ ) -> Tuple[Optional[List[SamplerOutput]], bool]:
18
+ raise NotImplementedError
19
+
20
+ def set_include_gpu_probs_tensor(self):
21
+ """Implementation optional"""
22
+ pass
23
+
24
+
25
+class NonLLMProposerWorkerBase(ProposerWorkerBase, ABC):
26
+ """Proposer worker which does not use a model with kvcache"""
27
+
28
+ def execute_model(
29
+ self,
30
+ execute_model_req: Optional[ExecuteModelRequest] = None
31
+ ) -> List[SamplerOutput]:
32
+ """get_spec_proposals is used to get the proposals"""
33
+ return []
34
+
35
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
36
+ """This is never called on the proposer, only the target model"""
37
+ raise NotImplementedError
38
+
39
+ def initialize_cache(self, num_gpu_blocks: int,
40
+ num_cpu_blocks: int) -> None:
41
+ pass
42
+
43
+ def get_cache_block_size_bytes(self) -> int:
44
+ return 0
vllm/spec_decode/spec_decode_worker.py
+3
-2
@@ -14,6 +14,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
14
from vllm.spec_decode.metrics import AsyncMetricsCollector
15
from vllm.spec_decode.multi_step_worker import MultiStepWorker
16
from vllm.spec_decode.ngram_worker import NGramWorker
17
+from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
18
from vllm.spec_decode.util import (create_sequence_group_output,
19
get_all_num_logprobs, get_all_seq_ids,
20
get_sampled_token_logprobs, nvtx_range,
@@ -117,7 +118,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
118
119
def __init__(
120
self,
120
- proposer_worker: WorkerBase,
121
+ proposer_worker: ProposerWorkerBase,
122
scorer_worker: WorkerBase,
123
rejection_sampler: RejectionSampler,
124
metrics_collector: Optional[AsyncMetricsCollector] = None,
@@ -260,7 +261,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
261
262
# This is required as if the number of draft model runs changes
263
# dynamically, the non-driver workers won't know unless we perform a
263
- # communication to inform then.
264
+ # communication to inform them.
265
broadcast_dict = dict(
266
num_lookahead_slots=num_lookahead_slots,
267
disable_all_speculation=disable_all_speculation,
vllm/spec_decode/top1_proposer.py
+3
-3
@@ -6,8 +6,8 @@ from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
6
SequenceGroupMetadata)
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeProposer)
9
+from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
10
from vllm.spec_decode.util import sampler_output_to_torch
10
-from vllm.worker.worker_base import WorkerBase
11
12
13
class Top1Proposer(SpeculativeProposer):
@@ -29,7 +29,7 @@ class Top1Proposer(SpeculativeProposer):
29
30
def __init__(
31
self,
32
- worker: WorkerBase,
32
+ worker: ProposerWorkerBase,
33
device: str,
34
vocab_size: int,
35
max_proposal_len: Optional[int] = None,
@@ -39,7 +39,7 @@ class Top1Proposer(SpeculativeProposer):
39
self.max_proposal_len = max_proposal_len
40
self._vocab_size = vocab_size
41
42
- def get_proposals(
42
+ def get_spec_proposals(
43
self,
44
execute_model_req: ExecuteModelRequest,
45
) -> SpeculativeProposals: