@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / faf71bcd4b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Speculative Decoding] Add `ProposerWorkerBase` abstract class (#5252)

Nick Hill committed Jun 5, 2024 at 14:53 UTC faf71bcd4b11b6d350431f432af08ccd9f30016f
9 files changed +91 -60
tests/spec_decode/test_dynamic_spec_decode.py
+2 -2
@@ -68,13 +68,13 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
68 if queue_size < disable_by_batch_size:
69 # Should raise exception when executing the mocked draft model.
70 with pytest.raises(ValueError, match=exception_secret):
71 - proposer.get_proposals(execute_model_req=ExecuteModelRequest(
71 + proposer.get_spec_proposals(execute_model_req=ExecuteModelRequest(
72 seq_group_metadata_list=seq_group_metadata_list,
73 num_lookahead_slots=k), )
74 else:
75 # Should not execute the draft model because spec decode is disabled
76 # for all requests. Accordingly, the proposal length should be 0.
77 - proposals = proposer.get_proposals(
77 + proposals = proposer.get_spec_proposals(
78 execute_model_req=ExecuteModelRequest(
79 seq_group_metadata_list=seq_group_metadata_list,
80 num_lookahead_slots=k), )
tests/spec_decode/test_multi_step_worker.py
+12 -9
@@ -307,9 +307,10 @@ def test_draft_proposals_full_speculation_len():
307
308 seq_group_metadata_list, _, _ = create_batch(batch_size, k)
309
310 - proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
311 - seq_group_metadata_list=seq_group_metadata_list,
312 - num_lookahead_slots=k), )
310 + proposals = proposer.get_spec_proposals(
311 + execute_model_req=ExecuteModelRequest(
312 + seq_group_metadata_list=seq_group_metadata_list,
313 + num_lookahead_slots=k), )
314
315 assert torch.is_tensor(proposals.proposal_token_ids)
316 assert torch.is_tensor(proposals.proposal_probs)
@@ -344,9 +345,10 @@ def test_draft_proposals_no_speculations():
345 k,
346 prompt_len=prompt_len)
347
347 - proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
348 - seq_group_metadata_list=seq_group_metadata_list,
349 - num_lookahead_slots=k), )
348 + proposals = proposer.get_spec_proposals(
349 + execute_model_req=ExecuteModelRequest(
350 + seq_group_metadata_list=seq_group_metadata_list,
351 + num_lookahead_slots=k), )
352
353 assert torch.is_tensor(proposals.proposal_token_ids)
354 assert torch.is_tensor(proposals.proposal_probs)
@@ -415,9 +417,10 @@ def test_draft_proposals_mixed_k():
417 prev_output_token_len=prev_output_token_len,
418 )
419
418 - proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
419 - seq_group_metadata_list=seq_group_metadata_list,
420 - num_lookahead_slots=k), )
420 + proposals = proposer.get_spec_proposals(
421 + execute_model_req=ExecuteModelRequest(
422 + seq_group_metadata_list=seq_group_metadata_list,
423 + num_lookahead_slots=k), )
424
425 assert torch.is_tensor(proposals.proposal_token_ids)
426 assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_ngram_worker.py
+12 -9
@@ -50,9 +50,10 @@ def test_ngram_algo_correctness_for_single_no_match():
50 block_size,
51 final_prompt_lens=final_prompt_lens)
52
53 - proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
54 - seq_group_metadata_list=seq_group_metadata_list,
55 - num_lookahead_slots=proposal_len), )
53 + proposals = proposer.get_spec_proposals(
54 + execute_model_req=ExecuteModelRequest(
55 + seq_group_metadata_list=seq_group_metadata_list,
56 + num_lookahead_slots=proposal_len), )
57
58 assert torch.is_tensor(proposals.proposal_token_ids)
59 assert torch.is_tensor(proposals.proposal_probs)
@@ -117,9 +118,10 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
118 block_size,
119 final_prompt_lens=final_prompt_lens)
120
120 - proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
121 - seq_group_metadata_list=seq_group_metadata_list,
122 - num_lookahead_slots=proposal_len), )
121 + proposals = proposer.get_spec_proposals(
122 + execute_model_req=ExecuteModelRequest(
123 + seq_group_metadata_list=seq_group_metadata_list,
124 + num_lookahead_slots=proposal_len), )
125
126 assert torch.is_tensor(proposals.proposal_token_ids)
127 assert torch.is_tensor(proposals.proposal_probs)
@@ -188,9 +190,10 @@ def test_ngram_algo_correctness_for_batches_match_all():
190 block_size,
191 final_prompt_lens=final_prompt_lens)
192
191 - proposals = proposer.get_proposals(execute_model_req=ExecuteModelRequest(
192 - seq_group_metadata_list=seq_group_metadata_list,
193 - num_lookahead_slots=proposal_len), )
193 + proposals = proposer.get_spec_proposals(
194 + execute_model_req=ExecuteModelRequest(
195 + seq_group_metadata_list=seq_group_metadata_list,
196 + num_lookahead_slots=proposal_len), )
197
198 assert torch.is_tensor(proposals.proposal_token_ids)
199 assert torch.is_tensor(proposals.proposal_probs)
vllm/spec_decode/interfaces.py
+1 -1
@@ -55,7 +55,7 @@ class SpeculativeScores:
55 class SpeculativeProposer(ABC):
56
57 @abstractmethod
58 - def get_proposals(
58 + def get_spec_proposals(
59 self,
60 execute_model_req: ExecuteModelRequest,
61 ) -> SpeculativeProposals:
vllm/spec_decode/multi_step_worker.py
+9 -6
@@ -7,11 +7,12 @@ import torch
7 from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
8 SequenceGroupMetadata)
9 from vllm.spec_decode.interfaces import SpeculativeProposals
10 +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
11 from vllm.spec_decode.top1_proposer import Top1Proposer
12 from vllm.worker.worker import Worker
13
14
14 -class MultiStepWorker(Worker):
15 +class MultiStepWorker(Worker, ProposerWorkerBase):
16 """The MultiStepWorker is equivalent to a Worker except that it allows
17 multiple forward passes in a single call, assuming the scheduler has
18 allocated enough space to store the additional KV. This reduces overhead
@@ -33,7 +34,7 @@ class MultiStepWorker(Worker):
34 super().init_device()
35
36 self._proposer = Top1Proposer(
36 - weakref.proxy(self),
37 + weakref.proxy(self), # type: ignore[arg-type]
38 self.device,
39 self.vocab_size,
40 max_proposal_len=self.max_model_len,
@@ -92,11 +93,12 @@ class MultiStepWorker(Worker):
93 speculative tokens per sequence is determined by max_proposal_len.
94 """
95
95 - return self._proposer.get_proposals(execute_model_req)
96 + return self._proposer.get_spec_proposals(execute_model_req)
97
98 + @staticmethod
99 def _append_new_tokens(
98 - self, model_output: SamplerOutput,
99 - seq_group_metadata_list: SequenceGroupMetadata) -> None:
100 + model_output: List[SamplerOutput],
101 + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
102 """Given model output from a single run, append the tokens to the
103 sequences. This is normally done outside of the worker, but it is
104 required if the worker is to perform multiple forward passes.
@@ -116,8 +118,9 @@ class MultiStepWorker(Worker):
118 seq.append_token_id(token_id, token_logprob.logprob)
119 seq.update_num_computed_tokens(1)
120
121 + @staticmethod
122 def _shallow_copy_inputs(
120 - self, seq_group_metadata_list: List[SequenceGroupMetadata]
123 + seq_group_metadata_list: List[SequenceGroupMetadata]
124 ) -> List[SequenceGroupMetadata]:
125 """Copy input data structures to remove side-effects when input data
126 structures are shared with other modules.
vllm/spec_decode/ngram_worker.py
+5 -28
@@ -5,15 +5,16 @@ import torch
5
6 from vllm.sequence import ExecuteModelRequest, SamplerOutput
7 from vllm.spec_decode.interfaces import SpeculativeProposals
8 +from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
9 from vllm.spec_decode.top1_proposer import Top1Proposer
10 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
11
12
12 -class NGramWorker(LoraNotSupportedWorkerBase):
13 +class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
14 """NGramWorker provides a light drafter without need for model.
15
16 Current NGramWorker only implement prompt lookup decoding,
16 - and in future we may also do RAG type drafter and other scenerios
17 + and in future we may also do RAG type drafter and other scenarios
18 which don't rely on LLM model to give proposals.
19 """
20
@@ -38,34 +39,11 @@ class NGramWorker(LoraNotSupportedWorkerBase):
39
40 # Current only support Top1Proposer
41 self._proposer = Top1Proposer(
41 - weakref.proxy(self),
42 + weakref.proxy(self), # type: ignore[arg-type]
43 device=self.device,
44 vocab_size=self.vocab_size,
45 )
46
46 - def set_include_gpu_probs_tensor(self):
47 - # NGram don't need gpu sampler
48 - pass
49 -
50 - def execute_model(
51 - self,
52 - execute_model_req: Optional[ExecuteModelRequest] = None) -> None:
53 - """NGram doesn't depend on model execution, just pass this function"""
54 - pass
55 -
56 - def determine_num_available_blocks(self) -> None:
57 - """NGram doesn't depend on model execution, no need to check blocks"""
58 - pass
59 -
60 - def initialize_cache(self, num_gpu_blocks: int,
61 - num_cpu_blocks: int) -> None:
62 - """As there is no cache need to handle, just pass this function"""
63 - pass
64 -
65 - def get_cache_block_size_bytes(self):
66 - """Return the size of a cache block in bytes."""
67 - return 0
68 -
47 def sampler_output(
48 self,
49 execute_model_req: ExecuteModelRequest,
@@ -97,7 +75,6 @@ class NGramWorker(LoraNotSupportedWorkerBase):
75 -1,
76 ):
77 ngram_tensor = input_ids[-ngram_size:]
100 - proposal_start_idx = None
78 if ngram_size == 1:
79 # Do not match itself and do not use unfold and all
80 matches = (input_ids[:-1] == ngram_tensor)
@@ -161,7 +138,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
138 speculative tokens per sequence is determined by max_proposal_len.
139 """
140
164 - return self._proposer.get_proposals(execute_model_req)
141 + return self._proposer.get_spec_proposals(execute_model_req)
142
143 def _raise_if_unsupported(
144 self,
vllm/spec_decode/proposer_worker_base.py new
+44
@@ -0,0 +1,44 @@
1 +from abc import ABC, abstractmethod
2 +from typing import List, Optional, Tuple
3 +
4 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
5 +from vllm.spec_decode.interfaces import SpeculativeProposer
6 +from vllm.worker.worker_base import WorkerBase
7 +
8 +
9 +class ProposerWorkerBase(WorkerBase, SpeculativeProposer):
10 + """Interface for proposer workers"""
11 +
12 + @abstractmethod
13 + def sampler_output(
14 + self,
15 + execute_model_req: ExecuteModelRequest,
16 + sample_len: int,
17 + ) -> Tuple[Optional[List[SamplerOutput]], bool]:
18 + raise NotImplementedError
19 +
20 + def set_include_gpu_probs_tensor(self):
21 + """Implementation optional"""
22 + pass
23 +
24 +
25 +class NonLLMProposerWorkerBase(ProposerWorkerBase, ABC):
26 + """Proposer worker which does not use a model with kvcache"""
27 +
28 + def execute_model(
29 + self,
30 + execute_model_req: Optional[ExecuteModelRequest] = None
31 + ) -> List[SamplerOutput]:
32 + """get_spec_proposals is used to get the proposals"""
33 + return []
34 +
35 + def determine_num_available_blocks(self) -> Tuple[int, int]:
36 + """This is never called on the proposer, only the target model"""
37 + raise NotImplementedError
38 +
39 + def initialize_cache(self, num_gpu_blocks: int,
40 + num_cpu_blocks: int) -> None:
41 + pass
42 +
43 + def get_cache_block_size_bytes(self) -> int:
44 + return 0
vllm/spec_decode/spec_decode_worker.py
+3 -2
@@ -14,6 +14,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
14 from vllm.spec_decode.metrics import AsyncMetricsCollector
15 from vllm.spec_decode.multi_step_worker import MultiStepWorker
16 from vllm.spec_decode.ngram_worker import NGramWorker
17 +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
18 from vllm.spec_decode.util import (create_sequence_group_output,
19 get_all_num_logprobs, get_all_seq_ids,
20 get_sampled_token_logprobs, nvtx_range,
@@ -117,7 +118,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
118
119 def __init__(
120 self,
120 - proposer_worker: WorkerBase,
121 + proposer_worker: ProposerWorkerBase,
122 scorer_worker: WorkerBase,
123 rejection_sampler: RejectionSampler,
124 metrics_collector: Optional[AsyncMetricsCollector] = None,
@@ -260,7 +261,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
261
262 # This is required as if the number of draft model runs changes
263 # dynamically, the non-driver workers won't know unless we perform a
263 - # communication to inform then.
264 + # communication to inform them.
265 broadcast_dict = dict(
266 num_lookahead_slots=num_lookahead_slots,
267 disable_all_speculation=disable_all_speculation,
vllm/spec_decode/top1_proposer.py
+3 -3
@@ -6,8 +6,8 @@ from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
6 SequenceGroupMetadata)
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeProposer)
9 +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
10 from vllm.spec_decode.util import sampler_output_to_torch
10 -from vllm.worker.worker_base import WorkerBase
11
12
13 class Top1Proposer(SpeculativeProposer):
@@ -29,7 +29,7 @@ class Top1Proposer(SpeculativeProposer):
29
30 def __init__(
31 self,
32 - worker: WorkerBase,
32 + worker: ProposerWorkerBase,
33 device: str,
34 vocab_size: int,
35 max_proposal_len: Optional[int] = None,
@@ -39,7 +39,7 @@ class Top1Proposer(SpeculativeProposer):
39 self.max_proposal_len = max_proposal_len
40 self._vocab_size = vocab_size
41
42 - def get_proposals(
42 + def get_spec_proposals(
43 self,
44 execute_model_req: ExecuteModelRequest,
45 ) -> SpeculativeProposals: