A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc][Speculative decoding] Typos and typing fixes (#6467)
Co-authored-by: caishangming.csm <caishangming.csm@alibaba-inc.com>
shangmingc committed
Jul 17, 2024 at 15:17 UTC
a19e8d372651abad75dc6a3939c18f23a1ae8d40
5 files changed
+7
-7
vllm/spec_decode/multi_step_worker.py
+1
-1
@@ -43,7 +43,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
43
)
44
45
def set_include_gpu_probs_tensor(self) -> None:
46
- # Need include_gpu_probs_tensor for multi_step_worker
46
+ # Need include_gpu_probs_tensor for MultiStepWorker
47
self.model_runner.model.sampler.include_gpu_probs_tensor = True
48
49
@torch.inference_mode()
vllm/spec_decode/ngram_worker.py
+2
-2
@@ -13,7 +13,7 @@ from vllm.worker.worker_base import LoraNotSupportedWorkerBase
13
class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
14
"""NGramWorker provides a light drafter without need for model.
15
16
- Current NGramWorker only implement prompt lookup decoding,
16
+ Current NGramWorker only implements prompt lookup decoding,
17
and in future we may also do RAG type drafter and other scenarios
18
which don't rely on LLM model to give proposals.
19
"""
@@ -37,7 +37,7 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
37
self.device = torch.device(f"cuda:{self.local_rank}")
38
self.load_model = lambda *args, **kwargs: None
39
40
- # Current only support Top1Proposer
40
+ # Current NGramWorker only supports Top1Proposer
41
self._proposer = Top1Proposer(
42
weakref.proxy(self), # type: ignore[arg-type]
43
device=self.device,
vllm/spec_decode/proposer_worker_base.py
+1
-1
@@ -24,7 +24,7 @@ class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
24
) -> Tuple[Optional[List[SamplerOutput]], bool]:
25
raise NotImplementedError
26
27
- def set_include_gpu_probs_tensor(self):
27
+ def set_include_gpu_probs_tensor(self) -> None:
28
"""Implementation optional"""
29
pass
30
vllm/spec_decode/spec_decode_worker.py
+1
-1
@@ -206,7 +206,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
206
207
self.probs_dtype = self.spec_decode_sampler.probs_dtype
208
self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
209
- # Lazy initiazliation.
209
+ # Lazy initialization.
210
self.scorer: SpeculativeScorer
211
212
# Hidden states from target model to pass to proposer
vllm/spec_decode/top1_proposer.py
+2
-2
@@ -138,7 +138,7 @@ class Top1Proposer(SpeculativeProposer):
138
139
# Currently only proposal lens of 0 or the global batch proposal len
140
# are supported.
141
- # If max_proposal_len is defined, then we shall no exccess this
141
+ # If max_proposal_len is defined, then we shall no exceed this
142
# quota for nonzero_proposal
143
new_k = 0
144
if (self.max_proposal_len is None
@@ -219,7 +219,7 @@ class Top1Proposer(SpeculativeProposer):
219
proposal_lens: List[int],
220
nonzero_proposal_len_indices: List[int],
221
sampler_transposed: bool,
222
- ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
222
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
223
"""After speculations are produced, merge the speculation results with
224
the skipped sequences.
225
"""