@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a19e8d3726
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc][Speculative decoding] Typos and typing fixes (#6467)

Co-authored-by: caishangming.csm <caishangming.csm@alibaba-inc.com>

shangmingc committed Jul 17, 2024 at 15:17 UTC a19e8d372651abad75dc6a3939c18f23a1ae8d40
5 files changed +7 -7
vllm/spec_decode/multi_step_worker.py
+1 -1
@@ -43,7 +43,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
43 )
44
45 def set_include_gpu_probs_tensor(self) -> None:
46 - # Need include_gpu_probs_tensor for multi_step_worker
46 + # Need include_gpu_probs_tensor for MultiStepWorker
47 self.model_runner.model.sampler.include_gpu_probs_tensor = True
48
49 @torch.inference_mode()
vllm/spec_decode/ngram_worker.py
+2 -2
@@ -13,7 +13,7 @@ from vllm.worker.worker_base import LoraNotSupportedWorkerBase
13 class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
14 """NGramWorker provides a light drafter without need for model.
15
16 - Current NGramWorker only implement prompt lookup decoding,
16 + Current NGramWorker only implements prompt lookup decoding,
17 and in future we may also do RAG type drafter and other scenarios
18 which don't rely on LLM model to give proposals.
19 """
@@ -37,7 +37,7 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
37 self.device = torch.device(f"cuda:{self.local_rank}")
38 self.load_model = lambda *args, **kwargs: None
39
40 - # Current only support Top1Proposer
40 + # Current NGramWorker only supports Top1Proposer
41 self._proposer = Top1Proposer(
42 weakref.proxy(self), # type: ignore[arg-type]
43 device=self.device,
vllm/spec_decode/proposer_worker_base.py
+1 -1
@@ -24,7 +24,7 @@ class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
24 ) -> Tuple[Optional[List[SamplerOutput]], bool]:
25 raise NotImplementedError
26
27 - def set_include_gpu_probs_tensor(self):
27 + def set_include_gpu_probs_tensor(self) -> None:
28 """Implementation optional"""
29 pass
30
vllm/spec_decode/spec_decode_worker.py
+1 -1
@@ -206,7 +206,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
206
207 self.probs_dtype = self.spec_decode_sampler.probs_dtype
208 self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
209 - # Lazy initiazliation.
209 + # Lazy initialization.
210 self.scorer: SpeculativeScorer
211
212 # Hidden states from target model to pass to proposer
vllm/spec_decode/top1_proposer.py
+2 -2
@@ -138,7 +138,7 @@ class Top1Proposer(SpeculativeProposer):
138
139 # Currently only proposal lens of 0 or the global batch proposal len
140 # are supported.
141 - # If max_proposal_len is defined, then we shall no exccess this
141 + # If max_proposal_len is defined, then we shall no exceed this
142 # quota for nonzero_proposal
143 new_k = 0
144 if (self.max_proposal_len is None
@@ -219,7 +219,7 @@ class Top1Proposer(SpeculativeProposer):
219 proposal_lens: List[int],
220 nonzero_proposal_len_indices: List[int],
221 sampler_transposed: bool,
222 - ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
222 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
223 """After speculations are produced, merge the speculation results with
224 the skipped sequences.
225 """