@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f6084f6324
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Speculative Decoding] Move indices to device before filtering output (#10850)

Co-authored-by: Yang Zheng(SW)(Alex) <you@example.com>

Yang Zheng committed Dec 3, 2024 at 17:01 UTC f6084f63248a89df52bed9d9c24d6604f87e51f3
1 file changed +6 -3
vllm/spec_decode/multi_step_worker.py
+6 -3
@@ -120,6 +120,9 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
120 indices_of_seq_with_bonus_tokens)
121 model_outputs.append(model_output)
122
123 + # move indices to device to avoid stream sync
124 + indices_of_seq_with_bonus_tokens = torch.tensor(
125 + indices_of_seq_with_bonus_tokens, device=self.device)
126 filtered_model_outputs = self._filter_model_output(
127 model_outputs, indices_of_seq_with_bonus_tokens)
128 return filtered_model_outputs, True
@@ -189,7 +192,7 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
192 @staticmethod
193 def _filter_model_output(
194 expanded_batch_outputs: List[SamplerOutput],
192 - output_indices_to_retain: List[int]) -> List[SamplerOutput]:
195 + output_indices_to_retain: torch.Tensor) -> List[SamplerOutput]:
196 """
197 Filters the model output to include only the specified sequence
198 outputs. This method contracts the expanded batch output from the
@@ -199,8 +202,8 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
202 Args:
203 expanded_batch_output (List[SamplerOutput]): The expanded output
204 batch from the model.
202 - output_indices_to_retain (List[int]): Indices of the model outputs
203 - to retain.
205 + output_indices_to_retain (torch.Tensor): Indices of the model
206 + outputs to retain.
207
208 Returns:
209 List[SamplerOutput]: A list containing the filtered model