A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Speculative Decoding] Move indices to device before filtering output (#10850)
Co-authored-by: Yang Zheng(SW)(Alex) <you@example.com>
Yang Zheng committed
Dec 3, 2024 at 17:01 UTC
f6084f63248a89df52bed9d9c24d6604f87e51f3
1 file changed
+6
-3
vllm/spec_decode/multi_step_worker.py
+6
-3
@@ -120,6 +120,9 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
120
indices_of_seq_with_bonus_tokens)
121
model_outputs.append(model_output)
122
123
+ # move indices to device to avoid stream sync
124
+ indices_of_seq_with_bonus_tokens = torch.tensor(
125
+ indices_of_seq_with_bonus_tokens, device=self.device)
126
filtered_model_outputs = self._filter_model_output(
127
model_outputs, indices_of_seq_with_bonus_tokens)
128
return filtered_model_outputs, True
@@ -189,7 +192,7 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
192
@staticmethod
193
def _filter_model_output(
194
expanded_batch_outputs: List[SamplerOutput],
192
- output_indices_to_retain: List[int]) -> List[SamplerOutput]:
195
+ output_indices_to_retain: torch.Tensor) -> List[SamplerOutput]:
196
"""
197
Filters the model output to include only the specified sequence
198
outputs. This method contracts the expanded batch output from the
@@ -199,8 +202,8 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
202
Args:
203
expanded_batch_output (List[SamplerOutput]): The expanded output
204
batch from the model.
202
- output_indices_to_retain (List[int]): Indices of the model outputs
203
- to retain.
205
+ output_indices_to_retain (torch.Tensor): Indices of the model
206
+ outputs to retain.
207
208
Returns:
209
List[SamplerOutput]: A list containing the filtered model