A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix type annotations in CPU model runner (#4256)
Woosuk Kwon committed
Apr 22, 2024 at 00:54 UTC
e73ed0f1c624f85d348c0709c256a0ae6627986b
1 file changed
+4
-3
vllm/worker/cpu_model_runner.py
+4
-3
@@ -73,7 +73,8 @@ class CPUModelRunner:
73
def _prepare_prompt(
74
self,
75
seq_group_metadata_list: List[SequenceGroupMetadata],
76
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int]]:
76
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
77
+ Optional[torch.Tensor]]:
78
assert len(seq_group_metadata_list) > 0
79
input_tokens: List[int] = []
80
input_positions: List[int] = []
@@ -347,8 +348,8 @@ class CPUModelRunner:
348
def prepare_input_tensors(
349
self,
350
seq_group_metadata_list: List[SequenceGroupMetadata],
350
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
351
- SamplingMetadata]:
351
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
352
+ Optional[torch.Tensor]]:
353
multi_modal_input = None
354
if self.is_driver_worker:
355
# NOTE: We assume that all sequences in the group are all prompts or