@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e73ed0f1c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix type annotations in CPU model runner (#4256)

Woosuk Kwon committed Apr 22, 2024 at 00:54 UTC e73ed0f1c624f85d348c0709c256a0ae6627986b
1 file changed +4 -3
vllm/worker/cpu_model_runner.py
+4 -3
@@ -73,7 +73,8 @@ class CPUModelRunner:
73 def _prepare_prompt(
74 self,
75 seq_group_metadata_list: List[SequenceGroupMetadata],
76 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int]]:
76 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
77 + Optional[torch.Tensor]]:
78 assert len(seq_group_metadata_list) > 0
79 input_tokens: List[int] = []
80 input_positions: List[int] = []
@@ -347,8 +348,8 @@ class CPUModelRunner:
348 def prepare_input_tensors(
349 self,
350 seq_group_metadata_list: List[SequenceGroupMetadata],
350 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
351 - SamplingMetadata]:
351 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
352 + Optional[torch.Tensor]]:
353 multi_modal_input = None
354 if self.is_driver_worker:
355 # NOTE: We assume that all sequences in the group are all prompts or