@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 343cea3dbc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add seq_ids to input metadata

Woosuk Kwon committed Feb 23, 2023 at 09:25 UTC 343cea3dbcc5d3ee44654fa6289262c2553486c5
2 files changed +6 -3
cacheflow/models/input_metadata.py
+3
@@ -7,12 +7,14 @@ class InputMetadata:
7
8 def __init__(
9 self,
10 + seq_ids: List[int],
11 prompt_lens: List[int],
12 slot_mapping: torch.Tensor,
13 context_lens: torch.Tensor,
14 max_context_len: int,
15 block_tables: torch.Tensor,
16 ) -> None:
17 + self.seq_ids = seq_ids
18 self.prompt_lens = prompt_lens
19 self.slot_mapping = slot_mapping
20 self.context_lens = context_lens
@@ -23,3 +25,4 @@ class InputMetadata:
25 self.num_generation_tokens = context_lens.shape[0]
26 self.max_num_blocks_per_seq = block_tables.shape[1]
27 assert self.num_generation_tokens == block_tables.shape[0]
28 + assert self.num_prompts + self.num_generation_tokens == len(seq_ids)
cacheflow/worker/worker.py
+3 -3
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Tuple
1 +from typing import Dict, List, Tuple, Union
2
3 import torch
4
@@ -120,6 +120,7 @@ class Worker:
120 padded_block_tables, dtype=int, device=self.device)
121
122 input_metadata = InputMetadata(
123 + seq_ids=prompt_seq_ids + generation_seq_ids,
124 prompt_lens=prompt_lens,
125 slot_mapping=slot_mapping_tensor,
126 context_lens=context_lens_tensor,
@@ -128,7 +129,6 @@ class Worker:
129 )
130 return tokens_tensor, positions_tensor, input_metadata
131
131 -
132 @torch.inference_mode()
133 def execute_stage(
134 self,
@@ -139,7 +139,7 @@ class Worker:
139 blocks_to_swap_in: Dict[int, int],
140 blocks_to_swap_out: Dict[int, int],
141 blocks_to_copy: Dict[int, int],
142 - ) -> torch.Tensor:
142 + ) -> Union[torch.Tensor, Dict[int, Tuple[int, int]]]:
143 # Issue cache operations.
144 command_issued = False
145 if blocks_to_swap_in: