A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add seq_ids to input metadata
Woosuk Kwon committed
Feb 23, 2023 at 09:25 UTC
343cea3dbcc5d3ee44654fa6289262c2553486c5
2 files changed
+6
-3
cacheflow/models/input_metadata.py
+3
@@ -7,12 +7,14 @@ class InputMetadata:
7
8
def __init__(
9
self,
10
+ seq_ids: List[int],
11
prompt_lens: List[int],
12
slot_mapping: torch.Tensor,
13
context_lens: torch.Tensor,
14
max_context_len: int,
15
block_tables: torch.Tensor,
16
) -> None:
17
+ self.seq_ids = seq_ids
18
self.prompt_lens = prompt_lens
19
self.slot_mapping = slot_mapping
20
self.context_lens = context_lens
@@ -23,3 +25,4 @@ class InputMetadata:
25
self.num_generation_tokens = context_lens.shape[0]
26
self.max_num_blocks_per_seq = block_tables.shape[1]
27
assert self.num_generation_tokens == block_tables.shape[0]
28
+ assert self.num_prompts + self.num_generation_tokens == len(seq_ids)
cacheflow/worker/worker.py
+3
-3
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Tuple
1
+from typing import Dict, List, Tuple, Union
2
3
import torch
4
@@ -120,6 +120,7 @@ class Worker:
120
padded_block_tables, dtype=int, device=self.device)
121
122
input_metadata = InputMetadata(
123
+ seq_ids=prompt_seq_ids + generation_seq_ids,
124
prompt_lens=prompt_lens,
125
slot_mapping=slot_mapping_tensor,
126
context_lens=context_lens_tensor,
@@ -128,7 +129,6 @@ class Worker:
129
)
130
return tokens_tensor, positions_tensor, input_metadata
131
131
-
132
@torch.inference_mode()
133
def execute_stage(
134
self,
@@ -139,7 +139,7 @@ class Worker:
139
blocks_to_swap_in: Dict[int, int],
140
blocks_to_swap_out: Dict[int, int],
141
blocks_to_copy: Dict[int, int],
142
- ) -> torch.Tensor:
142
+ ) -> Union[torch.Tensor, Dict[int, Tuple[int, int]]]:
143
# Issue cache operations.
144
command_issued = False
145
if blocks_to_swap_in: