A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add input metadata
Woosuk Kwon committed
Feb 22, 2023 at 19:01 UTC
7b6844e59044309b1c4014b02b95428018bfd8a5
2 files changed
+28
-2
cacheflow/models/__init__.py
+3
-2
@@ -1,7 +1,8 @@
1
-from cacheflow.worker.models.model_utils import get_model
1
+from cacheflow.models.input_metadata import InputMetadata
2
+from cacheflow.models.model_utils import get_model
3
4
5
__all__ = [
6
'get_model',
6
-
7
+ 'InputMetadata',
8
]
cacheflow/models/input_metadata.py
new
+25
@@ -0,0 +1,25 @@
1
+from typing import List
2
+
3
+import torch
4
+
5
+
6
+class InputMetadata:
7
+
8
+ def __init__(
9
+ self,
10
+ prompt_lens: List[int],
11
+ slot_mapping: torch.Tensor,
12
+ context_lens: torch.Tensor,
13
+ max_context_len: int,
14
+ block_tables: torch.Tensor,
15
+ ) -> None:
16
+ self.prompt_lens = prompt_lens
17
+ self.prompt_block_table = slot_mapping
18
+ self.context_lens = context_lens
19
+ self.max_context_len = max_context_len
20
+ self.block_tables = block_tables
21
+
22
+ self.num_prompts = len(prompt_lens)
23
+ self.num_generation_tokens = context_lens.shape[0]
24
+ self.max_num_blocks_per_seq = block_tables.shape[1]
25
+ assert self.num_generation_tokens == block_tables.shape[0]