@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7b6844e590
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add input metadata

Woosuk Kwon committed Feb 22, 2023 at 19:01 UTC 7b6844e59044309b1c4014b02b95428018bfd8a5
2 files changed +28 -2
cacheflow/models/__init__.py
+3 -2
@@ -1,7 +1,8 @@
1 -from cacheflow.worker.models.model_utils import get_model
1 +from cacheflow.models.input_metadata import InputMetadata
2 +from cacheflow.models.model_utils import get_model
3
4
5 __all__ = [
6 'get_model',
6 -
7 + 'InputMetadata',
8 ]
cacheflow/models/input_metadata.py new
+25
@@ -0,0 +1,25 @@
1 +from typing import List
2 +
3 +import torch
4 +
5 +
6 +class InputMetadata:
7 +
8 + def __init__(
9 + self,
10 + prompt_lens: List[int],
11 + slot_mapping: torch.Tensor,
12 + context_lens: torch.Tensor,
13 + max_context_len: int,
14 + block_tables: torch.Tensor,
15 + ) -> None:
16 + self.prompt_lens = prompt_lens
17 + self.prompt_block_table = slot_mapping
18 + self.context_lens = context_lens
19 + self.max_context_len = max_context_len
20 + self.block_tables = block_tables
21 +
22 + self.num_prompts = len(prompt_lens)
23 + self.num_generation_tokens = context_lens.shape[0]
24 + self.max_num_blocks_per_seq = block_tables.shape[1]
25 + assert self.num_generation_tokens == block_tables.shape[0]