@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b1ac23f53
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix slot mapping

Woosuk Kwon committed Feb 23, 2023 at 00:10 UTC 4b1ac23f53d0e714a4a48d2c8058438405c0fd07
2 files changed +16 -6
cacheflow/models/input_metadata.py
+1 -1
@@ -14,7 +14,7 @@ class InputMetadata:
14 block_tables: torch.Tensor,
15 ) -> None:
16 self.prompt_lens = prompt_lens
17 - self.prompt_block_table = slot_mapping
17 + self.slot_mapping = slot_mapping
18 self.context_lens = context_lens
19 self.max_context_len = max_context_len
20 self.block_tables = block_tables
cacheflow/worker/worker.py
+15 -5
@@ -83,12 +83,20 @@ class Worker:
83 generation_seq_ids = sorted(generation_tokens.keys())
84 for seq_id in generation_seq_ids:
85 input_tokens.append(generation_tokens[seq_id])
86 - input_positions.append(context_lens[seq_id] - 1)
87 - generation_block_tables.append(block_tables[seq_id])
86 + position_id = context_lens[seq_id] - 1
87 + input_positions.append(position_id)
88 +
89 + block_table = block_tables[seq_id]
90 + generation_block_tables.append(block_table)
91
92 max_context_len = max(max_context_len, context_lens[seq_id])
93 max_num_blocks_per_seq = max(
91 - max_num_blocks_per_seq, len(block_tables[seq_id]))
94 + max_num_blocks_per_seq, len(block_table))
95 +
96 + block_number = block_table[position_id // self.block_size]
97 + block_offset = position_id % self.block_size
98 + slot = block_number * self.block_size + block_offset
99 + slot_mapping.append(slot)
100
101 # Optimization: Pad the input length to be a multiple of 8.
102 # This is required for utilizing the Tensor Cores in NVIDIA GPUs.
@@ -105,9 +113,11 @@ class Worker:
113 context_lens_tensor = torch.tensor(
114 [context_lens[seq_id] for seq_id in generation_seq_ids],
115 dtype=torch.int, device=self.device)
116 + padded_block_tables = [
117 + _pad_to_max(block_table, max_num_blocks_per_seq)
118 + for block_table in generation_block_tables]
119 block_tables_tensor = torch.tensor(
109 - [_pad_to_max(block_table) for block_table in generation_block_tables],
110 - dtype=int, device=self.device)
120 + padded_block_tables, dtype=int, device=self.device)
121
122 input_metadata = InputMetadata(
123 prompt_lens=prompt_lens,