@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8290fce47d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add Worker class

Woosuk Kwon committed Feb 22, 2023 at 19:01 UTC 8290fce47d0d5efa02edc52d4c6961bea6043a31
1 file changed +169
cacheflow/worker/worker.py new
+169
@@ -0,0 +1,169 @@
1 +from typing import Dict, List, Tuple
2 +
3 +import torch
4 +
5 +from cacheflow.models import get_model
6 +from cacheflow.models import InputMetadata
7 +from cacheflow.worker.cache_engine import CacheEngine
8 +
9 +
10 +class Worker:
11 +
12 + def __init__(
13 + self,
14 + worker_id: int,
15 + gpu_id: int,
16 + model_name: str,
17 + block_size: int,
18 + num_gpu_blocks: int,
19 + num_cpu_blocks: int,
20 + ) -> None:
21 + self.worker_id = worker_id
22 + self.gpu_id = gpu_id
23 + self.block_size = block_size
24 +
25 + self.device = torch.device('cuda', index=gpu_id)
26 +
27 + # Initialize the model.
28 + # FIXME(woosuk): This is a hack.
29 + self.model = get_model(model_name).to(device=gpu_id)
30 + self.num_layers = self.model.config.num_hidden_layers
31 + self.num_heads = self.model.config.num_attention_heads
32 + self.head_size = self.model.config.hidden_size // self.num_heads
33 + self.dtype = self.model.dtype
34 +
35 + self.cache_engine = CacheEngine(
36 + worker_id=worker_id,
37 + gpu_id=gpu_id,
38 + num_layers=self.num_layers,
39 + num_heads=self.num_heads,
40 + head_size=self.head_size,
41 + block_size=block_size,
42 + num_gpu_blocks=num_gpu_blocks,
43 + num_cpu_blocks=num_cpu_blocks,
44 + dtype=self.dtype,
45 + )
46 + self.cache_events = self.cache_engine.events
47 + self.gpu_cache = self.cache_engine.gpu_cache
48 +
49 + def prepare_inputs(
50 + self,
51 + prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
52 + generation_tokens: Dict[int, int], # Seq id -> Input token id.
53 + context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
54 + block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
55 + ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
56 + # TODO(woosuk): Support interactive generation.
57 + # Add the prompt tokens.
58 + prompt_lens: List[int] = []
59 + input_tokens: List[int] = []
60 + input_positions: List[int] = []
61 + slot_mapping: List[int] = []
62 +
63 + prompt_seq_ids = sorted(prompt_tokens.keys())
64 + for seq_id in prompt_seq_ids:
65 + prompt_len = len(prompt_tokens[seq_id])
66 + prompt_lens.append(prompt_len)
67 +
68 + input_tokens.extend(prompt_tokens[seq_id])
69 + input_positions.extend(range(len(prompt_tokens[seq_id])))
70 +
71 + block_table = block_tables[seq_id]
72 + for i in range(prompt_len):
73 + block_number = block_table[i // self.block_size]
74 + block_offset = i % self.block_size
75 + slot = block_number * self.block_size + block_offset
76 + slot_mapping.append(slot)
77 +
78 + # Add the generation tokens.
79 + max_context_len = 0
80 + max_num_blocks_per_seq = 0
81 + generation_block_tables: List[List[int]] = []
82 +
83 + generation_seq_ids = sorted(generation_tokens.keys())
84 + for seq_id in generation_seq_ids:
85 + input_tokens.append(generation_tokens[seq_id])
86 + input_positions.append(context_lens[seq_id] - 1)
87 + generation_block_tables.append(block_tables[seq_id])
88 +
89 + max_context_len = max(max_context_len, context_lens[seq_id])
90 + max_num_blocks_per_seq = max(
91 + max_num_blocks_per_seq, len(block_tables[seq_id]))
92 +
93 + # Optimization: Pad the input length to be a multiple of 8.
94 + # This is required for utilizing the Tensor Cores in NVIDIA GPUs.
95 + input_tokens = _pad_to_alignment(input_tokens, multiple_of=8)
96 + input_positions = _pad_to_alignment(input_positions, multiple_of=8)
97 +
98 + # Convert to tensors.
99 + tokens_tensor = torch.tensor(
100 + input_tokens, dtype=torch.long, device=self.device)
101 + positions_tensor = torch.tensor(
102 + input_positions, dtype=torch.long, device=self.device)
103 + slot_mapping_tensor = torch.tensor(
104 + slot_mapping, dtype=torch.int, device=self.device)
105 + context_lens_tensor = torch.tensor(
106 + [context_lens[seq_id] for seq_id in generation_seq_ids],
107 + dtype=torch.int, device=self.device)
108 + block_tables_tensor = torch.tensor(
109 + [_pad_to_max(block_table) for block_table in generation_block_tables],
110 + dtype=int, device=self.device)
111 +
112 + input_metadata = InputMetadata(
113 + prompt_lens=prompt_lens,
114 + slot_mapping=slot_mapping_tensor,
115 + context_lens=context_lens_tensor,
116 + max_context_len=max_context_len,
117 + block_tables=block_tables_tensor,
118 + )
119 + return tokens_tensor, positions_tensor, input_metadata
120 +
121 +
122 + @torch.inference_mode()
123 + def execute_stage(
124 + self,
125 + prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
126 + generation_tokens: Dict[int, int], # Seq id -> Input token id.
127 + context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
128 + block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
129 + blocks_to_swap_in: Dict[int, int],
130 + blocks_to_swap_out: Dict[int, int],
131 + blocks_to_copy: Dict[int, int],
132 + ) -> torch.Tensor:
133 + # Issue cache operations.
134 + command_issued = False
135 + if blocks_to_swap_in:
136 + self.cache_engine.swap_in(blocks_to_swap_in)
137 + command_issued = True
138 + if blocks_to_swap_out:
139 + self.cache_engine.swap_out(blocks_to_swap_out)
140 + command_issued = True
141 + if blocks_to_copy:
142 + self.cache_engine.copy(blocks_to_copy)
143 + command_issued = True
144 +
145 + if command_issued:
146 + cache_events = self.cache_events
147 + else:
148 + cache_events = None
149 +
150 + # Prepare input tensors.
151 + input_tokens, input_positions, input_metadata = self.prepare_inputs(
152 + prompt_tokens, generation_tokens, context_lens, block_tables)
153 +
154 + # Execute the model.
155 + output = self.model(
156 + input_ids=input_tokens,
157 + positions=input_positions,
158 + input_metadata=input_metadata,
159 + cache_events=cache_events,
160 + )
161 + return output
162 +
163 +
164 +def _pad_to_alignment(x: List[int], multiple_of: int) -> List[int]:
165 + return x + [0] * ((-len(x)) % multiple_of)
166 +
167 +
168 +def _pad_to_max(x: List[int], max_len: int) -> List[int]:
169 + return x + [0] * (max_len - len(x))