A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add Worker class
Woosuk Kwon committed
Feb 22, 2023 at 19:01 UTC
8290fce47d0d5efa02edc52d4c6961bea6043a31
1 file changed
+169
cacheflow/worker/worker.py
new
+169
@@ -0,0 +1,169 @@
1
+from typing import Dict, List, Tuple
2
+
3
+import torch
4
+
5
+from cacheflow.models import get_model
6
+from cacheflow.models import InputMetadata
7
+from cacheflow.worker.cache_engine import CacheEngine
8
+
9
+
10
+class Worker:
11
+
12
+ def __init__(
13
+ self,
14
+ worker_id: int,
15
+ gpu_id: int,
16
+ model_name: str,
17
+ block_size: int,
18
+ num_gpu_blocks: int,
19
+ num_cpu_blocks: int,
20
+ ) -> None:
21
+ self.worker_id = worker_id
22
+ self.gpu_id = gpu_id
23
+ self.block_size = block_size
24
+
25
+ self.device = torch.device('cuda', index=gpu_id)
26
+
27
+ # Initialize the model.
28
+ # FIXME(woosuk): This is a hack.
29
+ self.model = get_model(model_name).to(device=gpu_id)
30
+ self.num_layers = self.model.config.num_hidden_layers
31
+ self.num_heads = self.model.config.num_attention_heads
32
+ self.head_size = self.model.config.hidden_size // self.num_heads
33
+ self.dtype = self.model.dtype
34
+
35
+ self.cache_engine = CacheEngine(
36
+ worker_id=worker_id,
37
+ gpu_id=gpu_id,
38
+ num_layers=self.num_layers,
39
+ num_heads=self.num_heads,
40
+ head_size=self.head_size,
41
+ block_size=block_size,
42
+ num_gpu_blocks=num_gpu_blocks,
43
+ num_cpu_blocks=num_cpu_blocks,
44
+ dtype=self.dtype,
45
+ )
46
+ self.cache_events = self.cache_engine.events
47
+ self.gpu_cache = self.cache_engine.gpu_cache
48
+
49
+ def prepare_inputs(
50
+ self,
51
+ prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
52
+ generation_tokens: Dict[int, int], # Seq id -> Input token id.
53
+ context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
54
+ block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
55
+ ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
56
+ # TODO(woosuk): Support interactive generation.
57
+ # Add the prompt tokens.
58
+ prompt_lens: List[int] = []
59
+ input_tokens: List[int] = []
60
+ input_positions: List[int] = []
61
+ slot_mapping: List[int] = []
62
+
63
+ prompt_seq_ids = sorted(prompt_tokens.keys())
64
+ for seq_id in prompt_seq_ids:
65
+ prompt_len = len(prompt_tokens[seq_id])
66
+ prompt_lens.append(prompt_len)
67
+
68
+ input_tokens.extend(prompt_tokens[seq_id])
69
+ input_positions.extend(range(len(prompt_tokens[seq_id])))
70
+
71
+ block_table = block_tables[seq_id]
72
+ for i in range(prompt_len):
73
+ block_number = block_table[i // self.block_size]
74
+ block_offset = i % self.block_size
75
+ slot = block_number * self.block_size + block_offset
76
+ slot_mapping.append(slot)
77
+
78
+ # Add the generation tokens.
79
+ max_context_len = 0
80
+ max_num_blocks_per_seq = 0
81
+ generation_block_tables: List[List[int]] = []
82
+
83
+ generation_seq_ids = sorted(generation_tokens.keys())
84
+ for seq_id in generation_seq_ids:
85
+ input_tokens.append(generation_tokens[seq_id])
86
+ input_positions.append(context_lens[seq_id] - 1)
87
+ generation_block_tables.append(block_tables[seq_id])
88
+
89
+ max_context_len = max(max_context_len, context_lens[seq_id])
90
+ max_num_blocks_per_seq = max(
91
+ max_num_blocks_per_seq, len(block_tables[seq_id]))
92
+
93
+ # Optimization: Pad the input length to be a multiple of 8.
94
+ # This is required for utilizing the Tensor Cores in NVIDIA GPUs.
95
+ input_tokens = _pad_to_alignment(input_tokens, multiple_of=8)
96
+ input_positions = _pad_to_alignment(input_positions, multiple_of=8)
97
+
98
+ # Convert to tensors.
99
+ tokens_tensor = torch.tensor(
100
+ input_tokens, dtype=torch.long, device=self.device)
101
+ positions_tensor = torch.tensor(
102
+ input_positions, dtype=torch.long, device=self.device)
103
+ slot_mapping_tensor = torch.tensor(
104
+ slot_mapping, dtype=torch.int, device=self.device)
105
+ context_lens_tensor = torch.tensor(
106
+ [context_lens[seq_id] for seq_id in generation_seq_ids],
107
+ dtype=torch.int, device=self.device)
108
+ block_tables_tensor = torch.tensor(
109
+ [_pad_to_max(block_table) for block_table in generation_block_tables],
110
+ dtype=int, device=self.device)
111
+
112
+ input_metadata = InputMetadata(
113
+ prompt_lens=prompt_lens,
114
+ slot_mapping=slot_mapping_tensor,
115
+ context_lens=context_lens_tensor,
116
+ max_context_len=max_context_len,
117
+ block_tables=block_tables_tensor,
118
+ )
119
+ return tokens_tensor, positions_tensor, input_metadata
120
+
121
+
122
+ @torch.inference_mode()
123
+ def execute_stage(
124
+ self,
125
+ prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
126
+ generation_tokens: Dict[int, int], # Seq id -> Input token id.
127
+ context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
128
+ block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
129
+ blocks_to_swap_in: Dict[int, int],
130
+ blocks_to_swap_out: Dict[int, int],
131
+ blocks_to_copy: Dict[int, int],
132
+ ) -> torch.Tensor:
133
+ # Issue cache operations.
134
+ command_issued = False
135
+ if blocks_to_swap_in:
136
+ self.cache_engine.swap_in(blocks_to_swap_in)
137
+ command_issued = True
138
+ if blocks_to_swap_out:
139
+ self.cache_engine.swap_out(blocks_to_swap_out)
140
+ command_issued = True
141
+ if blocks_to_copy:
142
+ self.cache_engine.copy(blocks_to_copy)
143
+ command_issued = True
144
+
145
+ if command_issued:
146
+ cache_events = self.cache_events
147
+ else:
148
+ cache_events = None
149
+
150
+ # Prepare input tensors.
151
+ input_tokens, input_positions, input_metadata = self.prepare_inputs(
152
+ prompt_tokens, generation_tokens, context_lens, block_tables)
153
+
154
+ # Execute the model.
155
+ output = self.model(
156
+ input_ids=input_tokens,
157
+ positions=input_positions,
158
+ input_metadata=input_metadata,
159
+ cache_events=cache_events,
160
+ )
161
+ return output
162
+
163
+
164
+def _pad_to_alignment(x: List[int], multiple_of: int) -> List[int]:
165
+ return x + [0] * ((-len(x)) % multiple_of)
166
+
167
+
168
+def _pad_to_max(x: List[int], max_len: int) -> List[int]:
169
+ return x + [0] * (max_len - len(x))