A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware][TPU] Support parallel sampling & Swapping (#5855)
Woosuk Kwon committed
Jun 26, 2024 at 11:07 UTC
cbc53b6b8d87b29949ce13d504750f63714df532
3 files changed
+147
-56
vllm/attention/backends/pallas.py
+22
-8
@@ -28,21 +28,35 @@ class PallasAttentionBackend(AttentionBackend):
28
) -> Tuple[int, ...]:
29
return (num_kv_heads, num_blocks, block_size, head_size)
30
31
+ @torch.compile(backend="openxla")
32
@staticmethod
33
def swap_blocks(
33
- src_kv_cache: torch.Tensor,
34
- dst_kv_cache: torch.Tensor,
35
- src_to_dst: Dict[int, int],
34
+ src_kv_cache: Tuple[torch.Tensor, torch.Tensor],
35
+ dst_kv_cache: Tuple[torch.Tensor, torch.Tensor],
36
+ src_to_dst: Tuple[torch.Tensor, torch.Tensor],
37
) -> None:
37
- raise NotImplementedError("swap_blocks is not implemented.")
38
+ src_k_cache, src_v_cache = src_kv_cache
39
+ dst_k_cache, dst_v_cache = dst_kv_cache
40
+ torch.ops.xla.dynamo_set_buffer_donor_(dst_k_cache, True)
41
+ torch.ops.xla.dynamo_set_buffer_donor_(dst_v_cache, True)
42
43
+ device = dst_k_cache.device
44
+ src_indices, dst_indices = src_to_dst
45
+ dst_k_cache[:, dst_indices] = src_k_cache[:, src_indices].to(device)
46
+ dst_v_cache[:, dst_indices] = src_v_cache[:, src_indices].to(device)
47
+
48
+ @torch.compile(backend="openxla")
49
@staticmethod
50
def copy_blocks(
41
- kv_caches: List[torch.Tensor],
42
- src_to_dists: Dict[int, List[int]],
51
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
52
+ src_to_dists: Tuple[torch.Tensor, torch.Tensor],
53
) -> None:
44
- # TODO(woosuk): Implement this.
45
- raise NotImplementedError("copy_blocks is not implemented.")
54
+ src_indices, dst_indices = src_to_dists
55
+ for k_cache, v_cache in kv_caches:
56
+ torch.ops.xla.dynamo_set_buffer_donor_(k_cache, True)
57
+ k_cache[:, dst_indices] = k_cache[:, src_indices]
58
+ torch.ops.xla.dynamo_set_buffer_donor_(v_cache, True)
59
+ v_cache[:, dst_indices] = v_cache[:, src_indices]
60
61
62
@dataclass
vllm/worker/tpu_model_runner.py
+50
-26
@@ -22,6 +22,9 @@ logger = init_logger(__name__)
22
_PAD_SLOT_ID = 0 # FIXME(woosuk)
23
# FIXME(woosuk): Temporarily disabled top-p sampling since it's too slow.
24
_ENABLE_TOP_P = False
25
+# FIXME(woosuk): A temporary hack to support `n > 1`.
26
+# This can significantly affect the performance if too large.
27
+_MAX_NUM_SAMPLES = 128
28
29
30
class TPUModelRunner:
@@ -143,8 +146,9 @@ class TPUModelRunner:
146
p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
147
148
# Dummy run.
149
+ num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
150
self.model(token_ids, position_ids, kv_caches, attn_metadata,
147
- input_lens, t, p)
151
+ input_lens, t, p, num_samples)
152
153
def warmup_model(
154
self,
@@ -268,14 +272,11 @@ class TPUModelRunner:
272
input_positions: List[List[int]] = []
273
slot_mapping: List[List[int]] = []
274
context_lens: List[int] = []
271
- num_seq_groups = len(seq_group_metadata_list)
272
- batch_size = _get_padded_batch_size(num_seq_groups)
275
274
- for i, seq_group_metadata in enumerate(seq_group_metadata_list):
276
+ batch_idx = 0
277
+ for seq_group_metadata in seq_group_metadata_list:
278
assert not seq_group_metadata.is_prompt
276
-
279
seq_ids = list(seq_group_metadata.seq_data.keys())
278
-
280
for seq_id in seq_ids:
281
seq_data = seq_group_metadata.seq_data[seq_id]
282
generation_token = seq_data.get_last_token_id()
@@ -288,14 +289,16 @@ class TPUModelRunner:
289
290
assert seq_group_metadata.block_tables is not None
291
block_table = seq_group_metadata.block_tables[seq_id]
291
- self.block_tables[i, :len(block_table)] = block_table
292
+ self.block_tables[batch_idx, :len(block_table)] = block_table
293
+ batch_idx += 1
294
295
block_number = block_table[position // self.block_size]
296
block_offset = position % self.block_size
297
slot = block_number * self.block_size + block_offset
298
slot_mapping.append([slot])
299
298
- num_paddings = batch_size - num_seq_groups
300
+ batch_size = _get_padded_batch_size(batch_idx)
301
+ num_paddings = batch_size - batch_idx
302
input_tokens = input_tokens + [[0]] * num_paddings
303
input_positions = input_positions + [[0]] * num_paddings
304
slot_mapping = slot_mapping + [[_PAD_SLOT_ID]] * num_paddings
@@ -333,14 +336,13 @@ class TPUModelRunner:
336
self,
337
seq_group_metadata_list: List[SequenceGroupMetadata],
338
padded_batch_size: int,
336
- ) -> Tuple[torch.Tensor, torch.Tensor]:
339
+ ) -> Tuple[torch.Tensor, torch.Tensor, List[int]]:
340
assert len(seq_group_metadata_list) > 0
341
t = []
342
p = []
343
+ best_of = []
344
for seq_group_metadata in seq_group_metadata_list:
341
- assert seq_group_metadata.sampling_params is not None
345
sampling_params = seq_group_metadata.sampling_params
343
-
346
# NOTE(woosuk): Here we mimic argmax sampling by applying a very
347
# low temperature. This is not accurate.
348
t.append(sampling_params.temperature
@@ -354,10 +356,11 @@ class TPUModelRunner:
356
raise NotImplementedError(
357
"Top-k sampling is currently disabled for the TPU backend "
358
"due to performance issues.")
357
- if sampling_params.best_of > 1:
359
+ if sampling_params.best_of > _MAX_NUM_SAMPLES:
360
raise NotImplementedError(
359
- "best_of > 1 is not currently supported by the TPU "
361
+ f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
362
"backend.")
363
+ best_of.append(sampling_params.best_of)
364
if sampling_params.use_beam_search:
365
raise NotImplementedError(
366
"Beam search is not supported by the TPU backend.")
@@ -369,13 +372,19 @@ class TPUModelRunner:
372
"prompt_logprobs is not currently supported by the TPU "
373
"backend.")
374
372
- num_paddings = padded_batch_size - len(seq_group_metadata_list)
375
+ # Repeat the sampling params if the seq group has multiple seqs.
376
+ num_seqs = len(seq_group_metadata.seq_data)
377
+ t += [t[-1]] * (num_seqs - 1)
378
+ p += [p[-1]] * (num_seqs - 1)
379
+ best_of += [best_of[-1]] * (num_seqs - 1)
380
+
381
+ num_paddings = padded_batch_size - len(t)
382
t += [1.0] * num_paddings
383
p += [1.0] * num_paddings
384
385
t = torch.tensor(t, dtype=torch.float32, device=self.device)
386
p = torch.tensor(p, dtype=torch.float32, device=self.device)
378
- return t, p
387
+ return t, p, best_of
388
389
def _execute_model(
390
self,
@@ -392,28 +401,41 @@ class TPUModelRunner:
401
else:
402
inputs = self._prepare_decode(seq_group_metadata_list)
403
padded_batch_size = inputs[0].shape[0]
395
- t, p = self._prepare_sample(seq_group_metadata_list, padded_batch_size)
404
+ t, p, best_of = self._prepare_sample(seq_group_metadata_list,
405
+ padded_batch_size)
406
+ num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
407
408
# Execute the model.
409
next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
399
- *inputs[2:], t, p)
410
+ *inputs[2:], t, p, num_samples)
411
# Retrieve the outputs to CPU.
412
next_token_ids = next_token_ids.cpu().tolist()
413
414
# NOTE(woosuk): Minimal code to construct the sampler outputs.
415
# The TPU backend does not reuse the sampler, since the TPU backend
416
# does not support the advanced sampling parameters such as logprobs.
406
- i = 0
417
+ zero_logprob = Logprob(0.0)
418
+ batch_idx = 0
419
sampler_outputs = []
420
for seq_group_metadata in seq_group_metadata_list:
421
seq_outputs = []
422
seq_ids = list(seq_group_metadata.seq_data.keys())
411
- for seq_id in seq_ids:
412
- next_token_id = next_token_ids[i]
413
- seq_outputs.append(
414
- SequenceOutput(seq_id, next_token_id,
415
- {next_token_id: Logprob(0.0)}))
416
- i += 1
423
+ if is_prompt:
424
+ assert len(seq_ids) == 1
425
+ seq_id = seq_ids[0]
426
+ for i in range(best_of[batch_idx]):
427
+ next_token_id = next_token_ids[batch_idx][i]
428
+ seq_outputs.append(
429
+ SequenceOutput(seq_id, next_token_id,
430
+ {next_token_id: zero_logprob}))
431
+ batch_idx += 1
432
+ else:
433
+ for seq_id in seq_ids:
434
+ next_token_id = next_token_ids[batch_idx][0]
435
+ seq_outputs.append(
436
+ SequenceOutput(seq_id, next_token_id,
437
+ {next_token_id: zero_logprob}))
438
+ batch_idx += 1
439
sampler_outputs.append(
440
CompletionSequenceGroupOutput(seq_outputs, None))
441
return sampler_outputs
@@ -458,6 +480,7 @@ class ModelWrapper(nn.Module):
480
input_lens: torch.Tensor,
481
t: torch.Tensor,
482
p: torch.Tensor,
483
+ num_samples: int,
484
) -> torch.Tensor:
485
"""Executes the forward pass of the model and samples the next token.
486
@@ -520,8 +543,9 @@ class ModelWrapper(nn.Module):
543
if _ENABLE_TOP_P:
544
logits = _apply_top_p(logits, p.unsqueeze(dim=1))
545
probs = torch.softmax(logits, dim=-1, dtype=torch.float32)
523
- # FIXME(woosuk): best_of > 1 is not supported.
524
- next_token_ids = torch.multinomial(probs, num_samples=1).squeeze(dim=1)
546
+ next_token_ids = torch.multinomial(probs,
547
+ num_samples,
548
+ replacement=True)
549
return next_token_ids
550
551
vllm/worker/tpu_worker.py
+75
-22
@@ -1,5 +1,5 @@
1
import os
2
-from typing import List, Optional, Tuple
2
+from typing import List, Optional, Tuple, Union
3
4
import torch
5
import torch_xla.core.xla_model as xm
@@ -117,19 +117,26 @@ class TPUWorker(LoraNotSupportedWorkerBase):
117
# Synchronize before measuring the memory usage.
118
xm.wait_device_ops()
119
120
+ dtype_btyes = get_dtype_size(self.cache_dtype)
121
+ block_size = self.cache_config.block_size
122
+ block_size_bytes = (dtype_btyes * block_size * num_layers * 2 *
123
+ head_size * num_kv_heads)
124
+
125
+ # Calculate the TPU KV cache size based on profiling.
126
m = xm.get_memory_info(self.device)
127
total_memory_size = m["bytes_limit"]
128
usable_memory_size = int(total_memory_size *
129
self.cache_config.gpu_memory_utilization)
130
profiled = m["bytes_used"] # Weights + intermediate activations.
125
- kv_cache_bytes = max(usable_memory_size - profiled, 0)
126
- dtype_btyes = get_dtype_size(self.cache_dtype)
127
- block_size = self.cache_config.block_size
128
- num_tpu_blocks = (kv_cache_bytes //
129
- (dtype_btyes * block_size * num_layers * 2 *
130
- head_size * num_kv_heads))
131
+ tpu_kv_cache_bytes = max(usable_memory_size - profiled, 0)
132
+ num_tpu_blocks = tpu_kv_cache_bytes // block_size_bytes
133
num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
132
- return num_tpu_blocks, 0
134
+
135
+ # Calculate the CPU KV cache size based on the config.
136
+ num_cpu_blocks = (self.cache_config.swap_space_bytes //
137
+ block_size_bytes)
138
+ num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
139
+ return num_tpu_blocks, num_cpu_blocks
140
141
def initialize_cache(
142
self,
@@ -145,15 +152,19 @@ class TPUWorker(LoraNotSupportedWorkerBase):
152
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
153
head_size = self.model_config.get_head_size()
154
155
+ self.cpu_cache = []
156
self.tpu_cache = []
157
tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
158
num_gpu_blocks, self.block_size, num_kv_heads, head_size)
159
for _ in range(num_layers):
152
- key_cache = torch.zeros(tpu_cache_shape,
153
- dtype=dtype,
154
- device=self.device)
155
- value_cache = torch.zeros_like(key_cache)
156
- self.tpu_cache.append((key_cache, value_cache))
160
+ tpu_k_cache = torch.zeros(tpu_cache_shape,
161
+ dtype=dtype,
162
+ device=self.device)
163
+ tpu_v_cache = torch.zeros_like(tpu_k_cache)
164
+ self.tpu_cache.append((tpu_k_cache, tpu_v_cache))
165
+ cpu_k_cache = torch.zeros_like(tpu_k_cache, device="cpu")
166
+ cpu_v_cache = torch.zeros_like(tpu_v_cache, device="cpu")
167
+ self.cpu_cache.append((cpu_k_cache, cpu_v_cache))
168
self._warmup_model()
169
170
def _warmup_model(self) -> None:
@@ -187,22 +198,48 @@ class TPUWorker(LoraNotSupportedWorkerBase):
198
if not self.is_driver_worker:
199
self._execute_model_non_driver()
200
return []
190
-
201
assert execute_model_req is not None
192
- # Currently, TPUWorker does not support swapping.
193
- # TODO(woosuk): Support block copying.
194
- assert len(execute_model_req.blocks_to_swap_in) == 0, (
195
- "Swapping is not supported for the TPU backend.")
196
- assert len(execute_model_req.blocks_to_swap_out) == 0, (
197
- "Swapping is not supported for the TPU backend.")
198
- assert len(execute_model_req.blocks_to_copy) == 0
199
-
202
+ # Issue cache operations.
203
+ self.cache_swap(
204
+ execute_model_req.blocks_to_swap_in,
205
+ execute_model_req.blocks_to_swap_out,
206
+ execute_model_req.blocks_to_copy,
207
+ )
208
+ # Run the model.
209
seq_group_metadata_list = execute_model_req.seq_group_metadata_list
210
assert len(seq_group_metadata_list) > 0
211
output = self.model_runner.execute_model(seq_group_metadata_list,
212
self.tpu_cache)
213
return [output]
214
215
+ def cache_swap(
216
+ self,
217
+ blocks_to_swap_in: List[Tuple[int, int]],
218
+ blocks_to_swap_out: List[Tuple[int, int]],
219
+ blocks_to_copy: List[Tuple[int, int]],
220
+ ) -> None:
221
+ attn_backend = self.model_runner.attn_backend
222
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
223
+
224
+ if blocks_to_swap_in:
225
+ # Swap from CPU to TPU.
226
+ src_to_dst = _make_src_to_dst(blocks_to_swap_in, "cpu",
227
+ self.device)
228
+ for i in range(num_layers):
229
+ attn_backend.swap_blocks(self.cpu_cache[i], self.tpu_cache[i],
230
+ src_to_dst)
231
+ if blocks_to_swap_out:
232
+ # Swap from TPU to CPU.
233
+ src_to_dst = _make_src_to_dst(blocks_to_swap_out, self.device,
234
+ "cpu")
235
+ for i in range(num_layers):
236
+ attn_backend.swap_blocks(self.tpu_cache[i], self.cpu_cache[i],
237
+ src_to_dst)
238
+ if blocks_to_copy:
239
+ src_to_dst = _make_src_to_dst(blocks_to_copy, self.device,
240
+ self.device)
241
+ attn_backend.copy_blocks(self.tpu_cache, src_to_dst)
242
+
243
def start_worker_execution_loop(self) -> None:
244
while self._execute_model_non_driver():
245
pass
@@ -210,3 +247,19 @@ class TPUWorker(LoraNotSupportedWorkerBase):
247
def _execute_model_non_driver(self) -> bool:
248
self.model_runner.execute_model(None, self.tpu_cache)
249
return True
250
+
251
+
252
+def _make_src_to_dst(
253
+ mapping: List[Tuple[int, int]],
254
+ src_device: Union[torch.device, str],
255
+ dst_device: Union[torch.device, str],
256
+) -> Tuple[torch.Tensor, torch.Tensor]:
257
+ src_indices = [i for i, _ in mapping]
258
+ dst_indices = [i for _, i in mapping]
259
+ src_indices = torch.tensor(src_indices,
260
+ device=src_device,
261
+ dtype=torch.int64)
262
+ dst_indices = torch.tensor(dst_indices,
263
+ device=dst_device,
264
+ dtype=torch.int64)
265
+ return src_indices, dst_indices