A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[TPU] Implement prefix caching for TPUs (#10307)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 20, 2024 at 13:54 UTC
2f77b6cfec32c8054f996aee4b021f511630ea6f
4 files changed
+182
-105
requirements-tpu.txt
+3
-3
@@ -16,8 +16,8 @@ ray[default]
16
--find-links https://storage.googleapis.com/libtpu-releases/index.html
17
--find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html
18
--find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
19
-torch==2.6.0.dev20241028+cpu
20
-torchvision==0.20.0.dev20241028+cpu
21
-torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241028-cp310-cp310-linux_x86_64.whl
19
+torch==2.6.0.dev20241114+cpu
20
+torchvision==0.20.0.dev20241114+cpu
21
+torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241114-cp310-cp310-linux_x86_64.whl
22
jaxlib==0.4.32.dev20240829
23
jax==0.4.32.dev20240829
vllm/attention/backends/pallas.py
+43
-23
@@ -65,6 +65,7 @@ class PallasMetadata(AttentionMetadata):
65
# or all decoding.
66
block_tables: Optional[torch.Tensor] = None
67
context_lens: Optional[torch.Tensor] = None
68
+ effective_query_lens: Optional[torch.Tensor] = None
69
70
@property
71
def prefill_metadata(self) -> Optional["PallasMetadata"]:
@@ -72,8 +73,6 @@ class PallasMetadata(AttentionMetadata):
73
return None
74
75
assert self.num_decode_tokens == 0
75
- assert self.block_tables is None
76
- assert self.context_lens is None
76
return self
77
78
@property
@@ -186,29 +185,50 @@ class PallasAttentionBackendImpl(AttentionImpl):
185
186
query = query * self.scale
187
if attn_metadata.num_prefills > 0:
189
- assert seq_len % 16 == 0, (
190
- "Pallas FlashAttention kernel requires seq_len to be a "
191
- f"multiple of 16 but got {seq_len}")
192
-
193
- # Handle GQA/MQA.
194
- if self.num_kv_heads != self.num_heads:
195
- key = key.repeat_interleave(self.num_queries_per_kv, dim=-2)
196
- key = key.view(batch_size, seq_len, self.num_heads,
197
- self.head_size)
198
- value = value.repeat_interleave(self.num_queries_per_kv,
188
+ if attn_metadata.block_tables is None:
189
+ # Prefill without paged KV cache.
190
+ assert seq_len % 16 == 0, (
191
+ "Pallas FlashAttention kernel requires seq_len to be a "
192
+ f"multiple of 16 but got {seq_len}")
193
+
194
+ # Handle GQA/MQA.
195
+ if self.num_kv_heads != self.num_heads:
196
+ key = key.repeat_interleave(self.num_queries_per_kv,
197
dim=-2)
200
- value = value.view(batch_size, seq_len, self.num_heads,
198
+ key = key.view(batch_size, seq_len, self.num_heads,
199
self.head_size)
202
- # FlashAttention requires [batch_size, num_heads, seq_len, d_model]
203
- # while the input is [batch_size, seq_len, num_heads, d_model].
204
- # Permute the input to match the required format.
205
- output = torch.ops.xla.flash_attention(
206
- query.permute(0, 2, 1, 3),
207
- key.permute(0, 2, 1, 3),
208
- value.permute(0, 2, 1, 3),
209
- True,
210
- )
211
- output = output.permute(0, 2, 1, 3)
200
+ value = value.repeat_interleave(self.num_queries_per_kv,
201
+ dim=-2)
202
+ value = value.view(batch_size, seq_len, self.num_heads,
203
+ self.head_size)
204
+ # FlashAttention kernel requires the input shape to be
205
+ # [batch_size, num_heads, seq_len, d_model]
206
+ # while the input is [batch_size, seq_len, num_heads, d_model].
207
+ # Permute the input to match the required format.
208
+ output = torch.ops.xla.flash_attention(
209
+ query.permute(0, 2, 1, 3),
210
+ key.permute(0, 2, 1, 3),
211
+ value.permute(0, 2, 1, 3),
212
+ True,
213
+ )
214
+ output = output.permute(0, 2, 1, 3)
215
+ else:
216
+ # Prefill with paged KV cache.
217
+ # TODO(woosuk): Tune the below knobs.
218
+ num_kv_pages_per_compute_block = 16
219
+ num_queries_per_compute_block = 16
220
+ assert seq_len % num_queries_per_compute_block == 0
221
+ output = torch.ops.xla.multi_queries_paged_attention(
222
+ query,
223
+ key_cache,
224
+ value_cache,
225
+ attn_metadata.context_lens,
226
+ attn_metadata.block_tables,
227
+ attn_metadata.effective_query_lens,
228
+ num_kv_pages_per_compute_block,
229
+ num_queries_per_compute_block,
230
+ use_kernel=True,
231
+ )
232
else:
233
# Decoding run.
234
assert kv_cache[0].numel() > 0
vllm/worker/tpu_model_runner.py
+134
-77
@@ -1,3 +1,4 @@
1
+import enum
2
import time
3
from dataclasses import dataclass
4
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
@@ -11,7 +12,6 @@ import torch_xla.core.xla_model as xm
12
import torch_xla.runtime as xr
13
14
from vllm.attention import AttentionMetadata, get_attn_backend
14
-from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
15
from vllm.config import VllmConfig
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -39,6 +39,15 @@ _ENABLE_TOP_P = False
39
_MAX_NUM_SAMPLES = 128
40
41
42
+class ExecutionMode(enum.Enum):
43
+ PREFILL = enum.auto()
44
+ DECODE = enum.auto()
45
+ PREFIX_PREFILL = enum.auto()
46
+
47
+ def is_prefill(self) -> bool:
48
+ return self in (ExecutionMode.PREFILL, ExecutionMode.PREFIX_PREFILL)
49
+
50
+
51
@dataclass(frozen=True)
52
class ModelInputForTPU(ModelRunnerInputBase):
53
token_ids: torch.Tensor
@@ -140,16 +149,21 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
149
model = get_model(vllm_config=self.vllm_config)
150
model = model.eval()
151
xm.wait_device_ops()
143
- self.model = ModelWrapper(model, self.vllm_config)
152
+ model = ModelWrapper(model)
153
+ self.model = torch.compile(model,
154
+ backend="openxla",
155
+ fullgraph=True,
156
+ dynamic=False)
157
158
def _dummy_run(
159
self,
160
batch_size: int,
161
seq_len: int,
162
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
150
- is_prompt: bool,
163
+ exec_mode: ExecutionMode,
164
) -> None:
152
- if is_prompt:
165
+ exec_mode = ExecutionMode(exec_mode)
166
+ if exec_mode.is_prefill():
167
seq_len = (seq_len + 15) // 16 * 16
168
token_ids = torch.zeros((batch_size, seq_len),
169
dtype=torch.int32,
@@ -160,18 +174,38 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
174
slot_mapping = torch.zeros((batch_size, seq_len),
175
dtype=torch.int64,
176
device=self.device)
163
- attn_metadata = self.attn_backend.make_metadata(
164
- num_prefills=batch_size,
165
- num_prefill_tokens=batch_size * seq_len,
166
- num_decode_tokens=0,
167
- slot_mapping=slot_mapping,
168
- multi_modal_placeholder_index_maps=None,
169
- block_tables=None,
170
- context_lens=None,
171
- )
177
input_lens = torch.ones((batch_size, ),
178
dtype=torch.int32,
179
device=self.device)
180
+ if exec_mode == ExecutionMode.PREFILL:
181
+ attn_metadata = self.attn_backend.make_metadata(
182
+ num_prefills=batch_size,
183
+ num_prefill_tokens=batch_size * seq_len,
184
+ num_decode_tokens=0,
185
+ slot_mapping=slot_mapping,
186
+ multi_modal_placeholder_index_maps=None,
187
+ block_tables=None,
188
+ context_lens=None,
189
+ effective_query_lens=None,
190
+ )
191
+ else:
192
+ context_lens = torch.ones((batch_size, ),
193
+ dtype=torch.int32,
194
+ device=self.device)
195
+ block_tables = torch.tensor(self.block_tables[:batch_size],
196
+ dtype=torch.int32,
197
+ device=self.device)
198
+ effective_query_lens = torch.ones_like(context_lens)
199
+ attn_metadata = self.attn_backend.make_metadata(
200
+ num_prefills=batch_size,
201
+ num_prefill_tokens=batch_size * seq_len,
202
+ num_decode_tokens=0,
203
+ slot_mapping=slot_mapping,
204
+ multi_modal_placeholder_index_maps=None,
205
+ block_tables=block_tables,
206
+ context_lens=context_lens,
207
+ effective_query_lens=effective_query_lens,
208
+ )
209
else:
210
assert seq_len == 1
211
token_ids = torch.zeros((batch_size, seq_len),
@@ -204,7 +238,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
238
)
239
t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
240
p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
207
- num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
241
+ num_samples = _MAX_NUM_SAMPLES if exec_mode.is_prefill() else 1
242
243
# NOTE(woosuk): There are two stages of compilation: torch.compile and
244
# XLA compilation. Using `mark_dynamic` can reduce the torch.compile
@@ -213,7 +247,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
247
# be re-compiled for every different shapes. This overhead is inevitable
248
# in the first run, but can be skipped afterwards as we cache the XLA
249
# graphs in the disk (VLLM_XLA_CACHE_PATH).
216
- if is_prompt:
250
+ if exec_mode.is_prefill():
251
# Prefll
252
torch._dynamo.mark_dynamic(token_ids, 1)
253
torch._dynamo.mark_dynamic(position_ids, 1)
@@ -229,15 +263,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
263
torch._dynamo.mark_dynamic(t, 0)
264
torch._dynamo.mark_dynamic(p, 0)
265
# Dummy run.
232
- self.model(token_ids,
233
- position_ids,
234
- attn_metadata,
235
- input_lens,
236
- t,
237
- p,
238
- num_samples,
239
- kv_caches,
240
- is_prompt=is_prompt)
266
+ self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
267
+ num_samples, kv_caches)
268
269
def warmup_model(
270
self,
@@ -248,13 +275,13 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
275
start = time.time()
276
for batch_size in [1]:
277
seq_len = 16
251
- while True:
252
- self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=True)
278
+ while seq_len <= self.model_config.max_model_len:
279
+ self._dummy_run(batch_size,
280
+ seq_len,
281
+ kv_caches,
282
+ exec_mode=ExecutionMode.PREFILL)
283
xm.wait_device_ops()
284
logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
255
-
256
- if seq_len >= self.model_config.max_model_len:
257
- break
285
num_tokens = batch_size * seq_len
286
if num_tokens >= self.scheduler_config.max_num_batched_tokens:
287
break
@@ -263,12 +290,39 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
290
end = time.time()
291
logger.info("Compilation for prefill done in %.2f s.", end - start)
292
293
+ # Prefix prefill
294
+ if self.cache_config.enable_prefix_caching:
295
+ logger.info("Compiling the model with different input shapes for "
296
+ "prefix prefill...")
297
+ start = time.time()
298
+ for batch_size in [1]:
299
+ seq_len = 16
300
+ while seq_len <= self.model_config.max_model_len:
301
+ self._dummy_run(batch_size,
302
+ seq_len,
303
+ kv_caches,
304
+ exec_mode=ExecutionMode.PREFIX_PREFILL)
305
+ xm.wait_device_ops()
306
+ logger.info("batch_size: %d, seq_len: %d", batch_size,
307
+ seq_len)
308
+ num_tokens = batch_size * seq_len
309
+ if (num_tokens >=
310
+ self.scheduler_config.max_num_batched_tokens):
311
+ break
312
+ seq_len = seq_len * 2
313
+ end = time.time()
314
+ logger.info("Compilation for prefix prefill done in %.2f s.",
315
+ end - start)
316
+
317
# Decode
318
start = time.time()
319
seq_len = 1
320
batch_size = 8 # Must be in sync with _get_padded_batch_size()
321
while True:
271
- self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False)
322
+ self._dummy_run(batch_size,
323
+ seq_len,
324
+ kv_caches,
325
+ exec_mode=ExecutionMode.DECODE)
326
xm.wait_device_ops()
327
logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
328
@@ -287,9 +341,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
341
input_tokens: List[int] = []
342
input_positions: List[int] = []
343
prompt_lens: List[int] = []
344
+ context_lens: List[int] = []
345
slot_mapping: List[int] = []
346
292
- for seq_group_metadata in seq_group_metadata_list:
347
+ for batch_idx, seq_group_metadata in enumerate(
348
+ seq_group_metadata_list):
349
assert seq_group_metadata.is_prompt
350
seq_ids = list(seq_group_metadata.seq_data.keys())
351
assert len(seq_ids) == 1
@@ -298,19 +354,31 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
354
seq_data = seq_group_metadata.seq_data[seq_id]
355
# Could include output tokens when a request is preempted.
356
prompt_tokens = seq_data.get_token_ids()
357
+ seq_len = len(prompt_tokens)
358
+
359
+ num_computed_blocks = len(seq_group_metadata.computed_block_nums)
360
+ num_computed_tokens = num_computed_blocks * self.block_size
361
+ if num_computed_tokens > 0:
362
+ prompt_tokens = prompt_tokens[num_computed_tokens:]
363
+ context_lens.append(seq_len)
364
+ else:
365
+ context_lens.append(0)
366
+
367
prompt_len = len(prompt_tokens)
368
prompt_lens.append(prompt_len)
369
370
input_tokens.extend(prompt_tokens)
305
- input_positions.extend(list(range(prompt_len)))
371
+ input_positions.extend(range(num_computed_tokens, seq_len))
372
373
assert seq_group_metadata.block_tables is not None
374
block_table = seq_group_metadata.block_tables[seq_id]
309
- for i in range(prompt_len):
375
+ for i in range(num_computed_tokens, seq_len):
376
block_number = block_table[i // self.block_size]
377
block_offset = i % self.block_size
378
slot = block_number * self.block_size + block_offset
379
slot_mapping.append(slot)
380
+ if num_computed_tokens > 0:
381
+ self.block_tables[batch_idx, :len(block_table)] = block_table
382
383
# Add paddings to EACH prompt to the smallest power of 2 that is
384
# greater than or equal to the prompt length.
@@ -338,14 +406,21 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
406
prompt_lens = torch.tensor(prompt_lens,
407
dtype=torch.int32,
408
device="cpu")
409
+ context_lens = torch.tensor(context_lens,
410
+ dtype=torch.int32,
411
+ device="cpu")
412
+ block_tables = torch.tensor(self.block_tables[:num_prefills],
413
+ dtype=torch.int32,
414
+ device="cpu")
415
attn_metadata = self.attn_backend.make_metadata(
416
num_prefills=num_prefills,
417
num_prefill_tokens=0, # NOTE: This is not used.
418
num_decode_tokens=0,
419
slot_mapping=slot_mapping,
420
multi_modal_placeholder_index_maps=None,
347
- block_tables=None,
348
- context_lens=None,
421
+ block_tables=block_tables,
422
+ context_lens=context_lens,
423
+ effective_query_lens=prompt_lens,
424
)
425
return input_tokens, input_positions, attn_metadata, prompt_lens
426
@@ -550,6 +625,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
625
# process them separately. This is a temporary hack that should be
626
# optimized by using SplashAttention.
627
orig_slot_mapping = model_input.attn_metadata.slot_mapping
628
+ orig_block_tables = model_input.attn_metadata.block_tables
629
+ orig_context_lens = model_input.attn_metadata.context_lens
630
+ orig_effective_query_lens = \
631
+ model_input.attn_metadata.effective_query_lens
632
batch_size = model_input.input_lens.shape[0]
633
start_idx = 0
634
next_token_ids = []
@@ -568,18 +647,24 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
647
attn_metadata.num_prefills = 1
648
attn_metadata.slot_mapping = orig_slot_mapping[
649
None, start_idx:end_idx].to(self.device)
650
+ if orig_context_lens[i].item() > 0:
651
+ attn_metadata.context_lens = orig_context_lens[i:i + 1].to(
652
+ self.device)
653
+ attn_metadata.block_tables = orig_block_tables[
654
+ i].unsqueeze(0).to(self.device)
655
+ attn_metadata.effective_query_lens = \
656
+ orig_effective_query_lens[i:i + 1].to(self.device)
657
+ else:
658
+ attn_metadata.context_lens = None
659
+ attn_metadata.block_tables = None
660
+ attn_metadata.effective_query_lens = None
661
input_lens = model_input.input_lens[i:i + 1].to(self.device)
662
t = model_input.t[i:i + 1].to(self.device)
663
p = model_input.p[i:i + 1].to(self.device)
574
- output_token_ids = self.model(token_ids,
575
- position_ids,
576
- attn_metadata,
577
- input_lens,
578
- t,
579
- p,
664
+ output_token_ids = self.model(token_ids, position_ids,
665
+ attn_metadata, input_lens, t, p,
666
model_input.num_samples,
581
- kv_caches,
582
- is_prompt=True)
667
+ kv_caches)
668
next_token_ids.append(output_token_ids[0])
669
start_idx = end_idx
670
@@ -624,15 +709,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
709
input_lens = model_input.input_lens.to(self.device)
710
for i in range(num_steps):
711
slot_mapping = attn_metadata.slot_mapping
627
- output_token_ids = self.model(token_ids,
628
- position_ids,
629
- attn_metadata,
630
- input_lens,
631
- t,
632
- p,
712
+ output_token_ids = self.model(token_ids, position_ids,
713
+ attn_metadata, input_lens, t, p,
714
model_input.num_samples,
634
- kv_caches,
635
- is_prompt=False)
715
+ kv_caches)
716
self.cached_step_outputs.append(output_token_ids)
717
718
if i < num_steps - 1:
@@ -667,34 +747,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
747
return [sampler_output]
748
749
670
-class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
750
+class ModelWrapper(nn.Module):
751
672
- def __init__(self, model: nn.Module, vllm_config: VllmConfig):
752
+ def __init__(self, model: nn.Module):
753
+ super().__init__()
754
self.model = model
674
- compiled_callable = torch.compile(self.forward,
675
- backend="openxla",
676
- fullgraph=True,
677
- dynamic=False)
678
- super().__init__(
679
- compiled_callable,
680
- compilation_level=vllm_config.compilation_config.level)
681
-
682
- def __call__(self, *args, is_prompt: bool, **kwargs):
683
- if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher:
684
- # not fully compiled yet, or not using the custom dispatcher,
685
- # let PyTorch handle it
686
- return self.compiled_callable(*args, **kwargs)
687
- # the 3 compiled codes are:
688
- # 0: for profiling
689
- # 1: for prompt
690
- # 2: for decode
691
- # dispatch to the compiled code directly, skip PyTorch
692
- if is_prompt:
693
- with self.dispatch_to_code(1):
694
- return self.forward(*args, **kwargs)
695
- else:
696
- with self.dispatch_to_code(2):
697
- return self.forward(*args, **kwargs)
755
756
def forward(
757
self,
vllm/worker/tpu_worker.py
+2
-2
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13
from vllm.model_executor import set_random_seed
14
from vllm.sequence import ExecuteModelRequest
15
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
16
-from vllm.worker.tpu_model_runner import TPUModelRunner
16
+from vllm.worker.tpu_model_runner import ExecutionMode, TPUModelRunner
17
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
18
LoraNotSupportedWorkerBase, WorkerBase,
19
WorkerInput)
@@ -112,7 +112,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
112
batch_size=1,
113
seq_len=self.scheduler_config.max_num_batched_tokens,
114
kv_caches=kv_caches,
115
- is_prompt=True,
115
+ exec_mode=ExecutionMode.PREFILL,
116
)
117
# Synchronize before measuring the memory usage.
118
xm.wait_device_ops()