@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f77b6cfec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[TPU] Implement prefix caching for TPUs (#10307)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 20, 2024 at 13:54 UTC 2f77b6cfec32c8054f996aee4b021f511630ea6f
4 files changed +182 -105
requirements-tpu.txt
+3 -3
@@ -16,8 +16,8 @@ ray[default]
16 --find-links https://storage.googleapis.com/libtpu-releases/index.html
17 --find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html
18 --find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
19 -torch==2.6.0.dev20241028+cpu
20 -torchvision==0.20.0.dev20241028+cpu
21 -torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241028-cp310-cp310-linux_x86_64.whl
19 +torch==2.6.0.dev20241114+cpu
20 +torchvision==0.20.0.dev20241114+cpu
21 +torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241114-cp310-cp310-linux_x86_64.whl
22 jaxlib==0.4.32.dev20240829
23 jax==0.4.32.dev20240829
vllm/attention/backends/pallas.py
+43 -23
@@ -65,6 +65,7 @@ class PallasMetadata(AttentionMetadata):
65 # or all decoding.
66 block_tables: Optional[torch.Tensor] = None
67 context_lens: Optional[torch.Tensor] = None
68 + effective_query_lens: Optional[torch.Tensor] = None
69
70 @property
71 def prefill_metadata(self) -> Optional["PallasMetadata"]:
@@ -72,8 +73,6 @@ class PallasMetadata(AttentionMetadata):
73 return None
74
75 assert self.num_decode_tokens == 0
75 - assert self.block_tables is None
76 - assert self.context_lens is None
76 return self
77
78 @property
@@ -186,29 +185,50 @@ class PallasAttentionBackendImpl(AttentionImpl):
185
186 query = query * self.scale
187 if attn_metadata.num_prefills > 0:
189 - assert seq_len % 16 == 0, (
190 - "Pallas FlashAttention kernel requires seq_len to be a "
191 - f"multiple of 16 but got {seq_len}")
192 -
193 - # Handle GQA/MQA.
194 - if self.num_kv_heads != self.num_heads:
195 - key = key.repeat_interleave(self.num_queries_per_kv, dim=-2)
196 - key = key.view(batch_size, seq_len, self.num_heads,
197 - self.head_size)
198 - value = value.repeat_interleave(self.num_queries_per_kv,
188 + if attn_metadata.block_tables is None:
189 + # Prefill without paged KV cache.
190 + assert seq_len % 16 == 0, (
191 + "Pallas FlashAttention kernel requires seq_len to be a "
192 + f"multiple of 16 but got {seq_len}")
193 +
194 + # Handle GQA/MQA.
195 + if self.num_kv_heads != self.num_heads:
196 + key = key.repeat_interleave(self.num_queries_per_kv,
197 dim=-2)
200 - value = value.view(batch_size, seq_len, self.num_heads,
198 + key = key.view(batch_size, seq_len, self.num_heads,
199 self.head_size)
202 - # FlashAttention requires [batch_size, num_heads, seq_len, d_model]
203 - # while the input is [batch_size, seq_len, num_heads, d_model].
204 - # Permute the input to match the required format.
205 - output = torch.ops.xla.flash_attention(
206 - query.permute(0, 2, 1, 3),
207 - key.permute(0, 2, 1, 3),
208 - value.permute(0, 2, 1, 3),
209 - True,
210 - )
211 - output = output.permute(0, 2, 1, 3)
200 + value = value.repeat_interleave(self.num_queries_per_kv,
201 + dim=-2)
202 + value = value.view(batch_size, seq_len, self.num_heads,
203 + self.head_size)
204 + # FlashAttention kernel requires the input shape to be
205 + # [batch_size, num_heads, seq_len, d_model]
206 + # while the input is [batch_size, seq_len, num_heads, d_model].
207 + # Permute the input to match the required format.
208 + output = torch.ops.xla.flash_attention(
209 + query.permute(0, 2, 1, 3),
210 + key.permute(0, 2, 1, 3),
211 + value.permute(0, 2, 1, 3),
212 + True,
213 + )
214 + output = output.permute(0, 2, 1, 3)
215 + else:
216 + # Prefill with paged KV cache.
217 + # TODO(woosuk): Tune the below knobs.
218 + num_kv_pages_per_compute_block = 16
219 + num_queries_per_compute_block = 16
220 + assert seq_len % num_queries_per_compute_block == 0
221 + output = torch.ops.xla.multi_queries_paged_attention(
222 + query,
223 + key_cache,
224 + value_cache,
225 + attn_metadata.context_lens,
226 + attn_metadata.block_tables,
227 + attn_metadata.effective_query_lens,
228 + num_kv_pages_per_compute_block,
229 + num_queries_per_compute_block,
230 + use_kernel=True,
231 + )
232 else:
233 # Decoding run.
234 assert kv_cache[0].numel() > 0
vllm/worker/tpu_model_runner.py
+134 -77
@@ -1,3 +1,4 @@
1 +import enum
2 import time
3 from dataclasses import dataclass
4 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
@@ -11,7 +12,6 @@ import torch_xla.core.xla_model as xm
12 import torch_xla.runtime as xr
13
14 from vllm.attention import AttentionMetadata, get_attn_backend
14 -from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
15 from vllm.config import VllmConfig
16 from vllm.logger import init_logger
17 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -39,6 +39,15 @@ _ENABLE_TOP_P = False
39 _MAX_NUM_SAMPLES = 128
40
41
42 +class ExecutionMode(enum.Enum):
43 + PREFILL = enum.auto()
44 + DECODE = enum.auto()
45 + PREFIX_PREFILL = enum.auto()
46 +
47 + def is_prefill(self) -> bool:
48 + return self in (ExecutionMode.PREFILL, ExecutionMode.PREFIX_PREFILL)
49 +
50 +
51 @dataclass(frozen=True)
52 class ModelInputForTPU(ModelRunnerInputBase):
53 token_ids: torch.Tensor
@@ -140,16 +149,21 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
149 model = get_model(vllm_config=self.vllm_config)
150 model = model.eval()
151 xm.wait_device_ops()
143 - self.model = ModelWrapper(model, self.vllm_config)
152 + model = ModelWrapper(model)
153 + self.model = torch.compile(model,
154 + backend="openxla",
155 + fullgraph=True,
156 + dynamic=False)
157
158 def _dummy_run(
159 self,
160 batch_size: int,
161 seq_len: int,
162 kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
150 - is_prompt: bool,
163 + exec_mode: ExecutionMode,
164 ) -> None:
152 - if is_prompt:
165 + exec_mode = ExecutionMode(exec_mode)
166 + if exec_mode.is_prefill():
167 seq_len = (seq_len + 15) // 16 * 16
168 token_ids = torch.zeros((batch_size, seq_len),
169 dtype=torch.int32,
@@ -160,18 +174,38 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
174 slot_mapping = torch.zeros((batch_size, seq_len),
175 dtype=torch.int64,
176 device=self.device)
163 - attn_metadata = self.attn_backend.make_metadata(
164 - num_prefills=batch_size,
165 - num_prefill_tokens=batch_size * seq_len,
166 - num_decode_tokens=0,
167 - slot_mapping=slot_mapping,
168 - multi_modal_placeholder_index_maps=None,
169 - block_tables=None,
170 - context_lens=None,
171 - )
177 input_lens = torch.ones((batch_size, ),
178 dtype=torch.int32,
179 device=self.device)
180 + if exec_mode == ExecutionMode.PREFILL:
181 + attn_metadata = self.attn_backend.make_metadata(
182 + num_prefills=batch_size,
183 + num_prefill_tokens=batch_size * seq_len,
184 + num_decode_tokens=0,
185 + slot_mapping=slot_mapping,
186 + multi_modal_placeholder_index_maps=None,
187 + block_tables=None,
188 + context_lens=None,
189 + effective_query_lens=None,
190 + )
191 + else:
192 + context_lens = torch.ones((batch_size, ),
193 + dtype=torch.int32,
194 + device=self.device)
195 + block_tables = torch.tensor(self.block_tables[:batch_size],
196 + dtype=torch.int32,
197 + device=self.device)
198 + effective_query_lens = torch.ones_like(context_lens)
199 + attn_metadata = self.attn_backend.make_metadata(
200 + num_prefills=batch_size,
201 + num_prefill_tokens=batch_size * seq_len,
202 + num_decode_tokens=0,
203 + slot_mapping=slot_mapping,
204 + multi_modal_placeholder_index_maps=None,
205 + block_tables=block_tables,
206 + context_lens=context_lens,
207 + effective_query_lens=effective_query_lens,
208 + )
209 else:
210 assert seq_len == 1
211 token_ids = torch.zeros((batch_size, seq_len),
@@ -204,7 +238,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
238 )
239 t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
240 p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
207 - num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
241 + num_samples = _MAX_NUM_SAMPLES if exec_mode.is_prefill() else 1
242
243 # NOTE(woosuk): There are two stages of compilation: torch.compile and
244 # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
@@ -213,7 +247,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
247 # be re-compiled for every different shapes. This overhead is inevitable
248 # in the first run, but can be skipped afterwards as we cache the XLA
249 # graphs in the disk (VLLM_XLA_CACHE_PATH).
216 - if is_prompt:
250 + if exec_mode.is_prefill():
251 # Prefll
252 torch._dynamo.mark_dynamic(token_ids, 1)
253 torch._dynamo.mark_dynamic(position_ids, 1)
@@ -229,15 +263,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
263 torch._dynamo.mark_dynamic(t, 0)
264 torch._dynamo.mark_dynamic(p, 0)
265 # Dummy run.
232 - self.model(token_ids,
233 - position_ids,
234 - attn_metadata,
235 - input_lens,
236 - t,
237 - p,
238 - num_samples,
239 - kv_caches,
240 - is_prompt=is_prompt)
266 + self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
267 + num_samples, kv_caches)
268
269 def warmup_model(
270 self,
@@ -248,13 +275,13 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
275 start = time.time()
276 for batch_size in [1]:
277 seq_len = 16
251 - while True:
252 - self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=True)
278 + while seq_len <= self.model_config.max_model_len:
279 + self._dummy_run(batch_size,
280 + seq_len,
281 + kv_caches,
282 + exec_mode=ExecutionMode.PREFILL)
283 xm.wait_device_ops()
284 logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
255 -
256 - if seq_len >= self.model_config.max_model_len:
257 - break
285 num_tokens = batch_size * seq_len
286 if num_tokens >= self.scheduler_config.max_num_batched_tokens:
287 break
@@ -263,12 +290,39 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
290 end = time.time()
291 logger.info("Compilation for prefill done in %.2f s.", end - start)
292
293 + # Prefix prefill
294 + if self.cache_config.enable_prefix_caching:
295 + logger.info("Compiling the model with different input shapes for "
296 + "prefix prefill...")
297 + start = time.time()
298 + for batch_size in [1]:
299 + seq_len = 16
300 + while seq_len <= self.model_config.max_model_len:
301 + self._dummy_run(batch_size,
302 + seq_len,
303 + kv_caches,
304 + exec_mode=ExecutionMode.PREFIX_PREFILL)
305 + xm.wait_device_ops()
306 + logger.info("batch_size: %d, seq_len: %d", batch_size,
307 + seq_len)
308 + num_tokens = batch_size * seq_len
309 + if (num_tokens >=
310 + self.scheduler_config.max_num_batched_tokens):
311 + break
312 + seq_len = seq_len * 2
313 + end = time.time()
314 + logger.info("Compilation for prefix prefill done in %.2f s.",
315 + end - start)
316 +
317 # Decode
318 start = time.time()
319 seq_len = 1
320 batch_size = 8 # Must be in sync with _get_padded_batch_size()
321 while True:
271 - self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False)
322 + self._dummy_run(batch_size,
323 + seq_len,
324 + kv_caches,
325 + exec_mode=ExecutionMode.DECODE)
326 xm.wait_device_ops()
327 logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
328
@@ -287,9 +341,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
341 input_tokens: List[int] = []
342 input_positions: List[int] = []
343 prompt_lens: List[int] = []
344 + context_lens: List[int] = []
345 slot_mapping: List[int] = []
346
292 - for seq_group_metadata in seq_group_metadata_list:
347 + for batch_idx, seq_group_metadata in enumerate(
348 + seq_group_metadata_list):
349 assert seq_group_metadata.is_prompt
350 seq_ids = list(seq_group_metadata.seq_data.keys())
351 assert len(seq_ids) == 1
@@ -298,19 +354,31 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
354 seq_data = seq_group_metadata.seq_data[seq_id]
355 # Could include output tokens when a request is preempted.
356 prompt_tokens = seq_data.get_token_ids()
357 + seq_len = len(prompt_tokens)
358 +
359 + num_computed_blocks = len(seq_group_metadata.computed_block_nums)
360 + num_computed_tokens = num_computed_blocks * self.block_size
361 + if num_computed_tokens > 0:
362 + prompt_tokens = prompt_tokens[num_computed_tokens:]
363 + context_lens.append(seq_len)
364 + else:
365 + context_lens.append(0)
366 +
367 prompt_len = len(prompt_tokens)
368 prompt_lens.append(prompt_len)
369
370 input_tokens.extend(prompt_tokens)
305 - input_positions.extend(list(range(prompt_len)))
371 + input_positions.extend(range(num_computed_tokens, seq_len))
372
373 assert seq_group_metadata.block_tables is not None
374 block_table = seq_group_metadata.block_tables[seq_id]
309 - for i in range(prompt_len):
375 + for i in range(num_computed_tokens, seq_len):
376 block_number = block_table[i // self.block_size]
377 block_offset = i % self.block_size
378 slot = block_number * self.block_size + block_offset
379 slot_mapping.append(slot)
380 + if num_computed_tokens > 0:
381 + self.block_tables[batch_idx, :len(block_table)] = block_table
382
383 # Add paddings to EACH prompt to the smallest power of 2 that is
384 # greater than or equal to the prompt length.
@@ -338,14 +406,21 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
406 prompt_lens = torch.tensor(prompt_lens,
407 dtype=torch.int32,
408 device="cpu")
409 + context_lens = torch.tensor(context_lens,
410 + dtype=torch.int32,
411 + device="cpu")
412 + block_tables = torch.tensor(self.block_tables[:num_prefills],
413 + dtype=torch.int32,
414 + device="cpu")
415 attn_metadata = self.attn_backend.make_metadata(
416 num_prefills=num_prefills,
417 num_prefill_tokens=0, # NOTE: This is not used.
418 num_decode_tokens=0,
419 slot_mapping=slot_mapping,
420 multi_modal_placeholder_index_maps=None,
347 - block_tables=None,
348 - context_lens=None,
421 + block_tables=block_tables,
422 + context_lens=context_lens,
423 + effective_query_lens=prompt_lens,
424 )
425 return input_tokens, input_positions, attn_metadata, prompt_lens
426
@@ -550,6 +625,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
625 # process them separately. This is a temporary hack that should be
626 # optimized by using SplashAttention.
627 orig_slot_mapping = model_input.attn_metadata.slot_mapping
628 + orig_block_tables = model_input.attn_metadata.block_tables
629 + orig_context_lens = model_input.attn_metadata.context_lens
630 + orig_effective_query_lens = \
631 + model_input.attn_metadata.effective_query_lens
632 batch_size = model_input.input_lens.shape[0]
633 start_idx = 0
634 next_token_ids = []
@@ -568,18 +647,24 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
647 attn_metadata.num_prefills = 1
648 attn_metadata.slot_mapping = orig_slot_mapping[
649 None, start_idx:end_idx].to(self.device)
650 + if orig_context_lens[i].item() > 0:
651 + attn_metadata.context_lens = orig_context_lens[i:i + 1].to(
652 + self.device)
653 + attn_metadata.block_tables = orig_block_tables[
654 + i].unsqueeze(0).to(self.device)
655 + attn_metadata.effective_query_lens = \
656 + orig_effective_query_lens[i:i + 1].to(self.device)
657 + else:
658 + attn_metadata.context_lens = None
659 + attn_metadata.block_tables = None
660 + attn_metadata.effective_query_lens = None
661 input_lens = model_input.input_lens[i:i + 1].to(self.device)
662 t = model_input.t[i:i + 1].to(self.device)
663 p = model_input.p[i:i + 1].to(self.device)
574 - output_token_ids = self.model(token_ids,
575 - position_ids,
576 - attn_metadata,
577 - input_lens,
578 - t,
579 - p,
664 + output_token_ids = self.model(token_ids, position_ids,
665 + attn_metadata, input_lens, t, p,
666 model_input.num_samples,
581 - kv_caches,
582 - is_prompt=True)
667 + kv_caches)
668 next_token_ids.append(output_token_ids[0])
669 start_idx = end_idx
670
@@ -624,15 +709,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
709 input_lens = model_input.input_lens.to(self.device)
710 for i in range(num_steps):
711 slot_mapping = attn_metadata.slot_mapping
627 - output_token_ids = self.model(token_ids,
628 - position_ids,
629 - attn_metadata,
630 - input_lens,
631 - t,
632 - p,
712 + output_token_ids = self.model(token_ids, position_ids,
713 + attn_metadata, input_lens, t, p,
714 model_input.num_samples,
634 - kv_caches,
635 - is_prompt=False)
715 + kv_caches)
716 self.cached_step_outputs.append(output_token_ids)
717
718 if i < num_steps - 1:
@@ -667,34 +747,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
747 return [sampler_output]
748
749
670 -class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
750 +class ModelWrapper(nn.Module):
751
672 - def __init__(self, model: nn.Module, vllm_config: VllmConfig):
752 + def __init__(self, model: nn.Module):
753 + super().__init__()
754 self.model = model
674 - compiled_callable = torch.compile(self.forward,
675 - backend="openxla",
676 - fullgraph=True,
677 - dynamic=False)
678 - super().__init__(
679 - compiled_callable,
680 - compilation_level=vllm_config.compilation_config.level)
681 -
682 - def __call__(self, *args, is_prompt: bool, **kwargs):
683 - if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher:
684 - # not fully compiled yet, or not using the custom dispatcher,
685 - # let PyTorch handle it
686 - return self.compiled_callable(*args, **kwargs)
687 - # the 3 compiled codes are:
688 - # 0: for profiling
689 - # 1: for prompt
690 - # 2: for decode
691 - # dispatch to the compiled code directly, skip PyTorch
692 - if is_prompt:
693 - with self.dispatch_to_code(1):
694 - return self.forward(*args, **kwargs)
695 - else:
696 - with self.dispatch_to_code(2):
697 - return self.forward(*args, **kwargs)
755
756 def forward(
757 self,
vllm/worker/tpu_worker.py
+2 -2
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13 from vllm.model_executor import set_random_seed
14 from vllm.sequence import ExecuteModelRequest
15 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
16 -from vllm.worker.tpu_model_runner import TPUModelRunner
16 +from vllm.worker.tpu_model_runner import ExecutionMode, TPUModelRunner
17 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
18 LoraNotSupportedWorkerBase, WorkerBase,
19 WorkerInput)
@@ -112,7 +112,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
112 batch_size=1,
113 seq_len=self.scheduler_config.max_num_batched_tokens,
114 kv_caches=kv_caches,
115 - is_prompt=True,
115 + exec_mode=ExecutionMode.PREFILL,
116 )
117 # Synchronize before measuring the memory usage.
118 xm.wait_device_ops()