@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7041de4384
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Flashinfer for prefill & decode, with Cudagraph support for decode (#4628)

Co-authored-by: LiuXiaoxuanPKU <llilyliupku@gmail.com>, bong-furiosa <bongwon.jang@furiosa.ai>

Lily Liu committed Jun 28, 2024 at 15:28 UTC 7041de43849fda7c8e931f0726f3db2a0d8015a4
7 files changed +313 -117
.buildkite/test-pipeline.yaml
+3
@@ -211,3 +211,6 @@ steps:
211 - pytest -v -s distributed/test_custom_all_reduce.py
212 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
213 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
214 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.5/flashinfer-0.0.5+cu121torch2.3-cp310-cp310-linux_x86_64.whl
215 + - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
216 + - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
\ No newline at end of file
requirements-test.txt
+1 -1
@@ -19,4 +19,4 @@ sentence-transformers # required for embedding
19 aiohttp
20
21 # quantization
22 -bitsandbytes==0.42.0
22 +bitsandbytes==0.42.0
\ No newline at end of file
tests/basic_correctness/test_basic_correctness.py
-6
@@ -2,7 +2,6 @@
2
3 Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4 """
5 -import os
5 import weakref
6
7 import pytest
@@ -13,7 +12,6 @@ MODELS = [
12 "facebook/opt-125m",
13 "meta-llama/Llama-2-7b-hf",
14 ]
16 -VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
15
16
17 def test_vllm_gc_ed():
@@ -39,10 +37,6 @@ def test_models(
37 max_tokens: int,
38 enforce_eager: bool,
39 ) -> None:
42 - backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
43 - if backend_by_env_var == "FLASHINFER" and enforce_eager is False:
44 - pytest.skip("Skipping non-eager test for FlashInferBackend.")
45 -
40 with hf_runner(model, dtype=dtype) as hf_model:
41 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
42
tests/distributed/test_basic_distributed_correctness.py
-5
@@ -21,7 +21,6 @@ MODELS = [
21 os.environ["TEST_DIST_MODEL"],
22 ]
23 DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
24 -VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
24
25
26 @pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -39,16 +38,12 @@ def test_models(
38 ) -> None:
39 distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
40
42 - backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
43 - enforce_eager = backend_by_env_var == "FLASHINFER"
44 -
41 with hf_runner(model, dtype=dtype) as hf_model:
42 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
43
44 with vllm_runner(model,
45 dtype=dtype,
46 tensor_parallel_size=2,
51 - enforce_eager=enforce_eager,
47 distributed_executor_backend=distributed_executor_backend
48 ) as vllm_model:
49 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
vllm/attention/backends/flashinfer.py
+58 -25
@@ -1,10 +1,16 @@
1 from dataclasses import dataclass
2 from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
4 -import flashinfer
4 +try:
5 + from flashinfer import BatchDecodeWithPagedKVCacheWrapper
6 + from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
7 + from vllm_flash_attn import flash_attn_varlen_func
8 +except ImportError:
9 + flash_attn_varlen_func = None
10 + BatchDecodeWithPagedKVCacheWrapper = None
11 + BatchPrefillWithPagedKVCacheWrapper = None
12 +
13 import torch
6 -from flashinfer import BatchDecodeWithPagedKVCacheWrapper
7 -from vllm_flash_attn import flash_attn_varlen_func
14
15 from vllm import _custom_ops as ops
16 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
@@ -60,19 +66,16 @@ class FlashInferMetadata(AttentionMetadata):
66 # requests only.
67 max_prefill_seq_len: int
68
63 - use_cuda_graph: bool = False
69 + use_cuda_graph: bool = True
70
71 + prefill_wrapper: Optional[BatchPrefillWithPagedKVCacheWrapper] = None
72 decode_wrapper: Optional[BatchDecodeWithPagedKVCacheWrapper] = None
73
67 - # Metadata for the prefill stage since we still
68 - # use flash attention for prefill.
74 + # Metadata for the prefill stage
75 seq_start_loc: Optional[torch.Tensor] = None
76 + query_start_loc: Optional[torch.Tensor] = None
77 block_tables: Optional[torch.Tensor] = None
78
72 - # Metadata for the decode stage
73 - # Workspace buffer required by the kernel, the buffer should not
74 - # be allocated/deacollated by the FalshInfermetadata object.
75 - workspace_buffer: Optional[torch.Tensor] = None
79 # An example for paged_kv_indices, paged_kv_indptr:
80 # request 1, page indices [0, 5, 8]
81 # request 2, page indices [1, 6, 7]
@@ -98,6 +101,7 @@ class FlashInferMetadata(AttentionMetadata):
101 page_size: Optional[int] = None
102 # The data type of the paged kv cache
103 data_type: torch.dtype = None
104 + device: torch.device = torch.device("cuda")
105
106 def __post_init__(self):
107 # Refer to
@@ -109,13 +113,35 @@ class FlashInferMetadata(AttentionMetadata):
113 f"Only {supported_head_sizes} are supported for head_dim,",
114 f"received {self.head_dim}.")
115
112 - # When using flashinfer, we are also creating the FlashInferMetadata,
113 - # which will also call post_init by default, here we want to skip the
114 - # post_init if it's the prefill phase.
115 - if self.num_prefills == 0:
116 - assert self.num_decode_tokens > 0
117 - self.decode_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
118 - self.workspace_buffer, "NHD")
116 + def begin_forward(self):
117 + if self.num_prefill_tokens > 0:
118 + if self.paged_kv_indices is None:
119 + return
120 +
121 + assert self.prefill_wrapper is not None
122 + assert self.paged_kv_indices is not None
123 + assert self.paged_kv_indptr is not None
124 + assert self.paged_kv_last_page_len is not None
125 + self.paged_kv_indices = self.paged_kv_indices.to(self.device)
126 + self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
127 + self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
128 + self.device)
129 + self.prefill_wrapper.begin_forward(
130 + self.query_start_loc, self.paged_kv_indptr,
131 + self.paged_kv_indices, self.paged_kv_last_page_len,
132 + self.num_qo_heads, self.num_kv_heads, self.head_dim,
133 + self.page_size)
134 + else:
135 + if not self.use_cuda_graph:
136 + assert self.paged_kv_indices is not None
137 + assert self.paged_kv_indptr is not None
138 + assert self.paged_kv_last_page_len is not None
139 + self.paged_kv_indices = self.paged_kv_indices.to(self.device)
140 + self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
141 + self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
142 + self.device)
143 +
144 + assert self.decode_wrapper is not None
145 self.decode_wrapper.begin_forward(
146 self.paged_kv_indptr,
147 self.paged_kv_indices,
@@ -133,8 +159,9 @@ class FlashInferMetadata(AttentionMetadata):
159 ) -> Dict[str, Any]:
160 if skip_fields is None:
161 skip_fields = set()
136 - # We need to skip the decode_wrapper field since it cannot be
162 + # We need to skip the prefill/decode_wrapper field since it cannot be
163 # broadcasted with nccl when TP is enabled.
164 + skip_fields.add('prefill_wrapper')
165 skip_fields.add('decode_wrapper')
166 return super().asdict_zerocopy(skip_fields)
167
@@ -168,6 +195,7 @@ class FlashInferImpl(AttentionImpl):
195 alibi_slopes: Optional[List[float]],
196 sliding_window: Optional[int],
197 kv_cache_dtype: str,
198 + blocksparse_params: Optional[Dict[str, Any]] = None,
199 ) -> None:
200 self.num_heads = num_heads
201 self.head_size = head_size
@@ -217,10 +245,14 @@ class FlashInferImpl(AttentionImpl):
245 self.kv_cache_dtype,
246 )
247
248 + query = query.contiguous(
249 + ) # Flashinfer requires query to be contiguous
250 if prefill_meta := attn_metadata.prefill_metadata:
221 - # Prompt run.
222 - assert prefill_meta.block_tables is not None
223 - if kv_cache is None or prefill_meta.block_tables.numel() == 0:
251 + # We will use flash attention for prefill
252 + # when kv_cache is not provided.
253 + # This happens when vllm runs the profiling to
254 + # determine the number of blocks.
255 + if kv_cache is None:
256 output = flash_attn_varlen_func(
257 q=query,
258 k=key,
@@ -235,13 +267,14 @@ class FlashInferImpl(AttentionImpl):
267 alibi_slopes=self.alibi_slopes,
268 )
269 else:
238 - raise NotImplementedError(
239 - "Prefix caching is not supported with flashinfer yet.")
270 + assert prefill_meta is not None
271 + assert prefill_meta.prefill_wrapper is not None
272 + output = prefill_meta.prefill_wrapper.forward(query,
273 + kv_cache,
274 + causal=True)
275 else:
276 assert attn_metadata.decode_metadata is not None
277 assert attn_metadata.decode_metadata.decode_wrapper is not None
243 - query = query.contiguous(
244 - ) # Flashinfer requires query to be contiguous
278 output = attn_metadata.decode_metadata.decode_wrapper.forward(
279 query,
280 kv_cache,
vllm/attention/selector.py
+3 -2
@@ -77,8 +77,9 @@ def get_attn_backend(
77 return IpexAttnBackend
78 elif backend == _Backend.FLASHINFER:
79 logger.info("Using Flashinfer backend.")
80 - logger.warning("Eager mode is required for the Flashinfer backend. "
81 - "Please make sure --enforce-eager is set.")
80 + logger.warning(("Flashinfer will be stuck on llma-2-7b,"
81 + " please avoid using Flashinfer as the"
82 + "backend when running on llma-2-7b."))
83 from vllm.attention.backends.flashinfer import FlashInferBackend
84 return FlashInferBackend
85 elif backend == _Backend.PALLAS:
vllm/worker/model_runner.py
+248 -78
@@ -10,6 +10,17 @@ import numpy as np
10 import torch
11 import torch.nn as nn
12
13 +try:
14 + from flashinfer import BatchDecodeWithPagedKVCacheWrapper
15 + from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
16 + from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
17 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
18 +except ImportError:
19 + BatchDecodeWithPagedKVCacheWrapper = None
20 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
21 + BatchPrefillWithPagedKVCacheWrapper = None
22 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
23 +
24 from vllm.attention import AttentionMetadata, get_attn_backend
25 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26 ModelConfig, ParallelConfig, SchedulerConfig,
@@ -198,11 +209,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
209
210 # Lazy initialization
211 self.model: nn.Module # Set after load_model
201 - # Set if the backend is flashinfer.
202 - self.flashinfer_workspace_buffer: torch.Tensor
212 # Set after load_model.
213 self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
214
215 + self.flashinfer_decode_workspace_buffer = None
216 + self.flashinfer_decode_wrapper = None
217 + self.flashinfer_prefill_workspace_buffer = None
218 + self.flashinfer_prefill_wrapper = None
219 +
220 def load_model(self) -> None:
221 with CudaMemoryProfiler() as m:
222 self.model = get_model(
@@ -450,15 +464,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
464 if curr_sliding_window_blocks is not None:
465 block_table = block_table[
466 -curr_sliding_window_blocks:]
453 - if self.attn_backend.get_name() == "flashinfer":
454 - paged_kv_indices.extend(block_table)
455 - paged_kv_indptr.append(paged_kv_indptr[-1] +
456 - len(block_table))
457 - last_page_len = seq_data.get_len(
458 - ) % self.block_size
459 - if last_page_len == 0:
460 - last_page_len = self.block_size
461 - paged_kv_last_page_len.append(last_page_len)
467 else:
468 # Only happens when memory profiling runs.
469 block_table = []
@@ -505,7 +510,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
510 for k, v in mm_kwargs.items():
511 multi_modal_kwargs_list[k].append(v)
512
508 - if _is_block_tables_empty(seq_group_metadata.block_tables):
513 + is_profile_run = _is_block_tables_empty(
514 + seq_group_metadata.block_tables)
515 + if is_profile_run:
516 # During memory profiling, the block tables are not
517 # initialized yet. In this case, we just use a dummy
518 # slot mapping.
@@ -544,6 +551,27 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
551 slot = block_number * self.block_size + block_offset
552 slot_mapping.append(slot)
553
554 + # Prepare input tensors for flashinfer
555 + if self.attn_backend.get_name() == "flashinfer":
556 + seq_len = seq_data.get_len()
557 + # Get the number of valid blocks based on sequence length.
558 + # If seq_len = 16, block_size = 16,
559 + # block_table_bound is 1 with 1 valid block.
560 + # If seq_len = 15, block_size = 16,
561 + # block_table_bound is 0 + 1 with 1 valid block.
562 + block_table_bound = seq_len // self.block_size + 1 \
563 + if seq_len % self.block_size != 0 \
564 + else seq_len // self.block_size
565 +
566 + paged_kv_indices.extend(block_table[:block_table_bound])
567 + paged_kv_indptr.append(paged_kv_indptr[-1] +
568 + block_table_bound)
569 +
570 + last_page_len = seq_len % self.block_size
571 + if last_page_len == 0:
572 + last_page_len = self.block_size
573 + paged_kv_last_page_len.append(last_page_len)
574 +
575 batch_size = len(input_tokens)
576 max_query_len = max(query_lens)
577 max_prefill_seq_len = max(prefill_seq_lens, default=0)
@@ -566,6 +594,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
594 seq_lens.append(1)
595 block_tables.append([])
596 lora_index_mapping.append(0)
597 +
598 + if self.attn_backend.get_name() == "flashinfer":
599 + last_paged_kv_indptr = paged_kv_indptr[-1]
600 + paged_kv_indptr.append(last_paged_kv_indptr)
601 + paged_kv_last_page_len.append(0)
602 +
603 batch_size = graph_batch_size
604 num_decode_tokens = batch_size
605
@@ -589,9 +623,19 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
623 )
624 assert max_query_len > 0, ("query_lens: {}".format(query_lens))
625
626 + context_lens_tensor = torch.tensor(context_lens,
627 + dtype=torch.int,
628 + device=self.device)
629 +
630 seq_lens_tensor = torch.tensor(seq_lens,
631 dtype=torch.int,
632 device=self.device)
633 + query_lens_tensor = torch.tensor(query_lens,
634 + dtype=torch.long,
635 + device=self.device)
636 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
637 + dtype=torch.int32,
638 + device=self.device)
639 seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
640 dtype=torch.int32,
641 device=self.device)
@@ -600,6 +644,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
644 dim=0,
645 dtype=seq_start_loc.dtype,
646 out=seq_start_loc[1:])
647 + torch.cumsum(query_lens_tensor,
648 + dim=0,
649 + dtype=query_start_loc.dtype,
650 + out=query_start_loc[1:])
651
652 input_tokens_tensor = torch.tensor(input_tokens,
653 dtype=torch.long,
@@ -612,30 +660,30 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
660 device=self.device)
661
662 if self.attn_backend.get_name() == "flashinfer":
615 - if not hasattr(self, "flashinfer_workspace_buffer"):
616 - # Allocate 16MB workspace buffer
617 - # Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html
618 - self.flashinfer_workspace_buffer = torch.empty(
619 - 16 * 1024 * 1024, dtype=torch.uint8, device=self.device)
620 - paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
621 - dtype=torch.int,
622 - device=self.device)
623 - paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
624 - dtype=torch.int,
625 - device=self.device)
626 - paged_kv_last_page_len_tensor = torch.tensor(
627 - paged_kv_last_page_len, dtype=torch.int, device=self.device)
663 + if len(paged_kv_indptr) > 0:
664 + paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
665 + device='cpu',
666 + dtype=torch.int)
667 + paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
668 + device='cpu',
669 + dtype=torch.int)
670 + paged_kv_last_page_len_tensor = torch.tensor(
671 + paged_kv_last_page_len, device='cpu', dtype=torch.int)
672 + else:
673 + paged_kv_indices_tensor = None
674 + paged_kv_indptr_tensor = None
675 + paged_kv_last_page_len_tensor = None
676 +
677 kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
678 self.model_config.dtype)
679 +
680 attn_metadata = self.attn_backend.make_metadata(
681 num_prefills=num_prefills,
682 slot_mapping=slot_mapping_tensor,
683 num_prefill_tokens=num_prefill_tokens,
684 num_decode_tokens=num_decode_tokens,
635 - use_cuda_graph=False,
685 max_prefill_seq_len=max_prefill_seq_len,
686 block_tables=block_tables,
638 - workspace_buffer=self.flashinfer_workspace_buffer,
687 paged_kv_indptr=paged_kv_indptr_tensor,
688 paged_kv_indices=paged_kv_indices_tensor,
689 paged_kv_last_page_len=paged_kv_last_page_len_tensor,
@@ -644,25 +692,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
692 num_kv_heads=self.model_config.get_num_kv_heads(
693 self.parallel_config),
694 head_dim=self.model_config.get_head_size(),
647 - page_size=16,
695 + page_size=self.block_size,
696 seq_start_loc=seq_start_loc,
649 - data_type=kv_cache_dtype)
650 - else:
651 - context_lens_tensor = torch.tensor(context_lens,
652 - dtype=torch.int,
653 - device=self.device)
654 - query_lens_tensor = torch.tensor(query_lens,
655 - dtype=torch.long,
656 - device=self.device)
657 - query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
658 - dtype=torch.int32,
659 - device=self.device)
660 -
661 - torch.cumsum(query_lens_tensor,
662 - dim=0,
663 - dtype=query_start_loc.dtype,
664 - out=query_start_loc[1:])
697 + query_start_loc=query_start_loc,
698 + device=self.device,
699 + data_type=kv_cache_dtype,
700 + use_cuda_graph=use_captured_graph)
701
702 + else:
703 attn_metadata = self.attn_backend.make_metadata(
704 num_prefills=num_prefills,
705 slot_mapping=slot_mapping_tensor,
@@ -854,27 +891,97 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
891 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
892 ]
893
894 + if self.attn_backend.get_name() == "flashinfer":
895 + # For flashinfer, different batch sizes will share the
896 + # same workspace buffer.
897 + decode_workspace_buffer = \
898 + torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE,
899 + dtype=torch.uint8,
900 + device=self.device)
901 + indices_buffer = torch.empty(max_batch_size *
902 + self.cache_config.num_gpu_blocks,
903 + dtype=torch.int32,
904 + device=self.device)
905 + indptr_buffer = torch.empty(max_batch_size + 1,
906 + dtype=torch.int32,
907 + device=self.device)
908 + last_page_len_buffer = torch.empty(max_batch_size,
909 + dtype=torch.int32,
910 + device=self.device)
911 +
912 with graph_capture() as graph_capture_context:
913 # NOTE: Capturing the largest batch size first may help reduce the
914 # memory usage of CUDA graph.
915 for batch_size in reversed(batch_size_capture_list):
861 - # Create dummy attn_metadata.
862 - attn_metadata = self.attn_backend.make_metadata(
863 - num_prefills=0,
864 - num_prefill_tokens=0,
865 - num_decode_tokens=batch_size,
866 - slot_mapping=slot_mapping[:batch_size],
867 - seq_lens=None,
868 - seq_lens_tensor=seq_lens[:batch_size],
869 - max_query_len=None,
870 - max_prefill_seq_len=0,
871 - max_decode_seq_len=self.max_seq_len_to_capture,
872 - query_start_loc=None,
873 - seq_start_loc=None,
874 - context_lens_tensor=None,
875 - block_tables=block_tables[:batch_size],
876 - use_cuda_graph=True,
877 - )
916 + if self.attn_backend.get_name() == "flashinfer":
917 + indptr_buffer = indptr_buffer[:batch_size + 1]
918 + last_page_len_buffer = last_page_len_buffer[:batch_size]
919 +
920 + num_qo_heads = self.model_config.get_num_attention_heads(
921 + self.parallel_config)
922 + num_kv_heads = self.model_config.get_num_kv_heads(
923 + self.parallel_config)
924 + if num_qo_heads // num_kv_heads >= 4:
925 + use_tensor_cores = True
926 + else:
927 + use_tensor_cores = False
928 + decode_wrapper = \
929 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
930 + decode_workspace_buffer, indptr_buffer, indices_buffer,
931 + last_page_len_buffer, "NHD", use_tensor_cores)
932 + kv_cache_dtype = get_kv_cache_torch_dtype(
933 + self.kv_cache_dtype, self.model_config.dtype)
934 +
935 + paged_kv_indptr_tensor_host = torch.arange(
936 + 0, batch_size + 1, dtype=torch.int32)
937 + paged_kv_indices_tensor_host = torch.arange(
938 + 0, batch_size, dtype=torch.int32)
939 + paged_kv_last_page_len_tensor_host = torch.full(
940 + (batch_size, ), self.block_size, dtype=torch.int32)
941 + query_start_loc_host = torch.arange(0,
942 + batch_size + 1,
943 + dtype=torch.int32)
944 +
945 + attn_metadata = self.attn_backend.make_metadata(
946 + num_prefills=0,
947 + slot_mapping=slot_mapping[:batch_size],
948 + num_prefill_tokens=0,
949 + num_decode_tokens=batch_size,
950 + max_prefill_seq_len=0,
951 + block_tables=block_tables,
952 + paged_kv_indptr=paged_kv_indptr_tensor_host,
953 + paged_kv_indices=paged_kv_indices_tensor_host,
954 + paged_kv_last_page_len=
955 + paged_kv_last_page_len_tensor_host,
956 + num_qo_heads=num_qo_heads,
957 + num_kv_heads=num_kv_heads,
958 + head_dim=self.model_config.get_head_size(),
959 + page_size=self.block_size,
960 + seq_start_loc=None,
961 + query_start_loc=query_start_loc_host,
962 + device=self.device,
963 + data_type=kv_cache_dtype,
964 + use_cuda_graph=True,
965 + decode_wrapper=decode_wrapper,
966 + prefill_wrapper=None)
967 + attn_metadata.begin_forward()
968 + else:
969 + attn_metadata = self.attn_backend.make_metadata(
970 + num_prefills=0,
971 + num_prefill_tokens=0,
972 + num_decode_tokens=batch_size,
973 + slot_mapping=slot_mapping[:batch_size],
974 + seq_lens=None,
975 + seq_lens_tensor=seq_lens[:batch_size],
976 + max_query_len=None,
977 + max_prefill_seq_len=0,
978 + max_decode_seq_len=self.max_seq_len_to_capture,
979 + query_start_loc=None,
980 + seq_start_loc=None,
981 + context_lens_tensor=None,
982 + block_tables=block_tables[:batch_size],
983 + use_cuda_graph=True,
984 + )
985
986 if self.lora_config:
987 lora_mapping = LoRAMapping(
@@ -883,8 +990,20 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
990 )
991 self.set_active_loras(set(), lora_mapping)
992
886 - graph_runner = CUDAGraphRunner(self.model)
887 - hidden_states = graph_runner.capture(
993 + graph_runner = CUDAGraphRunner(self.model,
994 + self.attn_backend.get_name())
995 +
996 + if self.attn_backend.get_name() == "flashinfer":
997 + graph_runner.flashinfer_indptr_buffer = indptr_buffer
998 + graph_runner.flashinfer_indices_buffer = indices_buffer
999 + graph_runner.flashinfer_last_page_len_buffer = \
1000 + last_page_len_buffer
1001 + graph_runner.flashinfer_decode_workspace_buffer = \
1002 + decode_workspace_buffer
1003 + graph_runner.flashinfer_decode_wrapper = \
1004 + decode_wrapper
1005 +
1006 + graph_runner.capture(
1007 input_tokens[:batch_size],
1008 input_positions[:batch_size],
1009 hidden_states[:batch_size]
@@ -918,11 +1037,12 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1037 self,
1038 tensor_dict: Dict[str, Any],
1039 ) -> ModelInputForGPUWithSamplingMetadata:
921 - return (
1040 + model_input = \
1041 ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
1042 tensor_dict,
1043 attn_backend=self.attn_backend,
925 - ))
1044 + )
1045 + return model_input
1046
1047 def prepare_model_input(
1048 self,
@@ -970,6 +1090,36 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1090 self.set_active_loras(model_input.lora_requests,
1091 model_input.lora_mapping)
1092
1093 + if self.attn_backend.get_name() == "flashinfer":
1094 + assert model_input.attn_metadata is not None
1095 + assert model_input.input_tokens is not None
1096 + if self.flashinfer_decode_workspace_buffer is None:
1097 + self.flashinfer_decode_workspace_buffer = torch.empty(
1098 + FLASHINFER_WORKSPACE_BUFFER_SIZE,
1099 + dtype=torch.uint8,
1100 + device=self.device)
1101 + self.flashinfer_decode_wrapper = \
1102 + BatchDecodeWithPagedKVCacheWrapper(
1103 + self.flashinfer_decode_workspace_buffer, "NHD")
1104 + self.flashinfer_prefill_workspace_buffer = torch.empty(
1105 + FLASHINFER_WORKSPACE_BUFFER_SIZE,
1106 + dtype=torch.uint8,
1107 + device=self.device)
1108 + self.flashinfer_prefill_wrapper = \
1109 + BatchPrefillWithPagedKVCacheWrapper(
1110 + self.flashinfer_prefill_workspace_buffer, "NHD")
1111 +
1112 + model_input.attn_metadata.prefill_wrapper = \
1113 + self.flashinfer_prefill_wrapper
1114 + if model_input.attn_metadata.use_cuda_graph:
1115 + batch_size = model_input.input_tokens.shape[0]
1116 + model_input.attn_metadata.decode_wrapper = self.graph_runners[
1117 + batch_size].flashinfer_decode_wrapper
1118 + else:
1119 + model_input.attn_metadata.decode_wrapper = \
1120 + self.flashinfer_decode_wrapper
1121 + model_input.attn_metadata.begin_forward()
1122 +
1123 # Currently cuda graph is only supported by the decode phase.
1124 assert model_input.attn_metadata is not None
1125 prefill_meta = model_input.attn_metadata.prefill_metadata
@@ -1020,13 +1170,22 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1170
1171 class CUDAGraphRunner:
1172
1023 - def __init__(self, model: nn.Module):
1173 + def __init__(self, model: nn.Module, backend_name: str):
1174 self.model = model
1175 + self.backend_name = backend_name
1176 +
1177 self.input_buffers: Dict[str, torch.Tensor] = {}
1178 self.output_buffers: Dict[str, torch.Tensor] = {}
1179
1180 self._graph: Optional[torch.cuda.CUDAGraph] = None
1181
1182 + self.flashinfer_decode_workspace_buffer: Optional[torch.Tensor] = None
1183 + self.flashinfer_indptr_buffer: Optional[torch.Tensor] = None
1184 + self.flashinfer_indices_buffer: Optional[torch.Tensor] = None
1185 + self.flashinfer_last_page_len_buffer: Optional[torch.Tensor] = None
1186 + self.flashinfer_decode_wrapper: Optional[
1187 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper] = None
1188 +
1189 @property
1190 def graph(self):
1191 assert self._graph is not None
@@ -1079,14 +1238,23 @@ class CUDAGraphRunner:
1238 torch.cuda.synchronize()
1239
1240 # Save the input and output buffers.
1082 - self.input_buffers = {
1083 - "input_ids": input_ids,
1084 - "positions": positions,
1085 - "kv_caches": kv_caches,
1086 - "slot_mapping": attn_metadata.slot_mapping,
1087 - "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
1088 - "block_tables": attn_metadata.decode_metadata.block_tables,
1089 - }
1241 + if self.backend_name == "flashinfer":
1242 + self.input_buffers = {
1243 + "input_ids": input_ids,
1244 + "positions": positions,
1245 + "kv_caches": kv_caches,
1246 + "slot_mapping": attn_metadata.slot_mapping,
1247 + }
1248 + else:
1249 + self.input_buffers = {
1250 + "input_ids": input_ids,
1251 + "positions": positions,
1252 + "kv_caches": kv_caches,
1253 + "slot_mapping": attn_metadata.slot_mapping,
1254 + "seq_lens_tensor":
1255 + attn_metadata.decode_metadata.seq_lens_tensor,
1256 + "block_tables": attn_metadata.decode_metadata.block_tables,
1257 + }
1258 self.output_buffers = {"hidden_states": hidden_states}
1259 return hidden_states
1260
@@ -1106,10 +1274,12 @@ class CUDAGraphRunner:
1274 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1275 self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1276 non_blocking=True)
1109 - self.input_buffers["seq_lens_tensor"].copy_(
1110 - attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
1111 - self.input_buffers["block_tables"].copy_(
1112 - attn_metadata.decode_metadata.block_tables, non_blocking=True)
1277 + if self.backend_name != "flashinfer":
1278 + self.input_buffers["seq_lens_tensor"].copy_(
1279 + attn_metadata.decode_metadata.seq_lens_tensor,
1280 + non_blocking=True)
1281 + self.input_buffers["block_tables"].copy_(
1282 + attn_metadata.decode_metadata.block_tables, non_blocking=True)
1283 # Run the graph.
1284 self.graph.replay()
1285