@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b682179dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Add `AttentionState` abstraction (#7663)

Antoni Baum committed Aug 20, 2024 at 11:50 UTC 3b682179dd4844fa346ef5e77706880c4d0b09da
16 files changed +372 -247
tests/worker/test_model_input.py
+6 -1
@@ -5,6 +5,7 @@ import torch
5
6 from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7 from vllm.attention.backends.abstract import AttentionBackend
8 +from vllm.attention.backends.utils import CommonAttentionState
9 from vllm.model_executor import SamplingMetadata
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.worker.embedding_model_runner import (
@@ -29,7 +30,11 @@ class MockAttentionBackend(AttentionBackend):
30
31 @staticmethod
32 def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
32 - raise AttentionMetadataBuilder
33 + return AttentionMetadataBuilder
34 +
35 + @staticmethod
36 + def get_state_cls() -> Type["CommonAttentionState"]:
37 + return CommonAttentionState
38
39 @staticmethod
40 def get_kv_cache_shape(
vllm/attention/__init__.py
+2 -1
@@ -1,7 +1,7 @@
1 from vllm.attention.backends.abstract import (AttentionBackend,
2 AttentionMetadata,
3 AttentionMetadataBuilder,
4 - AttentionType)
4 + AttentionState, AttentionType)
5 from vllm.attention.layer import Attention
6 from vllm.attention.selector import get_attn_backend
7
@@ -12,5 +12,6 @@ __all__ = [
12 "AttentionType",
13 "AttentionMetadataBuilder",
14 "Attention",
15 + "AttentionState",
16 "get_attn_backend",
17 ]
vllm/attention/backends/abstract.py
+50 -1
@@ -1,4 +1,5 @@
1 from abc import ABC, abstractmethod
2 +from contextlib import contextmanager
3 from dataclasses import dataclass, fields
4 from enum import Enum, auto
5 from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
@@ -7,7 +8,9 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
8 import torch
9
10 if TYPE_CHECKING:
10 - from vllm.worker.model_runner_base import ModelRunnerInputBuilderBase
11 + from vllm.worker.model_runner_base import (ModelRunnerBase,
12 + ModelRunnerInputBase,
13 + ModelRunnerInputBuilderBase)
14
15
16 class AttentionType(Enum):
@@ -34,6 +37,11 @@ class AttentionBackend(ABC):
37 def get_metadata_cls() -> Type["AttentionMetadata"]:
38 raise NotImplementedError
39
40 + @staticmethod
41 + @abstractmethod
42 + def get_state_cls() -> Type["AttentionState"]:
43 + raise NotImplementedError
44 +
45 @classmethod
46 def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":
47 return cls.get_metadata_cls()(*args, **kwargs)
@@ -126,6 +134,47 @@ class AttentionMetadata:
134 T = TypeVar("T", bound=AttentionMetadata)
135
136
137 +class AttentionState(ABC, Generic[T]):
138 + """Holds attention backend-specific objects reused during the
139 + lifetime of the model runner."""
140 +
141 + @abstractmethod
142 + def __init__(self, runner: "ModelRunnerBase"):
143 + ...
144 +
145 + @abstractmethod
146 + @contextmanager
147 + def graph_capture(self, max_batch_size: int):
148 + """Context manager used when capturing CUDA graphs."""
149 + yield
150 +
151 + @abstractmethod
152 + def graph_clone(self, batch_size: int) -> "AttentionState[T]":
153 + """Clone attention state to save in CUDA graph metadata."""
154 + ...
155 +
156 + @abstractmethod
157 + def graph_capture_get_metadata_for_batch(self, batch_size: int) -> T:
158 + """Get attention metadata for CUDA graph capture of batch_size."""
159 + ...
160 +
161 + @abstractmethod
162 + def get_graph_input_buffers(self, attn_metadata: T) -> Dict[str, Any]:
163 + """Get attention-specific input buffers for CUDA graph capture."""
164 + ...
165 +
166 + @abstractmethod
167 + def prepare_graph_input_buffers(self, input_buffers: Dict[str, Any],
168 + attn_metadata: T) -> None:
169 + """In-place modify input buffers dict for CUDA graph replay."""
170 + ...
171 +
172 + @abstractmethod
173 + def begin_forward(self, model_input: "ModelRunnerInputBase") -> None:
174 + """Prepare state for forward pass."""
175 + ...
176 +
177 +
178 class AttentionMetadataBuilder(ABC, Generic[T]):
179 """Abstract class for attention metadata builders."""
180
vllm/attention/backends/blocksparse_attn.py
+6 -1
@@ -5,7 +5,8 @@ import torch
5
6 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7 AttentionMetadata, AttentionType)
8 -from vllm.attention.backends.utils import CommonMetadataBuilder
8 +from vllm.attention.backends.utils import (CommonAttentionState,
9 + CommonMetadataBuilder)
10 from vllm.attention.ops.blocksparse_attention.interface import (
11 LocalStridedBlockSparseAttn, get_head_sliding_step)
12 from vllm.attention.ops.paged_attn import PagedAttention
@@ -98,6 +99,10 @@ class BlocksparseFlashAttentionBackend(AttentionBackend):
99 def get_builder_cls() -> Type["BlocksparseFlashAttentionMetadataBuilder"]:
100 return BlocksparseFlashAttentionMetadataBuilder
101
102 + @staticmethod
103 + def get_state_cls() -> Type["CommonAttentionState"]:
104 + return CommonAttentionState
105 +
106 @staticmethod
107 def get_kv_cache_shape(
108 num_blocks: int,
vllm/attention/backends/flash_attn.py
+6 -1
@@ -9,7 +9,8 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 AttentionMetadata,
10 AttentionMetadataBuilder,
11 AttentionType)
12 -from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
12 +from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
13 + compute_slot_mapping,
14 compute_slot_mapping_start_idx,
15 is_block_tables_empty)
16 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
@@ -142,6 +143,10 @@ class FlashAttentionBackend(AttentionBackend):
143 def get_builder_cls() -> Type["FlashAttentionMetadataBuilder"]:
144 return FlashAttentionMetadataBuilder
145
146 + @staticmethod
147 + def get_state_cls() -> Type["CommonAttentionState"]:
148 + return CommonAttentionState
149 +
150 @staticmethod
151 def get_kv_cache_shape(
152 num_blocks: int,
vllm/attention/backends/flashinfer.py
+164 -1
@@ -1,14 +1,19 @@
1 +from contextlib import contextmanager
2 from dataclasses import dataclass
3 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type
4
5 try:
6 from flashinfer import BatchDecodeWithPagedKVCacheWrapper
7 + from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
8 from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
9
10 import vllm.attention.backends.flash_attn # noqa
11 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
12 except ImportError:
13 BatchDecodeWithPagedKVCacheWrapper = None
14 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
15 BatchPrefillWithPagedKVCacheWrapper = None
16 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
17
18 import torch
19
@@ -16,7 +21,7 @@ from vllm import _custom_ops as ops
21 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
22 AttentionMetadata,
23 AttentionMetadataBuilder,
19 - AttentionType)
24 + AttentionState, AttentionType)
25 from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
26 compute_slot_mapping_start_idx,
27 is_block_tables_empty)
@@ -46,6 +51,10 @@ class FlashInferBackend(AttentionBackend):
51 def get_builder_cls() -> Type["FlashInferMetadataBuilder"]:
52 return FlashInferMetadataBuilder
53
54 + @staticmethod
55 + def get_state_cls() -> Type["FlashInferState"]:
56 + return FlashInferState
57 +
58 @staticmethod
59 def get_kv_cache_shape(
60 num_blocks: int,
@@ -75,6 +84,160 @@ class FlashInferBackend(AttentionBackend):
84 return [64, 128, 256]
85
86
87 +class FlashInferState(AttentionState):
88 +
89 + def __init__(self, runner):
90 + self.runner = runner
91 + self._is_graph_capturing = False
92 + self._workspace_buffer = None
93 + self._decode_wrapper = None
94 + self._prefill_wrapper = None
95 +
96 + def _get_workspace_buffer(self):
97 + if self._workspace_buffer is None:
98 + self._workspace_buffer = torch.empty(
99 + FLASHINFER_WORKSPACE_BUFFER_SIZE,
100 + dtype=torch.uint8,
101 + device=self.runner.device)
102 + return self._workspace_buffer
103 +
104 + def _get_prefill_wrapper(self):
105 + if self._prefill_wrapper is None:
106 + self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
107 + self._get_workspace_buffer(), "NHD")
108 + return self._prefill_wrapper
109 +
110 + def _get_decode_wrapper(self):
111 + if self._decode_wrapper is None:
112 + num_qo_heads = (self.runner.model_config.get_num_attention_heads(
113 + self.runner.parallel_config))
114 + num_kv_heads = self.runner.model_config.get_num_kv_heads(
115 + self.runner.parallel_config)
116 + use_tensor_cores = num_qo_heads // num_kv_heads >= 4
117 + self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
118 + self._get_workspace_buffer(),
119 + "NHD",
120 + use_tensor_cores=use_tensor_cores)
121 + return self._decode_wrapper
122 +
123 + @contextmanager
124 + def graph_capture(self, max_batch_size: int):
125 + self._is_graph_capturing = True
126 + self._graph_decode_wrapper = None
127 + self._graph_slot_mapping = torch.full((max_batch_size, ),
128 + PAD_SLOT_ID,
129 + dtype=torch.long,
130 + device=self.runner.device)
131 + self._graph_seq_lens = torch.ones(max_batch_size,
132 + dtype=torch.int32,
133 + device=self.runner.device)
134 + self._graph_block_tables = torch.from_numpy(
135 + self.runner.graph_block_tables).to(device=self.runner.device)
136 + self._graph_decode_workspace_buffer = self._get_workspace_buffer()
137 + self._graph_indices_buffer = torch.empty(
138 + max_batch_size * self.runner.cache_config.num_gpu_blocks,
139 + dtype=torch.int32,
140 + device=self.runner.device)
141 + self._graph_indptr_buffer = torch.empty(max_batch_size + 1,
142 + dtype=torch.int32,
143 + device=self.runner.device)
144 + self._graph_last_page_len_buffer = torch.empty(
145 + max_batch_size, dtype=torch.int32, device=self.runner.device)
146 + yield
147 + self._is_graph_capturing = False
148 + del self._graph_slot_mapping
149 + del self._graph_seq_lens
150 + del self._graph_block_tables
151 + del self._graph_decode_workspace_buffer
152 + del self._graph_indices_buffer
153 + del self._graph_indptr_buffer
154 + del self._graph_last_page_len_buffer
155 + del self._graph_decode_wrapper
156 +
157 + def graph_clone(self, batch_size: int):
158 + assert self._is_graph_capturing
159 + state = self.__class__(self.runner)
160 + state._workspace_buffer = self._graph_decode_workspace_buffer
161 + state._decode_wrapper = self._graph_decode_wrapper
162 + state._prefill_wrapper = self._get_prefill_wrapper()
163 + return state
164 +
165 + def graph_capture_get_metadata_for_batch(self, batch_size: int):
166 + assert self._is_graph_capturing
167 + _indptr_buffer = self._graph_indptr_buffer[:batch_size + 1]
168 + _last_page_len_buffer = self._graph_last_page_len_buffer[:batch_size]
169 +
170 + num_qo_heads = (self.runner.model_config.get_num_attention_heads(
171 + self.runner.parallel_config))
172 + num_kv_heads = self.runner.model_config.get_num_kv_heads(
173 + self.runner.parallel_config)
174 + use_tensor_cores = num_qo_heads // num_kv_heads >= 4
175 + self._graph_decode_wrapper = \
176 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
177 + self._graph_decode_workspace_buffer, _indptr_buffer,
178 + self._graph_indices_buffer, _last_page_len_buffer, "NHD",
179 + use_tensor_cores)
180 + kv_cache_dtype = get_kv_cache_torch_dtype(
181 + self.runner.kv_cache_dtype, self.runner.model_config.dtype)
182 +
183 + paged_kv_indptr_tensor_host = torch.arange(0,
184 + batch_size + 1,
185 + dtype=torch.int32)
186 + paged_kv_indices_tensor_host = torch.arange(0,
187 + batch_size,
188 + dtype=torch.int32)
189 + paged_kv_last_page_len_tensor_host = torch.full((batch_size, ),
190 + self.runner.block_size,
191 + dtype=torch.int32)
192 + query_start_loc_host = torch.arange(0,
193 + batch_size + 1,
194 + dtype=torch.int32)
195 +
196 + attn_metadata = self.runner.attn_backend.make_metadata(
197 + num_prefills=0,
198 + slot_mapping=self._graph_slot_mapping[:batch_size],
199 + num_prefill_tokens=0,
200 + num_decode_tokens=batch_size,
201 + max_prefill_seq_len=0,
202 + block_tables=self._graph_block_tables,
203 + paged_kv_indptr=paged_kv_indptr_tensor_host,
204 + paged_kv_indices=paged_kv_indices_tensor_host,
205 + paged_kv_last_page_len=paged_kv_last_page_len_tensor_host,
206 + num_qo_heads=num_qo_heads,
207 + num_kv_heads=num_kv_heads,
208 + head_dim=self.runner.model_config.get_head_size(),
209 + page_size=self.runner.block_size,
210 + seq_start_loc=None,
211 + query_start_loc=query_start_loc_host,
212 + device=self.runner.device,
213 + data_type=kv_cache_dtype,
214 + use_cuda_graph=True,
215 + decode_wrapper=self._graph_decode_wrapper,
216 + prefill_wrapper=None)
217 + attn_metadata.begin_forward()
218 + return attn_metadata
219 +
220 + def get_graph_input_buffers(self, attn_metadata):
221 + return {
222 + "slot_mapping": attn_metadata.slot_mapping,
223 + }
224 +
225 + def prepare_graph_input_buffers(self, input_buffers, attn_metadata):
226 + return
227 +
228 + def begin_forward(self, model_input):
229 + assert not self._is_graph_capturing
230 + state = self
231 + if model_input.attn_metadata.use_cuda_graph:
232 + batch_size = model_input.input_tokens.shape[0]
233 + state = (self.runner.graph_runners[model_input.virtual_engine]
234 + [batch_size].attn_state)
235 + model_input.attn_metadata.prefill_wrapper = state._get_prefill_wrapper(
236 + )
237 + model_input.attn_metadata.decode_wrapper = state._get_decode_wrapper()
238 + model_input.attn_metadata.begin_forward()
239 +
240 +
241 @dataclass
242 class FlashInferMetadata(AttentionMetadata):
243 # Maximum sequence length among prefill batch. 0 if there are decoding
vllm/attention/backends/ipex_attn.py
+5
@@ -8,6 +8,7 @@ import torch
8 from vllm._ipex_ops import ipex_ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata, AttentionType)
11 +from vllm.attention.backends.utils import CommonAttentionState
12 from vllm.attention.ops.paged_attn import (PagedAttention,
13 PagedAttentionMetadata)
14
@@ -28,6 +29,10 @@ class IpexAttnBackend(AttentionBackend):
29 def get_metadata_cls() -> Type["IpexAttnMetadata"]:
30 return IpexAttnMetadata
31
32 + @staticmethod
33 + def get_state_cls() -> Type["CommonAttentionState"]:
34 + return CommonAttentionState
35 +
36 @staticmethod
37 def get_kv_cache_shape(
38 num_blocks: int,
vllm/attention/backends/openvino.py
+6 -1
@@ -1,11 +1,12 @@
1 from dataclasses import dataclass
2 -from typing import List, Tuple
2 +from typing import List, Tuple, Type
3
4 import openvino as ov
5 import torch
6
7 from vllm.attention.backends.abstract import (AttentionBackend,
8 AttentionMetadata)
9 +from vllm.attention.backends.utils import CommonAttentionState
10
11
12 class OpenVINOAttentionBackend(AttentionBackend):
@@ -24,6 +25,10 @@ class OpenVINOAttentionBackend(AttentionBackend):
25 def make_metadata(*args, **kwargs) -> "AttentionMetadata":
26 raise NotImplementedError
27
28 + @staticmethod
29 + def get_state_cls() -> Type["CommonAttentionState"]:
30 + return CommonAttentionState
31 +
32 @staticmethod
33 def make_openvino_metadata(*args, **kwargs) -> "OpenVINOAttentionMetadata":
34 return OpenVINOAttentionMetadata(*args, **kwargs)
vllm/attention/backends/pallas.py
+5
@@ -6,6 +6,7 @@ import torch_xla.experimental.custom_kernel # Required to register custom ops.
6
7 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 AttentionMetadata, AttentionType)
9 +from vllm.attention.backends.utils import CommonAttentionState
10
11
12 class PallasAttentionBackend(AttentionBackend):
@@ -18,6 +19,10 @@ class PallasAttentionBackend(AttentionBackend):
19 def get_metadata_cls() -> Type["PallasMetadata"]:
20 return PallasMetadata
21
22 + @staticmethod
23 + def get_state_cls() -> Type["CommonAttentionState"]:
24 + return CommonAttentionState
25 +
26 @staticmethod
27 def get_kv_cache_shape(
28 num_blocks: int,
vllm/attention/backends/rocm_flash_attn.py
+6 -1
@@ -7,7 +7,8 @@ import torch
7 import vllm.envs as envs
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 AttentionMetadata, AttentionType)
10 -from vllm.attention.backends.utils import CommonMetadataBuilder
10 +from vllm.attention.backends.utils import (CommonAttentionState,
11 + CommonMetadataBuilder)
12 from vllm.attention.ops.paged_attn import (PagedAttention,
13 PagedAttentionMetadata)
14 from vllm.logger import init_logger
@@ -33,6 +34,10 @@ class ROCmFlashAttentionBackend(AttentionBackend):
34 def get_builder_cls() -> Type["ROCmFlashAttentionMetadataBuilder"]:
35 return ROCmFlashAttentionMetadataBuilder
36
37 + @staticmethod
38 + def get_state_cls() -> Type["CommonAttentionState"]:
39 + return CommonAttentionState
40 +
41 @staticmethod
42 def get_kv_cache_shape(
43 num_blocks: int,
vllm/attention/backends/torch_sdpa.py
+5
@@ -8,6 +8,7 @@ from torch.nn.functional import scaled_dot_product_attention
8
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata, AttentionType)
11 +from vllm.attention.backends.utils import CommonAttentionState
12 from vllm.attention.ops.paged_attn import PagedAttentionMetadata
13 from vllm.utils import is_cpu
14
@@ -34,6 +35,10 @@ class TorchSDPABackend(AttentionBackend):
35 def get_metadata_cls() -> Type["AttentionMetadata"]:
36 return TorchSDPAMetadata
37
38 + @staticmethod
39 + def get_state_cls() -> Type["CommonAttentionState"]:
40 + return CommonAttentionState
41 +
42 @staticmethod
43 def get_kv_cache_shape(
44 num_blocks: int,
vllm/attention/backends/utils.py
+73 -2
@@ -1,12 +1,17 @@
1 """Attention backend utils"""
2 -from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
2 +from contextlib import contextmanager
3 +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
4
5 import numpy as np
6 import torch
7
7 -from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
8 +from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
9 + AttentionState)
10 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
11
12 +if TYPE_CHECKING:
13 + from vllm.worker.model_runner_base import ModelRunnerBase
14 +
15 # Error string(s) for encoder/decoder
16 # unsupported attention scenarios
17 STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
@@ -269,3 +274,69 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
274 block_tables=block_tables,
275 use_cuda_graph=use_captured_graph,
276 )
277 +
278 +
279 +class CommonAttentionState(AttentionState):
280 +
281 + def __init__(self, runner: "ModelRunnerBase"):
282 + self.runner = runner
283 + self._is_graph_capturing = False
284 +
285 + @contextmanager
286 + def graph_capture(self, max_batch_size: int):
287 + self._is_graph_capturing = True
288 + self._graph_slot_mapping = torch.full((max_batch_size, ),
289 + PAD_SLOT_ID,
290 + dtype=torch.long,
291 + device=self.runner.device)
292 + self._graph_seq_lens = torch.ones(max_batch_size,
293 + dtype=torch.int32,
294 + device=self.runner.device)
295 + self._graph_block_tables = torch.from_numpy(
296 + self.runner.graph_block_tables).to(device=self.runner.device)
297 + yield
298 + self._is_graph_capturing = False
299 + del self._graph_slot_mapping
300 + del self._graph_seq_lens
301 + del self._graph_block_tables
302 +
303 + def graph_clone(self, batch_size: int) -> "CommonAttentionState":
304 + assert self._is_graph_capturing
305 + return self.__class__(self.runner)
306 +
307 + def graph_capture_get_metadata_for_batch(self, batch_size: int):
308 + assert self._is_graph_capturing
309 + attn_metadata = self.runner.attn_backend.make_metadata(
310 + num_prefills=0,
311 + num_prefill_tokens=0,
312 + num_decode_tokens=batch_size,
313 + slot_mapping=self._graph_slot_mapping[:batch_size],
314 + seq_lens=None,
315 + seq_lens_tensor=self._graph_seq_lens[:batch_size],
316 + max_query_len=None,
317 + max_prefill_seq_len=0,
318 + max_decode_seq_len=self.runner.max_seq_len_to_capture,
319 + query_start_loc=None,
320 + seq_start_loc=None,
321 + context_lens_tensor=None,
322 + block_tables=self._graph_block_tables[:batch_size],
323 + use_cuda_graph=True,
324 + )
325 + return attn_metadata
326 +
327 + def get_graph_input_buffers(self, attn_metadata) -> Dict[str, Any]:
328 + return {
329 + "slot_mapping": attn_metadata.slot_mapping,
330 + "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
331 + "block_tables": attn_metadata.decode_metadata.block_tables,
332 + }
333 +
334 + def prepare_graph_input_buffers(self, input_buffers,
335 + attn_metadata) -> None:
336 + input_buffers["seq_lens_tensor"].copy_(
337 + attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
338 + input_buffers["block_tables"].copy_(
339 + attn_metadata.decode_metadata.block_tables, non_blocking=True)
340 +
341 + def begin_forward(self, model_input) -> None:
342 + return
vllm/attention/backends/xformers.py
+6 -1
@@ -11,7 +11,8 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 AttentionMetadata, AttentionType)
14 -from vllm.attention.backends.utils import CommonMetadataBuilder
14 +from vllm.attention.backends.utils import (CommonAttentionState,
15 + CommonMetadataBuilder)
16 from vllm.attention.ops.paged_attn import (PagedAttention,
17 PagedAttentionMetadata)
18 from vllm.logger import init_logger
@@ -37,6 +38,10 @@ class XFormersBackend(AttentionBackend):
38 def get_builder_cls() -> Type["XFormersMetadataBuilder"]:
39 return XFormersMetadataBuilder
40
41 + @staticmethod
42 + def get_state_cls() -> Type["CommonAttentionState"]:
43 + return CommonAttentionState
44 +
45 @staticmethod
46 def get_kv_cache_shape(
47 num_blocks: int,
vllm/spec_decode/draft_model_runner.py
+1 -46
@@ -11,17 +11,6 @@ except ModuleNotFoundError:
11 from vllm.attention.backends.rocm_flash_attn import (
12 ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13
14 -try:
15 - from flashinfer import BatchDecodeWithPagedKVCacheWrapper
16 - from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
17 - from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
18 - FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
19 -except ImportError:
20 - BatchDecodeWithPagedKVCacheWrapper = None
21 - CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
22 - BatchPrefillWithPagedKVCacheWrapper = None
23 - FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24 -
14 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 ModelConfig, ObservabilityConfig, ParallelConfig,
16 PromptAdapterConfig, SchedulerConfig)
@@ -90,11 +79,6 @@ class TP1DraftModelRunner(ModelRunner):
79 observability_config=observability_config,
80 )
81
93 - self.flashinfer_decode_workspace_buffer = None
94 - self.flashinfer_decode_wrapper = None
95 - self.flashinfer_prefill_workspace_buffer = None
96 - self.flashinfer_prefill_wrapper = None
97 -
82 def _update_sampling_metadata(self, sampling_metadata, num_seqs,
83 num_queries):
84
@@ -270,36 +254,7 @@ class TP1DraftModelRunner(ModelRunner):
254 model_input.prompt_adapter_requests,
255 model_input.prompt_adapter_mapping)
256
273 - if self.attn_backend.get_name() == "flashinfer":
274 - assert model_input.attn_metadata is not None
275 - assert model_input.input_tokens is not None
276 - if self.flashinfer_decode_workspace_buffer is None:
277 - self.flashinfer_decode_workspace_buffer = torch.empty(
278 - FLASHINFER_WORKSPACE_BUFFER_SIZE,
279 - dtype=torch.uint8,
280 - device=self.device)
281 - self.flashinfer_decode_wrapper = \
282 - BatchDecodeWithPagedKVCacheWrapper(
283 - self.flashinfer_decode_workspace_buffer, "NHD")
284 - self.flashinfer_prefill_workspace_buffer = torch.empty(
285 - FLASHINFER_WORKSPACE_BUFFER_SIZE,
286 - dtype=torch.uint8,
287 - device=self.device)
288 - self.flashinfer_prefill_wrapper = \
289 - BatchPrefillWithPagedKVCacheWrapper(
290 - self.flashinfer_prefill_workspace_buffer, "NHD")
291 -
292 - model_input.attn_metadata.prefill_wrapper = \
293 - self.flashinfer_prefill_wrapper
294 - if model_input.attn_metadata.use_cuda_graph:
295 - batch_size = model_input.input_tokens.shape[0]
296 - model_input.attn_metadata.decode_wrapper = \
297 - self.graph_runners[model_input.
298 - virtual_engine][batch_size].flashinfer_decode_wrapper
299 - else:
300 - model_input.attn_metadata.decode_wrapper = \
301 - self.flashinfer_decode_wrapper
302 - model_input.attn_metadata.begin_forward()
257 + self.attn_state.begin_forward(model_input)
258
259 # Detect exec mode
260 assert model_input.attn_metadata is not None
vllm/worker/enc_dec_model_runner.py
+3 -2
@@ -6,6 +6,7 @@ import torch.distributed
6
7 from vllm.attention.backends.abstract import (AttentionBackend,
8 AttentionMetadata)
9 +from vllm.attention.backends.utils import PAD_SLOT_ID
10 from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
11 get_global_forced_attn_backend,
12 global_force_attn_backend)
@@ -20,7 +21,7 @@ from vllm.sampling_params import SamplingParams
21 from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
22 SequenceGroupMetadata)
23 from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
23 -from vllm.worker.model_runner import (_PAD_SLOT_ID, GPUModelRunnerBase,
24 +from vllm.worker.model_runner import (GPUModelRunnerBase,
25 ModelInputForGPUBuilder,
26 ModelInputForGPUWithSamplingMetadata)
27 from vllm.worker.model_runner_base import (
@@ -395,7 +396,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
396 # initialized yet. In this case, we just use a dummy
397 # slot mapping.
398 # In embeddings, the block tables are {seq_id: None}.
398 - cross_slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
399 + cross_slot_mapping.extend([PAD_SLOT_ID] * seq_len)
400 else:
401 for i in range(0, seq_len):
402 block_number = seq_group_metadata.cross_block_table[
vllm/worker/model_runner.py
+28 -188
@@ -13,19 +13,10 @@ import torch
13 import torch.distributed
14 import torch.nn as nn
15
16 -try:
17 - from flashinfer import BatchDecodeWithPagedKVCacheWrapper
18 - from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
19 - from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
20 - FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
21 -except ImportError:
22 - BatchDecodeWithPagedKVCacheWrapper = None
23 - CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
24 - BatchPrefillWithPagedKVCacheWrapper = None
25 - FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
26 -
16 import vllm.envs as envs
17 from vllm.attention import AttentionMetadata, get_attn_backend
18 +from vllm.attention.backends.abstract import AttentionState
19 +from vllm.attention.backends.utils import CommonAttentionState
20 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
21 ModelConfig, ObservabilityConfig, ParallelConfig,
22 PromptAdapterConfig, SchedulerConfig)
@@ -52,8 +43,7 @@ from vllm.sampling_params import SamplingParams
43 from vllm.sequence import (IntermediateTensors, SamplerOutput,
44 SequenceGroupMetadata)
45 from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
55 - flatten_2d_lists, get_kv_cache_torch_dtype, is_hip,
56 - is_pin_memory_available)
46 + flatten_2d_lists, is_hip, is_pin_memory_available)
47 from vllm.worker.model_runner_base import (
48 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
49 _add_attn_metadata_broadcastable_dict,
@@ -66,7 +56,6 @@ if TYPE_CHECKING:
56
57 logger = init_logger(__name__)
58
69 -_PAD_SLOT_ID = -1
59 LORA_WARMUP_RANK = 8
60 _BATCH_SIZE_ALIGNMENT = 8
61 # Capture graphs for token size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
@@ -858,6 +847,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
847 self.kv_cache_dtype,
848 self.block_size,
849 ) if num_attn_heads else None
850 + if self.attn_backend:
851 + self.attn_state = self.attn_backend.get_state_cls()(
852 + weakref.proxy(self))
853 + else:
854 + self.attn_state = CommonAttentionState(weakref.proxy(self))
855
856 # Multi-modal data support
857 self.input_registry = input_registry
@@ -872,11 +866,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
866 self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
867 self.prompt_adapter_manager: LRUCacheWorkerPromptAdapterManager = None
868
875 - self.flashinfer_decode_workspace_buffer = None
876 - self.flashinfer_decode_wrapper = None
877 - self.flashinfer_prefill_workspace_buffer = None
878 - self.flashinfer_prefill_wrapper = None
879 -
869 set_cpu_offload_max_bytes(
870 int(self.cache_config.cpu_offload_gb * 1024**3))
871
@@ -1203,10 +1192,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1192 max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
1193 input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1194 input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1206 - slot_mapping = torch.empty(max_batch_size, dtype=torch.long).cuda()
1207 - slot_mapping.fill_(_PAD_SLOT_ID)
1208 - seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
1209 - block_tables = torch.from_numpy(self.graph_block_tables).cuda()
1195 intermediate_inputs = None
1196 if not get_pp_group().is_first_rank:
1197 intermediate_inputs = self.model.make_empty_intermediate_tensors(
@@ -1226,102 +1211,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1211 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1212 ]
1213
1229 - if self.attn_backend.get_name() == "flashinfer":
1230 - # For flashinfer, different batch sizes will share the
1231 - # same workspace buffer.
1232 - decode_workspace_buffer = \
1233 - torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE,
1234 - dtype=torch.uint8,
1235 - device=self.device)
1236 - indices_buffer = torch.empty(max_batch_size *
1237 - self.cache_config.num_gpu_blocks,
1238 - dtype=torch.int32,
1239 - device=self.device)
1240 - indptr_buffer = torch.empty(max_batch_size + 1,
1241 - dtype=torch.int32,
1242 - device=self.device)
1243 - last_page_len_buffer = torch.empty(max_batch_size,
1244 - dtype=torch.int32,
1245 - device=self.device)
1246 -
1247 - with graph_capture() as graph_capture_context:
1214 + with self.attn_state.graph_capture(
1215 + max_batch_size), graph_capture() as graph_capture_context:
1216 # NOTE: Capturing the largest batch size first may help reduce the
1217 # memory usage of CUDA graph.
1218 for virtual_engine in range(
1219 self.parallel_config.pipeline_parallel_size):
1220 for batch_size in reversed(batch_size_capture_list):
1253 - if self.attn_backend.get_name() == "flashinfer":
1254 - _indptr_buffer = indptr_buffer[:batch_size + 1]
1255 - _last_page_len_buffer = last_page_len_buffer[:
1256 - batch_size]
1257 -
1258 - num_qo_heads = (
1259 - self.model_config.get_num_attention_heads(
1260 - self.parallel_config))
1261 - num_kv_heads = self.model_config.get_num_kv_heads(
1262 - self.parallel_config)
1263 - if num_qo_heads // num_kv_heads >= 4:
1264 - use_tensor_cores = True
1265 - else:
1266 - use_tensor_cores = False
1267 - decode_wrapper = \
1268 - CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
1269 - decode_workspace_buffer, _indptr_buffer,
1270 - indices_buffer, _last_page_len_buffer, "NHD",
1271 - use_tensor_cores)
1272 - kv_cache_dtype = get_kv_cache_torch_dtype(
1273 - self.kv_cache_dtype, self.model_config.dtype)
1274 -
1275 - paged_kv_indptr_tensor_host = torch.arange(
1276 - 0, batch_size + 1, dtype=torch.int32)
1277 - paged_kv_indices_tensor_host = torch.arange(
1278 - 0, batch_size, dtype=torch.int32)
1279 - paged_kv_last_page_len_tensor_host = torch.full(
1280 - (batch_size, ), self.block_size, dtype=torch.int32)
1281 - query_start_loc_host = torch.arange(0,
1282 - batch_size + 1,
1283 - dtype=torch.int32)
1284 -
1285 - attn_metadata = self.attn_backend.make_metadata(
1286 - num_prefills=0,
1287 - slot_mapping=slot_mapping[:batch_size],
1288 - num_prefill_tokens=0,
1289 - num_decode_tokens=batch_size,
1290 - max_prefill_seq_len=0,
1291 - block_tables=block_tables,
1292 - paged_kv_indptr=paged_kv_indptr_tensor_host,
1293 - paged_kv_indices=paged_kv_indices_tensor_host,
1294 - paged_kv_last_page_len=
1295 - paged_kv_last_page_len_tensor_host,
1296 - num_qo_heads=num_qo_heads,
1297 - num_kv_heads=num_kv_heads,
1298 - head_dim=self.model_config.get_head_size(),
1299 - page_size=self.block_size,
1300 - seq_start_loc=None,
1301 - query_start_loc=query_start_loc_host,
1302 - device=self.device,
1303 - data_type=kv_cache_dtype,
1304 - use_cuda_graph=True,
1305 - decode_wrapper=decode_wrapper,
1306 - prefill_wrapper=None)
1307 - attn_metadata.begin_forward()
1308 - else:
1309 - attn_metadata = self.attn_backend.make_metadata(
1310 - num_prefills=0,
1311 - num_prefill_tokens=0,
1312 - num_decode_tokens=batch_size,
1313 - slot_mapping=slot_mapping[:batch_size],
1314 - seq_lens=None,
1315 - seq_lens_tensor=seq_lens[:batch_size],
1316 - max_query_len=None,
1317 - max_prefill_seq_len=0,
1318 - max_decode_seq_len=self.max_seq_len_to_capture,
1319 - query_start_loc=None,
1320 - seq_start_loc=None,
1321 - context_lens_tensor=None,
1322 - block_tables=block_tables[:batch_size],
1323 - use_cuda_graph=True,
1324 - )
1221 + attn_metadata = (
1222 + self.attn_state.graph_capture_get_metadata_for_batch(
1223 + batch_size))
1224
1225 if self.lora_config:
1226 lora_mapping = LoRAMapping(
@@ -1339,17 +1238,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1238 set(), prompt_adapter_mapping)
1239
1240 graph_runner = CUDAGraphRunner(
1342 - self.model, self.attn_backend.get_name())
1343 -
1344 - if self.attn_backend.get_name() == "flashinfer":
1345 - graph_runner.flashinfer_indptr_buffer = _indptr_buffer
1346 - graph_runner.flashinfer_indices_buffer = indices_buffer
1347 - graph_runner.flashinfer_last_page_len_buffer = \
1348 - _last_page_len_buffer
1349 - graph_runner.flashinfer_decode_workspace_buffer = \
1350 - decode_workspace_buffer
1351 - graph_runner.flashinfer_decode_wrapper = \
1352 - decode_wrapper
1241 + self.model, self.attn_backend.get_name(),
1242 + self.attn_state.graph_clone(batch_size))
1243
1244 capture_inputs = {
1245 "input_ids":
@@ -1476,36 +1366,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1366 model_input.prompt_adapter_requests,
1367 model_input.prompt_adapter_mapping)
1368
1479 - if self.attn_backend.get_name() == "flashinfer":
1480 - assert model_input.attn_metadata is not None
1481 - assert model_input.input_tokens is not None
1482 - if self.flashinfer_decode_workspace_buffer is None:
1483 - self.flashinfer_decode_workspace_buffer = torch.empty(
1484 - FLASHINFER_WORKSPACE_BUFFER_SIZE,
1485 - dtype=torch.uint8,
1486 - device=self.device)
1487 - self.flashinfer_decode_wrapper = \
1488 - BatchDecodeWithPagedKVCacheWrapper(
1489 - self.flashinfer_decode_workspace_buffer, "NHD")
1490 - self.flashinfer_prefill_workspace_buffer = torch.empty(
1491 - FLASHINFER_WORKSPACE_BUFFER_SIZE,
1492 - dtype=torch.uint8,
1493 - device=self.device)
1494 - self.flashinfer_prefill_wrapper = \
1495 - BatchPrefillWithPagedKVCacheWrapper(
1496 - self.flashinfer_prefill_workspace_buffer, "NHD")
1497 -
1498 - model_input.attn_metadata.prefill_wrapper = \
1499 - self.flashinfer_prefill_wrapper
1500 - if model_input.attn_metadata.use_cuda_graph:
1501 - batch_size = model_input.input_tokens.shape[0]
1502 - model_input.attn_metadata.decode_wrapper = self.graph_runners[
1503 - model_input.
1504 - virtual_engine][batch_size].flashinfer_decode_wrapper
1505 - else:
1506 - model_input.attn_metadata.decode_wrapper = \
1507 - self.flashinfer_decode_wrapper
1508 - model_input.attn_metadata.begin_forward()
1369 + self.attn_state.begin_forward(model_input)
1370
1371 # Currently cuda graph is only supported by the decode phase.
1372 assert model_input.attn_metadata is not None
@@ -1613,22 +1474,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1474
1475 class CUDAGraphRunner:
1476
1616 - def __init__(self, model: nn.Module, backend_name: str):
1477 + def __init__(self, model: nn.Module, backend_name: str,
1478 + attn_state: AttentionState):
1479 self.model = model
1480 self.backend_name = backend_name
1481 + self.attn_state = attn_state
1482
1483 self.input_buffers: Dict[str, torch.Tensor] = {}
1484 self.output_buffers: Dict[str, torch.Tensor] = {}
1485
1486 self._graph: Optional[torch.cuda.CUDAGraph] = None
1487
1625 - self.flashinfer_decode_workspace_buffer: Optional[torch.Tensor] = None
1626 - self.flashinfer_indptr_buffer: Optional[torch.Tensor] = None
1627 - self.flashinfer_indices_buffer: Optional[torch.Tensor] = None
1628 - self.flashinfer_last_page_len_buffer: Optional[torch.Tensor] = None
1629 - self.flashinfer_decode_wrapper: Optional[
1630 - CUDAGraphBatchDecodeWithPagedKVCacheWrapper] = None
1631 -
1488 @property
1489 def graph(self):
1490 assert self._graph is not None
@@ -1693,25 +1549,13 @@ class CUDAGraphRunner:
1549 torch.cuda.synchronize()
1550
1551 # Save the input and output buffers.
1696 - if self.backend_name == "flashinfer":
1697 - self.input_buffers = {
1698 - "input_ids": input_ids,
1699 - "positions": positions,
1700 - "kv_caches": kv_caches,
1701 - "slot_mapping": attn_metadata.slot_mapping,
1702 - **kwargs,
1703 - }
1704 - else:
1705 - self.input_buffers = {
1706 - "input_ids": input_ids,
1707 - "positions": positions,
1708 - "kv_caches": kv_caches,
1709 - "slot_mapping": attn_metadata.slot_mapping,
1710 - "seq_lens_tensor":
1711 - attn_metadata.decode_metadata.seq_lens_tensor,
1712 - "block_tables": attn_metadata.decode_metadata.block_tables,
1713 - **kwargs,
1714 - }
1552 + self.input_buffers = {
1553 + "input_ids": input_ids,
1554 + "positions": positions,
1555 + "kv_caches": kv_caches,
1556 + **self.attn_state.get_graph_input_buffers(attn_metadata),
1557 + **kwargs,
1558 + }
1559 if intermediate_inputs is not None:
1560 self.input_buffers.update(intermediate_inputs.tensors)
1561 if get_pp_group().is_last_rank:
@@ -1739,12 +1583,8 @@ class CUDAGraphRunner:
1583 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1584 self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1585 non_blocking=True)
1742 - if self.backend_name != "flashinfer":
1743 - self.input_buffers["seq_lens_tensor"].copy_(
1744 - attn_metadata.decode_metadata.seq_lens_tensor,
1745 - non_blocking=True)
1746 - self.input_buffers["block_tables"].copy_(
1747 - attn_metadata.decode_metadata.block_tables, non_blocking=True)
1586 + self.attn_state.prepare_graph_input_buffers(self.input_buffers,
1587 + attn_metadata)
1588 if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1589 self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1590 **kwargs)