@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 925f3332ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] Refactor Attention Take 2 (#3462)

Woosuk Kwon committed Mar 24, 2024 at 21:39 UTC 925f3332cac488e5ad2dbc8f5c6d5f42d2556816
47 files changed +1269 -1118
tests/kernels/test_prefix_prefill.py
+1 -2
@@ -3,8 +3,7 @@ import pytest
3 import time
4
5 import torch
6 -from vllm.model_executor.layers.attention.ops.prefix_prefill import (
7 - context_attention_fwd)
6 +from vllm.attention.ops.prefix_prefill import context_attention_fwd
7 from xformers import ops as xops
8 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
tests/samplers/test_beam_search.py
+7
@@ -2,7 +2,10 @@
2
3 Run `pytest tests/samplers/test_beam_search.py --forked`.
4 """
5 +import gc
6 +
7 import pytest
8 +import torch
9
10 # FIXME(zhuohan): The test can not pass if we:
11 # 1. Increase max_tokens to 256.
@@ -36,6 +39,10 @@ def test_beam_search_single_input(
39 vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
40 max_tokens)
41 del vllm_model
42 + # NOTE(woosuk): For some reason, the following GC is required to avoid
43 + # GPU OOM errors in the following tests using `vllm_runner`.
44 + gc.collect()
45 + torch.cuda.empty_cache()
46
47 for i in range(len(example_prompts)):
48 hf_output_ids, _ = hf_outputs[i]
tests/worker/test_model_runner.py
+30 -30
@@ -34,19 +34,19 @@ def test_prepare_prompt(batch_size):
34 expected_selected_token_indices.append(selected_token_start_idx +
35 prompt_len - 1)
36 selected_token_start_idx += prompt_len
37 - (input_tokens, input_positions, input_metadata, return_prompt_lens, _, _,
38 - _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
37 + (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
38 + _) = (model_runner._prepare_prompt(seq_group_metadata_list))
39 assert return_prompt_lens == prompt_lens
40
41 # Verify input metadata is correct for prompts.
42 device = model_runner.device
43 - assert input_metadata.is_prompt is True
44 - assert torch.allclose(input_metadata.prompt_lens_tensor,
43 + assert attn_metadata.is_prompt is True
44 + assert torch.allclose(attn_metadata.prompt_lens_tensor,
45 torch.tensor(prompt_lens, device=device))
46 - assert input_metadata.prompt_lens == prompt_lens
47 - assert input_metadata.num_prompt_tokens == sum(prompt_lens)
48 - assert input_metadata.num_generation_tokens == 0
49 - assert input_metadata.max_seq_len == max(prompt_lens)
46 + assert attn_metadata.prompt_lens == prompt_lens
47 + assert attn_metadata.num_prompt_tokens == sum(prompt_lens)
48 + assert attn_metadata.num_generation_tokens == 0
49 + assert attn_metadata.max_prompt_len == max(prompt_lens)
50
51 # Test subquery start locs.
52 start_idx = 0
@@ -55,7 +55,7 @@ def test_prepare_prompt(batch_size):
55 start_idx += prompt_len
56 start_loc.append(start_idx)
57 assert torch.allclose(
58 - input_metadata.subquery_start_loc,
58 + attn_metadata.subquery_start_loc,
59 torch.tensor(start_loc, dtype=torch.int32, device=device))
60
61 # Test seq start locs. Note that for normal prefill it is
@@ -67,22 +67,22 @@ def test_prepare_prompt(batch_size):
67 seq_start_loc.append(start_idx)
68
69 assert torch.allclose(
70 - input_metadata.seq_start_loc,
70 + attn_metadata.seq_start_loc,
71 torch.tensor(start_loc, dtype=torch.int32, device=device))
72 - assert input_metadata.max_context_len is None
72 + assert attn_metadata.max_context_len is None
73 assert torch.allclose(
74 - input_metadata.context_lens,
75 - torch.zeros(input_metadata.context_lens.shape[0],
74 + attn_metadata.context_lens,
75 + torch.zeros(attn_metadata.context_lens.shape[0],
76 dtype=torch.int,
77 device=device))
78
79 expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
80 dtype=torch.int32,
81 device=model_runner.device)
82 - assert torch.allclose(input_metadata.block_tables, expected)
82 + assert torch.allclose(attn_metadata.block_tables, expected)
83 # Cuda graph should not be used for prerill.
84 - assert input_metadata.use_cuda_graph is False
85 - assert input_metadata.kv_cache_dtype == "auto"
84 + assert attn_metadata.use_cuda_graph is False
85 + assert attn_metadata.kv_cache_dtype == "auto"
86
87 assert input_tokens.shape == (sum(prompt_lens), )
88 assert input_positions.shape == (sum(prompt_lens), )
@@ -140,34 +140,34 @@ def test_prepare_decode_cuda_graph(batch_size):
140 block_tables={0: [1]},
141 ))
142
143 - input_tokens, input_positions, input_metadata, _, _, _ = (
143 + input_tokens, input_positions, attn_metadata, _, _, _ = (
144 model_runner._prepare_decode(seq_group_metadata_list))
145
146 expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
147 # Verify input metadata is correct for prompts.
148 device = model_runner.device
149 - assert input_metadata.is_prompt is False
150 - assert input_metadata.prompt_lens is None
151 - assert input_metadata.num_prompt_tokens == 0
152 - assert input_metadata.num_generation_tokens == expected_bs
153 - assert input_metadata.max_seq_len is None
154 - assert input_metadata.subquery_start_loc is None
155 - assert input_metadata.seq_start_loc is None
156 - assert input_metadata.max_context_len == max(prompt_lens)
149 + assert attn_metadata.is_prompt is False
150 + assert attn_metadata.prompt_lens is None
151 + assert attn_metadata.num_prompt_tokens == 0
152 + assert attn_metadata.num_generation_tokens == expected_bs
153 + assert attn_metadata.max_prompt_len is None
154 + assert attn_metadata.subquery_start_loc is None
155 + assert attn_metadata.seq_start_loc is None
156 + assert attn_metadata.max_context_len == max(prompt_lens)
157 assert torch.allclose(
158 - input_metadata.context_lens[:len(prompt_lens)],
158 + attn_metadata.context_lens[:len(prompt_lens)],
159 torch.tensor(prompt_lens, dtype=torch.int, device=device))
160
161 # block table's first index corresponds to each batch, meaning in
162 # decoding it is each token.
163 - assert input_metadata.block_tables.shape[0] == len(input_tokens)
163 + assert attn_metadata.block_tables.shape[0] == len(input_tokens)
164 # Block table's second dim correspondsd to each token's block number.
165 # It is padded up to
166 - assert input_metadata.block_tables.shape[1] == (
166 + assert attn_metadata.block_tables.shape[1] == (
167 model_runner.get_max_block_per_batch())
168 # Cuda graph should not be used for prerill.
169 - assert input_metadata.use_cuda_graph is True
170 - assert input_metadata.kv_cache_dtype == "auto"
169 + assert attn_metadata.use_cuda_graph is True
170 + assert attn_metadata.kv_cache_dtype == "auto"
171
172 assert input_tokens.shape == (expected_bs, )
173 assert input_positions.shape == (expected_bs, )
vllm/attention/__init__.py new
+10
@@ -0,0 +1,10 @@
1 +from vllm.attention.backends.abstract import AttentionBackend, AttentionMetadata
2 +from vllm.attention.layer import Attention
3 +from vllm.attention.selector import get_attn_backend
4 +
5 +__all__ = [
6 + "AttentionBackend",
7 + "AttentionMetadata",
8 + "Attention",
9 + "get_attn_backend",
10 +]
vllm/attention/backends/__init__.py renamed
vllm/attention/backends/abstract.py new
+85
@@ -0,0 +1,85 @@
1 +from abc import ABC, abstractmethod
2 +from dataclasses import dataclass, fields
3 +from typing import Any, Dict, List, Optional, Tuple, Type
4 +
5 +import torch
6 +
7 +
8 +class AttentionBackend(ABC):
9 + """Abstract class for attention backends."""
10 +
11 + @staticmethod
12 + @abstractmethod
13 + def get_impl_cls() -> Type["AttentionImpl"]:
14 + raise NotImplementedError
15 +
16 + @staticmethod
17 + @abstractmethod
18 + def make_metadata(*args, **kwargs) -> "AttentionMetadata":
19 + raise NotImplementedError
20 +
21 + @staticmethod
22 + @abstractmethod
23 + def get_kv_cache_shape(
24 + num_blocks: int,
25 + block_size: int,
26 + num_kv_heads: int,
27 + head_size: int,
28 + ) -> Tuple[int, ...]:
29 + raise NotImplementedError
30 +
31 + @staticmethod
32 + @abstractmethod
33 + def swap_blocks(
34 + src_kv_cache: torch.Tensor,
35 + dst_kv_cache: torch.Tensor,
36 + src_to_dst: Dict[int, int],
37 + ) -> None:
38 + raise NotImplementedError
39 +
40 + @staticmethod
41 + @abstractmethod
42 + def copy_blocks(
43 + kv_caches: List[torch.Tensor],
44 + src_to_dists: Dict[int, List[int]],
45 + ) -> None:
46 + raise NotImplementedError
47 +
48 +
49 +@dataclass
50 +class AttentionMetadata:
51 +
52 + def asdict_zerocopy(self) -> Dict[str, Any]:
53 + """Similar to dataclasses.asdict, but avoids deepcopying."""
54 + # Note that if we add dataclasses as fields, they will need
55 + # similar handling.
56 + return {
57 + field.name: getattr(self, field.name)
58 + for field in fields(self)
59 + }
60 +
61 +
62 +class AttentionImpl(ABC):
63 +
64 + @abstractmethod
65 + def __init__(
66 + self,
67 + num_heads: int,
68 + head_size: int,
69 + scale: float,
70 + num_kv_heads: Optional[int] = None,
71 + alibi_slopes: Optional[List[float]] = None,
72 + sliding_window: Optional[int] = None,
73 + ) -> None:
74 + raise NotImplementedError
75 +
76 + @abstractmethod
77 + def forward(
78 + self,
79 + query: torch.Tensor,
80 + key: torch.Tensor,
81 + value: torch.Tensor,
82 + kv_cache: torch.Tensor,
83 + attn_metadata: AttentionMetadata,
84 + ) -> torch.Tensor:
85 + raise NotImplementedError
vllm/attention/backends/flash_attn.py new
+238
@@ -0,0 +1,238 @@
1 +"""Attention layer with Flash and PagedAttention.
2 +
3 +NOTE(woosuk): At the moment, this file includes a lot of duplicated code from
4 +XFormers backend. The duplicated code will be removed once we use flash-attn or
5 +flashinfer for all the attention operations.
6 +"""
7 +from dataclasses import dataclass
8 +from typing import Dict, List, Optional, Tuple, Type
9 +
10 +from flash_attn import flash_attn_varlen_func
11 +import torch
12 +
13 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 + AttentionMetadata)
15 +from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
16 +
17 +
18 +class FlashAttentionBackend(AttentionBackend):
19 +
20 + @staticmethod
21 + def get_impl_cls() -> Type["FlashAttentionImpl"]:
22 + return FlashAttentionImpl
23 +
24 + @staticmethod
25 + def make_metadata(*args, **kwargs) -> "FlashAttentionMetadata":
26 + return FlashAttentionMetadata(*args, **kwargs)
27 +
28 + @staticmethod
29 + def get_kv_cache_shape(
30 + num_blocks: int,
31 + block_size: int,
32 + num_kv_heads: int,
33 + head_size: int,
34 + ) -> Tuple[int, ...]:
35 + return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
36 + num_kv_heads, head_size)
37 +
38 + @staticmethod
39 + def swap_blocks(
40 + src_kv_cache: torch.Tensor,
41 + dst_kv_cache: torch.Tensor,
42 + src_to_dst: Dict[int, int],
43 + ) -> None:
44 + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
45 +
46 + @staticmethod
47 + def copy_blocks(
48 + kv_caches: List[torch.Tensor],
49 + src_to_dists: Dict[int, List[int]],
50 + ) -> None:
51 + PagedAttention.copy_blocks(kv_caches, src_to_dists)
52 +
53 +
54 +@dataclass
55 +class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
56 + """Metadata for FlashAttentionBackend.
57 +
58 + NOTE: Any python object stored here is not updated when it is
59 + cuda-graph replayed. If you have values that need to be changed
60 + dynamically, it should be stored in tensor. The tensor has to be
61 + updated from `CUDAGraphRunner.forward` API.
62 + """
63 + # Currently, input sequences can only contain all prompts
64 + # or all decoding. True if all sequences are prompts.
65 + is_prompt: bool
66 + # (batch_size,). The prompt length per sequence. None if it is a decoding.
67 + prompt_lens: Optional[List[int]]
68 + # prompt_lens stored as a tensor.
69 + prompt_lens_tensor: Optional[torch.Tensor]
70 + # The number of prompt tokens. Doesn't include padding.
71 + num_prompt_tokens: int
72 + # The number of generation tokens. Doesn't include padding.
73 + num_generation_tokens: int
74 +
75 + # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
76 + # |---------- N-1 iteration --------|
77 + # |---------------- N iteration ---------------------|
78 + # |- tokenA -|......................|-- newTokens ---|
79 + # |---------- context_len ----------|
80 + # |-------------------- seqlen ----------------------|
81 + # |- subquery_len -|
82 +
83 + # WARNING(sang): context_len has different definition depending on if it is
84 + # prefill vs decoding. When it is prefill, it doesn't include new tokens.
85 + # When it is for decoding, it includes a new token.
86 +
87 + # Maximum subquery length in the batch.
88 + max_subquery_len: Optional[int]
89 + # Maximum prompt length in the batch.
90 + max_prompt_len: Optional[int]
91 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
92 + # the batch, used to index into subquery. E.g., if the subquery length
93 + # is [4, 6], it is [0, 4, 10].
94 + subquery_start_loc: Optional[torch.Tensor]
95 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
96 + # the batch, used to index into sequence. E.g., if the sequence length is
97 + # [4, 6], it is [0, 4, 10].
98 + seq_start_loc: Optional[torch.Tensor]
99 +
100 + # Whether or not if cuda graph is enabled.
101 + # Cuda-graph is currently enabled for decoding only.
102 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
103 + use_cuda_graph: bool
104 +
105 +
106 +class FlashAttentionImpl(AttentionImpl):
107 + """
108 + If the input tensors contain prompt tokens, the layout is as follows:
109 + |<--------------- num_prompt_tokens -------------->|
110 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
111 +
112 + Otherwise, the layout is as follows:
113 + |<------------------ num_generation_tokens (M) ----------------->|
114 + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
115 +
116 + Generation tokens can contain padding when cuda-graph is used.
117 + Currently, prompt tokens don't contain any padding.
118 +
119 + The prompts might have different lengths, while the generation tokens
120 + always have length 1.
121 + """
122 +
123 + def __init__(
124 + self,
125 + num_heads: int,
126 + head_size: int,
127 + scale: float,
128 + num_kv_heads: Optional[int] = None,
129 + alibi_slopes: Optional[List[float]] = None,
130 + sliding_window: Optional[int] = None,
131 + ) -> None:
132 + self.num_heads = num_heads
133 + self.head_size = head_size
134 + self.scale = float(scale)
135 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
136 + self.sliding_window = ((sliding_window, sliding_window)
137 + if sliding_window is not None else (-1, -1))
138 + if alibi_slopes is not None:
139 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
140 + self.alibi_slopes = alibi_slopes
141 +
142 + assert self.num_heads % self.num_kv_heads == 0
143 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
144 +
145 + suppored_head_sizes = PagedAttention.get_supported_head_sizes()
146 + if head_size not in suppored_head_sizes:
147 + raise ValueError(
148 + f"Head size {head_size} is not supported by PagedAttention. "
149 + f"Supported head sizes are: {suppored_head_sizes}.")
150 +
151 + def forward(
152 + self,
153 + query: torch.Tensor,
154 + key: torch.Tensor,
155 + value: torch.Tensor,
156 + kv_cache: torch.Tensor,
157 + attn_metadata: FlashAttentionMetadata,
158 + ) -> torch.Tensor:
159 + """Forward pass with FlashAttention and PagedAttention.
160 +
161 + Args:
162 + query: shape = [num_tokens, num_heads * head_size]
163 + key: shape = [num_tokens, num_kv_heads * head_size]
164 + value: shape = [num_tokens, num_kv_heads * head_size]
165 + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
166 + attn_metadata: Metadata for attention.
167 + Returns:
168 + shape = [num_tokens, num_heads * head_size]
169 + """
170 + num_tokens, hidden_size = query.shape
171 + # Reshape the query, key, and value tensors.
172 + query = query.view(-1, self.num_heads, self.head_size)
173 + key = key.view(-1, self.num_kv_heads, self.head_size)
174 + value = value.view(-1, self.num_kv_heads, self.head_size)
175 +
176 + if kv_cache is not None:
177 + key_cache, value_cache = PagedAttention.split_kv_cache(
178 + kv_cache, self.num_kv_heads, self.head_size)
179 +
180 + # Reshape the input keys and values and store them in the cache.
181 + # If kv_cache is not provided, the new key and value tensors are
182 + # not cached. This happens during the initial memory profiling run.
183 + PagedAttention.write_to_paged_cache(key, value, key_cache,
184 + value_cache,
185 + attn_metadata.slot_mapping,
186 + attn_metadata.kv_cache_dtype)
187 +
188 + if attn_metadata.is_prompt:
189 + # Prompt run.
190 + if kv_cache is None or attn_metadata.block_tables.numel() == 0:
191 + # normal attention
192 + # When block_tables are not filled, it means q and k are the
193 + # prompt, and they have the same length.
194 + output = flash_attn_varlen_func(
195 + q=query,
196 + k=key,
197 + v=value,
198 + cu_seqlens_q=attn_metadata.seq_start_loc,
199 + cu_seqlens_k=attn_metadata.seq_start_loc,
200 + max_seqlen_q=attn_metadata.max_prompt_len,
201 + max_seqlen_k=attn_metadata.max_prompt_len,
202 + softmax_scale=self.scale,
203 + causal=True,
204 + window_size=self.sliding_window,
205 + alibi_slopes=self.alibi_slopes,
206 + )
207 + else:
208 + # prefix-enabled attention
209 + output = PagedAttention.forward_prefix(
210 + query,
211 + key,
212 + value,
213 + key_cache,
214 + value_cache,
215 + attn_metadata.block_tables,
216 + attn_metadata.subquery_start_loc,
217 + attn_metadata.prompt_lens_tensor,
218 + attn_metadata.context_lens,
219 + attn_metadata.max_subquery_len,
220 + self.alibi_slopes,
221 + )
222 + else:
223 + # Decoding run.
224 + output = PagedAttention.forward_decode(
225 + query,
226 + key_cache,
227 + value_cache,
228 + attn_metadata.block_tables,
229 + attn_metadata.context_lens,
230 + attn_metadata.max_context_len,
231 + attn_metadata.kv_cache_dtype,
232 + self.num_kv_heads,
233 + self.scale,
234 + self.alibi_slopes,
235 + )
236 +
237 + # Reshape the output tensor.
238 + return output.view(num_tokens, hidden_size)
vllm/attention/backends/xformers.py renamed
+177 -55
@@ -1,19 +1,127 @@
1 """Attention layer with xFormers and PagedAttention."""
2 import importlib
3 -from typing import List, Optional
3 +from dataclasses import dataclass
4 +from typing import Dict, List, Optional, Tuple, Type
5
6 import torch
7 from xformers import ops as xops
7 -from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8 +from xformers.ops.fmha.attn_bias import (AttentionBias,
9 + BlockDiagonalCausalMask,
10 LowerTriangularMaskWithTensorBias)
11
10 -from vllm.model_executor.input_metadata import InputMetadata
11 -from vllm.model_executor.layers.attention.ops.paged_attn import (
12 - PagedAttentionImpl)
12 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 + AttentionMetadata)
14 +from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
15 +from vllm.logger import init_logger
16 from vllm.utils import is_hip
17
18 +logger = init_logger(__name__)
19
16 -class XFormersBackend:
20 +
21 +class XFormersBackend(AttentionBackend):
22 +
23 + @staticmethod
24 + def get_impl_cls() -> Type["XFormersImpl"]:
25 + return XFormersImpl
26 +
27 + @staticmethod
28 + def make_metadata(*args, **kwargs) -> "XFormersMetadata":
29 + return XFormersMetadata(*args, **kwargs)
30 +
31 + @staticmethod
32 + def get_kv_cache_shape(
33 + num_blocks: int,
34 + block_size: int,
35 + num_kv_heads: int,
36 + head_size: int,
37 + ) -> Tuple[int, ...]:
38 + return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
39 + num_kv_heads, head_size)
40 +
41 + @staticmethod
42 + def swap_blocks(
43 + src_kv_cache: torch.Tensor,
44 + dst_kv_cache: torch.Tensor,
45 + src_to_dst: Dict[int, int],
46 + ) -> None:
47 + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
48 +
49 + @staticmethod
50 + def copy_blocks(
51 + kv_caches: List[torch.Tensor],
52 + src_to_dists: Dict[int, List[int]],
53 + ) -> None:
54 + PagedAttention.copy_blocks(kv_caches, src_to_dists)
55 +
56 +
57 +@dataclass
58 +class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
59 + """Metadata for XFormersbackend.
60 +
61 + NOTE: Any python object stored here is not updated when it is
62 + cuda-graph replayed. If you have values that need to be changed
63 + dynamically, it should be stored in tensor. The tensor has to be
64 + updated from `CUDAGraphRunner.forward` API.
65 + """
66 + # Currently, input sequences can only contain all prompts
67 + # or all decoding. True if all sequences are prompts.
68 + is_prompt: bool
69 + # (num_tokens,). The indices of the token slots that input tokens will be
70 + # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
71 + # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
72 + # in block 0, and 1st slot in block 1, respectively.
73 + slot_mapping: torch.Tensor
74 + # (batch_size,). The prompt length per sequence. None if it is a decoding.
75 + prompt_lens: Optional[List[int]]
76 + # prompt_lens stored as a tensor.
77 + prompt_lens_tensor: Optional[torch.Tensor]
78 + # The number of prompt tokens. Doesn't include padding.
79 + num_prompt_tokens: int
80 + # The number of generation tokens. Doesn't include padding.
81 + num_generation_tokens: int
82 +
83 + # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
84 + # |---------- N-1 iteration --------|
85 + # |---------------- N iteration ---------------------|
86 + # |- tokenA -|......................|-- newTokens ---|
87 + # |---------- context_len ----------|
88 + # |-------------------- seqlen ----------------------|
89 + # |- subquery_len -|
90 +
91 + # WARNING(sang): context_len has different definition depending on if it is
92 + # prefill vs decoding. When it is prefill, it doesn't include new tokens.
93 + # When it is for decoding, it includes a new token.
94 +
95 + # Maximum subquery length in the batch.
96 + max_subquery_len: Optional[int]
97 + # FIXME: It is for flash attn.
98 + # Maximum prompt length in the batch.
99 + max_prompt_len: Optional[int]
100 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
101 + # the batch, used to index into subquery. E.g., if the subquery length
102 + # is [4, 6], it is [0, 4, 10].
103 + subquery_start_loc: Optional[torch.Tensor]
104 + # FIXME: It is for flash attn.
105 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
106 + # the batch, used to index into sequence. E.g., if the sequence length is
107 + # [4, 6], it is [0, 4, 10].
108 + seq_start_loc: Optional[torch.Tensor]
109 +
110 + # Whether or not if cuda graph is enabled.
111 + # Cuda-graph is currently enabled for decoding only.
112 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
113 + use_cuda_graph: bool
114 +
115 + def __post_init__(self):
116 + # Set during the execution of the first attention op.
117 + # It is a list because it is needed to set per prompt
118 + # when alibi slopes is used. It is because of the limitation
119 + # from xformer API.
120 + # will not appear in the __repr__ and __init__
121 + self.attn_bias: Optional[List[AttentionBias]] = None
122 +
123 +
124 +class XFormersImpl(AttentionImpl):
125 """
126 If the input tensors contain prompt tokens, the layout is as follows:
127 |<--------------- num_prompt_tokens --------------->|
@@ -50,22 +158,25 @@ class XFormersBackend:
158
159 assert self.num_heads % self.num_kv_heads == 0
160 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
53 - suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
161 +
162 + suppored_head_sizes = PagedAttention.get_supported_head_sizes()
163 if head_size not in suppored_head_sizes:
164 raise ValueError(
165 f"Head size {head_size} is not supported by PagedAttention. "
166 f"Supported head sizes are: {suppored_head_sizes}.")
167
59 - self.use_ref_attention = _check_use_ref_attention()
168 + # AMD Radeon 7900 series (gfx1100) currently does not support xFormers
169 + # nor FlashAttention. As a temporary workaround, we use naive PyTorch
170 + # implementation of attention.
171 + self.use_naive_attention = _check_use_naive_attention()
172
173 def forward(
174 self,
175 query: torch.Tensor,
176 key: torch.Tensor,
177 value: torch.Tensor,
66 - key_cache: Optional[torch.Tensor],
67 - value_cache: Optional[torch.Tensor],
68 - input_metadata: InputMetadata,
178 + kv_cache: Optional[torch.Tensor],
179 + attn_metadata: XFormersMetadata,
180 ) -> torch.Tensor:
181 """Forward pass with xFormers and PagedAttention.
182
@@ -73,11 +184,8 @@ class XFormersBackend:
184 query: shape = [num_tokens, num_heads * head_size]
185 key: shape = [num_tokens, num_kv_heads * head_size]
186 value: shape = [num_tokens, num_kv_heads * head_size]
76 - key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
77 - block_size, x]
78 - value_cache: shape = [num_blocks, num_kv_heads, head_size,
79 - block_size]
80 - input_metadata: metadata for the inputs.
187 + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
188 + attn_metadata: Metadata for attention.
189 Returns:
190 shape = [num_tokens, num_heads * head_size]
191 """
@@ -86,20 +194,24 @@ class XFormersBackend:
194 key = key.view(-1, self.num_kv_heads, self.head_size)
195 value = value.view(-1, self.num_kv_heads, self.head_size)
196
89 - # Reshape the keys and values and store them in the cache.
90 - # If key_cache and value_cache are not provided, the new key and value
91 - # vectors will not be cached. This happens during the initial memory
92 - # profiling run.
93 - if key_cache is not None and value_cache is not None:
94 - PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
95 - value_cache, input_metadata)
197 + if kv_cache is not None:
198 + key_cache, value_cache = PagedAttention.split_kv_cache(
199 + kv_cache, self.num_kv_heads, self.head_size)
200 +
201 + # Reshape the input keys and values and store them in the cache.
202 + # If kv_cache is not provided, the new key and value tensors are
203 + # not cached. This happens during the initial memory profiling run.
204 + PagedAttention.write_to_paged_cache(key, value, key_cache,
205 + value_cache,
206 + attn_metadata.slot_mapping,
207 + attn_metadata.kv_cache_dtype)
208
97 - if input_metadata.is_prompt:
209 + if attn_metadata.is_prompt:
210 # Prompt run.
99 - # key_cache and value_cache are None when it is a profiling run.
100 - # block tables are empty if the prompt has never been computed.
101 - if (key_cache is None or value_cache is None
102 - or input_metadata.block_tables.numel() == 0):
211 + if kv_cache is None or attn_metadata.block_tables.numel() == 0:
212 + # normal attention.
213 + # block tables are empty if the prompt does not have a cached
214 + # prefix.
215 if self.num_kv_heads != self.num_heads:
216 # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
217 # project the key and value tensors to the desired number of
@@ -118,13 +230,12 @@ class XFormersBackend:
230 self.num_queries_per_kv,
231 value.shape[-1])
232
121 - if self.use_ref_attention:
122 - print("ref attention used.")
233 + if self.use_naive_attention:
234 output = torch.empty_like(query)
235 start = 0
125 - for _, prompt_len in enumerate(input_metadata.prompt_lens):
236 + for _, prompt_len in enumerate(attn_metadata.prompt_lens):
237 end = start + prompt_len
127 - out = _ref_masked_attention(
238 + out = _naive_masked_attention(
239 query[None, start:end],
240 key[None, start:end],
241 value[None, start:end],
@@ -143,26 +254,33 @@ class XFormersBackend:
254 # Use reshape instead.
255 return output.reshape(num_tokens, hidden_size)
256
146 - output = self._run_memory_efficient_xformer_forward(
147 - query, key, value, input_metadata)
257 + output = self._run_memory_efficient_xformers_forward(
258 + query, key, value, attn_metadata)
259 else:
260 # prefix-enabled attention
150 - output = PagedAttentionImpl.forward_prefix(
261 + output = PagedAttention.forward_prefix(
262 query,
263 key,
264 value,
265 key_cache,
266 value_cache,
156 - input_metadata,
267 + attn_metadata.block_tables,
268 + attn_metadata.subquery_start_loc,
269 + attn_metadata.prompt_lens_tensor,
270 + attn_metadata.context_lens,
271 + attn_metadata.max_subquery_len,
272 self.alibi_slopes,
273 )
274 else:
275 # Decoding run.
161 - output = PagedAttentionImpl.forward_decode(
276 + output = PagedAttention.forward_decode(
277 query,
278 key_cache,
279 value_cache,
165 - input_metadata,
280 + attn_metadata.block_tables,
281 + attn_metadata.context_lens,
282 + attn_metadata.max_context_len,
283 + attn_metadata.kv_cache_dtype,
284 self.num_kv_heads,
285 self.scale,
286 self.alibi_slopes,
@@ -171,12 +289,12 @@ class XFormersBackend:
289 # Reshape the output tensor.
290 return output.view(-1, self.num_heads * self.head_size)
291
174 - def _run_memory_efficient_xformer_forward(
292 + def _run_memory_efficient_xformers_forward(
293 self,
294 query: torch.Tensor,
295 key: torch.Tensor,
296 value: torch.Tensor,
179 - input_metadata: InputMetadata,
297 + attn_metadata: XFormersMetadata,
298 ) -> torch.Tensor:
299 """Attention for 1D query of multiple prompts. Multiple prompt
300 tokens are flattened in to `query` input.
@@ -186,23 +304,23 @@ class XFormersBackend:
304 query: shape = [num_prompt_tokens, num_heads, head_size]
305 key: shape = [num_prompt_tokens, num_kv_heads, head_size]
306 value: shape = [num_prompt_tokens, num_kv_heads, head_size]
189 - input_metadata: metadata for paged attention.
307 + attn_metadata: Metadata for attention.
308 """
309 # Set attention bias if not provided. This typically happens at
310 # the very attention layer of every iteration.
311 # FIXME(woosuk): This is a hack.
194 - if input_metadata.attn_bias is None:
312 + if attn_metadata.attn_bias is None:
313 if self.alibi_slopes is None:
314 attn_bias = BlockDiagonalCausalMask.from_seqlens(
197 - input_metadata.prompt_lens)
315 + attn_metadata.prompt_lens)
316 if self.sliding_window is not None:
317 attn_bias = attn_bias.make_local_attention(
318 self.sliding_window)
201 - input_metadata.attn_bias = [attn_bias]
319 + attn_metadata.attn_bias = [attn_bias]
320 else:
203 - input_metadata.attn_bias = _make_alibi_bias(
321 + attn_metadata.attn_bias = _make_alibi_bias(
322 self.alibi_slopes, self.num_kv_heads, query.dtype,
205 - input_metadata)
323 + attn_metadata.prompt_lens)
324
325 op = xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if (
326 is_hip()) else None
@@ -217,7 +335,7 @@ class XFormersBackend:
335 query,
336 key,
337 value,
220 - attn_bias=input_metadata.attn_bias[0],
338 + attn_bias=attn_metadata.attn_bias[0],
339 p=0.0,
340 scale=self.scale,
341 op=op)
@@ -230,13 +348,13 @@ class XFormersBackend:
348 # one. This is inefficient, especially when we have many short prompts.
349 output = torch.empty_like(query)
350 start = 0
233 - for i, prompt_len in enumerate(input_metadata.prompt_lens):
351 + for i, prompt_len in enumerate(attn_metadata.prompt_lens):
352 end = start + prompt_len
353 out = xops.memory_efficient_attention_forward(
354 query[None, start:end],
355 key[None, start:end],
356 value[None, start:end],
239 - attn_bias=input_metadata.attn_bias[i],
357 + attn_bias=attn_metadata.attn_bias[i],
358 p=0.0,
359 scale=self.scale,
360 op=op)
@@ -250,10 +368,10 @@ def _make_alibi_bias(
368 alibi_slopes: torch.Tensor,
369 num_kv_heads: int,
370 dtype: torch.dtype,
253 - input_metadata: InputMetadata,
371 + prompt_lens: List[int],
372 ) -> LowerTriangularMaskWithTensorBias:
373 attn_biases = []
256 - for prompt_len in input_metadata.prompt_lens:
374 + for prompt_len in prompt_lens:
375 bias = torch.arange(prompt_len, dtype=dtype)
376 # NOTE(zhuohan): HF uses
377 # `bias = bias[None, :].repeat(prompt_len, 1)`
@@ -282,15 +400,19 @@ def _make_alibi_bias(
400 return attn_biases
401
402
285 -def _check_use_ref_attention() -> bool:
403 +def _check_use_naive_attention() -> bool:
404 if not is_hip():
405 return False
406 # For ROCm, check whether flash attention is installed or not.
289 - # if not, use_ref_attention needs to be True
290 - return importlib.util.find_spec("flash_attn") is None
407 + has_flash_attn = importlib.util.find_spec("flash_attn") is None
408 + if not has_flash_attn:
409 + logger.warning("flash_attn is not installed. Using naive attention. "
410 + "This will take significantly more GPU memory.")
411 + return True
412 + return False
413
414
293 -def _ref_masked_attention(
415 +def _naive_masked_attention(
416 query: torch.Tensor,
417 key: torch.Tensor,
418 value: torch.Tensor,
vllm/attention/layer.py new
+46
@@ -0,0 +1,46 @@
1 +"""Attention layer."""
2 +from typing import List, Optional
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from vllm.attention.backends.abstract import AttentionMetadata
8 +from vllm.attention.selector import get_attn_backend
9 +
10 +
11 +class Attention(nn.Module):
12 + """Attention layer.
13 +
14 + This class takes query, key, and value tensors as input. The input tensors
15 + can either contain prompt tokens or generation tokens.
16 + The class does the following:
17 +
18 + 1. Store the input key and value tensors in the KV cache.
19 + 2. Perform (multi-head/multi-query/grouped-query) attention.
20 + 3. Return the output tensor.
21 + """
22 +
23 + def __init__(
24 + self,
25 + num_heads: int,
26 + head_size: int,
27 + scale: float,
28 + num_kv_heads: Optional[int] = None,
29 + alibi_slopes: Optional[List[float]] = None,
30 + sliding_window: Optional[int] = None,
31 + ) -> None:
32 + super().__init__()
33 + self.backend = get_attn_backend(torch.get_default_dtype())
34 + impl_cls = self.backend.get_impl_cls()
35 + self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
36 + alibi_slopes, sliding_window)
37 +
38 + def forward(
39 + self,
40 + query: torch.Tensor,
41 + key: torch.Tensor,
42 + value: torch.Tensor,
43 + kv_cache: Optional[torch.Tensor],
44 + attn_metadata: AttentionMetadata,
45 + ) -> torch.Tensor:
46 + return self.impl.forward(query, key, value, kv_cache, attn_metadata)
vllm/attention/ops/__init__.py renamed
vllm/attention/ops/paged_attn.py new
+217
@@ -0,0 +1,217 @@
1 +from dataclasses import dataclass
2 +from typing import Dict, List, Optional, Tuple
3 +
4 +import torch
5 +
6 +from vllm._C import cache_ops
7 +from vllm._C import ops
8 +from vllm.attention.ops.prefix_prefill import context_attention_fwd
9 +
10 +# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
11 +_PARTITION_SIZE = 512
12 +
13 +
14 +@dataclass
15 +class PagedAttentionMetadata:
16 + """Metadata for PagedAttention."""
17 + # (num_tokens,). The indices of the token slots that input tokens will be
18 + # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
19 + # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
20 + # in block 0, and 1st slot in block 1, respectively.
21 + slot_mapping: torch.Tensor
22 + # (batch_size,). The length of context (tokens stored in KV cache) per
23 + # sequence. WARNING: When it is a prefill request, it doesn't include new
24 + # tokens. When it is for decoding, it includes a new token.
25 + context_lens: Optional[torch.Tensor]
26 + # Maximum context length in the batch.
27 + max_context_len: Optional[int]
28 + # (batch_size, max_blocks_per_seq).
29 + # Block addresses per sequence. (Seq id -> list of physical block)
30 + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
31 + # in the kv cache. Each block can contain up to block_size tokens.
32 + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
33 + # captured.
34 + block_tables: Optional[torch.Tensor]
35 + kv_cache_dtype: str
36 +
37 +
38 +class PagedAttention:
39 +
40 + @staticmethod
41 + def get_supported_head_sizes() -> List[int]:
42 + return [64, 80, 96, 112, 128, 256]
43 +
44 + @staticmethod
45 + def get_kv_cache_shape(
46 + num_blocks: int,
47 + block_size: int,
48 + num_kv_heads: int,
49 + head_size: int,
50 + ) -> Tuple[int, ...]:
51 + return (2, num_blocks, block_size * num_kv_heads * head_size)
52 +
53 + @staticmethod
54 + def split_kv_cache(
55 + kv_cache: torch.Tensor,
56 + num_kv_heads: int,
57 + head_size: int,
58 + ) -> Tuple[torch.Tensor, torch.Tensor]:
59 + x = 16 // kv_cache.element_size()
60 + num_blocks = kv_cache.shape[1]
61 +
62 + key_cache = kv_cache[0]
63 + key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x,
64 + -1, x)
65 + value_cache = kv_cache[1]
66 + value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1)
67 + return key_cache, value_cache
68 +
69 + @staticmethod
70 + def write_to_paged_cache(
71 + key: torch.Tensor,
72 + value: torch.Tensor,
73 + key_cache: torch.Tensor,
74 + value_cache: torch.Tensor,
75 + slot_mapping: torch.Tensor,
76 + kv_cache_dtype: str,
77 + ) -> None:
78 + cache_ops.reshape_and_cache(
79 + key,
80 + value,
81 + key_cache,
82 + value_cache,
83 + slot_mapping.flatten(),
84 + kv_cache_dtype,
85 + )
86 +
87 + @staticmethod
88 + def forward_decode(
89 + query: torch.Tensor,
90 + key_cache: torch.Tensor,
91 + value_cache: torch.Tensor,
92 + block_tables: torch.Tensor,
93 + context_lens: torch.Tensor,
94 + max_context_len: int,
95 + kv_cache_dtype: str,
96 + num_kv_heads: int,
97 + scale: float,
98 + alibi_slopes: Optional[torch.Tensor],
99 + ) -> torch.Tensor:
100 + output = torch.empty_like(query)
101 +
102 + block_size = value_cache.shape[3]
103 + num_seqs, num_heads, head_size = query.shape
104 + max_num_partitions = ((max_context_len + _PARTITION_SIZE - 1) //
105 + _PARTITION_SIZE)
106 + # NOTE(woosuk): We use a simple heuristic to decide whether to use
107 + # PagedAttention V1 or V2. If the number of partitions is 1, we use
108 + # V1 to avoid the overhead of reduction. Also, if the number of
109 + # sequences or heads is large, we use V1 since there is enough work
110 + # to parallelize.
111 + # TODO(woosuk): Tune this heuristic.
112 + # For context len > 8192, use V2 kernel to avoid shared memory shortage.
113 + use_v1 = (max_context_len <= 8192
114 + and (max_num_partitions == 1 or num_seqs * num_heads > 512))
115 + if use_v1:
116 + # Run PagedAttention V1.
117 + ops.paged_attention_v1(
118 + output,
119 + query,
120 + key_cache,
121 + value_cache,
122 + num_kv_heads,
123 + scale,
124 + block_tables,
125 + context_lens,
126 + block_size,
127 + max_context_len,
128 + alibi_slopes,
129 + kv_cache_dtype,
130 + )
131 + else:
132 + # Run PagedAttention V2.
133 + assert _PARTITION_SIZE % block_size == 0
134 + tmp_output = torch.empty(
135 + size=(num_seqs, num_heads, max_num_partitions, head_size),
136 + dtype=output.dtype,
137 + device=output.device,
138 + )
139 + exp_sums = torch.empty(
140 + size=(num_seqs, num_heads, max_num_partitions),
141 + dtype=torch.float32,
142 + device=output.device,
143 + )
144 + max_logits = torch.empty_like(exp_sums)
145 + ops.paged_attention_v2(
146 + output,
147 + exp_sums,
148 + max_logits,
149 + tmp_output,
150 + query,
151 + key_cache,
152 + value_cache,
153 + num_kv_heads,
154 + scale,
155 + block_tables,
156 + context_lens,
157 + block_size,
158 + max_context_len,
159 + alibi_slopes,
160 + kv_cache_dtype,
161 + )
162 + return output
163 +
164 + @staticmethod
165 + def forward_prefix(
166 + query: torch.Tensor,
167 + key: torch.Tensor,
168 + value: torch.Tensor,
169 + key_cache: torch.Tensor,
170 + value_cache: torch.Tensor,
171 + block_tables: torch.Tensor,
172 + subquery_start_loc: torch.Tensor,
173 + prompt_lens_tensor: torch.Tensor,
174 + context_lens: torch.Tensor,
175 + max_subquery_len: int,
176 + alibi_slopes: Optional[torch.Tensor],
177 + ) -> torch.Tensor:
178 + output = torch.empty_like(query)
179 + context_attention_fwd(
180 + query,
181 + key,
182 + value,
183 + output,
184 + key_cache,
185 + value_cache,
186 + block_tables,
187 + # subquery_start_loc is (batch_size + 1,)
188 + subquery_start_loc[:-1],
189 + prompt_lens_tensor,
190 + context_lens,
191 + max_subquery_len,
192 + alibi_slopes,
193 + )
194 + return output
195 +
196 + @staticmethod
197 + def swap_blocks(
198 + src_kv_cache: torch.Tensor,
199 + dst_kv_cache: torch.Tensor,
200 + src_to_dst: Dict[int, int],
201 + ) -> None:
202 + src_key_cache = src_kv_cache[0]
203 + dst_key_cache = dst_kv_cache[0]
204 + cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
205 +
206 + src_value_cache = src_kv_cache[1]
207 + dst_value_cache = dst_kv_cache[1]
208 + cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
209 +
210 + @staticmethod
211 + def copy_blocks(
212 + kv_caches: List[torch.Tensor],
213 + src_to_dists: Dict[int, List[int]],
214 + ) -> None:
215 + key_caches = [kv_cache[0] for kv_cache in kv_caches]
216 + value_caches = [kv_cache[1] for kv_cache in kv_caches]
217 + cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
vllm/attention/ops/prefix_prefill.py renamed
vllm/attention/selector.py new
+44
@@ -0,0 +1,44 @@
1 +from functools import lru_cache
2 +
3 +import torch
4 +
5 +from vllm.attention.backends.abstract import AttentionBackend
6 +from vllm.logger import init_logger
7 +from vllm.utils import is_hip
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +@lru_cache(maxsize=None)
13 +def get_attn_backend(dtype: torch.dtype) -> AttentionBackend:
14 + if _can_use_flash_attn(dtype):
15 + logger.info("Using FlashAttention backend.")
16 + from vllm.attention.backends.flash_attn import FlashAttentionBackend # noqa: F401
17 + return FlashAttentionBackend
18 + else:
19 + logger.info("Using XFormers backend.")
20 + from vllm.attention.backends.xformers import XFormersBackend # noqa: F401
21 + return XFormersBackend
22 +
23 +
24 +def _can_use_flash_attn(dtype: torch.dtype) -> bool:
25 + if is_hip():
26 + # AMD GPUs.
27 + logger.info("Cannot use FlashAttention backend for AMD GPUs.")
28 + return False
29 + if torch.cuda.get_device_capability()[0] < 8:
30 + # Volta and Turing NVIDIA GPUs.
31 + logger.info("Cannot use FlashAttention backend for Volta and Turing "
32 + "GPUs.")
33 + return False
34 + if dtype not in (torch.float16, torch.bfloat16):
35 + logger.info("Cannot use FlashAttention backend for dtype other than "
36 + "torch.float16 or torch.bfloat16.")
37 + return False
38 +
39 + try:
40 + import flash_attn # noqa: F401
41 + except ImportError:
42 + logger.info("flash_attn is not found.")
43 + return False
44 + return True
vllm/model_executor/__init__.py
-2
@@ -1,9 +1,7 @@
1 -from vllm.model_executor.input_metadata import InputMetadata
1 from vllm.model_executor.sampling_metadata import SamplingMetadata
2 from vllm.model_executor.utils import set_random_seed
3
4 __all__ = [
6 - "InputMetadata",
5 "SamplingMetadata",
6 "set_random_seed",
7 ]
vllm/model_executor/input_metadata.py deleted
-99
@@ -1,99 +0,0 @@
1 -from dataclasses import dataclass, fields
2 -from typing import TYPE_CHECKING, Optional, List, Any, Dict
3 -
4 -import torch
5 -if TYPE_CHECKING:
6 - from xformers.ops.fmha.attn_bias import AttentionBias
7 -
8 -
9 -@dataclass
10 -class InputMetadata:
11 - """Metadata for input sequences. Used in PagedAttention.
12 -
13 - NOTE: Any python object stored here is not updated when it is
14 - cuda-graph replayed. If you have values that need to be changed
15 - dynamically, it should be stored in tensor. The tensor has to be
16 - updated from `CUDAGraphRunner.forward` API.
17 - """
18 - # Currently, input sequences can only contain all prompts
19 - # or all decoding. True if all sequences are prompts.
20 - is_prompt: bool
21 - # (num_tokens,). The indices of the token slots that input tokens will be
22 - # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
23 - # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
24 - # in block 0, and 1st slot in block 1, respectively.
25 - slot_mapping: torch.Tensor
26 - # (batch_size,). The prompt length per sequence. None if it is a decoding.
27 - prompt_lens: Optional[List[int]]
28 - # prompt_lens stored as a tensor.
29 - prompt_lens_tensor: Optional[torch.Tensor]
30 - # The number of prompt tokens. Doesn't include padding.
31 - num_prompt_tokens: int
32 - # The number of generation tokens. Doesn't include padding.
33 - num_generation_tokens: int
34 - """
35 - Definition of context_len, subquery_len, and seqlen.
36 - |---------- N-1 iteration --------|
37 - |---------------- N iteration ---------------------|
38 - |- tokenA -|......................|-- newTokens ---|
39 - |---------- context_len ----------|
40 - |-------------------- seqlen ----------------------|
41 - |- subquery_len -|
42 -
43 - WARNING: context_len has different definition depending on if it is
44 - prefill vs decoding. When it is prefill, it doesn't include new
45 - tokens. When it is for decoding, it includes a new token.
46 - """
47 -
48 - # Maximum subquery length in the batch.
49 - max_subquery_len: Optional[int]
50 - # Maximum context length in the batch.
51 - max_context_len: Optional[int]
52 - # FIXME: It is for flash attn.
53 - # Maximum sequence length in the batch.
54 - max_seq_len: Optional[int]
55 - # (batch_size + 1,). The cumulative subquery lengths of the sequences in
56 - # the batch, used to index into subquery. E.g., if the subquery length
57 - # is [4, 6], it is [0, 4, 10].
58 - subquery_start_loc: Optional[torch.Tensor]
59 - # FIXME: It is for flash attn.
60 - # (batch_size + 1,). The cumulative sequence lengths of the sequences in
61 - # the batch, used to index into sequence. E.g., if the sequence length is
62 - # [4, 6], it is [0, 4, 10].
63 - seq_start_loc: Optional[torch.Tensor]
64 - # (batch_size,). The length of context (tokens stored in KV cache) per
65 - # sequence. WARNING: When it is a prefill request, it doesn't include new
66 - # tokens. When it is for decoding, it includes a new token.
67 - context_lens: Optional[torch.Tensor]
68 - # (batch_size, max_blocks_per_seq).
69 - # Block addresses per sequence. (Seq id -> list of physical block)
70 - # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
71 - # in the kv cache. Each block can contain up to block_size tokens.
72 - # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
73 - # captured.
74 - block_tables: Optional[torch.Tensor]
75 - # Whether or not if cuda graph is enabled.
76 - # Cuda-graph is currently enabled for decoding only.
77 - use_cuda_graph: bool
78 - kv_cache_dtype: str
79 -
80 - def __post_init__(self):
81 - # Set during the execution of the first attention op.
82 - # It is a list because it is needed to set per prompt
83 - # when alibi slopes is used. It is because of the limitation
84 - # from xformer API.
85 - # will not appear in the __repr__ and __init__
86 - self.attn_bias: Optional[List["AttentionBias"]] = None
87 -
88 - # Cuda graph is only used for decoding now.
89 - if self.use_cuda_graph:
90 - assert self.num_prompt_tokens == 0
91 -
92 - def asdict_zerocopy(self) -> Dict[str, Any]:
93 - """Similar to dataclasses.asdict, but avoids deepcopying."""
94 - # Note that if we add dataclasses as fields, they will need
95 - # similar handling.
96 - return {
97 - field.name: getattr(self, field.name)
98 - for field in fields(self)
99 - }
vllm/model_executor/layers/attention/__init__.py deleted
-5
@@ -1,5 +0,0 @@
1 -from vllm.model_executor.layers.attention.attention import Attention
2 -
3 -__all__ = [
4 - "Attention",
5 -]
vllm/model_executor/layers/attention/attention.py deleted
-85
@@ -1,85 +0,0 @@
1 -"""Attention layer."""
2 -from functools import lru_cache
3 -from typing import List, Optional
4 -
5 -import torch
6 -import torch.nn as nn
7 -
8 -from vllm.logger import init_logger
9 -from vllm.model_executor.input_metadata import InputMetadata
10 -from vllm.utils import is_hip
11 -
12 -logger = init_logger(__name__)
13 -
14 -
15 -class Attention(nn.Module):
16 - """Attention layer.
17 -
18 - This class takes query, key, and value tensors as input. The input tensors
19 - can either contain prompt tokens or generation tokens.
20 -
21 - The class does the following:
22 -
23 - 1. Store the input key and value tensors in the KV cache.
24 - 2. Perform (multi-head/multi-query/grouped-query) attention.
25 - 3. Output the output tensor.
26 - """
27 -
28 - def __init__(
29 - self,
30 - num_heads: int,
31 - head_size: int,
32 - scale: float,
33 - num_kv_heads: Optional[int] = None,
34 - alibi_slopes: Optional[List[float]] = None,
35 - sliding_window: Optional[int] = None,
36 - ) -> None:
37 - super().__init__()
38 - if _use_flash_attn():
39 - from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend # noqa: E501
40 - self.backend = FlashAttentionBackend(num_heads, head_size, scale,
41 - num_kv_heads, alibi_slopes,
42 - sliding_window)
43 - else:
44 - from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend # noqa: E501
45 - self.backend = XFormersBackend(num_heads, head_size, scale,
46 - num_kv_heads, alibi_slopes,
47 - sliding_window)
48 -
49 - def forward(
50 - self,
51 - query: torch.Tensor,
52 - key: torch.Tensor,
53 - value: torch.Tensor,
54 - key_cache: Optional[torch.Tensor],
55 - value_cache: Optional[torch.Tensor],
56 - input_metadata: InputMetadata,
57 - ) -> torch.Tensor:
58 - return self.backend.forward(query, key, value, key_cache, value_cache,
59 - input_metadata)
60 -
61 -
62 -@lru_cache(maxsize=1)
63 -def _use_flash_attn() -> bool:
64 - try:
65 - import flash_attn # noqa: F401
66 - except ImportError:
67 - logger.info("flash_attn is not found. Using xformers backend.")
68 - return False
69 -
70 - if is_hip():
71 - # AMD GPUs.
72 - return False
73 - if torch.cuda.get_device_capability()[0] < 8:
74 - # Volta and Turing NVIDIA GPUs.
75 - logger.info("flash_attn is not supported on Turing or older GPUs. "
76 - "Using xformers backend.")
77 - return False
78 - if torch.get_default_dtype() not in (torch.float16, torch.bfloat16):
79 - logger.info(
80 - "flash_attn only supports torch.float16 or torch.bfloat16. "
81 - "Using xformers backend.")
82 - return False
83 -
84 - logger.info("Using flash_attn backend.")
85 - return True
vllm/model_executor/layers/attention/backends/flash_attn.py deleted
-139
@@ -1,139 +0,0 @@
1 -"""Attention layer with Flash and PagedAttention."""
2 -from typing import List, Optional
3 -
4 -from flash_attn import flash_attn_varlen_func
5 -import torch
6 -
7 -from vllm.model_executor.input_metadata import InputMetadata
8 -from vllm.model_executor.layers.attention.ops.paged_attn import (
9 - PagedAttentionImpl)
10 -
11 -
12 -class FlashAttentionBackend:
13 - """
14 - If the input tensors contain prompt tokens, the layout is as follows:
15 - |<--------------- num_prompt_tokens -------------->|
16 - |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
17 -
18 - Otherwise, the layout is as follows:
19 - |<------------------ num_generation_tokens (M) ----------------->|
20 - |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
21 -
22 - Generation tokens can contain padding when cuda-graph is used.
23 - Currently, prompt tokens don't contain any padding.
24 -
25 - The prompts might have different lengths, while the generation tokens
26 - always have length 1.
27 - """
28 -
29 - def __init__(
30 - self,
31 - num_heads: int,
32 - head_size: int,
33 - scale: float,
34 - num_kv_heads: Optional[int] = None,
35 - alibi_slopes: Optional[List[float]] = None,
36 - sliding_window: Optional[int] = None,
37 - ) -> None:
38 - self.num_heads = num_heads
39 - self.head_size = head_size
40 - self.scale = float(scale)
41 - self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
42 - self.sliding_window = sliding_window
43 - if alibi_slopes is not None:
44 - alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
45 - self.alibi_slopes = alibi_slopes
46 -
47 - assert self.num_heads % self.num_kv_heads == 0
48 - self.num_queries_per_kv = self.num_heads // self.num_kv_heads
49 - suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
50 - if head_size not in suppored_head_sizes:
51 - raise ValueError(
52 - f"Head size {head_size} is not supported by PagedAttention. "
53 - f"Supported head sizes are: {suppored_head_sizes}.")
54 -
55 - self.sliding_window = ((self.sliding_window, self.sliding_window) if
56 - self.sliding_window is not None else (-1, -1))
57 -
58 - def forward(
59 - self,
60 - query: torch.Tensor,
61 - key: torch.Tensor,
62 - value: torch.Tensor,
63 - key_cache: Optional[torch.Tensor],
64 - value_cache: Optional[torch.Tensor],
65 - input_metadata: InputMetadata,
66 - ) -> torch.Tensor:
67 - """Forward pass with FlashAttention and PagedAttention.
68 -
69 - Args:
70 - query: shape = [num_tokens, num_heads * head_size]
71 - key: shape = [num_tokens, num_kv_heads * head_size]
72 - value: shape = [num_tokens, num_kv_heads * head_size]
73 - key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
74 - block_size, x]
75 - value_cache: shape = [num_blocks, num_kv_heads, head_size,
76 - block_size]
77 - input_metadata: metadata for the inputs.
78 - Returns:
79 - shape = [num_tokens, num_heads * head_size]
80 - """
81 - num_tokens, hidden_size = query.shape
82 - # Reshape the query, key, and value tensors.
83 - query = query.view(-1, self.num_heads, self.head_size)
84 - key = key.view(-1, self.num_kv_heads, self.head_size)
85 - value = value.view(-1, self.num_kv_heads, self.head_size)
86 -
87 - # Reshape the keys and values and store them in the cache.
88 - # If key_cache and value_cache are not provided, the new key and value
89 - # vectors will not be cached. This happens during the initial memory
90 - # profiling run.
91 - if key_cache is not None and value_cache is not None:
92 - PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
93 - value_cache, input_metadata)
94 -
95 - if input_metadata.is_prompt:
96 - # Prompt run.
97 - if (key_cache is None or value_cache is None
98 - or input_metadata.block_tables.numel() == 0):
99 - # normal attention
100 - # When block_tables are not filled, it means q and k are the
101 - # prompt, and they have the same length.
102 - output = flash_attn_varlen_func(
103 - q=query,
104 - k=key,
105 - v=value,
106 - cu_seqlens_q=input_metadata.seq_start_loc,
107 - cu_seqlens_k=input_metadata.seq_start_loc,
108 - max_seqlen_q=input_metadata.max_seq_len,
109 - max_seqlen_k=input_metadata.max_seq_len,
110 - softmax_scale=self.scale,
111 - causal=True,
112 - window_size=self.sliding_window,
113 - alibi_slopes=self.alibi_slopes,
114 - )
115 - else:
116 - # prefix-enabled attention
117 - output = PagedAttentionImpl.forward_prefix(
118 - query,
119 - key,
120 - value,
121 - key_cache,
122 - value_cache,
123 - input_metadata,
124 - self.alibi_slopes,
125 - )
126 - else:
127 - # Decoding run.
128 - output = PagedAttentionImpl.forward_decode(
129 - query,
130 - key_cache,
131 - value_cache,
132 - input_metadata,
133 - self.num_kv_heads,
134 - self.scale,
135 - self.alibi_slopes,
136 - )
137 -
138 - # Reshape the output tensor.
139 - return output.view(num_tokens, hidden_size)
vllm/model_executor/layers/attention/ops/paged_attn.py deleted
-139
@@ -1,139 +0,0 @@
1 -from typing import List, Optional
2 -
3 -import torch
4 -
5 -from vllm._C import cache_ops
6 -from vllm._C import ops
7 -from vllm.model_executor.input_metadata import InputMetadata
8 -from vllm.model_executor.layers.attention.ops.prefix_prefill import (
9 - context_attention_fwd)
10 -
11 -# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
12 -_PARTITION_SIZE = 512
13 -
14 -
15 -class PagedAttentionImpl:
16 -
17 - @staticmethod
18 - def get_supported_head_sizes() -> List[int]:
19 - return [64, 80, 96, 112, 128, 256]
20 -
21 - @staticmethod
22 - def reshape_and_cache(
23 - key: torch.Tensor,
24 - value: torch.Tensor,
25 - key_cache: torch.Tensor,
26 - value_cache: torch.Tensor,
27 - input_metadata: InputMetadata,
28 - ) -> None:
29 - cache_ops.reshape_and_cache(
30 - key,
31 - value,
32 - key_cache,
33 - value_cache,
34 - input_metadata.slot_mapping.flatten(),
35 - input_metadata.kv_cache_dtype,
36 - )
37 -
38 - @staticmethod
39 - def forward_decode(
40 - query: torch.Tensor,
41 - key_cache: torch.Tensor,
42 - value_cache: torch.Tensor,
43 - input_metadata: InputMetadata,
44 - num_kv_heads: int,
45 - scale: float,
46 - alibi_slopes: Optional[torch.Tensor],
47 - ) -> torch.Tensor:
48 - output = torch.empty_like(query)
49 -
50 - block_size = value_cache.shape[3]
51 - num_seqs, num_heads, head_size = query.shape
52 - max_num_partitions = (
53 - (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
54 - _PARTITION_SIZE)
55 - # NOTE(woosuk): We use a simple heuristic to decide whether to use
56 - # PagedAttention V1 or V2. If the number of partitions is 1, we use
57 - # V1 to avoid the overhead of reduction. Also, if the number of
58 - # sequences or heads is large, we use V1 since there is enough work
59 - # to parallelize.
60 - # TODO(woosuk): Tune this heuristic.
61 - # For context len > 8192, use V2 kernel to avoid shared memory shortage.
62 - use_v1 = input_metadata.max_context_len <= 8192 and (
63 - max_num_partitions == 1 or num_seqs * num_heads > 512)
64 - if use_v1:
65 - # Run PagedAttention V1.
66 - ops.paged_attention_v1(
67 - output,
68 - query,
69 - key_cache,
70 - value_cache,
71 - num_kv_heads,
72 - scale,
73 - input_metadata.block_tables,
74 - input_metadata.context_lens,
75 - block_size,
76 - input_metadata.max_context_len,
77 - alibi_slopes,
78 - input_metadata.kv_cache_dtype,
79 - )
80 - else:
81 - # Run PagedAttention V2.
82 - assert _PARTITION_SIZE % block_size == 0
83 - tmp_output = torch.empty(
84 - size=(num_seqs, num_heads, max_num_partitions, head_size),
85 - dtype=output.dtype,
86 - device=output.device,
87 - )
88 - exp_sums = torch.empty(
89 - size=(num_seqs, num_heads, max_num_partitions),
90 - dtype=torch.float32,
91 - device=output.device,
92 - )
93 - max_logits = torch.empty_like(exp_sums)
94 - ops.paged_attention_v2(
95 - output,
96 - exp_sums,
97 - max_logits,
98 - tmp_output,
99 - query,
100 - key_cache,
101 - value_cache,
102 - num_kv_heads,
103 - scale,
104 - input_metadata.block_tables,
105 - input_metadata.context_lens,
106 - block_size,
107 - input_metadata.max_context_len,
108 - alibi_slopes,
109 - input_metadata.kv_cache_dtype,
110 - )
111 - return output
112 -
113 - @staticmethod
114 - def forward_prefix(
115 - query: torch.Tensor,
116 - key: torch.Tensor,
117 - value: torch.Tensor,
118 - key_cache: torch.Tensor,
119 - value_cache: torch.Tensor,
120 - input_metadata: InputMetadata,
121 - alibi_slopes: Optional[torch.Tensor],
122 - ) -> torch.Tensor:
123 - output = torch.empty_like(query)
124 - context_attention_fwd(
125 - query,
126 - key,
127 - value,
128 - output,
129 - key_cache,
130 - value_cache,
131 - input_metadata.block_tables,
132 - # subquery_start_loc is (batch_size + 1,)
133 - input_metadata.subquery_start_loc[:-1],
134 - input_metadata.prompt_lens_tensor,
135 - input_metadata.context_lens,
136 - input_metadata.max_subquery_len,
137 - alibi_slopes,
138 - )
139 - return output
vllm/model_executor/models/baichuan.py
+13 -17
@@ -25,9 +25,8 @@ import torch
25 from torch import nn
26 from transformers import PretrainedConfig
27
28 -from vllm.model_executor.input_metadata import InputMetadata
28 +from vllm.attention import Attention, AttentionMetadata
29 from vllm.model_executor.layers.activation import SiluAndMul
30 -from vllm.model_executor.layers.attention import Attention
30 from vllm.model_executor.layers.layernorm import RMSNorm
31 from vllm.model_executor.layers.linear import (LinearMethodBase,
32 MergedColumnParallelLinear,
@@ -45,8 +44,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
44 hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
46
48 -KVCache = Tuple[torch.Tensor, torch.Tensor]
49 -
47
48 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
49 closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
@@ -170,15 +167,14 @@ class BaiChuanAttention(nn.Module):
167 self,
168 positions: torch.Tensor,
169 hidden_states: torch.Tensor,
173 - kv_cache: KVCache,
174 - input_metadata: InputMetadata,
170 + kv_cache: torch.Tensor,
171 + attn_metadata: AttentionMetadata,
172 ) -> torch.Tensor:
173 qkv, _ = self.W_pack(hidden_states)
174 q, k, v = qkv.chunk(chunks=3, dim=-1)
175 if self.postion_embedding != "ALIBI":
176 q, k = self.rotary_emb(positions, q, k)
180 - k_cache, v_cache = kv_cache
181 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
177 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
178 output, _ = self.o_proj(attn_output)
179 return output
180
@@ -217,8 +213,8 @@ class BaiChuanDecoderLayer(nn.Module):
213 self,
214 positions: torch.Tensor,
215 hidden_states: torch.Tensor,
220 - kv_cache: KVCache,
221 - input_metadata: InputMetadata,
216 + kv_cache: torch.Tensor,
217 + attn_metadata: AttentionMetadata,
218 residual: Optional[torch.Tensor],
219 ) -> Tuple[torch.Tensor, torch.Tensor]:
220 # Self Attention
@@ -232,7 +228,7 @@ class BaiChuanDecoderLayer(nn.Module):
228 positions=positions,
229 hidden_states=hidden_states,
230 kv_cache=kv_cache,
235 - input_metadata=input_metadata,
231 + attn_metadata=attn_metadata,
232 )
233
234 # Fully Connected
@@ -267,8 +263,8 @@ class BaiChuanModel(nn.Module):
263 self,
264 input_ids: torch.Tensor,
265 positions: torch.Tensor,
270 - kv_caches: List[KVCache],
271 - input_metadata: InputMetadata,
266 + kv_caches: List[torch.Tensor],
267 + attn_metadata: AttentionMetadata,
268 ) -> torch.Tensor:
269 hidden_states = self.embed_tokens(input_ids)
270 residual = None
@@ -278,7 +274,7 @@ class BaiChuanModel(nn.Module):
274 positions,
275 hidden_states,
276 kv_caches[i],
281 - input_metadata,
277 + attn_metadata,
278 residual,
279 )
280 hidden_states, _ = self.norm(hidden_states, residual)
@@ -303,11 +299,11 @@ class BaiChuanBaseForCausalLM(nn.Module):
299 self,
300 input_ids: torch.Tensor,
301 positions: torch.Tensor,
306 - kv_caches: List[KVCache],
307 - input_metadata: InputMetadata,
302 + kv_caches: List[torch.Tensor],
303 + attn_metadata: AttentionMetadata,
304 ) -> torch.Tensor:
305 hidden_states = self.model(input_ids, positions, kv_caches,
310 - input_metadata)
306 + attn_metadata)
307 return hidden_states
308
309 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/bloom.py
+14 -18
@@ -17,15 +17,14 @@
17 # limitations under the License.
18 """Inference-only BLOOM model compatible with HuggingFace weights."""
19 import math
20 -from typing import List, Optional, Tuple
20 +from typing import List, Optional
21
22 import torch
23 from torch import nn
24 from transformers import BloomConfig
25
26 -from vllm.model_executor.input_metadata import InputMetadata
26 +from vllm.attention import Attention, AttentionMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import Attention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
44 -KVCache = Tuple[torch.Tensor, torch.Tensor]
45 -
43
44 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
45 closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
@@ -117,14 +114,13 @@ class BloomAttention(nn.Module):
114 self,
115 position_ids: torch.Tensor,
116 hidden_states: torch.Tensor,
120 - kv_cache: KVCache,
121 - input_metadata: InputMetadata,
117 + kv_cache: torch.Tensor,
118 + attn_metadata: AttentionMetadata,
119 ) -> torch.Tensor:
120 del position_ids # Unused.
121 qkv, _ = self.query_key_value(hidden_states)
122 q, k, v = qkv.chunk(chunks=3, dim=-1)
126 - k_cache, v_cache = kv_cache
127 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
123 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
124 output, _ = self.dense(attn_output)
125 return output
126
@@ -181,8 +177,8 @@ class BloomBlock(nn.Module):
177 self,
178 position_ids: torch.Tensor,
179 hidden_states: torch.Tensor,
184 - kv_cache: KVCache,
185 - input_metadata: InputMetadata,
180 + kv_cache: torch.Tensor,
181 + attn_metadata: AttentionMetadata,
182 ) -> torch.Tensor:
183 # Layer norm at the beginning of the transformer layer.
184 layernorm_output = self.input_layernorm(hidden_states)
@@ -198,7 +194,7 @@ class BloomBlock(nn.Module):
194 position_ids=position_ids,
195 hidden_states=layernorm_output,
196 kv_cache=kv_cache,
201 - input_metadata=input_metadata,
197 + attn_metadata=attn_metadata,
198 )
199 attention_output = attention_output + residual
200 layernorm_output = self.post_attention_layernorm(attention_output)
@@ -245,8 +241,8 @@ class BloomModel(nn.Module):
241 self,
242 input_ids: torch.Tensor,
243 position_ids: torch.Tensor,
248 - kv_caches: List[KVCache],
249 - input_metadata: InputMetadata,
244 + kv_caches: List[torch.Tensor],
245 + attn_metadata: AttentionMetadata,
246 ) -> torch.Tensor:
247 hidden_states = self.word_embeddings(input_ids)
248 hidden_states = self.word_embeddings_layernorm(hidden_states)
@@ -256,7 +252,7 @@ class BloomModel(nn.Module):
252 position_ids,
253 hidden_states,
254 kv_caches[i],
259 - input_metadata,
255 + attn_metadata,
256 )
257 hidden_states = self.ln_f(hidden_states)
258 return hidden_states
@@ -281,11 +277,11 @@ class BloomForCausalLM(nn.Module):
277 self,
278 input_ids: torch.Tensor,
279 positions: torch.Tensor,
284 - kv_caches: List[KVCache],
285 - input_metadata: InputMetadata,
280 + kv_caches: List[torch.Tensor],
281 + attn_metadata: AttentionMetadata,
282 ) -> torch.Tensor:
283 hidden_states = self.transformer(input_ids, positions, kv_caches,
288 - input_metadata)
284 + attn_metadata)
285 return hidden_states
286
287 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/chatglm.py
+18 -23
@@ -2,15 +2,14 @@
2 # Adapted from
3 # https://github.com/THUDM/ChatGLM2-6B
4 """Inference-only ChatGLM model compatible with THUDM weights."""
5 -from typing import List, Optional, Tuple
5 +from typing import List, Optional
6
7 import torch
8 from torch import nn
9 from torch.nn import LayerNorm
10
11 -from vllm.model_executor.input_metadata import InputMetadata
11 +from vllm.attention import Attention, AttentionMetadata
12 from vllm.model_executor.layers.activation import SiluAndMul
13 -from vllm.model_executor.layers.attention import Attention
13 from vllm.model_executor.layers.layernorm import RMSNorm
14 from vllm.model_executor.layers.linear import (LinearMethodBase,
15 MergedColumnParallelLinear,
@@ -29,8 +28,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
28 from vllm.sequence import SamplerOutput
29 from vllm.transformers_utils.configs import ChatGLMConfig
30
32 -KVCache = Tuple[torch.Tensor, torch.Tensor]
33 -
31
32 class GLMAttention(nn.Module):
33
@@ -99,20 +96,18 @@ class GLMAttention(nn.Module):
96 self,
97 hidden_states: torch.Tensor,
98 position_ids: torch.Tensor,
102 - kv_cache: KVCache,
103 - input_metadata: InputMetadata,
99 + kv_cache: torch.Tensor,
100 + attn_metadata: AttentionMetadata,
101 ) -> torch.Tensor:
102 qkv, _ = self.query_key_value(hidden_states)
103 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
104 q, k = self.rotary_emb(position_ids, q, k)
108 - key_cache, value_cache = kv_cache
105 context_layer = self.attn(
106 q,
107 k,
108 v,
113 - key_cache,
114 - value_cache,
115 - input_metadata,
109 + kv_cache,
110 + attn_metadata,
111 )
112 attn_output, _ = self.dense(context_layer)
113 return attn_output
@@ -200,8 +195,8 @@ class GLMBlock(nn.Module):
195 self,
196 hidden_states: torch.Tensor,
197 position_ids: torch.Tensor,
203 - kv_cache: KVCache,
204 - input_metadata: InputMetadata,
198 + kv_cache: torch.Tensor,
199 + attn_metadata: AttentionMetadata,
200 ) -> torch.Tensor:
201 # hidden_states: [num_tokens, h]
202 # Layer norm at the beginning of the transformer layer.
@@ -211,7 +206,7 @@ class GLMBlock(nn.Module):
206 hidden_states=layernorm_output,
207 position_ids=position_ids,
208 kv_cache=kv_cache,
214 - input_metadata=input_metadata,
209 + attn_metadata=attn_metadata,
210 )
211
212 # Residual connection.
@@ -264,8 +259,8 @@ class GLMTransformer(nn.Module):
259 self,
260 hidden_states: torch.Tensor,
261 position_ids: torch.Tensor,
267 - kv_caches: List[KVCache],
268 - input_metadata: InputMetadata,
262 + kv_caches: List[torch.Tensor],
263 + attn_metadata: AttentionMetadata,
264 ) -> torch.Tensor:
265 for i in range(self.num_layers):
266 layer = self.layers[i]
@@ -273,7 +268,7 @@ class GLMTransformer(nn.Module):
268 hidden_states=hidden_states,
269 position_ids=position_ids,
270 kv_cache=kv_caches[i],
276 - input_metadata=input_metadata,
271 + attn_metadata=attn_metadata,
272 )
273 # Final layer norm.
274 if self.post_layer_norm:
@@ -306,8 +301,8 @@ class ChatGLMModel(nn.Module):
301 self,
302 input_ids: torch.Tensor,
303 position_ids: torch.Tensor,
309 - kv_caches: List[KVCache],
310 - input_metadata: InputMetadata,
304 + kv_caches: List[torch.Tensor],
305 + attn_metadata: AttentionMetadata,
306 ) -> torch.Tensor:
307 inputs_embeds = self.embedding(input_ids)
308
@@ -316,7 +311,7 @@ class ChatGLMModel(nn.Module):
311 hidden_states=inputs_embeds,
312 position_ids=position_ids,
313 kv_caches=kv_caches,
319 - input_metadata=input_metadata,
314 + attn_metadata=attn_metadata,
315 )
316 return hidden_states
317
@@ -340,11 +335,11 @@ class ChatGLMForCausalLM(nn.Module):
335 self,
336 input_ids: torch.Tensor,
337 positions: torch.Tensor,
343 - kv_caches: List[KVCache],
344 - input_metadata: InputMetadata,
338 + kv_caches: List[torch.Tensor],
339 + attn_metadata: AttentionMetadata,
340 ) -> torch.Tensor:
341 hidden_states = self.transformer(input_ids, positions, kv_caches,
347 - input_metadata)
342 + attn_metadata)
343 return hidden_states
344
345 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/deepseek.py
+14 -18
@@ -21,15 +21,14 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Deepseek model."""
24 -from typing import Any, Dict, List, Optional, Tuple
24 +from typing import Any, Dict, List, Optional
25
26 import torch
27 from torch import nn
28 from transformers import PretrainedConfig
29
30 -from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.attention import Attention, AttentionMetadata
31 from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import Attention
32 from vllm.model_executor.layers.fused_moe import fused_moe
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -51,8 +50,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
50 hf_model_weights_iterator)
51 from vllm.sequence import SamplerOutput
52
54 -KVCache = Tuple[torch.Tensor, torch.Tensor]
55 -
53
54 class DeepseekMLP(nn.Module):
55
@@ -239,14 +236,13 @@ class DeepseekAttention(nn.Module):
236 self,
237 positions: torch.Tensor,
238 hidden_states: torch.Tensor,
242 - kv_cache: KVCache,
243 - input_metadata: InputMetadata,
239 + kv_cache: torch.Tensor,
240 + attn_metadata: AttentionMetadata,
241 ) -> torch.Tensor:
242 qkv, _ = self.qkv_proj(hidden_states)
243 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
244 q, k = self.rotary_emb(positions, q, k)
248 - k_cache, v_cache = kv_cache
249 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
245 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
246 output, _ = self.o_proj(attn_output)
247 return output
248
@@ -294,8 +290,8 @@ class DeepseekDecoderLayer(nn.Module):
290 self,
291 positions: torch.Tensor,
292 hidden_states: torch.Tensor,
297 - kv_cache: KVCache,
298 - input_metadata: InputMetadata,
293 + kv_cache: torch.Tensor,
294 + attn_metadata: AttentionMetadata,
295 residual: Optional[torch.Tensor],
296 ) -> torch.Tensor:
297 # Self Attention
@@ -309,7 +305,7 @@ class DeepseekDecoderLayer(nn.Module):
305 positions=positions,
306 hidden_states=hidden_states,
307 kv_cache=kv_cache,
312 - input_metadata=input_metadata,
308 + attn_metadata=attn_metadata,
309 )
310
311 # Fully Connected
@@ -346,15 +342,15 @@ class DeepseekModel(nn.Module):
342 self,
343 input_ids: torch.Tensor,
344 positions: torch.Tensor,
349 - kv_caches: List[KVCache],
350 - input_metadata: InputMetadata,
345 + kv_caches: List[torch.Tensor],
346 + attn_metadata: AttentionMetadata,
347 ) -> torch.Tensor:
348 hidden_states = self.embed_tokens(input_ids)
349 residual = None
350 for i in range(len(self.layers)):
351 layer = self.layers[i]
352 hidden_states, residual = layer(positions, hidden_states,
357 - kv_caches[i], input_metadata,
353 + kv_caches[i], attn_metadata,
354 residual)
355 hidden_states, _ = self.norm(hidden_states, residual)
356 return hidden_states
@@ -379,11 +375,11 @@ class DeepseekForCausalLM(nn.Module):
375 self,
376 input_ids: torch.Tensor,
377 positions: torch.Tensor,
382 - kv_caches: List[KVCache],
383 - input_metadata: InputMetadata,
378 + kv_caches: List[torch.Tensor],
379 + attn_metadata: AttentionMetadata,
380 ) -> torch.Tensor:
381 hidden_states = self.model(input_ids, positions, kv_caches,
386 - input_metadata)
382 + attn_metadata)
383 return hidden_states
384
385 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/falcon.py
+14 -17
@@ -19,16 +19,15 @@
19 """PyTorch Falcon model."""
20
21 import math
22 -from typing import List, Optional, Tuple, Union
22 +from typing import List, Optional, Union
23
24 import torch
25 from torch import nn
26 from torch.nn import LayerNorm
27 from transformers import FalconConfig as HF_FalconConfig
28
29 -from vllm.model_executor.input_metadata import InputMetadata
29 +from vllm.attention import Attention, AttentionMetadata
30 from vllm.model_executor.layers.activation import get_act_fn
31 -from vllm.model_executor.layers.attention import Attention
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 LinearMethodBase,
33 QKVParallelLinear,
@@ -48,7 +47,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
47 from vllm.sequence import SamplerOutput
48 from vllm.transformers_utils.configs import RWConfig
49
51 -KVCache = Tuple[torch.Tensor, torch.Tensor]
50 FalconConfig = Union[HF_FalconConfig, RWConfig]
51
52
@@ -177,8 +175,8 @@ class FalconAttention(nn.Module):
175 self,
176 positions: torch.Tensor,
177 hidden_states: torch.Tensor,
180 - kv_cache: KVCache,
181 - input_metadata: InputMetadata,
178 + kv_cache: torch.Tensor,
179 + attn_metadata: AttentionMetadata,
180 ) -> torch.Tensor:
181 qkv, bias = self.query_key_value(hidden_states)
182 if bias is not None:
@@ -186,8 +184,7 @@ class FalconAttention(nn.Module):
184 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
185 if self.use_rotary:
186 q, k = self.rotary_emb(positions, q, k)
189 - k_cache, v_cache = kv_cache
190 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
187 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
188 attn_output, bias = self.dense(attn_output)
189 return attn_output, bias
190
@@ -263,8 +260,8 @@ class FalconDecoderLayer(nn.Module):
260 self,
261 positions: torch.Tensor,
262 hidden_states: torch.Tensor,
266 - kv_cache: KVCache,
267 - input_metadata: InputMetadata,
263 + kv_cache: torch.Tensor,
264 + attn_metadata: AttentionMetadata,
265 ) -> torch.Tensor:
266 residual = hidden_states
267
@@ -279,7 +276,7 @@ class FalconDecoderLayer(nn.Module):
276 positions=positions,
277 hidden_states=attention_layernorm_out,
278 kv_cache=kv_cache,
282 - input_metadata=input_metadata,
279 + attn_metadata=attn_metadata,
280 )
281 if self.reduce_row_parallel_results and attention_bias is not None:
282 attention_output += attention_bias
@@ -343,8 +340,8 @@ class FalconModel(nn.Module):
340 self,
341 input_ids: torch.LongTensor,
342 positions: torch.Tensor,
346 - kv_caches: List[KVCache],
347 - input_metadata: InputMetadata,
343 + kv_caches: List[torch.Tensor],
344 + attn_metadata: AttentionMetadata,
345 ) -> torch.Tensor:
346 hidden_states = self.word_embeddings(input_ids)
347 for i in range(len(self.h)):
@@ -353,7 +350,7 @@ class FalconModel(nn.Module):
350 positions,
351 hidden_states,
352 kv_caches[i],
356 - input_metadata,
353 + attn_metadata,
354 )
355 hidden_states = self.ln_f(hidden_states)
356 return hidden_states
@@ -378,14 +375,14 @@ class FalconForCausalLM(nn.Module):
375 self,
376 input_ids: torch.LongTensor,
377 positions: torch.Tensor,
381 - kv_caches: List[KVCache],
382 - input_metadata: InputMetadata,
378 + kv_caches: List[torch.Tensor],
379 + attn_metadata: AttentionMetadata,
380 ) -> torch.Tensor:
381 hidden_states = self.transformer(
382 input_ids,
383 positions,
384 kv_caches,
388 - input_metadata,
385 + attn_metadata,
386 )
387 return hidden_states
388
vllm/model_executor/models/gemma.py
+13 -17
@@ -20,10 +20,9 @@ import torch
20 from torch import nn
21 from transformers import GemmaConfig
22
23 +from vllm.attention import Attention, AttentionMetadata
24 from vllm.config import LoRAConfig
24 -from vllm.model_executor.input_metadata import InputMetadata
25 from vllm.model_executor.layers.activation import GeluAndMul
26 -from vllm.model_executor.layers.attention import Attention
26 from vllm.model_executor.layers.layernorm import RMSNorm
27 from vllm.model_executor.layers.linear import (LinearMethodBase,
28 MergedColumnParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
44 -KVCache = Tuple[torch.Tensor, torch.Tensor]
45 -
43
44 class GemmaMLP(nn.Module):
45
@@ -133,14 +130,13 @@ class GemmaAttention(nn.Module):
130 self,
131 positions: torch.Tensor,
132 hidden_states: torch.Tensor,
136 - kv_cache: KVCache,
137 - input_metadata: InputMetadata,
133 + kv_cache: torch.Tensor,
134 + attn_metadata: AttentionMetadata,
135 ) -> torch.Tensor:
136 qkv, _ = self.qkv_proj(hidden_states)
137 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138 q, k = self.rotary_emb(positions, q, k)
142 - k_cache, v_cache = kv_cache
143 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
139 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
140 output, _ = self.o_proj(attn_output)
141 return output
142
@@ -177,8 +173,8 @@ class GemmaDecoderLayer(nn.Module):
173 self,
174 positions: torch.Tensor,
175 hidden_states: torch.Tensor,
180 - kv_cache: KVCache,
181 - input_metadata: InputMetadata,
176 + kv_cache: torch.Tensor,
177 + attn_metadata: AttentionMetadata,
178 residual: Optional[torch.Tensor],
179 ) -> Tuple[torch.Tensor, torch.Tensor]:
180 # Self Attention
@@ -192,7 +188,7 @@ class GemmaDecoderLayer(nn.Module):
188 positions=positions,
189 hidden_states=hidden_states,
190 kv_cache=kv_cache,
195 - input_metadata=input_metadata,
191 + attn_metadata=attn_metadata,
192 )
193
194 # Fully Connected
@@ -226,8 +222,8 @@ class GemmaModel(nn.Module):
222 self,
223 input_ids: torch.Tensor,
224 positions: torch.Tensor,
229 - kv_caches: List[KVCache],
230 - input_metadata: InputMetadata,
225 + kv_caches: List[torch.Tensor],
226 + attn_metadata: AttentionMetadata,
227 ) -> torch.Tensor:
228 hidden_states = self.embed_tokens(input_ids)
229 # Normalize the embedding by sqrt(hidden_size)
@@ -240,7 +236,7 @@ class GemmaModel(nn.Module):
236 positions,
237 hidden_states,
238 kv_caches[i],
243 - input_metadata,
239 + attn_metadata,
240 residual,
241 )
242 hidden_states, _ = self.norm(hidden_states, residual)
@@ -290,11 +286,11 @@ class GemmaForCausalLM(nn.Module):
286 self,
287 input_ids: torch.Tensor,
288 positions: torch.Tensor,
293 - kv_caches: List[KVCache],
294 - input_metadata: InputMetadata,
289 + kv_caches: List[torch.Tensor],
290 + attn_metadata: AttentionMetadata,
291 ) -> torch.Tensor:
292 hidden_states = self.model(input_ids, positions, kv_caches,
297 - input_metadata)
293 + attn_metadata)
294 return hidden_states
295
296 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt2.py
+14 -19
@@ -17,15 +17,14 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """Inference-only GPT-2 model compatible with HuggingFace weights."""
20 -from typing import List, Optional, Tuple
20 +from typing import List, Optional
21
22 import torch
23 from torch import nn
24 from transformers import GPT2Config
25
26 -from vllm.model_executor.input_metadata import InputMetadata
26 +from vllm.attention import Attention, AttentionMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import Attention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
44 -KVCache = Tuple[torch.Tensor, torch.Tensor]
45 -
43
44 class GPT2Attention(nn.Module):
45
@@ -79,14 +76,12 @@ class GPT2Attention(nn.Module):
76 def forward(
77 self,
78 hidden_states: torch.Tensor,
82 - kv_cache: KVCache,
83 - input_metadata: InputMetadata,
79 + kv_cache: torch.Tensor,
80 + attn_metadata: AttentionMetadata,
81 ) -> torch.Tensor:
82 qkv, _ = self.c_attn(hidden_states)
83 q, k, v = qkv.chunk(chunks=3, dim=-1)
87 - key_cache, value_cache = kv_cache
88 - attn_output = self.attn(q, k, v, key_cache, value_cache,
89 - input_metadata)
84 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
85 attn_output, _ = self.c_proj(attn_output)
86 return attn_output
87
@@ -144,15 +139,15 @@ class GPT2Block(nn.Module):
139 def forward(
140 self,
141 hidden_states: torch.Tensor,
147 - kv_cache: KVCache,
148 - input_metadata: InputMetadata,
142 + kv_cache: torch.Tensor,
143 + attn_metadata: AttentionMetadata,
144 ) -> torch.Tensor:
145 residual = hidden_states
146 hidden_states = self.ln_1(hidden_states)
147 attn_output = self.attn(
148 hidden_states=hidden_states,
149 kv_cache=kv_cache,
155 - input_metadata=input_metadata,
150 + attn_metadata=attn_metadata,
151 )
152 # residual connection
153 hidden_states = attn_output + residual
@@ -190,8 +185,8 @@ class GPT2Model(nn.Module):
185 self,
186 input_ids: torch.Tensor,
187 position_ids: torch.Tensor,
193 - kv_caches: List[KVCache],
194 - input_metadata: InputMetadata,
188 + kv_caches: List[torch.Tensor],
189 + attn_metadata: AttentionMetadata,
190 ) -> torch.Tensor:
191 inputs_embeds = self.wte(input_ids)
192 position_embeds = self.wpe(position_ids)
@@ -199,7 +194,7 @@ class GPT2Model(nn.Module):
194
195 for i in range(len(self.h)):
196 layer = self.h[i]
202 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
197 + hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
198
199 hidden_states = self.ln_f(hidden_states)
200 return hidden_states
@@ -224,11 +219,11 @@ class GPT2LMHeadModel(nn.Module):
219 self,
220 input_ids: torch.Tensor,
221 positions: torch.Tensor,
227 - kv_caches: List[KVCache],
228 - input_metadata: InputMetadata,
222 + kv_caches: List[torch.Tensor],
223 + attn_metadata: AttentionMetadata,
224 ) -> torch.Tensor:
225 hidden_states = self.transformer(input_ids, positions, kv_caches,
231 - input_metadata)
226 + attn_metadata)
227 return hidden_states
228
229 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt_bigcode.py
+14 -19
@@ -18,15 +18,14 @@
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """Inference-only GPTBigCode model compatible with HuggingFace weights."""
21 -from typing import List, Optional, Tuple
21 +from typing import List, Optional
22
23 import torch
24 from torch import nn
25 from transformers import GPTBigCodeConfig
26
27 -from vllm.model_executor.input_metadata import InputMetadata
27 +from vllm.attention import Attention, AttentionMetadata
28 from vllm.model_executor.layers.activation import get_act_fn
29 -from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
@@ -42,8 +41,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
42 from vllm.sequence import SamplerOutput
43
45 -KVCache = Tuple[torch.Tensor, torch.Tensor]
46 -
44
45 class GPTBigCodeAttention(nn.Module):
46
@@ -94,8 +91,8 @@ class GPTBigCodeAttention(nn.Module):
91 def forward(
92 self,
93 hidden_states: torch.Tensor,
97 - kv_cache: KVCache,
98 - input_metadata: InputMetadata,
94 + kv_cache: torch.Tensor,
95 + attn_metadata: AttentionMetadata,
96 ) -> torch.Tensor:
97 qkv, _ = self.c_attn(hidden_states)
98 q, k, v = qkv.split(
@@ -105,9 +102,7 @@ class GPTBigCodeAttention(nn.Module):
102 ],
103 dim=-1,
104 )
108 - key_cache, value_cache = kv_cache
109 - attn_output = self.attn(q, k, v, key_cache, value_cache,
110 - input_metadata)
105 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
106 attn_output, _ = self.c_proj(attn_output)
107 return attn_output
108
@@ -165,15 +160,15 @@ class GPTBigCodeBlock(nn.Module):
160 def forward(
161 self,
162 hidden_states: torch.Tensor,
168 - kv_cache: KVCache,
169 - input_metadata: InputMetadata,
163 + kv_cache: torch.Tensor,
164 + attn_metadata: AttentionMetadata,
165 ) -> torch.Tensor:
166 residual = hidden_states
167 hidden_states = self.ln_1(hidden_states)
168 attn_output = self.attn(
169 hidden_states=hidden_states,
170 kv_cache=kv_cache,
176 - input_metadata=input_metadata,
171 + attn_metadata=attn_metadata,
172 )
173 # residual connection
174 hidden_states = attn_output + residual
@@ -211,8 +206,8 @@ class GPTBigCodeModel(nn.Module):
206 self,
207 input_ids: torch.Tensor,
208 position_ids: torch.Tensor,
214 - kv_caches: List[KVCache],
215 - input_metadata: InputMetadata,
209 + kv_caches: List[torch.Tensor],
210 + attn_metadata: AttentionMetadata,
211 ) -> torch.Tensor:
212 inputs_embeds = self.wte(input_ids)
213 position_embeds = self.wpe(position_ids)
@@ -220,7 +215,7 @@ class GPTBigCodeModel(nn.Module):
215
216 for i in range(len(self.h)):
217 layer = self.h[i]
223 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
218 + hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
219
220 hidden_states = self.ln_f(hidden_states)
221 return hidden_states
@@ -245,11 +240,11 @@ class GPTBigCodeForCausalLM(nn.Module):
240 self,
241 input_ids: torch.Tensor,
242 positions: torch.Tensor,
248 - kv_caches: List[KVCache],
249 - input_metadata: InputMetadata,
243 + kv_caches: List[torch.Tensor],
244 + attn_metadata: AttentionMetadata,
245 ) -> torch.Tensor:
246 hidden_states = self.transformer(input_ids, positions, kv_caches,
252 - input_metadata)
247 + attn_metadata)
248 return hidden_states
249
250 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt_j.py
+14 -18
@@ -16,15 +16,14 @@
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """Inference-only GPT-J model compatible with HuggingFace weights."""
19 -from typing import List, Optional, Tuple
19 +from typing import List, Optional
20
21 import torch
22 from torch import nn
23 from transformers import GPTJConfig
24
25 -from vllm.model_executor.input_metadata import InputMetadata
25 +from vllm.attention import Attention, AttentionMetadata
26 from vllm.model_executor.layers.activation import get_act_fn
27 -from vllm.model_executor.layers.attention import Attention
27 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28 LinearMethodBase,
29 QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
44 -KVCache = Tuple[torch.Tensor, torch.Tensor]
45 -
43
44 class GPTJAttention(nn.Module):
45
@@ -93,14 +90,13 @@ class GPTJAttention(nn.Module):
90 self,
91 position_ids: torch.Tensor,
92 hidden_states: torch.Tensor,
96 - kv_cache: KVCache,
97 - input_metadata: InputMetadata,
93 + kv_cache: torch.Tensor,
94 + attn_metadata: AttentionMetadata,
95 ) -> torch.Tensor:
96 qkv, _ = self.qkv_proj(hidden_states)
97 q, k, v = qkv.chunk(chunks=3, dim=-1)
98 q, k = self.rotary_emb(position_ids, q, k)
102 - k_cache, v_cache = kv_cache
103 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
99 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
100 attn_output, _ = self.out_proj(attn_output)
101 return attn_output
102
@@ -154,8 +150,8 @@ class GPTJBlock(nn.Module):
150 self,
151 position_ids: torch.Tensor,
152 hidden_states: torch.Tensor,
157 - kv_cache: KVCache,
158 - input_metadata: InputMetadata,
153 + kv_cache: torch.Tensor,
154 + attn_metadata: AttentionMetadata,
155 ) -> torch.Tensor:
156 residual = hidden_states
157 hidden_states = self.ln_1(hidden_states)
@@ -163,7 +159,7 @@ class GPTJBlock(nn.Module):
159 position_ids=position_ids,
160 hidden_states=hidden_states,
161 kv_cache=kv_cache,
166 - input_metadata=input_metadata,
162 + attn_metadata=attn_metadata,
163 )
164 mlp_output = self.mlp(hidden_states)
165 hidden_states = attn_output + mlp_output + residual
@@ -192,8 +188,8 @@ class GPTJModel(nn.Module):
188 self,
189 input_ids: torch.Tensor,
190 position_ids: torch.Tensor,
195 - kv_caches: List[KVCache],
196 - input_metadata: InputMetadata,
191 + kv_caches: List[torch.Tensor],
192 + attn_metadata: AttentionMetadata,
193 ) -> torch.Tensor:
194 hidden_states = self.wte(input_ids)
195 for i in range(len(self.h)):
@@ -202,7 +198,7 @@ class GPTJModel(nn.Module):
198 position_ids,
199 hidden_states,
200 kv_caches[i],
205 - input_metadata,
201 + attn_metadata,
202 )
203 hidden_states = self.ln_f(hidden_states)
204 return hidden_states
@@ -232,11 +228,11 @@ class GPTJForCausalLM(nn.Module):
228 self,
229 input_ids: torch.Tensor,
230 positions: torch.Tensor,
235 - kv_caches: List[KVCache],
236 - input_metadata: InputMetadata,
231 + kv_caches: List[torch.Tensor],
232 + attn_metadata: AttentionMetadata,
233 ) -> torch.Tensor:
234 hidden_states = self.transformer(input_ids, positions, kv_caches,
239 - input_metadata)
235 + attn_metadata)
236 return hidden_states
237
238 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt_neox.py
+14 -18
@@ -16,15 +16,14 @@
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """Inference-only GPT-NeoX model compatible with HuggingFace weights."""
19 -from typing import List, Optional, Tuple
19 +from typing import List, Optional
20
21 import torch
22 from torch import nn
23 from transformers import GPTNeoXConfig
24
25 -from vllm.model_executor.input_metadata import InputMetadata
25 +from vllm.attention import Attention, AttentionMetadata
26 from vllm.model_executor.layers.activation import get_act_fn
27 -from vllm.model_executor.layers.attention import Attention
27 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28 LinearMethodBase,
29 QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
44 -KVCache = Tuple[torch.Tensor, torch.Tensor]
45 -
43
44 class GPTNeoXAttention(nn.Module):
45
@@ -94,14 +91,13 @@ class GPTNeoXAttention(nn.Module):
91 self,
92 position_ids: torch.Tensor,
93 hidden_states: torch.Tensor,
97 - kv_cache: KVCache,
98 - input_metadata: InputMetadata,
94 + kv_cache: torch.Tensor,
95 + attn_metadata: AttentionMetadata,
96 ) -> torch.Tensor:
97 qkv, _ = self.query_key_value(hidden_states)
98 q, k, v = qkv.chunk(chunks=3, dim=-1)
99 q, k = self.rotary_emb(position_ids, q, k)
103 - k_cache, v_cache = kv_cache
104 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
100 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
101 output, _ = self.dense(attn_output)
102 return output
103
@@ -155,15 +151,15 @@ class GPTNeoXLayer(nn.Module):
151 self,
152 position_ids: torch.Tensor,
153 hidden_states: torch.Tensor,
158 - kv_cache: KVCache,
159 - input_metadata: InputMetadata,
154 + kv_cache: torch.Tensor,
155 + attn_metadata: AttentionMetadata,
156 ) -> torch.Tensor:
157 attn_input = self.input_layernorm(hidden_states)
158 attn_output = self.attention(
159 position_ids=position_ids,
160 hidden_states=attn_input,
161 kv_cache=kv_cache,
166 - input_metadata=input_metadata,
162 + attn_metadata=attn_metadata,
163 )
164
165 if self.use_parallel_residual:
@@ -208,8 +204,8 @@ class GPTNeoXModel(nn.Module):
204 self,
205 input_ids: torch.Tensor,
206 position_ids: torch.Tensor,
211 - kv_caches: List[KVCache],
212 - input_metadata: InputMetadata,
207 + kv_caches: List[torch.Tensor],
208 + attn_metadata: AttentionMetadata,
209 ) -> torch.Tensor:
210 hidden_states = self.embed_in(input_ids)
211 for i in range(len(self.layers)):
@@ -218,7 +214,7 @@ class GPTNeoXModel(nn.Module):
214 position_ids,
215 hidden_states,
216 kv_caches[i],
221 - input_metadata,
217 + attn_metadata,
218 )
219 hidden_states = self.final_layer_norm(hidden_states)
220 return hidden_states
@@ -246,11 +242,11 @@ class GPTNeoXForCausalLM(nn.Module):
242 self,
243 input_ids: torch.Tensor,
244 positions: torch.Tensor,
249 - kv_caches: List[KVCache],
250 - input_metadata: InputMetadata,
245 + kv_caches: List[torch.Tensor],
246 + attn_metadata: AttentionMetadata,
247 ) -> torch.Tensor:
248 hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
253 - input_metadata)
249 + attn_metadata)
250 return hidden_states
251
252 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/internlm2.py
+13 -17
@@ -5,9 +5,8 @@ import torch
5 from torch import nn
6 from transformers import PretrainedConfig
7
8 -from vllm.model_executor.input_metadata import InputMetadata
8 +from vllm.attention import Attention, AttentionMetadata
9 from vllm.model_executor.layers.activation import SiluAndMul
10 -from vllm.model_executor.layers.attention import Attention
10 from vllm.model_executor.layers.layernorm import RMSNorm
11 from vllm.model_executor.layers.linear import (LinearMethodBase,
12 MergedColumnParallelLinear,
@@ -25,8 +24,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
24 hf_model_weights_iterator)
25 from vllm.sequence import SamplerOutput
26
28 -KVCache = Tuple[torch.Tensor, torch.Tensor]
29 -
27
28 class InternLM2MLP(nn.Module):
29
@@ -124,14 +121,13 @@ class InternLM2Attention(nn.Module):
121 self,
122 positions: torch.Tensor,
123 hidden_states: torch.Tensor,
127 - kv_cache: KVCache,
128 - input_metadata: InputMetadata,
124 + kv_cache: torch.Tensor,
125 + attn_metadata: AttentionMetadata,
126 ) -> torch.Tensor:
127 qkv, _ = self.wqkv(hidden_states)
128 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
129 q, k = self.rotary_emb(positions, q, k)
133 - k_cache, v_cache = kv_cache
134 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
130 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
131 output, _ = self.wo(attn_output)
132 return output
133
@@ -172,8 +168,8 @@ class InternLMDecoderLayer(nn.Module):
168 self,
169 positions: torch.Tensor,
170 hidden_states: torch.Tensor,
175 - kv_cache: KVCache,
176 - input_metadata: InputMetadata,
171 + kv_cache: torch.Tensor,
172 + attn_metadata: AttentionMetadata,
173 residual: Optional[torch.Tensor],
174 ) -> Tuple[torch.Tensor, torch.Tensor]:
175 # Self Attention
@@ -187,7 +183,7 @@ class InternLMDecoderLayer(nn.Module):
183 positions=positions,
184 hidden_states=hidden_states,
185 kv_cache=kv_cache,
190 - input_metadata=input_metadata,
186 + attn_metadata=attn_metadata,
187 )
188
189 # Fully Connected
@@ -221,8 +217,8 @@ class InternLM2Model(nn.Module):
217 self,
218 input_ids: torch.Tensor,
219 positions: torch.Tensor,
224 - kv_caches: List[KVCache],
225 - input_metadata: InputMetadata,
220 + kv_caches: List[torch.Tensor],
221 + attn_metadata: AttentionMetadata,
222 ) -> torch.Tensor:
223 hidden_states = self.tok_embeddings(input_ids)
224 residual = None
@@ -232,7 +228,7 @@ class InternLM2Model(nn.Module):
228 positions,
229 hidden_states,
230 kv_caches[i],
235 - input_metadata,
231 + attn_metadata,
232 residual,
233 )
234 hidden_states, _ = self.norm(hidden_states, residual)
@@ -258,11 +254,11 @@ class InternLM2ForCausalLM(nn.Module):
254 self,
255 input_ids: torch.Tensor,
256 positions: torch.Tensor,
261 - kv_caches: List[KVCache],
262 - input_metadata: InputMetadata,
257 + kv_caches: List[torch.Tensor],
258 + attn_metadata: AttentionMetadata,
259 ) -> torch.Tensor:
260 hidden_states = self.model(input_ids, positions, kv_caches,
265 - input_metadata)
261 + attn_metadata)
262 return hidden_states
263
264 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/jais.py
+15 -20
@@ -20,14 +20,13 @@
20 """Inference-only Jais model compatible with HuggingFace weights."""
21
22 import math
23 -from typing import List, Optional, Tuple
23 +from typing import List, Optional
24
25 import torch
26 from torch import nn
27 from vllm.transformers_utils.configs import JAISConfig
28
29 -from vllm.model_executor.input_metadata import InputMetadata
30 -from vllm.model_executor.layers.attention import Attention
29 +from vllm.attention import Attention, AttentionMetadata
30 from vllm.model_executor.layers.linear import (
31 ColumnParallelLinear,
32 LinearMethodBase,
@@ -49,8 +48,6 @@ from vllm.model_executor.weight_utils import (
48 from vllm.sequence import SamplerOutput
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50
52 -KVCache = Tuple[torch.Tensor, torch.Tensor]
53 -
51
52 class SwiGLUActivation(nn.Module):
53
@@ -122,14 +119,12 @@ class JAISAttention(nn.Module):
119 def forward(
120 self,
121 hidden_states: torch.Tensor,
125 - kv_cache: KVCache,
126 - input_metadata: InputMetadata,
122 + kv_cache: torch.Tensor,
123 + attn_metadata: AttentionMetadata,
124 ) -> torch.Tensor:
125 qkv, _ = self.c_attn(hidden_states)
126 q, k, v = qkv.chunk(chunks=3, dim=-1)
130 - key_cache, value_cache = kv_cache
131 - attn_output = self.attn(q, k, v, key_cache, value_cache,
132 - input_metadata)
127 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
128 attn_output, _ = self.c_proj(attn_output)
129 return attn_output
130
@@ -196,15 +191,15 @@ class JAISBlock(nn.Module):
191 def forward(
192 self,
193 hidden_states: torch.Tensor,
199 - kv_cache: KVCache,
200 - input_metadata: InputMetadata,
194 + kv_cache: torch.Tensor,
195 + attn_metadata: AttentionMetadata,
196 ) -> torch.Tensor:
197 residual = hidden_states
198 hidden_states = self.ln_1(hidden_states)
199 attn_output = self.attn(
200 hidden_states=hidden_states,
201 kv_cache=kv_cache,
207 - input_metadata=input_metadata,
202 + attn_metadata=attn_metadata,
203 )
204 # residual connection
205 hidden_states = attn_output + residual
@@ -248,8 +243,8 @@ class JAISModel(nn.Module):
243 self,
244 input_ids: torch.Tensor,
245 position_ids: torch.Tensor,
251 - kv_caches: List[KVCache],
252 - input_metadata: InputMetadata,
246 + kv_caches: List[torch.Tensor],
247 + attn_metadata: AttentionMetadata,
248 ) -> torch.Tensor:
249 inputs_embeds = self.wte(input_ids)
250 if self.wpe is not None:
@@ -262,7 +257,7 @@ class JAISModel(nn.Module):
257
258 for i in range(len(self.h)):
259 layer = self.h[i]
265 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
260 + hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
261
262 hidden_states = self.ln_f(hidden_states)
263 return hidden_states
@@ -293,11 +288,11 @@ class JAISLMHeadModel(nn.Module):
288 self,
289 input_ids: torch.Tensor,
290 positions: torch.Tensor,
296 - kv_caches: List[KVCache],
297 - input_metadata: InputMetadata,
291 + kv_caches: List[torch.Tensor],
292 + attn_metadata: AttentionMetadata,
293 ) -> torch.Tensor:
294 hidden_states = self.transformer(input_ids, positions, kv_caches,
300 - input_metadata)
295 + attn_metadata)
296 return hidden_states
297
298 def compute_logits(self, hidden_states: torch.Tensor,
@@ -348,4 +343,4 @@ class JAISLMHeadModel(nn.Module):
343 loaded_weight = loaded_weight.t()
344 weight_loader = getattr(param, "weight_loader",
345 default_weight_loader)
351 - weight_loader(param, loaded_weight)
\ No newline at end of file
346 + weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+13 -17
@@ -27,10 +27,9 @@ import torch
27 from torch import nn
28 from transformers import LlamaConfig
29
30 +from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import LoRAConfig
31 -from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import SiluAndMul
33 -from vllm.model_executor.layers.attention import Attention
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
@@ -48,8 +47,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
47 hf_model_weights_iterator)
48 from vllm.sequence import SamplerOutput
49
51 -KVCache = Tuple[torch.Tensor, torch.Tensor]
52 -
50
51 class LlamaMLP(nn.Module):
52
@@ -150,14 +147,13 @@ class LlamaAttention(nn.Module):
147 self,
148 positions: torch.Tensor,
149 hidden_states: torch.Tensor,
153 - kv_cache: KVCache,
154 - input_metadata: InputMetadata,
150 + kv_cache: torch.Tensor,
151 + attn_metadata: AttentionMetadata,
152 ) -> torch.Tensor:
153 qkv, _ = self.qkv_proj(hidden_states)
154 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
155 q, k = self.rotary_emb(positions, q, k)
159 - k_cache, v_cache = kv_cache
160 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
156 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
157 output, _ = self.o_proj(attn_output)
158 return output
159
@@ -203,8 +199,8 @@ class LlamaDecoderLayer(nn.Module):
199 self,
200 positions: torch.Tensor,
201 hidden_states: torch.Tensor,
206 - kv_cache: KVCache,
207 - input_metadata: InputMetadata,
202 + kv_cache: torch.Tensor,
203 + attn_metadata: AttentionMetadata,
204 residual: Optional[torch.Tensor],
205 ) -> Tuple[torch.Tensor, torch.Tensor]:
206 # Self Attention
@@ -218,7 +214,7 @@ class LlamaDecoderLayer(nn.Module):
214 positions=positions,
215 hidden_states=hidden_states,
216 kv_cache=kv_cache,
221 - input_metadata=input_metadata,
217 + attn_metadata=attn_metadata,
218 )
219
220 # Fully Connected
@@ -258,8 +254,8 @@ class LlamaModel(nn.Module):
254 self,
255 input_ids: torch.Tensor,
256 positions: torch.Tensor,
261 - kv_caches: List[KVCache],
262 - input_metadata: InputMetadata,
257 + kv_caches: List[torch.Tensor],
258 + attn_metadata: AttentionMetadata,
259 ) -> torch.Tensor:
260 hidden_states = self.embed_tokens(input_ids)
261 residual = None
@@ -269,7 +265,7 @@ class LlamaModel(nn.Module):
265 positions,
266 hidden_states,
267 kv_caches[i],
272 - input_metadata,
268 + attn_metadata,
269 residual,
270 )
271 hidden_states, _ = self.norm(hidden_states, residual)
@@ -336,11 +332,11 @@ class LlamaForCausalLM(nn.Module):
332 self,
333 input_ids: torch.Tensor,
334 positions: torch.Tensor,
339 - kv_caches: List[KVCache],
340 - input_metadata: InputMetadata,
335 + kv_caches: List[torch.Tensor],
336 + attn_metadata: AttentionMetadata,
337 ) -> torch.Tensor:
338 hidden_states = self.model(input_ids, positions, kv_caches,
343 - input_metadata)
339 + attn_metadata)
340 return hidden_states
341
342 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/mixtral.py
+14 -18
@@ -21,15 +21,14 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Mixtral model."""
24 -from typing import List, Optional, Tuple
24 +from typing import List, Optional
25
26 import torch
27 from torch import nn
28 from transformers import MixtralConfig
29
30 +from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import LoRAConfig
31 -from vllm.model_executor.input_metadata import InputMetadata
32 -from vllm.model_executor.layers.attention import Attention
32 from vllm.model_executor.layers.fused_moe import fused_moe
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -51,8 +50,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
50 hf_model_weights_iterator)
51 from vllm.sequence import SamplerOutput
52
54 -KVCache = Tuple[torch.Tensor, torch.Tensor]
55 -
53
54 class MixtralMoE(nn.Module):
55 """A tensor-parallel MoE implementation for Mixtral that shards each expert
@@ -209,14 +206,13 @@ class MixtralAttention(nn.Module):
206 self,
207 positions: torch.Tensor,
208 hidden_states: torch.Tensor,
212 - kv_cache: KVCache,
213 - input_metadata: InputMetadata,
209 + kv_cache: torch.Tensor,
210 + attn_metadata: AttentionMetadata,
211 ) -> torch.Tensor:
212 qkv, _ = self.qkv_proj(hidden_states)
213 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
214 q, k = self.rotary_emb(positions, q, k)
218 - k_cache, v_cache = kv_cache
219 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
215 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
216 output, _ = self.o_proj(attn_output)
217 return output
218
@@ -254,8 +250,8 @@ class MixtralDecoderLayer(nn.Module):
250 self,
251 positions: torch.Tensor,
252 hidden_states: torch.Tensor,
257 - kv_cache: KVCache,
258 - input_metadata: InputMetadata,
253 + kv_cache: torch.Tensor,
254 + attn_metadata: AttentionMetadata,
255 residual: Optional[torch.Tensor],
256 ) -> torch.Tensor:
257 # Self Attention
@@ -269,7 +265,7 @@ class MixtralDecoderLayer(nn.Module):
265 positions=positions,
266 hidden_states=hidden_states,
267 kv_cache=kv_cache,
272 - input_metadata=input_metadata,
268 + attn_metadata=attn_metadata,
269 )
270
271 # Fully Connected
@@ -309,15 +305,15 @@ class MixtralModel(nn.Module):
305 self,
306 input_ids: torch.Tensor,
307 positions: torch.Tensor,
312 - kv_caches: List[KVCache],
313 - input_metadata: InputMetadata,
308 + kv_caches: List[torch.Tensor],
309 + attn_metadata: AttentionMetadata,
310 ) -> torch.Tensor:
311 hidden_states = self.embed_tokens(input_ids)
312 residual = None
313 for i in range(len(self.layers)):
314 layer = self.layers[i]
315 hidden_states, residual = layer(positions, hidden_states,
320 - kv_caches[i], input_metadata,
316 + kv_caches[i], attn_metadata,
317 residual)
318 hidden_states, _ = self.norm(hidden_states, residual)
319 return hidden_states
@@ -377,11 +373,11 @@ class MixtralForCausalLM(nn.Module):
373 self,
374 input_ids: torch.Tensor,
375 positions: torch.Tensor,
380 - kv_caches: List[KVCache],
381 - input_metadata: InputMetadata,
376 + kv_caches: List[torch.Tensor],
377 + attn_metadata: AttentionMetadata,
378 ) -> torch.Tensor:
379 hidden_states = self.model(input_ids, positions, kv_caches,
384 - input_metadata)
380 + attn_metadata)
381 return hidden_states
382
383 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/mixtral_quant.py
+14 -18
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Mixtral model."""
24 -from typing import List, Optional, Tuple
24 +from typing import List, Optional
25
26 import numpy as np
27
@@ -31,8 +31,7 @@ import torch.nn.functional as F
31 from torch import nn
32 from transformers import MixtralConfig
33
34 -from vllm.model_executor.input_metadata import InputMetadata
35 -from vllm.model_executor.layers.attention import Attention
34 +from vllm.attention import Attention, AttentionMetadata
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (LinearMethodBase,
37 ReplicatedLinear,
@@ -52,8 +51,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
51 hf_model_weights_iterator)
52 from vllm.sequence import SamplerOutput
53
55 -KVCache = Tuple[torch.Tensor, torch.Tensor]
56 -
54
55 class MixtralMLP(nn.Module):
56
@@ -227,14 +224,13 @@ class MixtralAttention(nn.Module):
224 self,
225 positions: torch.Tensor,
226 hidden_states: torch.Tensor,
230 - kv_cache: KVCache,
231 - input_metadata: InputMetadata,
227 + kv_cache: torch.Tensor,
228 + attn_metadata: AttentionMetadata,
229 ) -> torch.Tensor:
230 qkv, _ = self.qkv_proj(hidden_states)
231 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
232 q, k = self.rotary_emb(positions, q, k)
236 - k_cache, v_cache = kv_cache
237 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
233 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
234 output, _ = self.o_proj(attn_output)
235 return output
236
@@ -269,8 +265,8 @@ class MixtralDecoderLayer(nn.Module):
265 self,
266 positions: torch.Tensor,
267 hidden_states: torch.Tensor,
272 - kv_cache: KVCache,
273 - input_metadata: InputMetadata,
268 + kv_cache: torch.Tensor,
269 + attn_metadata: AttentionMetadata,
270 residual: Optional[torch.Tensor],
271 ) -> torch.Tensor:
272 # Self Attention
@@ -284,7 +280,7 @@ class MixtralDecoderLayer(nn.Module):
280 positions=positions,
281 hidden_states=hidden_states,
282 kv_cache=kv_cache,
287 - input_metadata=input_metadata,
283 + attn_metadata=attn_metadata,
284 )
285
286 # Fully Connected
@@ -319,15 +315,15 @@ class MixtralModel(nn.Module):
315 self,
316 input_ids: torch.Tensor,
317 positions: torch.Tensor,
322 - kv_caches: List[KVCache],
323 - input_metadata: InputMetadata,
318 + kv_caches: List[torch.Tensor],
319 + attn_metadata: AttentionMetadata,
320 ) -> torch.Tensor:
321 hidden_states = self.embed_tokens(input_ids)
322 residual = None
323 for i in range(len(self.layers)):
324 layer = self.layers[i]
325 hidden_states, residual = layer(positions, hidden_states,
330 - kv_caches[i], input_metadata,
326 + kv_caches[i], attn_metadata,
327 residual)
328 hidden_states, _ = self.norm(hidden_states, residual)
329 return hidden_states
@@ -352,11 +348,11 @@ class MixtralForCausalLM(nn.Module):
348 self,
349 input_ids: torch.Tensor,
350 positions: torch.Tensor,
355 - kv_caches: List[KVCache],
356 - input_metadata: InputMetadata,
351 + kv_caches: List[torch.Tensor],
352 + attn_metadata: AttentionMetadata,
353 ) -> torch.Tensor:
354 hidden_states = self.model(input_ids, positions, kv_caches,
359 - input_metadata)
355 + attn_metadata)
356 return hidden_states
357
358 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/mpt.py
+14 -18
@@ -1,14 +1,13 @@
1 # coding=utf-8
2 # Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
3 import math
4 -from typing import List, Optional, Tuple
4 +from typing import List, Optional
5
6 import torch
7 import torch.nn as nn
8
9 -from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.attention import Attention, AttentionMetadata
10 from vllm.model_executor.layers.activation import get_act_fn
11 -from vllm.model_executor.layers.attention import Attention
11 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
12 LinearMethodBase,
13 QKVParallelLinear,
@@ -25,8 +24,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
24 from vllm.sequence import SamplerOutput
25 from vllm.transformers_utils.configs.mpt import MPTConfig
26
28 -KVCache = Tuple[torch.Tensor, torch.Tensor]
29 -
27
28 def _get_alibi_slopes(
29 total_num_heads: int,
@@ -116,8 +113,8 @@ class MPTAttention(nn.Module):
113 self,
114 position_ids: torch.Tensor,
115 hidden_states: torch.Tensor,
119 - kv_cache: KVCache,
120 - input_metadata: InputMetadata,
116 + kv_cache: torch.Tensor,
117 + attn_metadata: AttentionMetadata,
118 ) -> torch.Tensor:
119 del position_ids # unused.
120 qkv, _ = self.Wqkv(hidden_states)
@@ -127,8 +124,7 @@ class MPTAttention(nn.Module):
124 if self.qk_ln:
125 q = self.q_ln(q)
126 k = self.k_ln(k)
130 - k_cache, v_cache = kv_cache
131 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
127 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
128 output, _ = self.out_proj(attn_output)
129 return output
130
@@ -184,15 +180,15 @@ class MPTBlock(nn.Module):
180 self,
181 position_ids: torch.Tensor,
182 hidden_states: torch.Tensor,
187 - kv_cache: KVCache,
188 - input_metadata: InputMetadata,
183 + kv_cache: torch.Tensor,
184 + attn_metadata: AttentionMetadata,
185 ) -> torch.Tensor:
186 x = self.norm_1(hidden_states)
187 x = self.attn(
188 position_ids=position_ids,
189 hidden_states=x,
190 kv_cache=kv_cache,
195 - input_metadata=input_metadata,
191 + attn_metadata=attn_metadata,
192 )
193 hidden_states = hidden_states + x
194 x = self.norm_2(hidden_states)
@@ -230,8 +226,8 @@ class MPTModel(nn.Module):
226 self,
227 input_ids: torch.Tensor,
228 position_ids: torch.Tensor,
233 - kv_caches: List[KVCache],
234 - input_metadata: InputMetadata,
229 + kv_caches: List[torch.Tensor],
230 + attn_metadata: AttentionMetadata,
231 ) -> torch.Tensor:
232 hidden_states = self.wte(input_ids)
233 for i in range(len(self.blocks)):
@@ -240,7 +236,7 @@ class MPTModel(nn.Module):
236 position_ids,
237 hidden_states,
238 kv_caches[i],
243 - input_metadata,
239 + attn_metadata,
240 )
241 hidden_states = self.norm_f(hidden_states)
242 return hidden_states
@@ -267,11 +263,11 @@ class MPTForCausalLM(nn.Module):
263 self,
264 input_ids: torch.Tensor,
265 positions: torch.Tensor,
270 - kv_caches: List[KVCache],
271 - input_metadata: InputMetadata,
266 + kv_caches: List[torch.Tensor],
267 + attn_metadata: AttentionMetadata,
268 ) -> torch.Tensor:
269 hidden_states = self.transformer(input_ids, positions, kv_caches,
274 - input_metadata)
270 + attn_metadata)
271 return hidden_states
272
273 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/olmo.py
+13 -17
@@ -42,8 +42,7 @@ import torch
42 import torch.nn.functional as F
43 from torch import nn
44
45 -from vllm.model_executor.input_metadata import InputMetadata
46 -from vllm.model_executor.layers.attention import Attention
45 +from vllm.attention import Attention, AttentionMetadata
46 from vllm.model_executor.layers.linear import (
47 ColumnParallelLinear,
48 LinearMethodBase,
@@ -67,8 +66,6 @@ from vllm.sequence import SamplerOutput
66 # this model must need this dependency
67 from hf_olmo import OLMoConfig
68
70 -KVCache = Tuple[torch.Tensor, torch.Tensor]
71 -
69
70 class SwiGLU(nn.Module):
71
@@ -146,16 +143,15 @@ class OlmoAttention(nn.Module):
143 self,
144 positions: torch.Tensor,
145 hidden_states: torch.Tensor,
149 - kv_cache: KVCache,
150 - input_metadata: InputMetadata,
146 + kv_cache: torch.Tensor,
147 + attn_metadata: AttentionMetadata,
148 ) -> torch.Tensor:
149 hidden_states = self.attn_norm(hidden_states)
150 qkv, _ = self.att_proj(hidden_states)
151 q, k, v = qkv.chunk(chunks=3, dim=-1)
152 if self.config.rope:
153 q, k = self.rotary_emb(positions, q, k)
157 - k_cache, v_cache = kv_cache
158 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
154 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
155 output, _ = self.attn_out(attn_output)
156 return output
157
@@ -241,12 +237,12 @@ class OlmoBlock(nn.Module):
237 self,
238 positions: torch.Tensor,
239 hidden_states: torch.Tensor,
244 - kv_cache: KVCache,
245 - input_metadata: InputMetadata,
240 + kv_cache: torch.Tensor,
241 + attn_metadata: AttentionMetadata,
242 ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
243 # Attention block.
244 og_x = hidden_states
249 - x = self.attn(positions, hidden_states, kv_cache, input_metadata)
245 + x = self.attn(positions, hidden_states, kv_cache, attn_metadata)
246 x = x + og_x
247
248 # MLP block.
@@ -296,8 +292,8 @@ class OlmoModel(nn.Module):
292 self,
293 input_ids: torch.Tensor,
294 positions: torch.Tensor,
299 - kv_caches: List[KVCache],
300 - input_metadata: InputMetadata,
295 + kv_caches: List[torch.Tensor],
296 + attn_metadata: AttentionMetadata,
297 ) -> torch.Tensor:
298 """
299 :param input_ids: A tensor of shape `(batch_size, seq_len)`.
@@ -313,7 +309,7 @@ class OlmoModel(nn.Module):
309 positions,
310 x,
311 kv_caches[block_idx],
316 - input_metadata,
312 + attn_metadata,
313 )
314
315 # Apply final layer norm.
@@ -344,14 +340,14 @@ class OLMoForCausalLM(nn.Module):
340 self,
341 input_ids: torch.Tensor,
342 positions: torch.Tensor,
347 - kv_caches: List[KVCache],
348 - input_metadata: InputMetadata,
343 + kv_caches: List[torch.Tensor],
344 + attn_metadata: AttentionMetadata,
345 ) -> torch.Tensor:
346 hidden_states = self.model(
347 input_ids=input_ids,
348 positions=positions,
349 kv_caches=kv_caches,
354 - input_metadata=input_metadata,
350 + attn_metadata=attn_metadata,
351 )
352 return hidden_states
353
vllm/model_executor/models/opt.py
+17 -22
@@ -17,15 +17,14 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """Inference-only OPT model compatible with HuggingFace weights."""
20 -from typing import List, Optional, Tuple
20 +from typing import List, Optional
21
22 import torch
23 from torch import nn
24 from transformers import OPTConfig
25
26 -from vllm.model_executor.input_metadata import InputMetadata
26 +from vllm.attention import Attention, AttentionMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import Attention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
@@ -42,8 +41,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
42 from vllm.sequence import SamplerOutput
43
45 -KVCache = Tuple[torch.Tensor, torch.Tensor]
46 -
44
45 class OPTLearnedPositionalEmbedding(nn.Embedding):
46
@@ -97,14 +94,12 @@ class OPTAttention(nn.Module):
94 def forward(
95 self,
96 hidden_states: torch.Tensor,
100 - kv_cache: KVCache,
101 - input_metadata: InputMetadata,
97 + kv_cache: torch.Tensor,
98 + attn_metadata: AttentionMetadata,
99 ) -> torch.Tensor:
100 qkv, _ = self.qkv_proj(hidden_states)
101 q, k, v = qkv.chunk(chunks=3, dim=-1)
105 - key_cache, value_cache = kv_cache
106 - attn_output = self.attn(q, k, v, key_cache, value_cache,
107 - input_metadata)
102 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
103 output, _ = self.out_proj(attn_output)
104 return output
105
@@ -152,8 +147,8 @@ class OPTDecoderLayer(nn.Module):
147 def forward(
148 self,
149 hidden_states: torch.Tensor,
155 - kv_cache: KVCache,
156 - input_metadata: InputMetadata,
150 + kv_cache: torch.Tensor,
151 + attn_metadata: AttentionMetadata,
152 ) -> torch.Tensor:
153 # Self Attention
154 residual = hidden_states
@@ -162,7 +157,7 @@ class OPTDecoderLayer(nn.Module):
157 hidden_states = self.self_attn_layer_norm(hidden_states)
158 hidden_states = self.self_attn(hidden_states=hidden_states,
159 kv_cache=kv_cache,
165 - input_metadata=input_metadata)
160 + attn_metadata=attn_metadata)
161 hidden_states = residual + hidden_states
162 # 350m applies layer norm AFTER attention
163 if not self.do_layer_norm_before:
@@ -241,8 +236,8 @@ class OPTDecoder(nn.Module):
236 self,
237 input_ids: torch.Tensor,
238 positions: torch.Tensor,
244 - kv_caches: List[KVCache],
245 - input_metadata: InputMetadata,
239 + kv_caches: List[torch.Tensor],
240 + attn_metadata: AttentionMetadata,
241 ) -> torch.Tensor:
242 inputs_embeds = self.embed_tokens(input_ids)
243 pos_embeds = self.embed_positions(positions)
@@ -252,7 +247,7 @@ class OPTDecoder(nn.Module):
247
248 for i in range(len(self.layers)):
249 layer = self.layers[i]
255 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
250 + hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
251
252 if self.final_layer_norm is not None:
253 hidden_states = self.final_layer_norm(hidden_states)
@@ -275,10 +270,10 @@ class OPTModel(nn.Module):
270 self,
271 input_ids: torch.Tensor,
272 positions: torch.Tensor,
278 - kv_caches: List[KVCache],
279 - input_metadata: InputMetadata,
273 + kv_caches: List[torch.Tensor],
274 + attn_metadata: AttentionMetadata,
275 ) -> torch.Tensor:
281 - return self.decoder(input_ids, positions, kv_caches, input_metadata)
276 + return self.decoder(input_ids, positions, kv_caches, attn_metadata)
277
278
279 class OPTForCausalLM(nn.Module):
@@ -300,11 +295,11 @@ class OPTForCausalLM(nn.Module):
295 self,
296 input_ids: torch.Tensor,
297 positions: torch.Tensor,
303 - kv_caches: List[KVCache],
304 - input_metadata: InputMetadata,
298 + kv_caches: List[torch.Tensor],
299 + attn_metadata: AttentionMetadata,
300 ) -> torch.Tensor:
301 hidden_states = self.model(input_ids, positions, kv_caches,
307 - input_metadata)
302 + attn_metadata)
303 return hidden_states
304
305 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/orion.py
+13 -17
@@ -10,9 +10,8 @@ import torch
10 from torch import nn
11 from transformers import PretrainedConfig
12
13 -from vllm.model_executor.input_metadata import InputMetadata
13 +from vllm.attention import Attention, AttentionMetadata
14 from vllm.model_executor.layers.activation import SiluAndMul
15 -from vllm.model_executor.layers.attention import Attention
15 from vllm.model_executor.layers.linear import (LinearMethodBase,
16 MergedColumnParallelLinear,
17 QKVParallelLinear,
@@ -29,8 +28,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
28 hf_model_weights_iterator)
29 from vllm.sequence import SamplerOutput
30
32 -KVCache = Tuple[torch.Tensor, torch.Tensor]
33 -
31
32 class OrionMLP(nn.Module):
33
@@ -128,14 +125,13 @@ class OrionAttention(nn.Module):
125 self,
126 positions: torch.Tensor,
127 hidden_states: torch.Tensor,
131 - kv_cache: KVCache,
132 - input_metadata: InputMetadata,
128 + kv_cache: torch.Tensor,
129 + attn_metadata: AttentionMetadata,
130 ) -> torch.Tensor:
131 qkv, _ = self.qkv_proj(hidden_states)
132 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
133 q, k = self.rotary_emb(positions, q, k)
137 - k_cache, v_cache = kv_cache
138 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
135 output, _ = self.o_proj(attn_output)
136 return output
137
@@ -178,8 +174,8 @@ class OrionDecoderLayer(nn.Module):
174 self,
175 positions: torch.Tensor,
176 hidden_states: torch.Tensor,
181 - kv_cache: KVCache,
182 - input_metadata: InputMetadata,
177 + kv_cache: torch.Tensor,
178 + attn_metadata: AttentionMetadata,
179 residual: Optional[torch.Tensor],
180 ) -> Tuple[torch.Tensor, torch.Tensor]:
181 # Self Attention
@@ -189,7 +185,7 @@ class OrionDecoderLayer(nn.Module):
185 positions=positions,
186 hidden_states=hidden_states,
187 kv_cache=kv_cache,
192 - input_metadata=input_metadata,
188 + attn_metadata=attn_metadata,
189 )
190
191 hidden_states = residual + hidden_states
@@ -227,8 +223,8 @@ class OrionModel(nn.Module):
223 self,
224 input_ids: torch.Tensor,
225 positions: torch.Tensor,
230 - kv_caches: List[KVCache],
231 - input_metadata: InputMetadata,
226 + kv_caches: List[torch.Tensor],
227 + attn_metadata: AttentionMetadata,
228 ) -> torch.Tensor:
229 hidden_states = self.embed_tokens(input_ids)
230 residual = None
@@ -238,7 +234,7 @@ class OrionModel(nn.Module):
234 positions,
235 hidden_states,
236 kv_caches[i],
241 - input_metadata,
237 + attn_metadata,
238 residual,
239 )
240 hidden_states = self.norm(hidden_states)
@@ -264,11 +260,11 @@ class OrionForCausalLM(nn.Module):
260 self,
261 input_ids: torch.Tensor,
262 positions: torch.Tensor,
267 - kv_caches: List[KVCache],
268 - input_metadata: InputMetadata,
263 + kv_caches: List[torch.Tensor],
264 + attn_metadata: AttentionMetadata,
265 ) -> torch.Tensor:
266 hidden_states = self.model(input_ids, positions, kv_caches,
271 - input_metadata)
267 + attn_metadata)
268 return hidden_states
269
270 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/phi.py
+14 -18
@@ -35,15 +35,14 @@
35 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 """Inference-only Phi-1.5 model compatible with HuggingFace weights."""
38 -from typing import List, Optional, Tuple
38 +from typing import List, Optional
39
40 import torch
41 from torch import nn
42 from transformers import PretrainedConfig
43
44 -from vllm.model_executor.input_metadata import InputMetadata
44 +from vllm.attention import Attention, AttentionMetadata
45 from vllm.model_executor.layers.activation import get_act_fn
46 -from vllm.model_executor.layers.attention import Attention
46 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
47 LinearMethodBase,
48 QKVParallelLinear,
@@ -60,8 +59,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
59 hf_model_weights_iterator)
60 from vllm.sequence import SamplerOutput
61
63 -KVCache = Tuple[torch.Tensor, torch.Tensor]
64 -
62
63 class PhiAttention(nn.Module):
64
@@ -115,14 +112,13 @@ class PhiAttention(nn.Module):
112 self,
113 position_ids: torch.Tensor,
114 hidden_states: torch.Tensor,
118 - kv_cache: KVCache,
119 - input_metadata: InputMetadata,
115 + kv_cache: torch.Tensor,
116 + attn_metadata: AttentionMetadata,
117 ) -> torch.Tensor:
118 qkv, _ = self.qkv_proj(hidden_states)
119 q, k, v = qkv.chunk(chunks=3, dim=-1)
120 q, k = self.rotary_emb(position_ids, q, k)
124 - k_cache, v_cache = kv_cache
125 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
122 output, _ = self.dense(attn_output)
123 return output
124
@@ -172,8 +168,8 @@ class PhiLayer(nn.Module):
168 self,
169 position_ids: torch.Tensor,
170 hidden_states: torch.Tensor,
175 - kv_cache: KVCache,
176 - input_metadata: InputMetadata,
171 + kv_cache: torch.Tensor,
172 + attn_metadata: AttentionMetadata,
173 ) -> torch.Tensor:
174 residual = hidden_states
175 hidden_states = self.input_layernorm(hidden_states)
@@ -181,7 +177,7 @@ class PhiLayer(nn.Module):
177 position_ids=position_ids,
178 hidden_states=hidden_states,
179 kv_cache=kv_cache,
184 - input_metadata=input_metadata,
180 + attn_metadata=attn_metadata,
181 )
182 feed_forward_hidden_states = self.mlp(hidden_states)
183 hidden_states = attn_outputs + feed_forward_hidden_states + residual
@@ -209,8 +205,8 @@ class PhiModel(nn.Module):
205 self,
206 input_ids: torch.Tensor,
207 positions: torch.Tensor,
212 - kv_caches: List[KVCache],
213 - input_metadata: InputMetadata,
208 + kv_caches: List[torch.Tensor],
209 + attn_metadata: AttentionMetadata,
210 ) -> torch.Tensor:
211 hidden_states = self.embed_tokens(input_ids)
212 for i in range(self.config.num_hidden_layers):
@@ -219,7 +215,7 @@ class PhiModel(nn.Module):
215 positions,
216 hidden_states,
217 kv_caches[i],
222 - input_metadata,
218 + attn_metadata,
219 )
220
221 hidden_states = self.final_layernorm(hidden_states)
@@ -248,11 +244,11 @@ class PhiForCausalLM(nn.Module):
244 self,
245 input_ids: torch.Tensor,
246 positions: torch.Tensor,
251 - kv_caches: List[KVCache],
252 - input_metadata: InputMetadata,
247 + kv_caches: List[torch.Tensor],
248 + attn_metadata: AttentionMetadata,
249 ) -> torch.Tensor:
250 hidden_states = self.model(input_ids, positions, kv_caches,
255 - input_metadata)
251 + attn_metadata)
252
253 return hidden_states
254
vllm/model_executor/models/qwen.py
+13 -18
@@ -10,9 +10,8 @@ import torch
10 from torch import nn
11 from transformers import PretrainedConfig
12
13 -from vllm.model_executor.input_metadata import InputMetadata
13 +from vllm.attention import Attention, AttentionMetadata
14 from vllm.model_executor.layers.activation import SiluAndMul
15 -from vllm.model_executor.layers.attention import Attention
15 from vllm.model_executor.layers.layernorm import RMSNorm
16 from vllm.model_executor.layers.linear import (LinearMethodBase,
17 MergedColumnParallelLinear,
@@ -30,8 +29,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
29 hf_model_weights_iterator)
30 from vllm.sequence import SamplerOutput
31
33 -KVCache = Tuple[torch.Tensor, torch.Tensor]
34 -
32
33 class QWenMLP(nn.Module):
34
@@ -111,15 +108,13 @@ class QWenAttention(nn.Module):
108 self,
109 positions: torch.Tensor,
110 hidden_states: torch.Tensor,
114 - kv_cache: KVCache,
115 - input_metadata: InputMetadata,
111 + kv_cache: torch.Tensor,
112 + attn_metadata: AttentionMetadata,
113 ) -> torch.Tensor:
114 qkv, _ = self.c_attn(hidden_states)
115 q, k, v = qkv.chunk(chunks=3, dim=-1)
116 q, k = self.rotary_emb(positions, q, k)
120 - k_cache, v_cache = kv_cache
121 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
122 -
117 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
118 output, _ = self.c_proj(attn_output)
119 return output
120
@@ -153,8 +148,8 @@ class QWenBlock(nn.Module):
148 self,
149 positions: torch.Tensor,
150 hidden_states: torch.Tensor,
156 - kv_cache: KVCache,
157 - input_metadata: InputMetadata,
151 + kv_cache: torch.Tensor,
152 + attn_metadata: AttentionMetadata,
153 residual: Optional[torch.Tensor],
154 ) -> Tuple[torch.Tensor, torch.Tensor]:
155 # Self Attention
@@ -167,7 +162,7 @@ class QWenBlock(nn.Module):
162 positions=positions,
163 hidden_states=hidden_states,
164 kv_cache=kv_cache,
170 - input_metadata=input_metadata,
165 + attn_metadata=attn_metadata,
166 )
167
168 # Fully Connected
@@ -201,8 +196,8 @@ class QWenModel(nn.Module):
196 self,
197 input_ids: torch.Tensor,
198 positions: torch.Tensor,
204 - kv_caches: List[KVCache],
205 - input_metadata: InputMetadata,
199 + kv_caches: List[torch.Tensor],
200 + attn_metadata: AttentionMetadata,
201 ) -> torch.Tensor:
202 hidden_states = self.wte(input_ids)
203 residual = None
@@ -212,7 +207,7 @@ class QWenModel(nn.Module):
207 positions,
208 hidden_states,
209 kv_caches[i],
215 - input_metadata,
210 + attn_metadata,
211 residual,
212 )
213 hidden_states, _ = self.ln_f(hidden_states, residual)
@@ -238,11 +233,11 @@ class QWenLMHeadModel(nn.Module):
233 self,
234 input_ids: torch.Tensor,
235 positions: torch.Tensor,
241 - kv_caches: List[KVCache],
242 - input_metadata: InputMetadata,
236 + kv_caches: List[torch.Tensor],
237 + attn_metadata: AttentionMetadata,
238 ) -> torch.Tensor:
239 hidden_states = self.transformer(input_ids, positions, kv_caches,
245 - input_metadata)
240 + attn_metadata)
241 return hidden_states
242
243 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/qwen2.py
+13 -17
@@ -28,9 +28,8 @@ import torch
28 from torch import nn
29 from transformers import Qwen2Config
30
31 -from vllm.model_executor.input_metadata import InputMetadata
31 +from vllm.attention import Attention, AttentionMetadata
32 from vllm.model_executor.layers.activation import SiluAndMul
33 -from vllm.model_executor.layers.attention import Attention
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
@@ -49,8 +48,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
48 from vllm.sequence import SamplerOutput
49 from vllm.config import LoRAConfig
50
52 -KVCache = Tuple[torch.Tensor, torch.Tensor]
53 -
51
52 class Qwen2MLP(nn.Module):
53
@@ -147,14 +144,13 @@ class Qwen2Attention(nn.Module):
144 self,
145 positions: torch.Tensor,
146 hidden_states: torch.Tensor,
150 - kv_cache: KVCache,
151 - input_metadata: InputMetadata,
147 + kv_cache: torch.Tensor,
148 + attn_metadata: AttentionMetadata,
149 ) -> torch.Tensor:
150 qkv, _ = self.qkv_proj(hidden_states)
151 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 q, k = self.rotary_emb(positions, q, k)
156 - k_cache, v_cache = kv_cache
157 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
153 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
154 output, _ = self.o_proj(attn_output)
155 return output
156
@@ -197,8 +193,8 @@ class Qwen2DecoderLayer(nn.Module):
193 self,
194 positions: torch.Tensor,
195 hidden_states: torch.Tensor,
200 - kv_cache: KVCache,
201 - input_metadata: InputMetadata,
196 + kv_cache: torch.Tensor,
197 + attn_metadata: AttentionMetadata,
198 residual: Optional[torch.Tensor],
199 ) -> Tuple[torch.Tensor, torch.Tensor]:
200 # Self Attention
@@ -212,7 +208,7 @@ class Qwen2DecoderLayer(nn.Module):
208 positions=positions,
209 hidden_states=hidden_states,
210 kv_cache=kv_cache,
215 - input_metadata=input_metadata,
211 + attn_metadata=attn_metadata,
212 )
213
214 # Fully Connected
@@ -248,8 +244,8 @@ class Qwen2Model(nn.Module):
244 self,
245 input_ids: torch.Tensor,
246 positions: torch.Tensor,
251 - kv_caches: List[KVCache],
252 - input_metadata: InputMetadata,
247 + kv_caches: List[torch.Tensor],
248 + attn_metadata: AttentionMetadata,
249 ) -> torch.Tensor:
250 hidden_states = self.embed_tokens(input_ids)
251 residual = None
@@ -259,7 +255,7 @@ class Qwen2Model(nn.Module):
255 positions,
256 hidden_states,
257 kv_caches[i],
262 - input_metadata,
258 + attn_metadata,
259 residual,
260 )
261 hidden_states, _ = self.norm(hidden_states, residual)
@@ -315,11 +311,11 @@ class Qwen2ForCausalLM(nn.Module):
311 self,
312 input_ids: torch.Tensor,
313 positions: torch.Tensor,
318 - kv_caches: List[KVCache],
319 - input_metadata: InputMetadata,
314 + kv_caches: List[torch.Tensor],
315 + attn_metadata: AttentionMetadata,
316 ) -> torch.Tensor:
317 hidden_states = self.model(input_ids, positions, kv_caches,
322 - input_metadata)
318 + attn_metadata)
319 return hidden_states
320
321 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/stablelm.py
+13 -17
@@ -25,9 +25,8 @@ import torch
25 from torch import nn
26 from transformers import PretrainedConfig
27
28 -from vllm.model_executor.input_metadata import InputMetadata
28 +from vllm.attention import Attention, AttentionMetadata
29 from vllm.model_executor.layers.activation import SiluAndMul
30 -from vllm.model_executor.layers.attention import Attention
30 from vllm.model_executor.layers.linear import (LinearMethodBase,
31 MergedColumnParallelLinear,
32 QKVParallelLinear,
@@ -44,8 +43,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
43 hf_model_weights_iterator)
44 from vllm.sequence import SamplerOutput
45
47 -KVCache = Tuple[torch.Tensor, torch.Tensor]
48 -
46
47 class StablelmMLP(nn.Module):
48
@@ -134,14 +131,13 @@ class StablelmAttention(nn.Module):
131 self,
132 positions: torch.Tensor,
133 hidden_states: torch.Tensor,
137 - kv_cache: KVCache,
138 - input_metadata: InputMetadata,
134 + kv_cache: torch.Tensor,
135 + attn_metadata: AttentionMetadata,
136 ) -> torch.Tensor:
137 qkv, _ = self.qkv_proj(hidden_states)
138 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
139 q, k = self.rotary_emb(positions, q, k)
143 - k_cache, v_cache = kv_cache
144 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
140 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
141 output, _ = self.o_proj(attn_output)
142 return output
143
@@ -166,8 +162,8 @@ class StablelmDecoderLayer(nn.Module):
162 self,
163 positions: torch.Tensor,
164 hidden_states: torch.Tensor,
169 - kv_cache: KVCache,
170 - input_metadata: InputMetadata,
165 + kv_cache: torch.Tensor,
166 + attn_metadata: AttentionMetadata,
167 ) -> Tuple[torch.Tensor, torch.Tensor]:
168 # Self Attention
169 residual = hidden_states
@@ -176,7 +172,7 @@ class StablelmDecoderLayer(nn.Module):
172 positions=positions,
173 hidden_states=hidden_states,
174 kv_cache=kv_cache,
179 - input_metadata=input_metadata,
175 + attn_metadata=attn_metadata,
176 )
177 hidden_states = residual + hidden_states
178
@@ -211,8 +207,8 @@ class StableLMEpochModel(nn.Module):
207 self,
208 input_ids: torch.Tensor,
209 positions: torch.Tensor,
214 - kv_caches: List[KVCache],
215 - input_metadata: InputMetadata,
210 + kv_caches: List[torch.Tensor],
211 + attn_metadata: AttentionMetadata,
212 ) -> torch.Tensor:
213 hidden_states = self.embed_tokens(input_ids)
214 for i in range(len(self.layers)):
@@ -221,7 +217,7 @@ class StableLMEpochModel(nn.Module):
217 positions,
218 hidden_states,
219 kv_caches[i],
224 - input_metadata,
220 + attn_metadata,
221 )
222 hidden_states = self.norm(hidden_states)
223 return hidden_states
@@ -246,11 +242,11 @@ class StablelmForCausalLM(nn.Module):
242 self,
243 input_ids: torch.Tensor,
244 positions: torch.Tensor,
249 - kv_caches: List[KVCache],
250 - input_metadata: InputMetadata,
245 + kv_caches: List[torch.Tensor],
246 + attn_metadata: AttentionMetadata,
247 ) -> torch.Tensor:
248 hidden_states = self.model(input_ids, positions, kv_caches,
253 - input_metadata)
249 + attn_metadata)
250 return hidden_states
251
252 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/starcoder2.py
+14 -18
@@ -18,15 +18,14 @@
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """ PyTorch Starcoder2 model."""
21 -from typing import List, Optional, Tuple
21 +from typing import List, Optional
22
23 import torch
24 from torch import nn
25 from transformers import Starcoder2Config
26
27 -from vllm.model_executor.input_metadata import InputMetadata
27 +from vllm.attention import Attention, AttentionMetadata
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 -from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.rotary_embedding import get_rope
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -43,8 +42,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
42 hf_model_weights_iterator)
43 from vllm.sequence import SamplerOutput
44
46 -KVCache = Tuple[torch.Tensor, torch.Tensor]
47 -
45
46 class Starcoder2Attention(nn.Module):
47
@@ -111,14 +108,13 @@ class Starcoder2Attention(nn.Module):
108 self,
109 positions: torch.Tensor,
110 hidden_states: torch.Tensor,
114 - kv_cache: KVCache,
115 - input_metadata: InputMetadata,
111 + kv_cache: torch.Tensor,
112 + attn_metadata: AttentionMetadata,
113 ) -> torch.Tensor:
114 qkv, _ = self.qkv_proj(hidden_states)
115 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
116 q, k = self.rotary_emb(positions, q, k)
120 - k_cache, v_cache = kv_cache
121 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
117 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
118 output, _ = self.o_proj(attn_output)
119 return output
120
@@ -171,8 +167,8 @@ class Starcoder2DecoderLayer(nn.Module):
167 self,
168 positions: torch.Tensor,
169 hidden_states: torch.Tensor,
174 - kv_cache: KVCache,
175 - input_metadata: InputMetadata,
170 + kv_cache: torch.Tensor,
171 + attn_metadata: AttentionMetadata,
172 ) -> torch.Tensor:
173 # Self Attention
174 residual = hidden_states
@@ -181,7 +177,7 @@ class Starcoder2DecoderLayer(nn.Module):
177 positions=positions,
178 hidden_states=hidden_states,
179 kv_cache=kv_cache,
184 - input_metadata=input_metadata,
180 + attn_metadata=attn_metadata,
181 )
182 hidden_states = residual + hidden_states
183
@@ -217,14 +213,14 @@ class Starcoder2Model(nn.Module):
213 self,
214 input_ids: torch.Tensor,
215 positions: torch.Tensor,
220 - kv_caches: List[KVCache],
221 - input_metadata: InputMetadata,
216 + kv_caches: List[torch.Tensor],
217 + attn_metadata: AttentionMetadata,
218 ) -> torch.Tensor:
219 hidden_states = self.embed_tokens(input_ids)
220 for i in range(len(self.layers)):
221 layer = self.layers[i]
222 hidden_states = layer(positions, hidden_states, kv_caches[i],
227 - input_metadata)
223 + attn_metadata)
224 hidden_states = self.norm(hidden_states)
225 return hidden_states
226
@@ -258,11 +254,11 @@ class Starcoder2ForCausalLM(nn.Module):
254 self,
255 input_ids: torch.Tensor,
256 positions: torch.Tensor,
261 - kv_caches: List[KVCache],
262 - input_metadata: InputMetadata,
257 + kv_caches: List[torch.Tensor],
258 + attn_metadata: AttentionMetadata,
259 ) -> torch.Tensor:
260 hidden_states = self.model(input_ids, positions, kv_caches,
265 - input_metadata)
261 + attn_metadata)
262 return hidden_states
263
264 def compute_logits(self, hidden_states: torch.Tensor,
vllm/sequence.py
+1 -1
@@ -431,7 +431,7 @@ class SequenceGroup:
431
432
433 class SequenceGroupMetadata:
434 - """Metadata for a sequence group. Used to create `InputMetadata`.
434 + """Metadata for a sequence group. Used to create `AttentionMetadata`.
435
436 Args:
437 request_id: The ID of the request.
vllm/worker/cache_engine.py
+31 -84
@@ -1,16 +1,15 @@
1 """CacheEngine class for managing the KV cache."""
2 -from typing import Dict, List, Tuple
2 +from typing import Dict, List
3
4 import torch
5
6 +from vllm.attention import get_attn_backend
7 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8 from vllm.logger import init_logger
9 from vllm.utils import is_pin_memory_available, STR_DTYPE_TO_TORCH_DTYPE
10
11 logger = init_logger(__name__)
12
12 -KVCache = Tuple[torch.Tensor, torch.Tensor]
13 -
13
14 class CacheEngine:
15 """Manages the KV cache.
@@ -43,95 +42,43 @@ class CacheEngine:
42 else:
43 self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
44
45 + # Get attention backend.
46 + self.attn_backend = get_attn_backend(model_config.dtype)
47 +
48 # Initialize the cache.
47 - self.gpu_cache = self.allocate_gpu_cache()
48 - self.cpu_cache = self.allocate_cpu_cache()
49 -
50 - def get_key_block_shape(self) -> Tuple[int, int, int, int]:
51 - element_size = torch.tensor([], dtype=self.dtype).element_size()
52 - x = 16 // element_size
53 - return (
54 - self.num_heads,
55 - self.head_size // x,
56 - self.block_size,
57 - x,
58 - )
59 -
60 - def get_value_block_shape(self) -> Tuple[int, int, int]:
61 - return (
62 - self.num_heads,
63 - self.head_size,
64 - self.block_size,
65 - )
66 -
67 - def allocate_gpu_cache(self) -> List[KVCache]:
68 - gpu_cache: List[KVCache] = []
69 - key_block_shape = self.get_key_block_shape()
70 - value_block_shape = self.get_value_block_shape()
71 - for _ in range(self.num_layers):
72 - key_blocks = torch.empty(
73 - size=(self.num_gpu_blocks, *key_block_shape),
74 - dtype=self.dtype,
75 - device="cuda",
76 - )
77 - value_blocks = torch.empty(
78 - size=(self.num_gpu_blocks, *value_block_shape),
79 - dtype=self.dtype,
80 - device="cuda",
81 - )
82 - gpu_cache.append((key_blocks, value_blocks))
83 - return gpu_cache
84 -
85 - def allocate_cpu_cache(self) -> List[KVCache]:
86 - cpu_cache: List[KVCache] = []
87 - key_block_shape = self.get_key_block_shape()
88 - value_block_shape = self.get_value_block_shape()
89 - pin_memory = is_pin_memory_available()
90 - for _ in range(self.num_layers):
91 - key_blocks = torch.empty(
92 - size=(self.num_cpu_blocks, *key_block_shape),
93 - dtype=self.dtype,
94 - pin_memory=pin_memory,
95 - device="cpu",
96 - )
97 - value_blocks = torch.empty(
98 - size=(self.num_cpu_blocks, *value_block_shape),
99 - dtype=self.dtype,
100 - pin_memory=pin_memory,
101 - device="cpu",
102 - )
103 - cpu_cache.append((key_blocks, value_blocks))
104 - return cpu_cache
105 -
106 - def _swap(
107 - self,
108 - src: List[KVCache],
109 - dst: List[KVCache],
110 - src_to_dst: Dict[int, int],
111 - ) -> None:
112 - from vllm._C import cache_ops
49 + self.gpu_cache = self._allocate_kv_cache(self.num_gpu_blocks, "cuda")
50 + self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks, "cpu")
51
114 - for i in range(self.num_layers):
115 - src_key_cache, src_value_cache = src[i]
116 - dst_key_cache, dst_value_cache = dst[i]
117 - # Copy the key blocks.
118 - cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
119 - # Copy the value blocks.
120 - cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
52 + def _allocate_kv_cache(
53 + self,
54 + num_blocks: int,
55 + device: str,
56 + ) -> List[torch.Tensor]:
57 + """Allocates KV cache on the specified device."""
58 + kv_cache_shape = self.attn_backend.get_kv_cache_shape(
59 + num_blocks, self.block_size, self.num_heads, self.head_size)
60 + pin_memory = is_pin_memory_available() if device == "cpu" else False
61 + kv_cache: List[torch.Tensor] = []
62 + for _ in range(self.num_layers):
63 + kv_cache.append(
64 + torch.empty(kv_cache_shape,
65 + dtype=self.dtype,
66 + pin_memory=pin_memory,
67 + device=device))
68 + return kv_cache
69
70 def swap_in(self, src_to_dst: Dict[int, int]) -> None:
123 - self._swap(self.cpu_cache, self.gpu_cache, src_to_dst)
71 + for i in range(self.num_layers):
72 + self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
73 + src_to_dst)
74
75 def swap_out(self, src_to_dst: Dict[int, int]) -> None:
126 - self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
76 + for i in range(self.num_layers):
77 + self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
78 + src_to_dst)
79
80 def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
129 - from vllm._C import cache_ops
130 -
131 - key_caches = [key_cache for key_cache, _ in self.gpu_cache]
132 - value_caches = [value_cache for _, value_cache in self.gpu_cache]
133 - # NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
134 - cache_ops.copy_blocks(key_caches, value_caches, src_to_dsts)
81 + self.attn_backend.copy_blocks(self.gpu_cache, src_to_dsts)
82
83 @staticmethod
84 def get_cache_block_size(
vllm/worker/model_runner.py
+44 -41
@@ -6,10 +6,11 @@ import numpy as np
6 import torch
7 import torch.nn as nn
8
9 +from vllm.attention import AttentionMetadata, get_attn_backend
10 from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
11 SchedulerConfig)
12 from vllm.logger import init_logger
12 -from vllm.model_executor import InputMetadata, SamplingMetadata
13 +from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader import get_model
15 from vllm.model_executor.parallel_utils import cupy_utils
16 from vllm.model_executor.parallel_utils.communication_op import (
@@ -28,7 +29,6 @@ from vllm.utils import (async_tensor_h2d, CudaMemoryProfiler,
29
30 logger = init_logger(__name__)
31
31 -KVCache = Tuple[torch.Tensor, torch.Tensor]
32 _PAD_SLOT_ID = -1
33 LORA_WARMUP_RANK = 8
34 _BATCH_SIZE_ALIGNMENT = 8
@@ -85,6 +85,9 @@ class ModelRunner:
85 self.pin_memory = is_pin_memory_available()
86 self.kv_cache_dtype = kv_cache_dtype
87
88 + self.attn_backend = get_attn_backend(
89 + self.model_config.dtype if model_config is not None else None)
90 +
91 def load_model(self) -> None:
92 with CudaMemoryProfiler() as m:
93 self.model = get_model(self.model_config,
@@ -127,8 +130,8 @@ class ModelRunner:
130 def _prepare_prompt(
131 self,
132 seq_group_metadata_list: List[SequenceGroupMetadata],
130 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
131 - List[int], List[int], Set[LoRARequest]]:
133 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
134 + List[int], List[int], List[int], Set[LoRARequest]]:
135 assert len(seq_group_metadata_list) > 0
136 input_tokens: List[int] = []
137 input_positions: List[int] = []
@@ -216,7 +219,7 @@ class ModelRunner:
219 slot_mapping.append(slot)
220
221 max_subquery_len = max(subquery_lens)
219 - max_seq_len = max(prompt_lens)
222 + max_prompt_len = max(prompt_lens)
223 num_prompt_tokens = len(input_tokens)
224 assert max_subquery_len > 0
225
@@ -270,7 +273,7 @@ class ModelRunner:
273 dtype=seq_start_loc.dtype,
274 out=seq_start_loc[1:])
275
273 - input_metadata = InputMetadata(
276 + attn_metadata = self.attn_backend.make_metadata(
277 is_prompt=True,
278 slot_mapping=slot_mapping,
279 prompt_lens=prompt_lens,
@@ -279,7 +282,7 @@ class ModelRunner:
282 num_generation_tokens=0,
283 max_subquery_len=max_subquery_len,
284 max_context_len=None,
282 - max_seq_len=max_seq_len,
285 + max_prompt_len=max_prompt_len,
286 subquery_start_loc=subquery_start_loc,
287 seq_start_loc=seq_start_loc,
288 context_lens=context_lens_tensor,
@@ -287,15 +290,15 @@ class ModelRunner:
290 use_cuda_graph=False,
291 kv_cache_dtype=self.kv_cache_dtype,
292 )
290 - return (input_tokens, input_positions, input_metadata, prompt_lens,
293 + return (input_tokens, input_positions, attn_metadata, prompt_lens,
294 subquery_lens, lora_index_mapping, lora_prompt_mapping,
295 lora_requests)
296
297 def _prepare_decode(
298 self,
299 seq_group_metadata_list: List[SequenceGroupMetadata],
297 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
298 - Set[LoRARequest]]:
300 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
301 + List[int], Set[LoRARequest]]:
302 assert len(seq_group_metadata_list) > 0
303 input_tokens: List[int] = []
304 input_positions: List[int] = []
@@ -401,7 +404,7 @@ class ModelRunner:
404 device=self.device,
405 )
406
404 - input_metadata = InputMetadata(
407 + attn_metadata = self.attn_backend.make_metadata(
408 is_prompt=False,
409 slot_mapping=slot_mapping,
410 prompt_lens=None,
@@ -410,7 +413,7 @@ class ModelRunner:
413 num_generation_tokens=len(input_tokens),
414 max_subquery_len=None,
415 max_context_len=max_context_len,
413 - max_seq_len=None,
416 + max_prompt_len=None,
417 subquery_start_loc=None,
418 seq_start_loc=None,
419 context_lens=context_lens,
@@ -418,7 +421,7 @@ class ModelRunner:
421 use_cuda_graph=use_captured_graph,
422 kv_cache_dtype=self.kv_cache_dtype,
423 )
421 - return (input_tokens, input_positions, input_metadata,
424 + return (input_tokens, input_positions, attn_metadata,
425 lora_index_mapping, lora_prompt_mapping, lora_requests)
426
427 def _prepare_sample(
@@ -522,7 +525,7 @@ class ModelRunner:
525 def prepare_input_tensors(
526 self,
527 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
525 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata,
528 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
529 Set[int], LoRAMapping]:
530 if self.is_driver_worker:
531 # NOTE: We assume that all sequences in the group are all prompts or
@@ -530,11 +533,11 @@ class ModelRunner:
533 is_prompt = seq_group_metadata_list[0].is_prompt
534 # Prepare input tensors.
535 if is_prompt:
533 - (input_tokens, input_positions, input_metadata, prompt_lens,
536 + (input_tokens, input_positions, attn_metadata, prompt_lens,
537 subquery_lens, lora_index_mapping, lora_prompt_mapping,
538 lora_requests) = self._prepare_prompt(seq_group_metadata_list)
539 else:
537 - (input_tokens, input_positions, input_metadata,
540 + (input_tokens, input_positions, attn_metadata,
541 lora_index_mapping, lora_prompt_mapping,
542 lora_requests) = self._prepare_decode(seq_group_metadata_list)
543 prompt_lens = []
@@ -560,7 +563,7 @@ class ModelRunner:
563 "lora_requests": lora_requests,
564 "lora_mapping": lora_mapping,
565 }
563 - metadata_dict.update(input_metadata.asdict_zerocopy())
566 + metadata_dict.update(attn_metadata.asdict_zerocopy())
567 broadcast_tensor_dict(metadata_dict, src=0)
568 else:
569 metadata_dict = broadcast_tensor_dict(src=0)
@@ -570,7 +573,7 @@ class ModelRunner:
573 "selected_token_indices")
574 lora_mapping = metadata_dict.pop("lora_mapping")
575 lora_requests = metadata_dict.pop("lora_requests")
573 - input_metadata = InputMetadata(**metadata_dict)
576 + attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
577 sampling_metadata = SamplingMetadata(
578 seq_groups=None,
579 seq_data=None,
@@ -581,16 +584,16 @@ class ModelRunner:
584 perform_sampling=False,
585 )
586
584 - return (input_tokens, input_positions, input_metadata,
587 + return (input_tokens, input_positions, attn_metadata,
588 sampling_metadata, lora_requests, lora_mapping)
589
590 @torch.inference_mode()
591 def execute_model(
592 self,
593 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
591 - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
594 + kv_caches: List[torch.Tensor],
595 ) -> Optional[SamplerOutput]:
593 - (input_tokens, input_positions, input_metadata, sampling_metadata,
596 + (input_tokens, input_positions, attn_metadata, sampling_metadata,
597 lora_requests,
598 lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
599
@@ -598,7 +601,7 @@ class ModelRunner:
601 self.set_active_loras(lora_requests, lora_mapping)
602
603 # Execute the model.
601 - if input_metadata.use_cuda_graph:
604 + if attn_metadata.use_cuda_graph:
605 graph_batch_size = input_tokens.shape[0]
606 model_executable = self.graph_runners[graph_batch_size]
607 else:
@@ -607,7 +610,7 @@ class ModelRunner:
610 input_ids=input_tokens,
611 positions=input_positions,
612 kv_caches=kv_caches,
610 - input_metadata=input_metadata,
613 + attn_metadata=attn_metadata,
614 )
615
616 # Compute the logits.
@@ -673,7 +676,7 @@ class ModelRunner:
676
677 # Run the model with the dummy inputs.
678 num_layers = self.model_config.get_num_layers(self.parallel_config)
676 - kv_caches = [(None, None)] * num_layers
679 + kv_caches = [None] * num_layers
680 self.execute_model(seqs, kv_caches)
681 torch.cuda.synchronize()
682 return
@@ -705,7 +708,7 @@ class ModelRunner:
708 return self.lora_manager.list_loras()
709
710 @torch.inference_mode()
708 - def capture_model(self, kv_caches: List[KVCache]) -> None:
711 + def capture_model(self, kv_caches: List[torch.Tensor]) -> None:
712 """Cuda graph capture a model.
713
714 Note that CUDA graph's performance gain is negligible if number
@@ -759,8 +762,8 @@ class ModelRunner:
762 # NOTE: Capturing the largest batch size first may help reduce the
763 # memory usage of CUDA graph.
764 for batch_size in reversed(batch_size_capture_list):
762 - # Create dummy input_metadata.
763 - input_metadata = InputMetadata(
765 + # Create dummy attn_metadata.
766 + attn_metadata = self.attn_backend.make_metadata(
767 is_prompt=False,
768 slot_mapping=slot_mapping[:batch_size],
769 prompt_lens=None,
@@ -769,7 +772,7 @@ class ModelRunner:
772 num_generation_tokens=batch_size,
773 max_subquery_len=None,
774 max_context_len=self.max_context_len_to_capture,
772 - max_seq_len=None,
775 + max_prompt_len=None,
776 subquery_start_loc=None,
777 seq_start_loc=None,
778 context_lens=context_lens[:batch_size],
@@ -790,7 +793,7 @@ class ModelRunner:
793 input_tokens[:batch_size],
794 input_positions[:batch_size],
795 kv_caches,
793 - input_metadata,
796 + attn_metadata,
797 memory_pool=self.graph_memory_pool,
798 )
799 self.graph_memory_pool = graph_runner.graph.pool()
@@ -826,8 +829,8 @@ class CUDAGraphRunner:
829 self,
830 input_ids: torch.Tensor,
831 positions: torch.Tensor,
829 - kv_caches: List[KVCache],
830 - input_metadata: InputMetadata,
832 + kv_caches: List[torch.Tensor],
833 + attn_metadata: AttentionMetadata,
834 memory_pool,
835 ) -> None:
836 assert self.graph is None
@@ -839,7 +842,7 @@ class CUDAGraphRunner:
842 input_ids,
843 positions,
844 kv_caches,
842 - input_metadata,
845 + attn_metadata,
846 )
847 torch.cuda.synchronize()
848
@@ -853,7 +856,7 @@ class CUDAGraphRunner:
856 input_ids,
857 positions,
858 kv_caches,
856 - input_metadata,
859 + attn_metadata,
860 )
861 torch.cuda.synchronize()
862
@@ -862,9 +865,9 @@ class CUDAGraphRunner:
865 "input_ids": input_ids,
866 "positions": positions,
867 "kv_caches": kv_caches,
865 - "slot_mapping": input_metadata.slot_mapping,
866 - "context_lens": input_metadata.context_lens,
867 - "block_tables": input_metadata.block_tables,
868 + "slot_mapping": attn_metadata.slot_mapping,
869 + "context_lens": attn_metadata.context_lens,
870 + "block_tables": attn_metadata.block_tables,
871 }
872 self.output_buffers = {"hidden_states": hidden_states}
873 return
@@ -873,8 +876,8 @@ class CUDAGraphRunner:
876 self,
877 input_ids: torch.Tensor,
878 positions: torch.Tensor,
876 - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
877 - input_metadata: InputMetadata,
879 + kv_caches: List[torch.Tensor],
880 + attn_metadata: AttentionMetadata,
881 ) -> torch.Tensor:
882 # KV caches are fixed tensors, so we don't need to copy them.
883 del kv_caches
@@ -882,11 +885,11 @@ class CUDAGraphRunner:
885 # Copy the input tensors to the input buffers.
886 self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
887 self.input_buffers["positions"].copy_(positions, non_blocking=True)
885 - self.input_buffers["slot_mapping"].copy_(input_metadata.slot_mapping,
888 + self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
889 non_blocking=True)
887 - self.input_buffers["context_lens"].copy_(input_metadata.context_lens,
890 + self.input_buffers["context_lens"].copy_(attn_metadata.context_lens,
891 non_blocking=True)
889 - self.input_buffers["block_tables"].copy_(input_metadata.block_tables,
892 + self.input_buffers["block_tables"].copy_(attn_metadata.block_tables,
893 non_blocking=True)
894 # Run the graph.
895 self.graph.replay()
vllm/worker/worker.py
+3
@@ -128,6 +128,9 @@ class Worker:
128 # NOTE(woosuk): Here we assume that the other processes using the same
129 # GPU did not change their memory usage during the profiling.
130 peak_memory = self.init_gpu_memory - free_gpu_memory
131 + assert peak_memory > 0, (
132 + "Error in memory profiling. This happens when the GPU memory was "
133 + "not properly cleaned up before initializing the vLLM instance.")
134
135 cache_block_size = self.get_cache_block_size_bytes(
136 block_size, cache_dtype)