A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Attention] MLA decode optimizations (#12528)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: simon-mo <xmo@berkeley.edu> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Alexander Matveev <59768536+alexm-neuralmagic@users.noreply.github.com> Co-authored-by: simon-mo <xmo@berkeley.edu>
Lucas Wilkinson committed
Jan 31, 2025 at 02:49 UTC
cabaf4eff3c7df30d785769d5a0a1fa1a1c48a8a
31 files changed
+2266
-32
csrc/cache.h
+5
@@ -28,6 +28,11 @@ void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
28
const std::string& kv_cache_dtype,
29
torch::Tensor& k_scale, torch::Tensor& v_scale);
30
31
+void concat_and_cache_mla(torch::Tensor& kv_c, torch::Tensor& k_pe,
32
+ torch::Tensor& kv_cache, torch::Tensor& slot_mapping,
33
+ const std::string& kv_cache_dtype,
34
+ torch::Tensor& scale);
35
+
36
// Just for unittest
37
void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
38
const double scale, const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+95
@@ -245,6 +245,51 @@ __global__ void reshape_and_cache_flash_kernel(
245
}
246
}
247
}
248
+
249
+template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
250
+__global__ void concat_and_cache_mla_kernel(
251
+ const scalar_t* __restrict__ kv_c, // [num_tokens, kv_lora_rank]
252
+ const scalar_t* __restrict__ k_pe, // [num_tokens, pe_dim]
253
+ cache_t* __restrict__ kv_cache, // [num_blocks, block_size, (kv_lora_rank
254
+ // + pe_dim)]
255
+ const int64_t* __restrict__ slot_mapping, // [num_tokens]
256
+ const int block_stride, //
257
+ const int kv_c_stride, //
258
+ const int k_pe_stride, //
259
+ const int kv_lora_rank, //
260
+ const int pe_dim, //
261
+ const int block_size, //
262
+ const float* scale //
263
+) {
264
+ const int64_t token_idx = blockIdx.x;
265
+ const int64_t slot_idx = slot_mapping[token_idx];
266
+ // NOTE: slot_idx can be -1 if the token is padded
267
+ if (slot_idx < 0) {
268
+ return;
269
+ }
270
+ const int64_t block_idx = slot_idx / block_size;
271
+ const int64_t block_offset = slot_idx % block_size;
272
+
273
+ auto copy = [&](const scalar_t* __restrict__ src, cache_t* __restrict__ dst,
274
+ int src_stride, int dst_stride, int size, int offset) {
275
+ for (int i = threadIdx.x; i < size; i += blockDim.x) {
276
+ const int64_t src_idx = token_idx * src_stride + i;
277
+ const int64_t dst_idx = block_idx * block_stride +
278
+ block_offset * (kv_lora_rank + pe_dim) + i +
279
+ offset;
280
+ if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
281
+ dst[dst_idx] = src[src_idx];
282
+ } else {
283
+ dst[dst_idx] =
284
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(src[src_idx], *scale);
285
+ }
286
+ }
287
+ };
288
+
289
+ copy(kv_c, kv_cache, kv_c_stride, block_stride, kv_lora_rank, 0);
290
+ copy(k_pe, kv_cache, k_pe_stride, block_stride, pe_dim, kv_lora_rank);
291
+}
292
+
293
} // namespace vllm
294
295
// KV_T is the stored data type of kv-cache.
@@ -343,6 +388,56 @@ void reshape_and_cache_flash(
388
CALL_RESHAPE_AND_CACHE_FLASH);
389
}
390
391
+// KV_T is the stored data type of kv-cache.
392
+// CACHE_T is the data type of key and value tensors.
393
+// KV_DTYPE is the real data type of kv-cache.
394
+#define CALL_CONCAT_AND_CACHE_MLA(KV_T, CACHE_T, KV_DTYPE) \
395
+ vllm::concat_and_cache_mla_kernel<KV_T, CACHE_T, KV_DTYPE> \
396
+ <<<grid, block, 0, stream>>>( \
397
+ reinterpret_cast<KV_T*>(kv_c.data_ptr()), \
398
+ reinterpret_cast<KV_T*>(k_pe.data_ptr()), \
399
+ reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
400
+ slot_mapping.data_ptr<int64_t>(), block_stride, kv_c_stride, \
401
+ k_pe_stride, kv_lora_rank, pe_dim, block_size, \
402
+ reinterpret_cast<const float*>(scale.data_ptr()));
403
+
404
+void concat_and_cache_mla(
405
+ torch::Tensor& kv_c, // [num_tokens, kv_lora_rank]
406
+ torch::Tensor& k_pe, // [num_tokens, pe_dim]
407
+ torch::Tensor& kv_cache, // [num_blocks, block_size, (kv_lora_rank +
408
+ // pe_dim)]
409
+ torch::Tensor& slot_mapping, // [num_tokens] or [num_actual_tokens]
410
+ const std::string& kv_cache_dtype, torch::Tensor& scale) {
411
+ // NOTE(woosuk): In vLLM V1, key.size(0) can be different from
412
+ // slot_mapping.size(0) because of padding for CUDA graphs.
413
+ // In vLLM V0, key.size(0) is always equal to slot_mapping.size(0) because
414
+ // both include padding.
415
+ // In vLLM V1, however, key.size(0) can be larger than slot_mapping.size(0)
416
+ // since key includes padding for CUDA graphs, while slot_mapping does not.
417
+ // In this case, slot_mapping.size(0) represents the actual number of tokens
418
+ // before padding.
419
+ // For compatibility with both cases, we use slot_mapping.size(0) as the
420
+ // number of tokens.
421
+ int num_tokens = slot_mapping.size(0);
422
+ int kv_lora_rank = kv_c.size(1);
423
+ int pe_dim = k_pe.size(1);
424
+ int block_size = kv_cache.size(1);
425
+
426
+ TORCH_CHECK(kv_cache.size(2) == kv_lora_rank + pe_dim);
427
+
428
+ int kv_c_stride = kv_c.stride(0);
429
+ int k_pe_stride = k_pe.stride(0);
430
+ int block_stride = kv_cache.stride(0);
431
+
432
+ dim3 grid(num_tokens);
433
+ dim3 block(std::min(kv_lora_rank, 512));
434
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(kv_c));
435
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
436
+
437
+ DISPATCH_BY_KV_CACHE_DTYPE(kv_c.dtype(), kv_cache_dtype,
438
+ CALL_CONCAT_AND_CACHE_MLA);
439
+}
440
+
441
namespace vllm {
442
443
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
csrc/torch_bindings.cpp
+9
@@ -463,6 +463,15 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
463
cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
464
&reshape_and_cache_flash);
465
466
+ // Concat kv_c and k_pe and cache them.
467
+ cache_ops.def(
468
+ "concat_and_cache_mla(Tensor kv_c, Tensor k_pe,"
469
+ " Tensor! kv_cache,"
470
+ " Tensor slot_mapping,"
471
+ " str kv_cache_dtype,"
472
+ " Tensor scale) -> ()");
473
+ cache_ops.impl("concat_and_cache_mla", torch::kCUDA, &concat_and_cache_mla);
474
+
475
// Convert the key and value cache to fp8 data type.
476
cache_ops.def(
477
"convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, "
tests/kernels/test_triton_decode_attention.py
new
+89
@@ -0,0 +1,89 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
5
+
6
+
7
+def cdiv(a, b):
8
+ return (a + b - 1) // b
9
+
10
+
11
+@pytest.mark.parametrize("B", [3, 5])
12
+@pytest.mark.parametrize("L", [1027, 1025])
13
+@pytest.mark.parametrize("H_Q", [32])
14
+@pytest.mark.parametrize("H_KV", [32, 8])
15
+@pytest.mark.parametrize("D_QK", [128, 192, 576])
16
+@pytest.mark.parametrize("D_V", [128, 512])
17
+@pytest.mark.parametrize("CACHE_SIZE", [16384])
18
+@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
19
+def test_decode_attention(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
20
+ assert CACHE_SIZE % PAGE_SIZE == 0
21
+ dtype = torch.bfloat16
22
+ seq_len = L # This represents the number of tokens already in the sequence
23
+ sm_scale = 1.0 / (D_QK**0.5)
24
+ num_kv_splits = 8
25
+
26
+ num_pages_per_batch = cdiv(seq_len, PAGE_SIZE)
27
+ req_to_page = torch.randint(0,
28
+ CACHE_SIZE // PAGE_SIZE,
29
+ (B, num_pages_per_batch, 1),
30
+ device="cuda")
31
+ req_to_token = req_to_page * PAGE_SIZE
32
+ req_to_token = req_to_token.expand(B, num_pages_per_batch, PAGE_SIZE)
33
+ req_to_token = req_to_token + torch.arange(PAGE_SIZE, device="cuda").view(
34
+ 1, 1, -1)
35
+ req_to_token = req_to_token.view(B, -1)
36
+ req_to_token = req_to_token[:, :seq_len].contiguous()
37
+
38
+ # q represents the new token being generated, one per batch
39
+ q = torch.randn(B, H_Q, D_QK, dtype=dtype, device="cuda")
40
+
41
+ # k_buffer and v_buffer represent all previous tokens
42
+ # Page size is 1.
43
+ k_buffer = torch.randn(CACHE_SIZE, H_KV, D_QK, dtype=dtype, device="cuda")
44
+ v_buffer = torch.randn(CACHE_SIZE, H_KV, D_V, dtype=dtype, device="cuda")
45
+
46
+ # o will have the same shape as q
47
+ o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda")
48
+
49
+ b_seq_len = torch.full((B, ), seq_len, device="cuda")
50
+
51
+ attn_logits = torch.empty(
52
+ (B, H_Q, num_kv_splits, D_V + 1),
53
+ dtype=torch.float32,
54
+ device="cuda",
55
+ )
56
+
57
+ # Call the original implementation.
58
+ decode_attention_fwd(
59
+ q,
60
+ k_buffer,
61
+ v_buffer,
62
+ o,
63
+ req_to_token,
64
+ b_seq_len,
65
+ attn_logits,
66
+ num_kv_splits,
67
+ sm_scale,
68
+ )
69
+
70
+ # Page size can be larger than 1.
71
+ k_buffer = k_buffer.view(CACHE_SIZE // PAGE_SIZE, PAGE_SIZE, H_KV, D_QK)
72
+ v_buffer = v_buffer.view(CACHE_SIZE // PAGE_SIZE, PAGE_SIZE, H_KV, D_V)
73
+
74
+ o1 = torch.zeros_like(o)
75
+
76
+ decode_attention_fwd(
77
+ q,
78
+ k_buffer,
79
+ v_buffer,
80
+ o1,
81
+ req_to_page,
82
+ b_seq_len,
83
+ attn_logits,
84
+ num_kv_splits,
85
+ sm_scale,
86
+ PAGE_SIZE,
87
+ )
88
+
89
+ assert torch.allclose(o, o1)
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/dummy_platform.py
+1
-1
@@ -5,5 +5,5 @@ class DummyPlatform(CudaPlatform):
5
device_name = "DummyDevice"
6
7
def get_attn_backend_cls(self, backend_name, head_size, dtype,
8
- kv_cache_dtype, block_size, use_v1):
8
+ kv_cache_dtype, block_size, use_v1, use_mla):
9
return "vllm_add_dummy_platform.dummy_attention_backend.DummyAttentionBackend" # noqa E501
tests/weight_loading/models.txt
+1
-1
@@ -20,7 +20,7 @@ compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
20
compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22
compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23
-compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
23
+#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
24
compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-FP8-Dynamic-testing, main, 90
25
compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-W8A8-testing, main, 90
26
awq, casperhansen/mixtral-instruct-awq, main
tests/weight_loading/run_model_weight_loading_test.sh
+7
-2
@@ -3,7 +3,7 @@ SUCCESS=0
3
4
while getopts "c:" OPT; do
5
case ${OPT} in
6
- c )
6
+ c )
7
CONFIG="$OPTARG"
8
;;
9
\? )
@@ -18,9 +18,14 @@ IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
18
19
for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
20
do
21
+ if [[ $MODEL_CONFIG == \#* ]]; then
22
+ echo "=== SKIPPING MODEL: $MODEL_CONFIG ==="
23
+ continue
24
+ fi
25
+
26
LOCAL_SUCCESS=0
27
IFS=', ' read -r -a array <<< "$MODEL_CONFIG"
23
-
28
+
29
echo "=== RUNNING MODEL: $MODEL_CONFIG ==="
30
31
export QUANTIZATION=${array[0]}
vllm/_custom_ops.py
+13
@@ -1002,6 +1002,19 @@ def reshape_and_cache_flash(
1002
v_scale)
1003
1004
1005
+def concat_and_cache_mla(
1006
+ kv_c: torch.Tensor,
1007
+ k_pe: torch.Tensor,
1008
+ kv_cache: torch.Tensor,
1009
+ slot_mapping: torch.Tensor,
1010
+ kv_cache_dtype: str,
1011
+ scale: torch.Tensor,
1012
+) -> None:
1013
+ torch.ops._C_cache_ops.concat_and_cache_mla(kv_c, k_pe, kv_cache,
1014
+ slot_mapping, kv_cache_dtype,
1015
+ scale)
1016
+
1017
+
1018
def copy_blocks(key_caches: List[torch.Tensor],
1019
value_caches: List[torch.Tensor],
1020
block_mapping: torch.Tensor) -> None:
vllm/attention/backends/abstract.py
+16
@@ -276,3 +276,19 @@ class AttentionImpl(ABC, Generic[T]):
276
output: Optional[torch.Tensor] = None,
277
) -> torch.Tensor:
278
raise NotImplementedError
279
+
280
+
281
+class MLAAttentionImpl(AttentionImpl[T], Generic[T]):
282
+
283
+ @abstractmethod
284
+ def forward(
285
+ self,
286
+ layer: AttentionLayer,
287
+ hidden_states_or_cq: torch.Tensor,
288
+ kv_c_normed: torch.Tensor,
289
+ k_pe: torch.Tensor,
290
+ kv_cache: torch.Tensor,
291
+ attn_metadata: T,
292
+ output: Optional[torch.Tensor] = None,
293
+ ) -> torch.Tensor:
294
+ raise NotImplementedError
vllm/attention/backends/mla/__init__.py
vllm/attention/backends/mla/utils.py
new
+365
@@ -0,0 +1,365 @@
1
+from abc import abstractmethod
2
+from dataclasses import dataclass
3
+from typing import Any, Dict, Generic, List, Optional
4
+
5
+import torch
6
+
7
+from vllm import _custom_ops as ops
8
+from vllm import envs
9
+from vllm.attention.backends.abstract import (AttentionLayer,
10
+ AttentionMetadata,
11
+ MLAAttentionImpl, T)
12
+from vllm.distributed import get_tensor_model_parallel_world_size
13
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
+ RowParallelLinear)
15
+from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
16
+from vllm.vllm_flash_attn import flash_attn_varlen_func
17
+
18
+
19
+@dataclass
20
+class MLACommonMetadata(AttentionMetadata):
21
+ # Input positions for rotrary embeddings since for MLA the rotary
22
+ # position embeddings are applied inside the attention backend
23
+ input_positions: torch.Tensor
24
+
25
+
26
+class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
27
+ """
28
+ Common class for implementing repeated parts
29
+
30
+ Main reference: DeepseekV2 paper, and FlashInfer Implementation
31
+ (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
32
+
33
+ Deepseek's MLA attention works the following way:
34
+ * Use a single latent vector to represent the entire KV cache.
35
+ * The attention "simulates" a multi-head attention, while the compute is
36
+ similar to multi-query attention.
37
+ * The dataflow is as follows,
38
+
39
+ * B: batch/sequence length
40
+ * H: hidden size
41
+ * N: number of attention heads
42
+ * Lq: latent dimension for Q
43
+ * Lkv: latent dimension for K/V
44
+ * P: nope dimension, P+R is the actual head_dim in common attention.
45
+ * R: rope dimension, this slide of the head_dim goes through rope.
46
+ * V: V head dim.
47
+ * kv_c: latent/compressed KV
48
+ * q_c: latent/compressed Q
49
+
50
+ #
51
+ # Outside the MLA attention backend
52
+ #
53
+
54
+ 1. The hidden states (B, H) are projected down into cq (B, Lq) and
55
+ kv_c_k_pe (B, Lkv+R).
56
+ 2. The kv_c_k_pe is split into kv_c (B, Lkv) and k_pe (B, R). cq
57
+ and kv_c are normalized.
58
+
59
+ #
60
+ # Inside the MLA attention backend
61
+ #
62
+
63
+ * if prefill:
64
+
65
+ 3. The q_c is then projected up into the multi-head version.
66
+ * q_c goes from (B, Lq) to (B, N, (P+R)), which is split into q_nope
67
+ (B, N, P) and q_pe (B, N, R).
68
+ 4. q_pe, k_pe are then passed through rotary embeddings.
69
+ 5. kv_c and k_pe are concatenated and inserted into the cache
70
+ 6. The kv_c is then projected up into the multi-head version.
71
+ * kv_c goes from (B, Lkv) to (B, N, (P+V)) which has the nope
72
+ dimensions for K and V, which is split into k_nope (B, N, P)
73
+ and v (B, N, V).
74
+ 7. q (B, N, (P+R)) and k (B, N, (P+R)) matrices are assembled from
75
+ q_nope, q_pe, k_nope, k_pe.
76
+ 8. Attention is computued with q, k, v.
77
+ 9. The attention computation returns (B, N, V), which is projected back
78
+ to (B, H) using out projection.
79
+
80
+ * if decode:
81
+
82
+ 3. Here's the change, we do not perform up the full up projection for
83
+ q_c, and there is no up projection at all for kv_c. This is
84
+ achieved by the technique of "weight absorption". The paper says
85
+ "Fortunately, due to the associative law of matrix multiplication,
86
+ we can absorb WUK into WUQ, and WUV into WO"
87
+ * The q up projection turns (B, Lq) into (B, N, (P+R)), we split it
88
+ into W_UQ (Lq, N, P) and W_QR (Lq, N, R).
89
+ * The kv_c up projection turns (B, Lkv) into (B, N, (P+V)), we split
90
+ it into W_UK (Lkv, N, P) and W_UV (Lkv, N, V).
91
+ * The out projection shape W_O (N*V, H) turns (B, N, V) into (B, H).
92
+ * We can precompute the product of W_UQ and W_UK into
93
+ W_UQ_UK (Lq, N, Lkv), which is possible due to QK^T operation in
94
+ attention.
95
+ * We can precompute the product of W_UV and W_O into
96
+ W_UV_O (N, Lkv, H), which is possible due to V@O as the
97
+ "epilogue" of attention
98
+ 4. We still need to compute q_pe (B, N, R) by applying W_QR to q_latent.
99
+ 5. q_pe, k_pe are then passed through rotary embeddings.
100
+ 6. kv_c and k_pe are concatenated and inserted into the cache
101
+ 7. By applying W_UQ_UK to q_latent, we have the new q_nope of shape
102
+ (B, N, Lkv).
103
+ 8. q (B, N, (Lkv+R)), k (B, (Lkv+R)) are assembled from q_nope, q_pe,
104
+ kv_a, k_pe. v (B, Lkv) is exactly the same vector as kv_a.
105
+ 9. The attention is computed with q, k, v. Note that we just performed
106
+ a MQA attention with (LKv+R) as our head dim.
107
+ 10. The KV cache is updated using the new entries k (B, N, (Lkv+R)),
108
+ which included the v and rope values.
109
+ 11. The attention computation returns (B, N, Lkv), which is projected
110
+ back to (B, H) using W_UV_O.
111
+
112
+ From @tsu-bin's calculation, we only want to use the absorption technique
113
+ for decode. The prefill algorithm should still use the up-projected MHA
114
+ for less flops and memory usage.
115
+
116
+ """
117
+
118
+ def __init__(
119
+ self,
120
+ num_heads: int,
121
+ head_size: int,
122
+ scale: float,
123
+ num_kv_heads: int,
124
+ alibi_slopes: Optional[List[float]],
125
+ sliding_window: Optional[int],
126
+ kv_cache_dtype: str,
127
+ blocksparse_params: Optional[Dict[str, Any]],
128
+ logits_soft_cap: Optional[float],
129
+ attn_type: str,
130
+ # MLA Specific Arguments
131
+ q_lora_rank: Optional[int],
132
+ kv_lora_rank: int,
133
+ qk_nope_head_dim: int,
134
+ qk_rope_head_dim: int,
135
+ qk_head_dim: int,
136
+ v_head_dim: int,
137
+ rotary_emb: RotaryEmbedding,
138
+ # q_proj should be q_b_proj if q_lora_rank is not None, but from an
139
+ # attention backend perspective we rely on the layer to pass in the
140
+ # correct matrix
141
+ q_proj: ColumnParallelLinear,
142
+ kv_b_proj: ColumnParallelLinear,
143
+ o_proj: RowParallelLinear,
144
+ ) -> None:
145
+ self.num_heads = num_heads
146
+ self.head_size = head_size
147
+ self.scale = float(scale)
148
+ self.num_kv_heads = num_kv_heads
149
+ self.kv_cache_dtype = kv_cache_dtype
150
+
151
+ self.q_lora_rank = q_lora_rank
152
+ self.kv_lora_rank = kv_lora_rank
153
+ self.qk_nope_head_dim = qk_nope_head_dim
154
+ self.qk_rope_head_dim = qk_rope_head_dim
155
+ self.qk_head_dim = qk_head_dim
156
+ self.v_head_dim = v_head_dim
157
+
158
+ self.rotary_emb = rotary_emb
159
+ self.q_proj = q_proj
160
+ self.kv_b_proj = kv_b_proj
161
+ self.o_proj = o_proj
162
+
163
+ def _v_up_proj_and_o_proj(self, x):
164
+ if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
165
+ return self.o_proj_absorbed(
166
+ x.reshape(-1, self.num_heads * self.kv_lora_rank))[0]
167
+ else:
168
+ x = torch.einsum("bnl,lnv->bnv", x, self.W_UV)
169
+ return self.o_proj(x.reshape(-1,
170
+ self.num_heads * self.v_head_dim))[0]
171
+
172
+ def _q_proj_and_k_up_proj(self, x):
173
+ if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
174
+ return torch.matmul(x, self.W_Q_UK)\
175
+ .view(-1, self.num_heads, self.kv_lora_rank)
176
+ else:
177
+ x = torch.matmul(x, self.W_Q)\
178
+ .view(-1, self.num_heads, self.qk_nope_head_dim)
179
+ return torch.einsum("bnp,lnp->bnl", x, self.W_UK)\
180
+ .view(-1, self.num_heads, self.kv_lora_rank)
181
+
182
+ def process_weights_after_loading(self):
183
+ kv_b_proj_weight = self.kv_b_proj.weight.T
184
+ assert kv_b_proj_weight.shape == (
185
+ self.kv_lora_rank,
186
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim)), (
187
+ f"{kv_b_proj_weight.shape=}, "
188
+ f"{self.kv_lora_rank=}, "
189
+ f"{self.num_heads=}, "
190
+ f"{self.qk_nope_head_dim=}, "
191
+ f"{self.v_head_dim=}")
192
+ kv_b_proj_weight = kv_b_proj_weight.view(
193
+ self.kv_lora_rank,
194
+ self.num_heads,
195
+ self.qk_nope_head_dim + self.v_head_dim,
196
+ )
197
+
198
+ W_UK, W_UV = kv_b_proj_weight.split(
199
+ [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
200
+
201
+ q_proj = self.q_proj.weight.T\
202
+ .view(-1, self.num_heads, self.qk_head_dim)
203
+
204
+ # can be W_Q or W_UQ depending q_lora_rank, the former if
205
+ # q_lora_rank is None, the latter otherwise. From the Attention backend
206
+ # perspective though we call these both W_Q and rely on the layer
207
+ # to pass in the correct matrix
208
+ W_Q = q_proj[..., :self.qk_nope_head_dim]
209
+ self.W_QR = q_proj[..., self.qk_nope_head_dim:]\
210
+ .flatten(start_dim=1).contiguous()
211
+
212
+ if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
213
+ #
214
+ # Perform matrix-absorption following
215
+ # https://github.com/flashinfer-ai/flashinfer/pull/551
216
+ # for decode, as a result we end up with absorbed weights for decode
217
+ # and another copy of raw weights for prefill.
218
+ #
219
+ self.W_UK, self.W_UV = kv_b_proj_weight.split(
220
+ [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
221
+ # We absorb `W_UK` into `W_Q` resulting in either W_Q_UK or W_UQ_UK
222
+ # depending q_lora_rank, the former if q_lora_rank is None, the
223
+ # latter otherwise
224
+ # basically if q_lora_rank is none we are absorbing into q_proj
225
+ # instead of UQ
226
+ self.W_Q_UK = torch.einsum("qnd,lnd -> qnl", W_Q, W_UK)\
227
+ .flatten(start_dim=1).contiguous()
228
+
229
+ W_O = self.o_proj.weight\
230
+ .view(-1, self.num_heads, self.v_head_dim)
231
+ self.W_UV_O = torch.einsum("lnd,hnd -> nlh", W_UV, W_O)\
232
+ .flatten(start_dim=0, end_dim=1).contiguous()
233
+
234
+ tp_size = get_tensor_model_parallel_world_size()
235
+ self.o_proj_absorbed = RowParallelLinear(
236
+ self.W_UV_O.shape[0] * tp_size,
237
+ self.W_UV_O.shape[1],
238
+ bias=False,
239
+ # TODO(lucas) figure out how to properly forward quant_method
240
+ #quant_config=self.o_proj.quant_method,
241
+ )
242
+
243
+ self.o_proj_absorbed.weight = torch.nn.Parameter(self.W_UV_O.T)
244
+ else:
245
+ self.W_UV = W_UV
246
+ self.W_UK = W_UK
247
+ self.W_Q = W_Q.flatten(start_dim=1)
248
+
249
+ @abstractmethod
250
+ def _forward_prefill(
251
+ self,
252
+ q: torch.Tensor,
253
+ kv_c_normed: torch.Tensor,
254
+ k_pe: torch.Tensor,
255
+ attn_metadata: T,
256
+ ) -> torch.Tensor:
257
+ raise NotImplementedError
258
+
259
+ @abstractmethod
260
+ def _forward_decode(
261
+ self,
262
+ q_nope: torch.Tensor,
263
+ q_pe: torch.Tensor,
264
+ kv_cache: torch.Tensor,
265
+ attn_metadata: T,
266
+ ) -> torch.Tensor:
267
+ raise NotImplementedError
268
+
269
+ def forward(
270
+ self,
271
+ layer: AttentionLayer,
272
+ hidden_states_or_q_c: torch.Tensor, # query in unified attn
273
+ k_c_normed: torch.Tensor, # key in unified attn
274
+ k_pe: torch.Tensor, # value in unified attn
275
+ kv_cache: torch.Tensor,
276
+ attn_metadata: T,
277
+ output: Optional[torch.Tensor] = None,
278
+ ) -> torch.Tensor:
279
+ if output is not None:
280
+ raise NotImplementedError(
281
+ "output is not yet supported for MLAImplBase")
282
+
283
+ is_decode = attn_metadata.decode_metadata is not None
284
+ is_prefill = attn_metadata.prefill_metadata is not None
285
+
286
+ if (is_decode and is_prefill):
287
+ raise NotImplementedError(
288
+ "chunked prefill is not supported for MLAImplBase")
289
+
290
+ # Restore head dim (for rotary embedding)
291
+ k_pe = k_pe.unsqueeze(1)
292
+ assert hasattr(attn_metadata, "input_positions")
293
+
294
+ if is_decode:
295
+ q_nope = self._q_proj_and_k_up_proj(hidden_states_or_q_c)
296
+ q_pe = torch.matmul(hidden_states_or_q_c, self.W_QR)\
297
+ .view(-1, self.num_heads, self.qk_rope_head_dim)
298
+ q_pe, k_pe = \
299
+ self.rotary_emb(attn_metadata.input_positions, q_pe, k_pe)
300
+ else:
301
+ assert is_prefill
302
+ q = self.q_proj(hidden_states_or_q_c)[0]\
303
+ .view(-1, self.num_heads, self.qk_head_dim)
304
+
305
+ # TODO(lucas): there must be a nicer way to write this line
306
+ q[..., self.qk_nope_head_dim:], k_pe = \
307
+ self.rotary_emb(
308
+ attn_metadata.input_positions,
309
+ q[..., self.qk_nope_head_dim:], k_pe)
310
+
311
+ # write the latent and rope to kv cache
312
+ if kv_cache.numel() > 0:
313
+ ops.concat_and_cache_mla(
314
+ k_c_normed,
315
+ k_pe.squeeze(1),
316
+ kv_cache,
317
+ attn_metadata.slot_mapping.flatten(),
318
+ kv_cache_dtype=self.kv_cache_dtype,
319
+ scale=layer._k_scale,
320
+ )
321
+
322
+ if attn_metadata.prefill_metadata is not None:
323
+ return self._forward_prefill(q, k_c_normed, k_pe, attn_metadata)
324
+
325
+ if attn_metadata.decode_metadata is not None:
326
+ return self._forward_decode(q_nope, q_pe, kv_cache, attn_metadata)
327
+
328
+ # Optional common flash-attn based prefill
329
+ def _forward_prefill_flash(
330
+ self,
331
+ q: torch.Tensor,
332
+ k_c_normed: torch.Tensor,
333
+ k_pe: torch.Tensor,
334
+ seq_start_loc: torch.Tensor,
335
+ max_prefill_seq_len: int,
336
+ ) -> torch.Tensor:
337
+
338
+ kv_nope = self.kv_b_proj(k_c_normed)[0]\
339
+ .view(-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
340
+ k_nope, v = kv_nope\
341
+ .split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
342
+
343
+ k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1)
344
+
345
+ # For MLA the v head dim is smaller than qk head dim so we pad out
346
+ # v with 0s to match the qk head dim
347
+ v_padded = torch.nn.functional.pad(v, [0, q.shape[-1] - v.shape[-1]],
348
+ value=0)
349
+
350
+ attn_output = flash_attn_varlen_func(
351
+ q=q,
352
+ k=k,
353
+ v=v_padded,
354
+ cu_seqlens_q=seq_start_loc,
355
+ cu_seqlens_k=seq_start_loc,
356
+ max_seqlen_q=max_prefill_seq_len,
357
+ max_seqlen_k=max_prefill_seq_len,
358
+ softmax_scale=self.scale,
359
+ causal=True,
360
+ )
361
+ attn_output = attn_output\
362
+ .view(-1, self.num_heads, q.shape[-1])[..., :v.shape[-1]]\
363
+ .reshape(-1, self.num_heads * v.shape[-1])
364
+
365
+ return self.o_proj(attn_output)[0]
vllm/attention/backends/triton_mla.py
new
+749
@@ -0,0 +1,749 @@
1
+from collections import defaultdict
2
+from contextlib import contextmanager
3
+from dataclasses import dataclass
4
+from itertools import accumulate
5
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
6
+
7
+from vllm.multimodal import MultiModalPlaceholderMap
8
+
9
+try:
10
+ from flashinfer import BatchDecodeMlaWithPagedKVCacheWrapper
11
+ FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
12
+except ImportError:
13
+ BatchDecodeMlaWithPagedKVCacheWrapper = None
14
+ FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
15
+
16
+import torch
17
+
18
+from vllm import _custom_ops as ops
19
+from vllm.attention.backends.abstract import (AttentionBackend,
20
+ AttentionMetadata,
21
+ AttentionMetadataBuilder,
22
+ AttentionState, AttentionType)
23
+from vllm.attention.backends.mla.utils import MLACommonImpl, MLACommonMetadata
24
+from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
25
+ compute_slot_mapping_start_idx,
26
+ is_block_tables_empty)
27
+from vllm.attention.ops.paged_attn import PagedAttention
28
+from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
29
+from vllm.utils import async_tensor_h2d, make_tensor_with_pad
30
+
31
+if TYPE_CHECKING:
32
+ from vllm.worker.model_runner import (ModelInputForGPUBuilder,
33
+ ModelInputForGPUWithSamplingMetadata)
34
+
35
+
36
+class TritonMLABackend(AttentionBackend):
37
+
38
+ @staticmethod
39
+ def get_name() -> str:
40
+ return "TRITON_MLA"
41
+
42
+ @staticmethod
43
+ def get_impl_cls() -> Type["TritonMLAImpl"]:
44
+ return TritonMLAImpl
45
+
46
+ @staticmethod
47
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
48
+ return TritonMLAMetadata
49
+
50
+ @staticmethod
51
+ def get_builder_cls() -> Type["TritonMLAMetadataBuilder"]:
52
+ return TritonMLAMetadataBuilder
53
+
54
+ @staticmethod
55
+ def get_state_cls() -> Type["TritonMLAState"]:
56
+ return TritonMLAState
57
+
58
+ @staticmethod
59
+ def get_kv_cache_shape(
60
+ num_blocks: int,
61
+ block_size: int,
62
+ num_kv_heads: int, # assumed to be 1 for MLA
63
+ kv_lora_rank: int, # passed via head_size
64
+ ) -> Tuple[int, ...]:
65
+ # TODO(lucas): remove hardcoding k_pe size as 1/8th of kv_lora_rank
66
+ k_pe_size = kv_lora_rank // 8
67
+ return (num_blocks, block_size, kv_lora_rank + k_pe_size)
68
+
69
+ @staticmethod
70
+ def swap_blocks(
71
+ src_kv_cache: torch.Tensor,
72
+ dst_kv_cache: torch.Tensor,
73
+ src_to_dst: torch.Tensor,
74
+ ) -> None:
75
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
76
+
77
+ @staticmethod
78
+ def copy_blocks(
79
+ kv_caches: List[torch.Tensor],
80
+ src_to_dists: torch.Tensor,
81
+ ) -> None:
82
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
83
+
84
+ @staticmethod
85
+ def get_supported_head_sizes() -> List[int]:
86
+ return [512]
87
+
88
+
89
+class TritonMLAState(AttentionState):
90
+
91
+ def __init__(self, runner):
92
+ self.runner = runner
93
+ self._is_graph_capturing = False
94
+
95
+ @contextmanager
96
+ def graph_capture(self, max_batch_size: int):
97
+ self._is_graph_capturing = True
98
+
99
+ self._graph_slot_mapping = torch.full((max_batch_size, ),
100
+ PAD_SLOT_ID,
101
+ dtype=torch.long,
102
+ device=self.runner.device)
103
+ self._graph_seq_lens = torch.ones(max_batch_size,
104
+ dtype=torch.int32,
105
+ device=self.runner.device)
106
+ self._graph_block_tables = torch.from_numpy(
107
+ self.runner.graph_block_tables).to(device=self.runner.device)
108
+
109
+ self._positions = torch.zeros((max_batch_size, ),
110
+ dtype=torch.long,
111
+ device=self.runner.device)
112
+
113
+ yield
114
+
115
+ self._is_graph_capturing = False
116
+ del self._graph_slot_mapping
117
+ del self._graph_seq_lens
118
+ del self._graph_block_tables
119
+ del self._positions
120
+
121
+ def graph_clone(self, batch_size: int):
122
+ assert self._is_graph_capturing
123
+ return self.__class__(self.runner)
124
+
125
+ def graph_capture_get_metadata_for_batch(
126
+ self, batch_size: int, is_encoder_decoder_model: bool = False):
127
+ assert self._is_graph_capturing
128
+
129
+ attn_metadata = self.runner.attn_backend.make_metadata(
130
+ num_prefills=0,
131
+ num_prefill_tokens=0,
132
+ num_decode_tokens=batch_size,
133
+ slot_mapping=self._graph_slot_mapping[:batch_size],
134
+ multi_modal_placeholder_index_maps=None,
135
+ enable_kv_scales_calculation=True,
136
+ seq_lens=None,
137
+ seq_lens_tensor=self._graph_seq_lens[:batch_size],
138
+ max_query_len=1,
139
+ max_decode_query_len=1,
140
+ max_prefill_seq_len=0,
141
+ max_decode_seq_len=self.runner.max_seq_len_to_capture,
142
+ query_start_loc=None,
143
+ seq_start_loc=None,
144
+ context_lens_tensor=None,
145
+ block_tables=self._graph_block_tables[:batch_size],
146
+ use_cuda_graph=True,
147
+ input_positions=self._positions[:batch_size],
148
+ head_dim=self.runner.model_config.get_head_size())
149
+
150
+ if is_encoder_decoder_model:
151
+ raise NotImplementedError(
152
+ "TritonMLAState does not support encoder/decoder yet")
153
+
154
+ return attn_metadata
155
+
156
+ def get_graph_input_buffers(self,
157
+ attn_metadata,
158
+ is_encoder_decoder_model: bool = False):
159
+ input_buffers = {
160
+ "slot_mapping": attn_metadata.slot_mapping,
161
+ "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
162
+ "block_tables": attn_metadata.decode_metadata.block_tables,
163
+ "input_positions": attn_metadata.decode_metadata.input_positions,
164
+ }
165
+ if is_encoder_decoder_model:
166
+ raise NotImplementedError(
167
+ "TritonMLAState does not support encoder/decoder yet")
168
+
169
+ return input_buffers
170
+
171
+ def prepare_graph_input_buffers(self,
172
+ input_buffers,
173
+ attn_metadata,
174
+ is_encoder_decoder_model: bool = False):
175
+ input_positions = attn_metadata.input_positions
176
+ num_positions = input_positions.shape[0]
177
+ input_buffers["seq_lens_tensor"].copy_(
178
+ attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
179
+ input_buffers["block_tables"].copy_(
180
+ attn_metadata.decode_metadata.block_tables, non_blocking=True)
181
+ # CUDA graph buffer is padded so only perform a partial copy based on
182
+ # num_positions
183
+ input_buffers["input_positions"][:num_positions].copy_(
184
+ input_positions, non_blocking=True)
185
+ if is_encoder_decoder_model:
186
+ raise NotImplementedError(
187
+ "TritonMLAState does not support encoder/decoder yet")
188
+
189
+ def begin_forward(self, model_input):
190
+ return
191
+
192
+
193
+@dataclass
194
+class TritonMLAMetadata(MLACommonMetadata):
195
+ """Metadata for TritonMLAMetadata.
196
+
197
+ NOTE: Any python object stored here is not updated when it is
198
+ cuda-graph replayed. If you have values that need to be changed
199
+ dynamically, it should be stored in tensor. The tensor has to be
200
+ updated from `CUDAGraphRunner.forward` API.
201
+ """
202
+ # (batch_size,). The sequence length per sequence. Sequence length means
203
+ # the computed tokens + new tokens None if it is a decoding.
204
+ seq_lens: Optional[List[int]]
205
+ # seq_lens stored as a tensor.
206
+ seq_lens_tensor: Optional[torch.Tensor]
207
+
208
+ # NOTE(sang): Definition of context_len, query_len, and seq_len.
209
+ # |---------- N-1 iteration --------|
210
+ # |---------------- N iteration ---------------------|
211
+ # |- tokenA -|......................|-- newTokens ---|
212
+ # |---------- context_len ----------|
213
+ # |-------------------- seq_len ---------------------|
214
+ # |-- query_len ---|
215
+
216
+ # Maximum sequence length among prefill batch. 0 if there are decoding
217
+ # requests only.
218
+ max_prefill_seq_len: int
219
+ # Maximum sequence length among decode batch. 0 if there are prefill
220
+ # requests only.
221
+ max_decode_seq_len: int
222
+ # (batch_size,) A tensor of context lengths (tokens that are computed
223
+ # so far).
224
+ context_lens_tensor: Optional[torch.Tensor]
225
+
226
+ # (batch_size, max_blocks_per_seq).
227
+ # Block addresses per sequence. (Seq id -> list of physical block)
228
+ # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
229
+ # in the kv cache. Each block can contain up to block_size tokens.
230
+ # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
231
+ # captured.
232
+ block_tables: Optional[torch.Tensor]
233
+
234
+ # Whether or not if cuda graph is enabled.
235
+ # Cuda-graph is currently enabled for decoding only.
236
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
237
+
238
+ use_cuda_graph: bool
239
+
240
+ # Maximum query length in the batch.
241
+ max_query_len: Optional[int] = None
242
+
243
+ # Max number of query tokens among request in the batch.
244
+ max_decode_query_len: Optional[int] = None
245
+
246
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
247
+ # the batch, used to index into subquery. E.g., if the subquery length
248
+ # is [4, 6], it is [0, 4, 10].
249
+ query_start_loc: Optional[torch.Tensor] = None
250
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
251
+ # the batch, used to index into sequence. E.g., if the sequence length is
252
+ # [4, 6], it is [0, 4, 10].
253
+ seq_start_loc: Optional[torch.Tensor] = None
254
+
255
+ _cached_prefill_metadata: Optional["TritonMLAMetadata"] = None
256
+ _cached_decode_metadata: Optional["TritonMLAMetadata"] = None
257
+
258
+ num_prefill_tokens: int
259
+
260
+ num_kv_splits: int = 4 # TODO(lucas) add heuristic
261
+ attn_logits: Optional[torch.Tensor] = None
262
+ req_idx: Optional[torch.Tensor] = None
263
+
264
+ # The dimension of the attention heads
265
+ head_dim: Optional[int] = None
266
+
267
+ def __post_init__(self):
268
+ supported_head_sizes = TritonMLABackend.get_supported_head_sizes()
269
+ if self.head_dim is not None and self.head_dim \
270
+ not in supported_head_sizes:
271
+ raise ValueError(
272
+ f"Only {supported_head_sizes} are supported for head_dim,",
273
+ f"received {self.head_dim}.")
274
+
275
+ @property
276
+ def prefill_metadata(self) -> Optional["TritonMLAMetadata"]:
277
+ if self.num_prefills == 0:
278
+ return None
279
+
280
+ if self._cached_prefill_metadata is not None:
281
+ return self._cached_prefill_metadata
282
+
283
+ assert self.seq_lens is not None
284
+ assert self.seq_lens_tensor is not None
285
+
286
+ # Compute some attn_metadata fields which default to None
287
+ query_start_loc = (None if self.query_start_loc is None else
288
+ self.query_start_loc[:self.num_prefills + 1])
289
+ slot_mapping = (None if self.slot_mapping is None else
290
+ self.slot_mapping[:self.num_prefill_tokens])
291
+ seq_lens = (None if self.seq_lens is None else
292
+ self.seq_lens[:self.num_prefills])
293
+ seq_lens_tensor = (None if self.seq_lens_tensor is None else
294
+ self.seq_lens_tensor[:self.num_prefills])
295
+ seq_start_loc = (None if self.seq_start_loc is None else
296
+ self.seq_start_loc[:self.num_prefills + 1])
297
+ context_lens_tensor = (None if self.context_lens_tensor is None else
298
+ self.context_lens_tensor[:self.num_prefills])
299
+ block_tables = (None if self.block_tables is None else
300
+ self.block_tables[:self.num_prefills])
301
+ input_positions = (None if self.input_positions is None else
302
+ self.input_positions[:self.num_prefill_tokens])
303
+
304
+ self._cached_prefill_metadata = TritonMLAMetadata(
305
+ num_prefills=self.num_prefills,
306
+ num_prefill_tokens=self.num_prefill_tokens,
307
+ num_decode_tokens=0,
308
+ slot_mapping=slot_mapping,
309
+ multi_modal_placeholder_index_maps=self.
310
+ multi_modal_placeholder_index_maps,
311
+ enable_kv_scales_calculation=self.enable_kv_scales_calculation,
312
+ input_positions=input_positions,
313
+ seq_lens=seq_lens,
314
+ seq_lens_tensor=seq_lens_tensor,
315
+ max_query_len=self.max_query_len,
316
+ max_prefill_seq_len=self.max_prefill_seq_len,
317
+ max_decode_query_len=0,
318
+ max_decode_seq_len=0,
319
+ query_start_loc=query_start_loc,
320
+ seq_start_loc=seq_start_loc,
321
+ context_lens_tensor=context_lens_tensor,
322
+ block_tables=block_tables,
323
+ use_cuda_graph=False,
324
+ head_dim=self.head_dim)
325
+ return self._cached_prefill_metadata
326
+
327
+ @property
328
+ def decode_metadata(self) -> Optional["TritonMLAMetadata"]:
329
+ if self.num_decode_tokens == 0:
330
+ return None
331
+
332
+ if self._cached_decode_metadata is not None:
333
+ return self._cached_decode_metadata
334
+ assert self.seq_lens_tensor is not None
335
+
336
+ # Compute some attn_metadata fields which default to None
337
+ slot_mapping = (None if self.slot_mapping is None else
338
+ self.slot_mapping[self.num_prefill_tokens:])
339
+ seq_lens_tensor = (None if self.seq_lens_tensor is None else
340
+ self.seq_lens_tensor[self.num_prefills:])
341
+ block_tables = (None if self.block_tables is None else
342
+ self.block_tables[self.num_prefills:])
343
+ input_positions = (None if self.input_positions is None else
344
+ self.input_positions[self.num_prefill_tokens:])
345
+
346
+ self._cached_decode_metadata = TritonMLAMetadata(
347
+ num_prefills=0,
348
+ num_prefill_tokens=0,
349
+ num_decode_tokens=self.num_decode_tokens,
350
+ slot_mapping=slot_mapping,
351
+ multi_modal_placeholder_index_maps=None,
352
+ enable_kv_scales_calculation=True,
353
+ seq_lens=None,
354
+ seq_lens_tensor=seq_lens_tensor,
355
+ max_decode_query_len=self.max_decode_query_len,
356
+ max_query_len=self.max_query_len,
357
+ max_prefill_seq_len=0,
358
+ max_decode_seq_len=self.max_decode_seq_len,
359
+ # Batch may be composed of prefill|decodes, adjust query start
360
+ # indices to refer to the start of decodes. E.g.
361
+ # in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
362
+ query_start_loc=(self.query_start_loc[self.num_prefills:] -
363
+ self.query_start_loc[self.num_prefills])
364
+ if self.query_start_loc is not None else None,
365
+ seq_start_loc=self.seq_start_loc[self.num_prefills:]
366
+ if self.seq_start_loc is not None else None,
367
+ context_lens_tensor=None,
368
+ block_tables=block_tables,
369
+ use_cuda_graph=self.use_cuda_graph,
370
+ input_positions=input_positions,
371
+ head_dim=self.head_dim)
372
+ return self._cached_decode_metadata
373
+
374
+ def advance_step(self,
375
+ model_input: "ModelInputForGPUWithSamplingMetadata",
376
+ sampled_token_ids: Optional[torch.Tensor],
377
+ block_size: int,
378
+ num_seqs: int,
379
+ num_queries: int,
380
+ turn_prefills_into_decodes: bool = False):
381
+ """
382
+ Update metadata in-place to advance one decode step.
383
+ """
384
+ # When using cudagraph, the num_seqs is padded to the next captured
385
+ # batch sized, but num_queries tracks the actual number of requests in
386
+ # the batch. For --enforce-eager mode, num_seqs == num_queries
387
+ if num_seqs != num_queries:
388
+ assert num_seqs > num_queries
389
+ assert self.use_cuda_graph
390
+
391
+ if turn_prefills_into_decodes:
392
+ # When Mutli-Step is enabled with Chunked-Prefill, prefills and
393
+ # decodes are scheduled together. In the first step, all the
394
+ # prefills turn into decodes. This update reflects that
395
+ # conversion.
396
+ assert self.num_decode_tokens + self.num_prefills == num_seqs
397
+ self.num_decode_tokens += self.num_prefills
398
+ self.num_prefills = 0
399
+ self.num_prefill_tokens = 0
400
+ self.max_prefill_seq_len = 0
401
+ self.max_query_len = 1
402
+
403
+ self.slot_mapping = self.slot_mapping[:num_seqs]
404
+ else:
405
+ assert self.seq_lens is not None
406
+ assert self.max_decode_seq_len == max(self.seq_lens)
407
+
408
+ assert self.num_prefills == 0
409
+ assert self.num_prefill_tokens == 0
410
+ assert self.num_decode_tokens == num_seqs
411
+ assert self.slot_mapping.shape == (num_seqs, )
412
+
413
+ assert self.seq_lens is not None
414
+ assert len(self.seq_lens) == num_seqs
415
+ assert self.seq_lens_tensor is not None
416
+ assert self.seq_lens_tensor.shape == (num_seqs, )
417
+ assert self.max_query_len == 1
418
+ assert self.max_prefill_seq_len == 0
419
+
420
+ assert self.query_start_loc is not None
421
+ assert self.query_start_loc.shape == (num_queries + 1, )
422
+ assert self.seq_start_loc is not None
423
+ assert self.seq_start_loc.shape == (num_seqs + 1, )
424
+
425
+ assert self.context_lens_tensor is not None
426
+ assert self.context_lens_tensor.shape == (num_queries, )
427
+
428
+ assert self.block_tables is not None
429
+ assert self.block_tables.shape[0] == num_seqs
430
+
431
+ # Update query lengths. Note that we update only queries and not seqs,
432
+ # since tensors may be padded due to captured cuda graph batch size
433
+ for i in range(num_queries):
434
+ self.seq_lens[i] += 1
435
+ self.max_decode_seq_len = max(self.seq_lens)
436
+
437
+ ops.advance_step_flashattn(num_seqs=num_seqs,
438
+ num_queries=num_queries,
439
+ block_size=block_size,
440
+ input_tokens=model_input.input_tokens,
441
+ sampled_token_ids=sampled_token_ids,
442
+ input_positions=model_input.input_positions,
443
+ seq_lens=self.seq_lens_tensor,
444
+ slot_mapping=self.slot_mapping,
445
+ block_tables=self.block_tables)
446
+
447
+
448
+class TritonMLAMetadataBuilder(AttentionMetadataBuilder[TritonMLAMetadata]):
449
+
450
+ def __init__(self, input_builder: "ModelInputForGPUBuilder"):
451
+ self.input_builder = input_builder
452
+ self.runner = input_builder.runner
453
+ self.sliding_window = input_builder.sliding_window
454
+ self.block_size = input_builder.block_size
455
+
456
+ def prepare(self):
457
+ self.slot_mapping: List[int] = []
458
+ self.prefill_seq_lens: List[int] = []
459
+ self.context_lens: List[int] = []
460
+ self.block_tables: List[List[int]] = []
461
+ self.curr_seq_lens: List[int] = []
462
+ self.input_positions: List[int] = []
463
+ self.multimodal_placeholder_maps: Dict[
464
+ str,
465
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
466
+ self.num_prefills = 0
467
+ self.num_prefill_tokens = 0
468
+ self.num_decode_tokens = 0
469
+ self.has_prefix_cache_hit = False
470
+
471
+ def _add_seq_group(
472
+ self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
473
+ chunked_prefill_enabled: bool, prefix_cache_hit: bool):
474
+ """Add a sequence group to the metadata. Specifically update/append
475
+ 1. context length.
476
+ 2. block table.
477
+ 3. slot mapping.
478
+ """
479
+ is_prompt = inter_data.is_prompt
480
+ block_tables = inter_data.block_tables
481
+
482
+ for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
483
+ curr_sliding_window_block, input_positions) in zip(
484
+ inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
485
+ inter_data.orig_seq_lens, inter_data.seq_lens,
486
+ inter_data.query_lens, inter_data.context_lens,
487
+ inter_data.curr_sliding_window_blocks,
488
+ inter_data.input_positions):
489
+ self.input_positions.extend(input_positions)
490
+ self.context_lens.append(context_len)
491
+ if is_prompt:
492
+ mm_maps = inter_data.multi_modal_placeholder_maps
493
+ if mm_maps:
494
+ for modality, placeholders in mm_maps.items():
495
+ self.multimodal_placeholder_maps[modality].extend(
496
+ placeholders)
497
+
498
+ self.num_prefills += 1
499
+ self.num_prefill_tokens += token_len
500
+ self.prefill_seq_lens.append(seq_len)
501
+ else:
502
+ self.num_decode_tokens += query_len
503
+ self.curr_seq_lens.append(curr_seq_len)
504
+
505
+ # Compute block table.
506
+ # TODO(sang): Combine chunked prefill and prefix caching by
507
+ # only allowing multiple of block_size chunk size.
508
+ # NOTE: This only works for oooooooxxx style attention.
509
+ block_table = []
510
+ if prefix_cache_hit:
511
+ # NOTE(woosuk): For flash-attn, the block table should
512
+ # include the entries for the incoming prefill tokens.
513
+ block_table = block_tables[seq_id]
514
+ elif ((chunked_prefill_enabled or not is_prompt)
515
+ and block_tables is not None):
516
+ if curr_sliding_window_block == 0:
517
+ block_table = block_tables[seq_id]
518
+ else:
519
+ block_table = block_tables[seq_id][
520
+ -curr_sliding_window_block:]
521
+ self.block_tables.append(block_table)
522
+
523
+ # Compute slot mapping.
524
+ is_profile_run = is_block_tables_empty(block_tables)
525
+ start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
526
+ context_len,
527
+ self.sliding_window)
528
+ compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
529
+ seq_len, context_len, start_idx,
530
+ self.block_size, inter_data.block_tables)
531
+
532
+ def _get_graph_runner_block_tables(
533
+ self, num_seqs: int,
534
+ block_tables: List[List[int]]) -> torch.Tensor:
535
+ # The shape of graph_block_tables is
536
+ # [max batch size, max context len // block size].
537
+ max_batch_size, max_blocks = self.runner.graph_block_tables.shape
538
+ assert max_batch_size >= num_seqs
539
+
540
+ graph_block_tables = self.runner.graph_block_tables[:num_seqs]
541
+ for i, block_table in enumerate(block_tables):
542
+ if block_table:
543
+ num_blocks = len(block_table)
544
+ if num_blocks <= max_blocks:
545
+ graph_block_tables[i, :num_blocks] = block_table
546
+ else:
547
+ # It may be possible to have more blocks allocated due
548
+ # to lookahead slots of multi-step, however, they are
549
+ # not used anyway, so can be safely ignored.
550
+ graph_block_tables[
551
+ i, :max_blocks] = block_table[:max_blocks]
552
+
553
+ return torch.from_numpy(graph_block_tables).to(
554
+ device=self.runner.device, non_blocking=True)
555
+
556
+ def build(self, seq_lens: List[int], query_lens: List[int],
557
+ cuda_graph_pad_size: int, batch_size: int):
558
+ """Build attention metadata with on-device tensors.
559
+
560
+ Args:
561
+ seq_lens: The maybe padded sequence lengths of the input sequences.
562
+ query_lens: The query lengths of the input sequences.
563
+ cuda_graph_pad_size: The padding size for cuda graph.
564
+ -1 if cuda graph is not used.
565
+ batch_size: The maybe padded batch size.
566
+ """
567
+ prefix_cache_hit = any([
568
+ inter_data.prefix_cache_hit
569
+ for inter_data in self.input_builder.inter_data_list
570
+ ])
571
+ for inter_data in self.input_builder.inter_data_list:
572
+ self._add_seq_group(inter_data,
573
+ self.input_builder.chunked_prefill_enabled,
574
+ prefix_cache_hit)
575
+
576
+ device = self.runner.device
577
+ use_captured_graph = cuda_graph_pad_size != -1
578
+
579
+ max_query_len = max(query_lens)
580
+ decode_query_lens = query_lens[self.num_prefills:]
581
+ if len(decode_query_lens) > 0:
582
+ max_decode_query_len = max(decode_query_lens)
583
+ else:
584
+ max_decode_query_len = 1
585
+ max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
586
+ max_decode_seq_len = max(self.curr_seq_lens, default=0)
587
+ num_decode_tokens = self.num_decode_tokens
588
+ query_start_loc = list(accumulate(query_lens, initial=0))
589
+ seq_start_loc = list(accumulate(seq_lens, initial=0))
590
+
591
+ num_seqs = len(seq_lens)
592
+ if use_captured_graph:
593
+ self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
594
+ self.block_tables.extend([] * cuda_graph_pad_size)
595
+ num_decode_tokens = batch_size - self.num_prefill_tokens
596
+ block_tables = self._get_graph_runner_block_tables(
597
+ num_seqs, self.block_tables)
598
+ else:
599
+ block_tables = make_tensor_with_pad(
600
+ self.block_tables,
601
+ pad=0,
602
+ dtype=torch.int,
603
+ device=device,
604
+ )
605
+ assert max_query_len > 0, ("query_lens: {}".format(query_lens))
606
+
607
+ assert device is not None
608
+ context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int,
609
+ device, self.runner.pin_memory)
610
+ seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
611
+ self.runner.pin_memory)
612
+ input_positions = async_tensor_h2d(self.input_positions, torch.long,
613
+ device, self.runner.pin_memory)
614
+ slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
615
+ device, self.runner.pin_memory)
616
+ query_start_loc_tensor = async_tensor_h2d(query_start_loc, torch.int32,
617
+ device,
618
+ self.runner.pin_memory)
619
+ seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
620
+ device, self.runner.pin_memory)
621
+ placeholder_index_maps = {
622
+ modality: placeholder_map.index_map()
623
+ for modality, placeholder_map in
624
+ self.multimodal_placeholder_maps.items()
625
+ }
626
+
627
+ num_kv_splits = 8
628
+
629
+ return TritonMLAMetadata(
630
+ num_prefills=self.num_prefills,
631
+ slot_mapping=slot_mapping_tensor,
632
+ num_prefill_tokens=self.num_prefill_tokens,
633
+ num_decode_tokens=num_decode_tokens,
634
+ seq_lens=seq_lens,
635
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
636
+ enable_kv_scales_calculation=True,
637
+ input_positions=input_positions,
638
+ seq_lens_tensor=seq_lens_tensor,
639
+ max_query_len=max_query_len,
640
+ max_decode_query_len=max_decode_query_len,
641
+ max_prefill_seq_len=max_prefill_seq_len,
642
+ max_decode_seq_len=max_decode_seq_len,
643
+ query_start_loc=query_start_loc_tensor,
644
+ seq_start_loc=seq_start_loc_tensor,
645
+ context_lens_tensor=context_lens_tensor,
646
+ block_tables=block_tables,
647
+ use_cuda_graph=use_captured_graph,
648
+ num_kv_splits=num_kv_splits,
649
+ head_dim=self.runner.model_config.get_head_size(),
650
+ )
651
+
652
+
653
+class TritonMLAImpl(MLACommonImpl[TritonMLAMetadata]):
654
+
655
+ def __init__(
656
+ self,
657
+ num_heads: int,
658
+ head_size: int,
659
+ scale: float,
660
+ num_kv_heads: int,
661
+ alibi_slopes: Optional[List[float]],
662
+ sliding_window: Optional[int],
663
+ kv_cache_dtype: str,
664
+ blocksparse_params: Optional[Dict[str, Any]],
665
+ logits_soft_cap: Optional[float],
666
+ attn_type: str,
667
+ # MLA Specific Arguments
668
+ **kwargs) -> None:
669
+ super().__init__(num_heads, head_size, scale, num_kv_heads,
670
+ alibi_slopes, sliding_window, kv_cache_dtype,
671
+ blocksparse_params, logits_soft_cap, attn_type,
672
+ **kwargs)
673
+
674
+ unsupported_features = [
675
+ alibi_slopes, sliding_window, blocksparse_params, logits_soft_cap
676
+ ]
677
+ if any(unsupported_features):
678
+ raise NotImplementedError(
679
+ "TritonMLAImpl does not support one of the following: "
680
+ "alibi_slopes, sliding_window, blocksparse_params, "
681
+ "logits_soft_cap")
682
+
683
+ if attn_type != AttentionType.DECODER:
684
+ raise NotImplementedError("Encoder self-attention and "
685
+ "encoder/decoder cross-attention "
686
+ "are not implemented for "
687
+ "TritonMLAImpl")
688
+
689
+ def _forward_prefill(
690
+ self,
691
+ q: torch.Tensor,
692
+ kv_c_normed: torch.Tensor,
693
+ k_pe: torch.Tensor,
694
+ attn_metadata: TritonMLAMetadata,
695
+ ) -> torch.Tensor:
696
+ assert isinstance(attn_metadata, TritonMLAMetadata)
697
+ return self._forward_prefill_flash(q, kv_c_normed, k_pe,
698
+ attn_metadata.seq_start_loc,
699
+ attn_metadata.max_prefill_seq_len)
700
+
701
+ def _forward_decode(
702
+ self,
703
+ q_nope: torch.Tensor,
704
+ q_pe: torch.Tensor,
705
+ kv_c_and_k_pe_cache: torch.Tensor,
706
+ attn_metadata: TritonMLAMetadata,
707
+ ) -> torch.Tensor:
708
+ assert kv_c_and_k_pe_cache.numel() > 0
709
+ if self.kv_cache_dtype.startswith("fp8"):
710
+ raise NotImplementedError("FP8 Triton MLA not yet supported")
711
+
712
+ decode_meta = attn_metadata.decode_metadata
713
+ assert decode_meta is not None
714
+ B = q_nope.shape[0]
715
+
716
+ q = torch.cat([q_nope, q_pe], dim=-1)
717
+ o = torch.zeros(B,
718
+ self.num_heads,
719
+ self.kv_lora_rank,
720
+ dtype=q.dtype,
721
+ device=q.device)
722
+
723
+ # TODO(lucas) Allocate ahead of time
724
+ attn_logits = torch.empty(
725
+ (
726
+ B,
727
+ self.num_heads,
728
+ attn_metadata.num_kv_splits,
729
+ # NOTE(lucas) idk why the +1 is here but sglang has it so we
730
+ # just mirror that
731
+ self.kv_lora_rank + 1,
732
+ ),
733
+ dtype=torch.float32,
734
+ device=q.device,
735
+ )
736
+
737
+ # Add a head dim of 1
738
+ kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.unsqueeze(2)
739
+ kv_c_cache = kv_c_and_k_pe_cache[..., :self.kv_lora_rank]
740
+ PAGE_SIZE = kv_c_and_k_pe_cache.size(1)
741
+
742
+ # Run MQA
743
+ decode_attention_fwd(q, kv_c_and_k_pe_cache, kv_c_cache, o,
744
+ decode_meta.block_tables,
745
+ decode_meta.seq_lens_tensor, attn_logits,
746
+ attn_metadata.num_kv_splits, self.scale,
747
+ PAGE_SIZE)
748
+
749
+ return self._v_up_proj_and_o_proj(o)
vllm/attention/backends/utils.py
+4
@@ -289,7 +289,9 @@ class CommonAttentionState(AttentionState):
289
290
@contextmanager
291
def graph_capture(self, max_batch_size: int):
292
+
293
self._is_graph_capturing = True
294
+
295
self._graph_slot_mapping = torch.full((max_batch_size, ),
296
PAD_SLOT_ID,
297
dtype=torch.long,
@@ -299,7 +301,9 @@ class CommonAttentionState(AttentionState):
301
device=self.runner.device)
302
self._graph_block_tables = torch.from_numpy(
303
self.runner.graph_block_tables).to(device=self.runner.device)
304
+
305
yield
306
+
307
self._is_graph_capturing = False
308
del self._graph_slot_mapping
309
del self._graph_seq_lens
vllm/attention/layer.py
+15
-4
@@ -41,8 +41,10 @@ class Attention(nn.Module):
41
blocksparse_params: Optional[Dict[str, Any]] = None,
42
logits_soft_cap: Optional[float] = None,
43
per_layer_sliding_window: Optional[int] = None,
44
+ use_mla: bool = False,
45
prefix: str = "",
46
attn_type: str = AttentionType.DECODER,
47
+ **extra_impl_args,
48
) -> None:
49
super().__init__()
50
if per_layer_sliding_window is not None:
@@ -101,13 +103,18 @@ class Attention(nn.Module):
103
# During model initialization, the default dtype is set as the model
104
# weight and activation dtype.
105
dtype = torch.get_default_dtype()
104
- attn_backend = get_attn_backend(head_size, dtype, kv_cache_dtype,
105
- block_size, is_attention_free,
106
- blocksparse_params is not None)
106
+ attn_backend = get_attn_backend(head_size,
107
+ dtype,
108
+ kv_cache_dtype,
109
+ block_size,
110
+ is_attention_free,
111
+ blocksparse_params is not None,
112
+ use_mla=use_mla)
113
impl_cls = attn_backend.get_impl_cls()
114
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
115
alibi_slopes, sliding_window, kv_cache_dtype,
110
- blocksparse_params, logits_soft_cap, attn_type)
116
+ blocksparse_params, logits_soft_cap, attn_type,
117
+ **extra_impl_args)
118
self.num_heads = num_heads
119
self.head_size = head_size
120
self.num_kv_heads = num_kv_heads
@@ -193,6 +200,10 @@ class Attention(nn.Module):
200
s += f", backend={self.impl.__class__.__name__}"
201
return s
202
203
+ def process_weights_after_loading(self):
204
+ if hasattr(self.impl, "process_weights_after_loading"):
205
+ self.impl.process_weights_after_loading()
206
+
207
208
class MultiHeadAttention(nn.Module):
209
"""Multi-headed attention without any cache, used for ViT."""
vllm/attention/ops/triton_decode_attention.py
new
+667
@@ -0,0 +1,667 @@
1
+# Adapted from
2
+# https://github.com/sgl-project/sglang/blob/9f635ea50de920aa507f486daafba26a5b837574/python/sglang/srt/layers/attention/triton_ops/decode_attention.py
3
+# which was originally adapted from
4
+# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage1.py
5
+# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage2.py
6
+
7
+# Changes:
8
+# - Add support for page size >= 1.
9
+
10
+# Copyright 2025 vLLM Team
11
+# Copyright 2023-2024 SGLang Team
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+# ==============================================================================
24
+"""
25
+Memory-efficient attention for decoding.
26
+It supports page size >= 1.
27
+"""
28
+
29
+import logging
30
+
31
+import triton
32
+import triton.language as tl
33
+
34
+from vllm.platforms import current_platform
35
+
36
+is_hip_ = current_platform.is_rocm()
37
+
38
+logger = logging.getLogger(__name__)
39
+
40
+# TODO: Remove this when triton>=3.2.0. This issue will not affect performance
41
+# and accuracy.
42
+logger.warning(
43
+ "The following error message 'operation scheduled before its operands' "
44
+ "can be ignored.")
45
+
46
+
47
+@triton.jit
48
+def tanh(x):
49
+ # Tanh is just a scaled sigmoid
50
+ return 2 * tl.sigmoid(2 * x) - 1
51
+
52
+
53
+@triton.jit
54
+def _fwd_kernel_stage1(
55
+ Q,
56
+ K_Buffer,
57
+ V_Buffer,
58
+ sm_scale,
59
+ Req_to_tokens,
60
+ B_Seqlen,
61
+ Att_Out,
62
+ stride_req_to_tokens_b,
63
+ stride_qbs,
64
+ stride_qh,
65
+ stride_buf_kbs,
66
+ stride_buf_kh,
67
+ stride_buf_vbs,
68
+ stride_buf_vh,
69
+ stride_mid_ob,
70
+ stride_mid_oh,
71
+ stride_mid_os,
72
+ kv_group_num: tl.constexpr,
73
+ BLOCK_DMODEL: tl.constexpr,
74
+ BLOCK_DV: tl.constexpr,
75
+ BLOCK_N: tl.constexpr,
76
+ NUM_KV_SPLITS: tl.constexpr,
77
+ PAGE_SIZE: tl.constexpr,
78
+ logit_cap: tl.constexpr,
79
+ Lk: tl.constexpr,
80
+ Lv: tl.constexpr,
81
+):
82
+ cur_batch = tl.program_id(0)
83
+ cur_head = tl.program_id(1)
84
+ split_kv_id = tl.program_id(2)
85
+
86
+ cur_kv_head = cur_head // kv_group_num
87
+
88
+ offs_d = tl.arange(0, BLOCK_DMODEL)
89
+ offs_dv = tl.arange(0, BLOCK_DV)
90
+ mask_d = offs_d < Lk
91
+ mask_dv = offs_dv < Lv
92
+ cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
93
+ cur_batch_req_idx = cur_batch
94
+
95
+ off_q = cur_batch * stride_qbs + cur_head * stride_qh + offs_d
96
+ q = tl.load(Q + off_q, mask=mask_d, other=0.0)
97
+
98
+ kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS)
99
+ split_kv_start = kv_len_per_split * split_kv_id
100
+ split_kv_end = tl.minimum(split_kv_start + kv_len_per_split,
101
+ cur_batch_seq_len)
102
+
103
+ e_max = -float("inf")
104
+ e_sum = 0.0
105
+ acc = tl.zeros([BLOCK_DV], dtype=tl.float32)
106
+
107
+ if split_kv_end > split_kv_start:
108
+ for start_n in range(split_kv_start, split_kv_end, BLOCK_N):
109
+ offs_n = start_n + tl.arange(0, BLOCK_N)
110
+ kv_page_number = tl.load(
111
+ Req_to_tokens + stride_req_to_tokens_b * cur_batch_req_idx +
112
+ offs_n // PAGE_SIZE,
113
+ mask=offs_n < split_kv_end,
114
+ other=0,
115
+ )
116
+ kv_loc = kv_page_number * PAGE_SIZE + offs_n % PAGE_SIZE
117
+ offs_buf_k = (kv_loc[:, None] * stride_buf_kbs +
118
+ cur_kv_head * stride_buf_kh + offs_d[None, :])
119
+ k = tl.load(
120
+ K_Buffer + offs_buf_k,
121
+ mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]),
122
+ other=0.0,
123
+ )
124
+ qk = tl.sum(q[None, :] * k, 1)
125
+ qk *= sm_scale
126
+
127
+ if logit_cap > 0:
128
+ qk = logit_cap * tanh(qk / logit_cap)
129
+
130
+ qk = tl.where(offs_n < split_kv_end, qk, float("-inf"))
131
+
132
+ offs_buf_v = (kv_loc[:, None] * stride_buf_vbs +
133
+ cur_kv_head * stride_buf_vh + offs_dv[None, :])
134
+ v = tl.load(
135
+ V_Buffer + offs_buf_v,
136
+ mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]),
137
+ other=0.0,
138
+ )
139
+
140
+ n_e_max = tl.maximum(tl.max(qk, 0), e_max)
141
+ re_scale = tl.exp(e_max - n_e_max)
142
+ p = tl.exp(qk - n_e_max)
143
+ acc *= re_scale
144
+ acc += tl.sum(p[:, None] * v, 0)
145
+
146
+ e_sum = e_sum * re_scale + tl.sum(p, 0)
147
+ e_max = n_e_max
148
+
149
+ offs_mid_o = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh +
150
+ split_kv_id * stride_mid_os + offs_dv)
151
+
152
+ tl.store(
153
+ Att_Out + offs_mid_o,
154
+ acc / e_sum,
155
+ mask=(mask_dv),
156
+ )
157
+
158
+ offs_mid_o_1 = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh +
159
+ split_kv_id * stride_mid_os + Lv)
160
+
161
+ tl.store(
162
+ Att_Out + offs_mid_o_1,
163
+ e_max + tl.log(e_sum),
164
+ )
165
+
166
+
167
+def _decode_att_m_fwd(
168
+ q,
169
+ k_buffer,
170
+ v_buffer,
171
+ att_out,
172
+ Req_to_tokens,
173
+ B_Seqlen,
174
+ num_kv_splits,
175
+ sm_scale,
176
+ page_size,
177
+ logit_cap,
178
+):
179
+ BLOCK = 64
180
+ NUM_KV_SPLITS = num_kv_splits
181
+ Lk = k_buffer.shape[-1]
182
+ Lv = v_buffer.shape[-1]
183
+
184
+ batch, head_num = q.shape[0], q.shape[1]
185
+
186
+ grid = (batch, head_num, NUM_KV_SPLITS)
187
+ kv_group_num = q.shape[1] // k_buffer.shape[-2]
188
+
189
+ num_warps = 4 if kv_group_num == 1 else 2
190
+
191
+ BLOCK_DMODEL = triton.next_power_of_2(Lk)
192
+ BLOCK_DV = triton.next_power_of_2(Lv)
193
+
194
+ _fwd_kernel_stage1[grid](
195
+ q,
196
+ k_buffer,
197
+ v_buffer,
198
+ sm_scale,
199
+ Req_to_tokens,
200
+ B_Seqlen,
201
+ att_out,
202
+ Req_to_tokens.stride(0),
203
+ q.stride(0),
204
+ q.stride(1),
205
+ k_buffer.stride(-2),
206
+ k_buffer.stride(-1),
207
+ v_buffer.stride(-2),
208
+ v_buffer.stride(-1),
209
+ att_out.stride(0),
210
+ att_out.stride(1),
211
+ att_out.stride(2),
212
+ kv_group_num=kv_group_num,
213
+ BLOCK_DMODEL=BLOCK_DMODEL,
214
+ BLOCK_DV=BLOCK_DV,
215
+ BLOCK_N=BLOCK,
216
+ NUM_KV_SPLITS=NUM_KV_SPLITS,
217
+ PAGE_SIZE=page_size,
218
+ logit_cap=logit_cap,
219
+ num_warps=num_warps,
220
+ num_stages=2,
221
+ Lk=Lk,
222
+ Lv=Lv,
223
+ )
224
+
225
+
226
+@triton.jit
227
+def _fwd_grouped_kernel_stage1(
228
+ Q,
229
+ K_Buffer,
230
+ V_Buffer,
231
+ sm_scale,
232
+ Req_to_tokens,
233
+ B_Seqlen,
234
+ Att_Out,
235
+ stride_req_to_tokens_b,
236
+ stride_qbs,
237
+ stride_qh,
238
+ stride_buf_kbs,
239
+ stride_buf_kh,
240
+ stride_buf_vbs,
241
+ stride_buf_vh,
242
+ stride_mid_ob,
243
+ stride_mid_oh,
244
+ stride_mid_os,
245
+ kv_group_num: tl.constexpr,
246
+ q_head_num: tl.constexpr,
247
+ BLOCK_DMODEL: tl.constexpr,
248
+ BLOCK_DPE: tl.constexpr,
249
+ BLOCK_DV: tl.constexpr,
250
+ BLOCK_N: tl.constexpr,
251
+ BLOCK_H: tl.constexpr,
252
+ NUM_KV_SPLITS: tl.constexpr,
253
+ PAGE_SIZE: tl.constexpr,
254
+ logit_cap: tl.constexpr,
255
+ Lk: tl.constexpr,
256
+ Lv: tl.constexpr,
257
+):
258
+ cur_batch = tl.program_id(0)
259
+ cur_head_id = tl.program_id(1)
260
+ cur_kv_head = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H)
261
+ split_kv_id = tl.program_id(2)
262
+
263
+ if kv_group_num > BLOCK_H:
264
+ VALID_BLOCK_H: tl.constexpr = BLOCK_H
265
+ else:
266
+ VALID_BLOCK_H: tl.constexpr = kv_group_num
267
+ cur_head = cur_head_id * VALID_BLOCK_H + tl.arange(0, BLOCK_H)
268
+ mask_h = cur_head < (cur_head_id + 1) * VALID_BLOCK_H
269
+ mask_h = mask_h & (cur_head < q_head_num)
270
+
271
+ offs_d = tl.arange(0, BLOCK_DMODEL)
272
+ offs_dv = tl.arange(0, BLOCK_DV)
273
+ mask_d = offs_d < Lk
274
+ mask_dv = offs_dv < Lv
275
+ cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
276
+ cur_batch_req_idx = cur_batch
277
+
278
+ offs_q = cur_batch * stride_qbs + cur_head[:, None] * stride_qh + offs_d[
279
+ None, :]
280
+ q = tl.load(Q + offs_q,
281
+ mask=(mask_h[:, None]) & (mask_d[None, :]),
282
+ other=0.0)
283
+
284
+ if BLOCK_DPE > 0:
285
+ offs_dpe = BLOCK_DMODEL + tl.arange(0, BLOCK_DPE)
286
+ mask_dpe = offs_dpe < Lk
287
+ off_qpe = (cur_batch * stride_qbs + cur_head[:, None] * stride_qh +
288
+ offs_dpe[None, :])
289
+ qpe = tl.load(Q + off_qpe,
290
+ mask=(mask_h[:, None]) & (mask_dpe[None, :]),
291
+ other=0.0)
292
+
293
+ kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS)
294
+ split_kv_start = kv_len_per_split * split_kv_id
295
+ split_kv_end = tl.minimum(split_kv_start + kv_len_per_split,
296
+ cur_batch_seq_len)
297
+
298
+ e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - float("inf")
299
+ e_sum = tl.zeros([BLOCK_H], dtype=tl.float32)
300
+ acc = tl.zeros([BLOCK_H, BLOCK_DV], dtype=tl.float32)
301
+
302
+ if split_kv_end > split_kv_start:
303
+ for start_n in range(split_kv_start, split_kv_end, BLOCK_N):
304
+ offs_n = start_n + tl.arange(0, BLOCK_N)
305
+ kv_page_number = tl.load(
306
+ Req_to_tokens + stride_req_to_tokens_b * cur_batch_req_idx +
307
+ offs_n // PAGE_SIZE,
308
+ mask=offs_n < split_kv_end,
309
+ other=0,
310
+ )
311
+ kv_loc = kv_page_number * PAGE_SIZE + offs_n % PAGE_SIZE
312
+ offs_buf_k = (kv_loc[None, :] * stride_buf_kbs +
313
+ cur_kv_head * stride_buf_kh + offs_d[:, None])
314
+ k = tl.load(
315
+ K_Buffer + offs_buf_k,
316
+ mask=(offs_n[None, :] < split_kv_end) & (mask_d[:, None]),
317
+ other=0.0,
318
+ )
319
+ qk = tl.dot(q, k.to(q.dtype))
320
+ if BLOCK_DPE > 0:
321
+ offs_buf_kpe = (kv_loc[None, :] * stride_buf_kbs +
322
+ cur_kv_head * stride_buf_kh +
323
+ offs_dpe[:, None])
324
+ kpe = tl.load(
325
+ K_Buffer + offs_buf_kpe,
326
+ mask=(offs_n[None, :] < split_kv_end) &
327
+ (mask_dpe[:, None]),
328
+ other=0.0,
329
+ )
330
+ qk += tl.dot(qpe, kpe.to(qpe.dtype))
331
+ qk *= sm_scale
332
+
333
+ if logit_cap > 0:
334
+ qk = logit_cap * tanh(qk / logit_cap)
335
+
336
+ qk = tl.where(mask_h[:, None] & (offs_n[None, :] < split_kv_end),
337
+ qk, float("-inf"))
338
+
339
+ offs_buf_v = (kv_loc[:, None] * stride_buf_vbs +
340
+ cur_kv_head * stride_buf_vh + offs_dv[None, :])
341
+ v = tl.load(
342
+ V_Buffer + offs_buf_v,
343
+ mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]),
344
+ other=0.0,
345
+ )
346
+
347
+ n_e_max = tl.maximum(tl.max(qk, 1), e_max)
348
+ re_scale = tl.exp(e_max - n_e_max)
349
+ p = tl.exp(qk - n_e_max[:, None])
350
+ acc *= re_scale[:, None]
351
+ acc += tl.dot(p.to(v.dtype), v)
352
+
353
+ e_sum = e_sum * re_scale + tl.sum(p, 1)
354
+ e_max = n_e_max
355
+
356
+ offs_mid_o = (cur_batch * stride_mid_ob +
357
+ cur_head[:, None] * stride_mid_oh +
358
+ split_kv_id * stride_mid_os + offs_dv[None, :])
359
+
360
+ tl.store(
361
+ Att_Out + offs_mid_o,
362
+ acc / e_sum[:, None],
363
+ mask=(mask_h[:, None]) & (mask_dv[None, :]),
364
+ )
365
+
366
+ offs_mid_o_1 = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh +
367
+ split_kv_id * stride_mid_os + Lv)
368
+
369
+ tl.store(
370
+ Att_Out + offs_mid_o_1,
371
+ e_max + tl.log(e_sum),
372
+ mask=mask_h,
373
+ )
374
+
375
+
376
+def _decode_grouped_att_m_fwd(
377
+ q,
378
+ k_buffer,
379
+ v_buffer,
380
+ att_out,
381
+ Req_to_tokens,
382
+ B_Seqlen,
383
+ num_kv_splits,
384
+ sm_scale,
385
+ page_size,
386
+ logit_cap,
387
+):
388
+ BLOCK = 32
389
+ Lk = k_buffer.shape[-1]
390
+ Lv = v_buffer.shape[-1]
391
+
392
+ # [TODO] work around shmem limit on MI3xx
393
+ if is_hip_ and Lk >= 576:
394
+ BLOCK = 16
395
+
396
+ if Lk == 576:
397
+ BLOCK_DMODEL = 512
398
+ BLOCK_DPE = 64
399
+ elif Lk == 288:
400
+ BLOCK_DMODEL = 256
401
+ BLOCK_DPE = 32
402
+ else:
403
+ BLOCK_DMODEL = triton.next_power_of_2(Lk)
404
+ BLOCK_DPE = 0
405
+ BLOCK_DV = triton.next_power_of_2(Lv)
406
+
407
+ batch, head_num = q.shape[0], q.shape[1]
408
+ kv_group_num = q.shape[1] // k_buffer.shape[-2]
409
+
410
+ BLOCK_H = 16
411
+ NUM_KV_SPLITS = num_kv_splits
412
+ grid = (
413
+ batch,
414
+ triton.cdiv(head_num, min(BLOCK_H, kv_group_num)),
415
+ NUM_KV_SPLITS,
416
+ )
417
+
418
+ extra_kargs = {}
419
+ if is_hip_:
420
+ # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html
421
+ # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py
422
+ extra_kargs = {
423
+ "waves_per_eu": 4,
424
+ "matrix_instr_nonkdim": 16,
425
+ "kpack": 2
426
+ }
427
+
428
+ _fwd_grouped_kernel_stage1[grid](
429
+ q,
430
+ k_buffer,
431
+ v_buffer,
432
+ sm_scale,
433
+ Req_to_tokens,
434
+ B_Seqlen,
435
+ att_out,
436
+ Req_to_tokens.stride(0),
437
+ q.stride(0),
438
+ q.stride(1),
439
+ k_buffer.stride(-2),
440
+ k_buffer.stride(-1),
441
+ v_buffer.stride(-2),
442
+ v_buffer.stride(-1),
443
+ att_out.stride(0),
444
+ att_out.stride(1),
445
+ att_out.stride(2),
446
+ kv_group_num=kv_group_num,
447
+ q_head_num=head_num,
448
+ BLOCK_DMODEL=BLOCK_DMODEL,
449
+ BLOCK_DPE=BLOCK_DPE,
450
+ BLOCK_DV=BLOCK_DV,
451
+ BLOCK_N=BLOCK,
452
+ BLOCK_H=BLOCK_H,
453
+ NUM_KV_SPLITS=NUM_KV_SPLITS,
454
+ PAGE_SIZE=page_size,
455
+ logit_cap=logit_cap,
456
+ num_warps=4,
457
+ num_stages=2,
458
+ Lk=Lk,
459
+ Lv=Lv,
460
+ **extra_kargs,
461
+ )
462
+
463
+
464
+@triton.jit
465
+def _fwd_kernel_stage2(
466
+ Mid_O,
467
+ o,
468
+ B_Seqlen,
469
+ stride_mid_ob,
470
+ stride_mid_oh,
471
+ stride_mid_os,
472
+ stride_obs,
473
+ stride_oh,
474
+ NUM_KV_SPLITS: tl.constexpr,
475
+ BLOCK_DV: tl.constexpr,
476
+ Lv: tl.constexpr,
477
+):
478
+ cur_batch = tl.program_id(0)
479
+ cur_head = tl.program_id(1)
480
+
481
+ cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
482
+
483
+ offs_d = tl.arange(0, BLOCK_DV)
484
+ mask_d = offs_d < Lv
485
+
486
+ e_sum = 0.0
487
+ e_max = -float("inf")
488
+ acc = tl.zeros([BLOCK_DV], dtype=tl.float32)
489
+
490
+ offs_v = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + offs_d
491
+ offs_logic = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + Lv
492
+
493
+ for split_kv_id in range(0, NUM_KV_SPLITS):
494
+ kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS)
495
+ split_kv_start = kv_len_per_split * split_kv_id
496
+ split_kv_end = tl.minimum(split_kv_start + kv_len_per_split,
497
+ cur_batch_seq_len)
498
+
499
+ if split_kv_end > split_kv_start:
500
+ tv = tl.load(Mid_O + offs_v + split_kv_id * stride_mid_os,
501
+ mask=mask_d,
502
+ other=0.0)
503
+ tlogic = tl.load(Mid_O + offs_logic + split_kv_id * stride_mid_os)
504
+ n_e_max = tl.maximum(tlogic, e_max)
505
+
506
+ old_scale = tl.exp(e_max - n_e_max)
507
+ acc *= old_scale
508
+ exp_logic = tl.exp(tlogic - n_e_max)
509
+ acc += exp_logic * tv
510
+
511
+ e_sum = e_sum * old_scale + exp_logic
512
+ e_max = n_e_max
513
+
514
+ tl.store(
515
+ o + cur_batch * stride_obs + cur_head * stride_oh + offs_d,
516
+ acc / e_sum,
517
+ mask=mask_d,
518
+ )
519
+
520
+
521
+def _decode_softmax_reducev_fwd(
522
+ logits,
523
+ q,
524
+ o,
525
+ v_buffer,
526
+ b_seq_len,
527
+ num_kv_splits,
528
+):
529
+ batch, head_num = q.shape[0], q.shape[1]
530
+ Lv = v_buffer.shape[-1]
531
+ BLOCK_DV = triton.next_power_of_2(Lv)
532
+
533
+ NUM_KV_SPLITS = num_kv_splits
534
+
535
+ extra_kargs = {}
536
+ if is_hip_:
537
+ # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html
538
+ # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py
539
+ extra_kargs = {
540
+ "waves_per_eu": 4,
541
+ "matrix_instr_nonkdim": 16,
542
+ "kpack": 2
543
+ }
544
+
545
+ grid = (batch, head_num)
546
+ _fwd_kernel_stage2[grid](
547
+ logits,
548
+ o,
549
+ b_seq_len,
550
+ logits.stride(0),
551
+ logits.stride(1),
552
+ logits.stride(2),
553
+ o.stride(0),
554
+ o.stride(1),
555
+ NUM_KV_SPLITS=NUM_KV_SPLITS,
556
+ BLOCK_DV=BLOCK_DV,
557
+ Lv=Lv,
558
+ num_warps=4,
559
+ num_stages=2,
560
+ **extra_kargs,
561
+ )
562
+
563
+
564
+def decode_attention_fwd_normal(
565
+ q,
566
+ k_buffer,
567
+ v_buffer,
568
+ o,
569
+ req_to_token,
570
+ b_seq_len,
571
+ attn_logits,
572
+ num_kv_splits,
573
+ sm_scale,
574
+ page_size,
575
+ logit_cap=0.0,
576
+):
577
+ _decode_att_m_fwd(
578
+ q,
579
+ k_buffer,
580
+ v_buffer,
581
+ attn_logits,
582
+ req_to_token,
583
+ b_seq_len,
584
+ num_kv_splits,
585
+ sm_scale,
586
+ page_size,
587
+ logit_cap,
588
+ )
589
+ _decode_softmax_reducev_fwd(attn_logits, q, o, v_buffer, b_seq_len,
590
+ num_kv_splits)
591
+
592
+
593
+def decode_attention_fwd_grouped(
594
+ q,
595
+ k_buffer,
596
+ v_buffer,
597
+ o,
598
+ req_to_token,
599
+ b_seq_len,
600
+ attn_logits,
601
+ num_kv_splits,
602
+ sm_scale,
603
+ page_size,
604
+ logit_cap=0.0,
605
+):
606
+ _decode_grouped_att_m_fwd(
607
+ q,
608
+ k_buffer,
609
+ v_buffer,
610
+ attn_logits,
611
+ req_to_token,
612
+ b_seq_len,
613
+ num_kv_splits,
614
+ sm_scale,
615
+ page_size,
616
+ logit_cap,
617
+ )
618
+ _decode_softmax_reducev_fwd(attn_logits, q, o, v_buffer, b_seq_len,
619
+ num_kv_splits)
620
+
621
+
622
+def decode_attention_fwd(
623
+ q,
624
+ k_buffer,
625
+ v_buffer,
626
+ o,
627
+ req_to_token,
628
+ b_seq_len,
629
+ attn_logits,
630
+ num_kv_splits,
631
+ sm_scale,
632
+ page_size=1,
633
+ logit_cap=0.0,
634
+):
635
+ assert num_kv_splits == attn_logits.shape[2]
636
+ kv_group_num = q.shape[1] // v_buffer.shape[-2]
637
+
638
+ if kv_group_num == 1:
639
+ # MHA
640
+ decode_attention_fwd_normal(
641
+ q,
642
+ k_buffer,
643
+ v_buffer,
644
+ o,
645
+ req_to_token,
646
+ b_seq_len,
647
+ attn_logits,
648
+ num_kv_splits,
649
+ sm_scale,
650
+ page_size,
651
+ logit_cap,
652
+ )
653
+ else:
654
+ # GQA/MQA/MLA
655
+ decode_attention_fwd_grouped(
656
+ q,
657
+ k_buffer,
658
+ v_buffer,
659
+ o,
660
+ req_to_token,
661
+ b_seq_len,
662
+ attn_logits,
663
+ num_kv_splits,
664
+ sm_scale,
665
+ page_size,
666
+ logit_cap,
667
+ )
vllm/attention/selector.py
+5
-1
@@ -83,6 +83,7 @@ def get_attn_backend(
83
block_size: int,
84
is_attention_free: bool,
85
is_blocksparse: bool = False,
86
+ use_mla: bool = False,
87
) -> Type[AttentionBackend]:
88
"""Selects which attention backend to use and lazily imports it."""
89
# Accessing envs.* behind an @lru_cache decorator can cause the wrong
@@ -97,6 +98,7 @@ def get_attn_backend(
98
is_attention_free=is_attention_free,
99
is_blocksparse=is_blocksparse,
100
use_v1=envs.VLLM_USE_V1,
101
+ use_mla=use_mla,
102
)
103
104
@@ -109,6 +111,7 @@ def _cached_get_attn_backend(
111
is_attention_free: bool,
112
is_blocksparse: bool = False,
113
use_v1: bool = False,
114
+ use_mla: bool = False,
115
) -> Type[AttentionBackend]:
116
if is_blocksparse:
117
logger.info("Using BlocksparseFlashAttention backend.")
@@ -141,7 +144,8 @@ def _cached_get_attn_backend(
144
145
# get device-specific attn_backend
146
attention_cls = current_platform.get_attn_backend_cls(
144
- selected_backend, head_size, dtype, kv_cache_dtype, block_size, use_v1)
147
+ selected_backend, head_size, dtype, kv_cache_dtype, block_size, use_v1,
148
+ use_mla)
149
if not attention_cls:
150
raise ValueError(
151
f"Invalid attention backend for {current_platform.device_name}")
vllm/config.py
+26
-9
@@ -736,17 +736,25 @@ class ModelConfig:
736
def get_hidden_size(self) -> int:
737
return self.hf_text_config.hidden_size
738
739
+ @property
740
+ def is_deepseek_mla(self) -> bool:
741
+ # TODO add deepseek_v3
742
+ return hasattr(self.hf_text_config,
743
+ "model_type") and (self.hf_text_config.model_type
744
+ in ('deepseek_v2'))
745
+
746
def get_head_size(self) -> int:
747
# TODO remove hard code
741
- if hasattr(self.hf_text_config,
742
- "model_type") and (self.hf_text_config.model_type
743
- in ('deepseek_v2', 'deepseek_v3')):
744
- qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
745
- 0)
746
- qk_nope_head_dim = getattr(self.hf_text_config, "qk_nope_head_dim",
747
- 0)
748
- if qk_rope_head_dim and qk_nope_head_dim:
749
- return qk_rope_head_dim + qk_nope_head_dim
748
+ if self.is_deepseek_mla:
749
+ if self.use_mla:
750
+ return self.hf_text_config.kv_lora_rank
751
+ else:
752
+ qk_rope_head_dim = getattr(self.hf_text_config,
753
+ "qk_rope_head_dim", 0)
754
+ qk_nope_head_dim = getattr(self.hf_text_config,
755
+ "qk_nope_head_dim", 0)
756
+ if qk_rope_head_dim and qk_nope_head_dim:
757
+ return qk_rope_head_dim + qk_nope_head_dim
758
759
if self.is_attention_free:
760
return 0
@@ -805,6 +813,10 @@ class ModelConfig:
813
814
def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
815
"""Returns the number of KV heads per GPU."""
816
+ if self.use_mla:
817
+ # When using MLA during decode it becomes MQA
818
+ return 1
819
+
820
total_num_kv_heads = self.get_total_num_kv_heads()
821
# If tensor parallelism is used, we divide the number of KV heads by
822
# the tensor parallel size. We will replicate the KV heads in the
@@ -955,6 +967,11 @@ class ModelConfig:
967
architectures = getattr(self.hf_config, "architectures", [])
968
return ModelRegistry.is_cross_encoder_model(architectures)
969
970
+ @property
971
+ def use_mla(self) -> bool:
972
+ use_mla = (self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE)
973
+ return use_mla
974
+
975
@property
976
def supported_runner_types(self) -> Set[RunnerType]:
977
return {_TASK_RUNNER[task] for task in self.supported_tasks}
vllm/engine/arg_utils.py
-1
@@ -931,7 +931,6 @@ class EngineArgs:
931
type=str,
932
default="auto",
933
help='The worker class to use for distributed execution.')
934
-
934
parser.add_argument(
935
"--generation-config",
936
type=nullable_str,
vllm/envs.py
+14
@@ -77,6 +77,8 @@ if TYPE_CHECKING:
77
V_SCALE_CONSTANT: int = 100
78
VLLM_SERVER_DEV_MODE: bool = False
79
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
80
+ VLLM_MLA_DISABLE: bool = False
81
+ VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
82
83
84
def get_default_cache_root():
@@ -506,6 +508,18 @@ environment_variables: Dict[str, Callable[[], Any]] = {
508
# TTFT and overall throughput.
509
"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE":
510
lambda: int(os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128")),
511
+
512
+ # If set, vLLM will disable the MLA attention optimizations.
513
+ "VLLM_MLA_DISABLE":
514
+ lambda: bool(int(os.getenv("VLLM_MLA_DISABLE", "0"))),
515
+
516
+ # Flag that can control whether or not we perform matrix-absorption for MLA
517
+ # decode, i.e. absorb W_UK into W_Q/W_UK and W_UV into W_O, absorbing the
518
+ # matrices reduces the runtime FLOPs needed to compute MLA but requires
519
+ # storing more weights, W_Q_UK and W_UV_O, so can increase memory usage,
520
+ # the is enabled by default
521
+ "VLLM_MLA_PERFORM_MATRIX_ABSORPTION":
522
+ lambda: bool(int(os.getenv("VLLM_MLA_PERFORM_MATRIX_ABSORPTION", "1")))
523
}
524
525
# end-env-vars-definition
vllm/model_executor/model_loader/loader.py
+6
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import AutoModelForCausalLM
24
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
25
26
+from vllm.attention import Attention
27
from vllm.config import (LoadConfig, LoadFormat, ModelConfig, ParallelConfig,
28
VllmConfig, set_current_vllm_config)
29
from vllm.distributed import (get_tensor_model_parallel_rank,
@@ -397,6 +398,11 @@ class DefaultModelLoader(BaseModelLoader):
398
# parameters onto device for processing and back off after.
399
with device_loading_context(module, target_device):
400
quant_method.process_weights_after_loading(module)
401
+ elif isinstance(module, Attention) and \
402
+ hasattr(module, "process_weights_after_loading"):
403
+ # When attention modules need to process weights after
404
+ # currently only used by MLA
405
+ module.process_weights_after_loading()
406
return model.eval()
407
408
vllm/model_executor/models/deepseek_v2.py
+152
-2
@@ -28,7 +28,7 @@ from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, VllmConfig
31
+from vllm.config import CacheConfig, ModelConfig, VllmConfig
32
from vllm.distributed import (get_pp_group,
33
get_tensor_model_parallel_world_size,
34
tensor_model_parallel_all_reduce)
@@ -326,12 +326,156 @@ class DeepseekV2Attention(nn.Module):
326
return output
327
328
329
+class DeepseekV2MLAAttention(nn.Module):
330
+ """
331
+ Main reference: DeepseekV2 paper, and FlashInfer Implementation
332
+ (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
333
+
334
+ For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py
335
+ """
336
+
337
+ def __init__(
338
+ self,
339
+ config: PretrainedConfig,
340
+ hidden_size: int,
341
+ num_heads: int,
342
+ qk_nope_head_dim: int,
343
+ qk_rope_head_dim: int,
344
+ v_head_dim: int,
345
+ q_lora_rank: Optional[int],
346
+ kv_lora_rank: int,
347
+ rope_theta: float = 10000,
348
+ rope_scaling: Optional[Dict[str, Any]] = None,
349
+ max_position_embeddings: int = 8192,
350
+ cache_config: Optional[CacheConfig] = None,
351
+ quant_config: Optional[QuantizationConfig] = None,
352
+ prefix: str = "",
353
+ ) -> None:
354
+ super().__init__()
355
+ self.hidden_size = hidden_size
356
+ self.qk_nope_head_dim = qk_nope_head_dim
357
+ self.qk_rope_head_dim = qk_rope_head_dim
358
+ self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
359
+ self.v_head_dim = v_head_dim
360
+
361
+ self.q_lora_rank = q_lora_rank
362
+ self.kv_lora_rank = kv_lora_rank
363
+
364
+ self.num_heads = num_heads
365
+ tp_size = get_tensor_model_parallel_world_size()
366
+ assert num_heads % tp_size == 0
367
+ self.num_local_heads = num_heads // tp_size
368
+
369
+ self.scaling = self.qk_head_dim**-0.5
370
+ self.rope_theta = rope_theta
371
+ self.max_position_embeddings = max_position_embeddings
372
+
373
+ if self.q_lora_rank is not None:
374
+ self.q_a_proj = ReplicatedLinear(self.hidden_size,
375
+ self.q_lora_rank,
376
+ bias=False,
377
+ quant_config=quant_config,
378
+ prefix=f"{prefix}.q_a_proj")
379
+ self.q_a_layernorm = RMSNorm(self.q_lora_rank,
380
+ eps=config.rms_norm_eps)
381
+ self.q_b_proj = ColumnParallelLinear(q_lora_rank,
382
+ self.num_heads *
383
+ self.qk_head_dim,
384
+ bias=False,
385
+ quant_config=quant_config,
386
+ prefix=f"{prefix}.q_b_proj")
387
+ else:
388
+ self.q_proj = ColumnParallelLinear(self.hidden_size,
389
+ self.num_heads *
390
+ self.qk_head_dim,
391
+ bias=False,
392
+ quant_config=quant_config,
393
+ prefix=f"{prefix}.q_proj")
394
+
395
+ self.kv_a_proj_with_mqa = ReplicatedLinear(
396
+ self.hidden_size,
397
+ self.kv_lora_rank + self.qk_rope_head_dim,
398
+ bias=False,
399
+ quant_config=quant_config,
400
+ prefix=f"{prefix}.kv_a_proj_with_mqa")
401
+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
402
+ eps=config.rms_norm_eps)
403
+ self.kv_b_proj = ColumnParallelLinear(
404
+ self.kv_lora_rank,
405
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
406
+ bias=False,
407
+ quant_config=quant_config,
408
+ prefix=f"{prefix}.kv_b_proj")
409
+ self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
410
+ self.hidden_size,
411
+ bias=False,
412
+ quant_config=quant_config,
413
+ prefix=f"{prefix}.o_proj")
414
+
415
+ rope_scaling["rope_type"] = 'deepseek_yarn'
416
+ self.rotary_emb = get_rope(qk_rope_head_dim,
417
+ rotary_dim=qk_rope_head_dim,
418
+ max_position=max_position_embeddings,
419
+ base=rope_theta,
420
+ rope_scaling=rope_scaling,
421
+ is_neox_style=False)
422
+ if rope_scaling:
423
+ mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
424
+ scaling_factor = rope_scaling["factor"]
425
+ mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
426
+ self.scaling = self.scaling * mscale * mscale
427
+
428
+ self.mla_attn = Attention(
429
+ num_heads=self.num_local_heads,
430
+ head_size=self.kv_lora_rank,
431
+ scale=self.scaling,
432
+ num_kv_heads=1,
433
+ cache_config=cache_config,
434
+ quant_config=quant_config,
435
+ prefix=f"{prefix}.attn",
436
+ use_mla=True,
437
+ # MLA Args
438
+ q_lora_rank=self.q_lora_rank,
439
+ kv_lora_rank=self.kv_lora_rank,
440
+ qk_nope_head_dim=self.qk_nope_head_dim,
441
+ qk_rope_head_dim=self.qk_rope_head_dim,
442
+ qk_head_dim=self.qk_head_dim,
443
+ v_head_dim=self.v_head_dim,
444
+ rotary_emb=self.rotary_emb,
445
+ q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj,
446
+ kv_b_proj=self.kv_b_proj,
447
+ o_proj=self.o_proj,
448
+ )
449
+
450
+ self.prefix = prefix
451
+ self.debug_layer_idx = int(self.prefix.split(".")[-2])
452
+
453
+ def forward(
454
+ self,
455
+ positions: torch.Tensor,
456
+ hidden_states: torch.Tensor,
457
+ kv_cache: torch.Tensor,
458
+ attn_metadata: AttentionMetadata,
459
+ ) -> torch.Tensor:
460
+ if self.q_lora_rank is not None:
461
+ ckq = self.q_a_proj(hidden_states)[0]
462
+ hidden_states_or_q_c = self.q_a_layernorm(ckq)
463
+ else:
464
+ hidden_states_or_q_c = hidden_states
465
+ kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states)[0].split(
466
+ [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
467
+ kv_c_normed = self.kv_a_layernorm(kv_c.contiguous())
468
+ return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, kv_cache,
469
+ attn_metadata)
470
+
471
+
472
class DeepseekV2DecoderLayer(nn.Module):
473
474
def __init__(
475
self,
476
config: PretrainedConfig,
477
prefix: str,
478
+ model_config: ModelConfig,
479
cache_config: Optional[CacheConfig] = None,
480
quant_config: Optional[QuantizationConfig] = None,
481
) -> None:
@@ -344,7 +488,11 @@ class DeepseekV2DecoderLayer(nn.Module):
488
# DecoderLayers are created with `make_layers` which passes the prefix
489
# with the layer's index.
490
layer_idx = int(prefix.split(sep='.')[-1])
347
- self.self_attn = DeepseekV2Attention(
491
+ if model_config.use_mla:
492
+ attn_cls = DeepseekV2MLAAttention
493
+ else:
494
+ attn_cls = DeepseekV2Attention
495
+ self.self_attn = attn_cls(
496
config=config,
497
hidden_size=self.hidden_size,
498
num_heads=config.num_attention_heads,
@@ -421,6 +569,7 @@ class DeepseekV2Model(nn.Module):
569
super().__init__()
570
571
config = vllm_config.model_config.hf_config
572
+ model_config = vllm_config.model_config
573
cache_config = vllm_config.cache_config
574
quant_config = vllm_config.quant_config
575
@@ -440,6 +589,7 @@ class DeepseekV2Model(nn.Module):
589
lambda prefix: DeepseekV2DecoderLayer(
590
config,
591
prefix,
592
+ model_config=model_config,
593
cache_config=cache_config,
594
quant_config=quant_config,
595
),
vllm/platforms/cpu.py
+2
-1
@@ -31,7 +31,8 @@ class CpuPlatform(Platform):
31
@classmethod
32
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
33
dtype: torch.dtype, kv_cache_dtype: Optional[str],
34
- block_size: int, use_v1: bool) -> str:
34
+ block_size: int, use_v1: bool,
35
+ use_mla: bool) -> str:
36
if selected_backend != _Backend.TORCH_SDPA:
37
logger.info("Cannot use %s backend on CPU.", selected_backend)
38
logger.info("Using Torch SDPA backend.")
vllm/platforms/cuda.py
+7
-2
@@ -157,10 +157,14 @@ class CudaPlatformBase(Platform):
157
158
@classmethod
159
def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
160
- kv_cache_dtype, block_size, use_v1) -> str:
160
+ kv_cache_dtype, block_size, use_v1,
161
+ use_mla) -> str:
162
if use_v1:
163
logger.info("Using Flash Attention backend on V1 engine.")
164
return "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend"
165
+ if use_mla:
166
+ logger.info("Using Triton MLA backend.")
167
+ return "vllm.attention.backends.triton_mla.TritonMLABackend"
168
if selected_backend == _Backend.FLASHINFER:
169
logger.info("Using FlashInfer backend.")
170
return "vllm.attention.backends.flashinfer.FlashInferBackend"
@@ -171,7 +175,8 @@ class CudaPlatformBase(Platform):
175
pass
176
elif selected_backend:
177
raise ValueError(
174
- f"Invalid attention backend for {cls.device_name}")
178
+ f"Invalid attention backend for {cls.device_name}, "
179
+ f"with use_v1: {use_v1} use_mla: {use_mla}")
180
181
target_backend = _Backend.FLASH_ATTN
182
if not cls.has_device_capability(80):
vllm/platforms/hpu.py
+2
-1
@@ -27,7 +27,8 @@ class HpuPlatform(Platform):
27
@classmethod
28
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
29
dtype: torch.dtype, kv_cache_dtype: Optional[str],
30
- block_size: int, use_v1: bool) -> str:
30
+ block_size: int, use_v1: bool,
31
+ use_mla: bool) -> str:
32
logger.info("Using HPUAttention backend.")
33
return "vllm.attention.backends.hpu_attn.HPUAttentionBackend"
34
vllm/platforms/interface.py
+3
-1
@@ -30,6 +30,7 @@ class _Backend(enum.Enum):
30
TORCH_SDPA = enum.auto()
31
OPENVINO = enum.auto()
32
FLASHINFER = enum.auto()
33
+ TRITON_MLA = enum.auto()
34
HPU_ATTN = enum.auto()
35
PALLAS = enum.auto()
36
IPEX = enum.auto()
@@ -139,7 +140,8 @@ class Platform:
140
@classmethod
141
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
142
dtype: torch.dtype, kv_cache_dtype: Optional[str],
142
- block_size: int, use_v1: bool) -> str:
143
+ block_size: int, use_v1: bool,
144
+ use_mla: bool) -> str:
145
"""Get the attention backend class of a device."""
146
return ""
147
vllm/platforms/openvino.py
+2
-1
@@ -30,7 +30,8 @@ class OpenVinoPlatform(Platform):
30
@classmethod
31
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
32
dtype: torch.dtype, kv_cache_dtype: Optional[str],
33
- block_size: int, use_v1: bool) -> str:
33
+ block_size: int, use_v1: bool,
34
+ use_mla: bool) -> str:
35
if selected_backend != _Backend.OPENVINO:
36
logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
37
logger.info("Using OpenVINO Attention backend.")
vllm/platforms/rocm.py
+2
-1
@@ -75,7 +75,8 @@ class RocmPlatform(Platform):
75
76
@classmethod
77
def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
78
- kv_cache_dtype, block_size, use_v1) -> str:
78
+ kv_cache_dtype, block_size, use_v1,
79
+ use_mla) -> str:
80
selected_backend = (_Backend.ROCM_FLASH if selected_backend
81
== _Backend.FLASH_ATTN else selected_backend)
82
if selected_backend == _Backend.ROCM_FLASH:
vllm/platforms/tpu.py
+2
-1
@@ -29,7 +29,8 @@ class TpuPlatform(Platform):
29
@classmethod
30
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
31
dtype: torch.dtype, kv_cache_dtype: Optional[str],
32
- block_size: int, use_v1: bool) -> str:
32
+ block_size: int, use_v1: bool,
33
+ use_mla: bool) -> str:
34
if selected_backend != _Backend.PALLAS:
35
logger.info("Cannot use %s backend on TPU.", selected_backend)
36
logger.info("Using Pallas backend.")
vllm/platforms/xpu.py
+2
-1
@@ -27,7 +27,8 @@ class XPUPlatform(Platform):
27
@classmethod
28
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
29
dtype: torch.dtype, kv_cache_dtype: Optional[str],
30
- block_size: int, use_v1: bool) -> str:
30
+ block_size: int, use_v1: bool,
31
+ use_mla: bool) -> str:
32
if selected_backend != _Backend.IPEX:
33
logger.info("Cannot use %s backend on XPU.", selected_backend)
34
logger.info("Using IPEX attention backend.")
vllm/worker/cache_engine.py
+2
-1
@@ -56,7 +56,8 @@ class CacheEngine:
56
model_config.dtype,
57
cache_config.cache_dtype,
58
self.block_size,
59
- model_config.is_attention_free)
59
+ model_config.is_attention_free,
60
+ use_mla=model_config.use_mla)
61
62
# Initialize the cache.
63
self.gpu_cache = self._allocate_kv_cache(
vllm/worker/model_runner.py
+3
-1
@@ -1066,6 +1066,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1066
self.kv_cache_dtype,
1067
self.block_size,
1068
self.model_config.is_attention_free,
1069
+ use_mla=self.model_config.use_mla,
1070
) if needs_attn_backend else None
1071
if self.attn_backend:
1072
self.attn_state = self.attn_backend.get_state_cls()(
@@ -1973,7 +1974,8 @@ class CUDAGraphRunner(nn.Module):
1974
1975
# Copy the input tensors to the input buffers.
1976
self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
1976
- self.input_buffers["positions"].copy_(positions, non_blocking=True)
1977
+ if positions is not None:
1978
+ self.input_buffers["positions"].copy_(positions, non_blocking=True)
1979
1980
if self.backend_name != "NO_ATTENTION":
1981
self.input_buffers["slot_mapping"].copy_(