@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cabaf4eff3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Attention] MLA decode optimizations (#12528)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: simon-mo <xmo@berkeley.edu> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Alexander Matveev <59768536+alexm-neuralmagic@users.noreply.github.com> Co-authored-by: simon-mo <xmo@berkeley.edu>

Lucas Wilkinson committed Jan 31, 2025 at 02:49 UTC cabaf4eff3c7df30d785769d5a0a1fa1a1c48a8a
31 files changed +2266 -32
csrc/cache.h
+5
@@ -28,6 +28,11 @@ void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
28 const std::string& kv_cache_dtype,
29 torch::Tensor& k_scale, torch::Tensor& v_scale);
30
31 +void concat_and_cache_mla(torch::Tensor& kv_c, torch::Tensor& k_pe,
32 + torch::Tensor& kv_cache, torch::Tensor& slot_mapping,
33 + const std::string& kv_cache_dtype,
34 + torch::Tensor& scale);
35 +
36 // Just for unittest
37 void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
38 const double scale, const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+95
@@ -245,6 +245,51 @@ __global__ void reshape_and_cache_flash_kernel(
245 }
246 }
247 }
248 +
249 +template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
250 +__global__ void concat_and_cache_mla_kernel(
251 + const scalar_t* __restrict__ kv_c, // [num_tokens, kv_lora_rank]
252 + const scalar_t* __restrict__ k_pe, // [num_tokens, pe_dim]
253 + cache_t* __restrict__ kv_cache, // [num_blocks, block_size, (kv_lora_rank
254 + // + pe_dim)]
255 + const int64_t* __restrict__ slot_mapping, // [num_tokens]
256 + const int block_stride, //
257 + const int kv_c_stride, //
258 + const int k_pe_stride, //
259 + const int kv_lora_rank, //
260 + const int pe_dim, //
261 + const int block_size, //
262 + const float* scale //
263 +) {
264 + const int64_t token_idx = blockIdx.x;
265 + const int64_t slot_idx = slot_mapping[token_idx];
266 + // NOTE: slot_idx can be -1 if the token is padded
267 + if (slot_idx < 0) {
268 + return;
269 + }
270 + const int64_t block_idx = slot_idx / block_size;
271 + const int64_t block_offset = slot_idx % block_size;
272 +
273 + auto copy = [&](const scalar_t* __restrict__ src, cache_t* __restrict__ dst,
274 + int src_stride, int dst_stride, int size, int offset) {
275 + for (int i = threadIdx.x; i < size; i += blockDim.x) {
276 + const int64_t src_idx = token_idx * src_stride + i;
277 + const int64_t dst_idx = block_idx * block_stride +
278 + block_offset * (kv_lora_rank + pe_dim) + i +
279 + offset;
280 + if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
281 + dst[dst_idx] = src[src_idx];
282 + } else {
283 + dst[dst_idx] =
284 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(src[src_idx], *scale);
285 + }
286 + }
287 + };
288 +
289 + copy(kv_c, kv_cache, kv_c_stride, block_stride, kv_lora_rank, 0);
290 + copy(k_pe, kv_cache, k_pe_stride, block_stride, pe_dim, kv_lora_rank);
291 +}
292 +
293 } // namespace vllm
294
295 // KV_T is the stored data type of kv-cache.
@@ -343,6 +388,56 @@ void reshape_and_cache_flash(
388 CALL_RESHAPE_AND_CACHE_FLASH);
389 }
390
391 +// KV_T is the stored data type of kv-cache.
392 +// CACHE_T is the data type of key and value tensors.
393 +// KV_DTYPE is the real data type of kv-cache.
394 +#define CALL_CONCAT_AND_CACHE_MLA(KV_T, CACHE_T, KV_DTYPE) \
395 + vllm::concat_and_cache_mla_kernel<KV_T, CACHE_T, KV_DTYPE> \
396 + <<<grid, block, 0, stream>>>( \
397 + reinterpret_cast<KV_T*>(kv_c.data_ptr()), \
398 + reinterpret_cast<KV_T*>(k_pe.data_ptr()), \
399 + reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
400 + slot_mapping.data_ptr<int64_t>(), block_stride, kv_c_stride, \
401 + k_pe_stride, kv_lora_rank, pe_dim, block_size, \
402 + reinterpret_cast<const float*>(scale.data_ptr()));
403 +
404 +void concat_and_cache_mla(
405 + torch::Tensor& kv_c, // [num_tokens, kv_lora_rank]
406 + torch::Tensor& k_pe, // [num_tokens, pe_dim]
407 + torch::Tensor& kv_cache, // [num_blocks, block_size, (kv_lora_rank +
408 + // pe_dim)]
409 + torch::Tensor& slot_mapping, // [num_tokens] or [num_actual_tokens]
410 + const std::string& kv_cache_dtype, torch::Tensor& scale) {
411 + // NOTE(woosuk): In vLLM V1, key.size(0) can be different from
412 + // slot_mapping.size(0) because of padding for CUDA graphs.
413 + // In vLLM V0, key.size(0) is always equal to slot_mapping.size(0) because
414 + // both include padding.
415 + // In vLLM V1, however, key.size(0) can be larger than slot_mapping.size(0)
416 + // since key includes padding for CUDA graphs, while slot_mapping does not.
417 + // In this case, slot_mapping.size(0) represents the actual number of tokens
418 + // before padding.
419 + // For compatibility with both cases, we use slot_mapping.size(0) as the
420 + // number of tokens.
421 + int num_tokens = slot_mapping.size(0);
422 + int kv_lora_rank = kv_c.size(1);
423 + int pe_dim = k_pe.size(1);
424 + int block_size = kv_cache.size(1);
425 +
426 + TORCH_CHECK(kv_cache.size(2) == kv_lora_rank + pe_dim);
427 +
428 + int kv_c_stride = kv_c.stride(0);
429 + int k_pe_stride = k_pe.stride(0);
430 + int block_stride = kv_cache.stride(0);
431 +
432 + dim3 grid(num_tokens);
433 + dim3 block(std::min(kv_lora_rank, 512));
434 + const at::cuda::OptionalCUDAGuard device_guard(device_of(kv_c));
435 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
436 +
437 + DISPATCH_BY_KV_CACHE_DTYPE(kv_c.dtype(), kv_cache_dtype,
438 + CALL_CONCAT_AND_CACHE_MLA);
439 +}
440 +
441 namespace vllm {
442
443 template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
csrc/torch_bindings.cpp
+9
@@ -463,6 +463,15 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
463 cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
464 &reshape_and_cache_flash);
465
466 + // Concat kv_c and k_pe and cache them.
467 + cache_ops.def(
468 + "concat_and_cache_mla(Tensor kv_c, Tensor k_pe,"
469 + " Tensor! kv_cache,"
470 + " Tensor slot_mapping,"
471 + " str kv_cache_dtype,"
472 + " Tensor scale) -> ()");
473 + cache_ops.impl("concat_and_cache_mla", torch::kCUDA, &concat_and_cache_mla);
474 +
475 // Convert the key and value cache to fp8 data type.
476 cache_ops.def(
477 "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, "
tests/kernels/test_triton_decode_attention.py new
+89
@@ -0,0 +1,89 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
5 +
6 +
7 +def cdiv(a, b):
8 + return (a + b - 1) // b
9 +
10 +
11 +@pytest.mark.parametrize("B", [3, 5])
12 +@pytest.mark.parametrize("L", [1027, 1025])
13 +@pytest.mark.parametrize("H_Q", [32])
14 +@pytest.mark.parametrize("H_KV", [32, 8])
15 +@pytest.mark.parametrize("D_QK", [128, 192, 576])
16 +@pytest.mark.parametrize("D_V", [128, 512])
17 +@pytest.mark.parametrize("CACHE_SIZE", [16384])
18 +@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
19 +def test_decode_attention(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
20 + assert CACHE_SIZE % PAGE_SIZE == 0
21 + dtype = torch.bfloat16
22 + seq_len = L # This represents the number of tokens already in the sequence
23 + sm_scale = 1.0 / (D_QK**0.5)
24 + num_kv_splits = 8
25 +
26 + num_pages_per_batch = cdiv(seq_len, PAGE_SIZE)
27 + req_to_page = torch.randint(0,
28 + CACHE_SIZE // PAGE_SIZE,
29 + (B, num_pages_per_batch, 1),
30 + device="cuda")
31 + req_to_token = req_to_page * PAGE_SIZE
32 + req_to_token = req_to_token.expand(B, num_pages_per_batch, PAGE_SIZE)
33 + req_to_token = req_to_token + torch.arange(PAGE_SIZE, device="cuda").view(
34 + 1, 1, -1)
35 + req_to_token = req_to_token.view(B, -1)
36 + req_to_token = req_to_token[:, :seq_len].contiguous()
37 +
38 + # q represents the new token being generated, one per batch
39 + q = torch.randn(B, H_Q, D_QK, dtype=dtype, device="cuda")
40 +
41 + # k_buffer and v_buffer represent all previous tokens
42 + # Page size is 1.
43 + k_buffer = torch.randn(CACHE_SIZE, H_KV, D_QK, dtype=dtype, device="cuda")
44 + v_buffer = torch.randn(CACHE_SIZE, H_KV, D_V, dtype=dtype, device="cuda")
45 +
46 + # o will have the same shape as q
47 + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda")
48 +
49 + b_seq_len = torch.full((B, ), seq_len, device="cuda")
50 +
51 + attn_logits = torch.empty(
52 + (B, H_Q, num_kv_splits, D_V + 1),
53 + dtype=torch.float32,
54 + device="cuda",
55 + )
56 +
57 + # Call the original implementation.
58 + decode_attention_fwd(
59 + q,
60 + k_buffer,
61 + v_buffer,
62 + o,
63 + req_to_token,
64 + b_seq_len,
65 + attn_logits,
66 + num_kv_splits,
67 + sm_scale,
68 + )
69 +
70 + # Page size can be larger than 1.
71 + k_buffer = k_buffer.view(CACHE_SIZE // PAGE_SIZE, PAGE_SIZE, H_KV, D_QK)
72 + v_buffer = v_buffer.view(CACHE_SIZE // PAGE_SIZE, PAGE_SIZE, H_KV, D_V)
73 +
74 + o1 = torch.zeros_like(o)
75 +
76 + decode_attention_fwd(
77 + q,
78 + k_buffer,
79 + v_buffer,
80 + o1,
81 + req_to_page,
82 + b_seq_len,
83 + attn_logits,
84 + num_kv_splits,
85 + sm_scale,
86 + PAGE_SIZE,
87 + )
88 +
89 + assert torch.allclose(o, o1)
tests/plugins/vllm_add_dummy_platform/vllm_add_dummy_platform/dummy_platform.py
+1 -1
@@ -5,5 +5,5 @@ class DummyPlatform(CudaPlatform):
5 device_name = "DummyDevice"
6
7 def get_attn_backend_cls(self, backend_name, head_size, dtype,
8 - kv_cache_dtype, block_size, use_v1):
8 + kv_cache_dtype, block_size, use_v1, use_mla):
9 return "vllm_add_dummy_platform.dummy_attention_backend.DummyAttentionBackend" # noqa E501
tests/weight_loading/models.txt
+1 -1
@@ -20,7 +20,7 @@ compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
20 compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22 compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23 -compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
23 +#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
24 compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-FP8-Dynamic-testing, main, 90
25 compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-W8A8-testing, main, 90
26 awq, casperhansen/mixtral-instruct-awq, main
tests/weight_loading/run_model_weight_loading_test.sh
+7 -2
@@ -3,7 +3,7 @@ SUCCESS=0
3
4 while getopts "c:" OPT; do
5 case ${OPT} in
6 - c )
6 + c )
7 CONFIG="$OPTARG"
8 ;;
9 \? )
@@ -18,9 +18,14 @@ IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
18
19 for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
20 do
21 + if [[ $MODEL_CONFIG == \#* ]]; then
22 + echo "=== SKIPPING MODEL: $MODEL_CONFIG ==="
23 + continue
24 + fi
25 +
26 LOCAL_SUCCESS=0
27 IFS=', ' read -r -a array <<< "$MODEL_CONFIG"
23 -
28 +
29 echo "=== RUNNING MODEL: $MODEL_CONFIG ==="
30
31 export QUANTIZATION=${array[0]}
vllm/_custom_ops.py
+13
@@ -1002,6 +1002,19 @@ def reshape_and_cache_flash(
1002 v_scale)
1003
1004
1005 +def concat_and_cache_mla(
1006 + kv_c: torch.Tensor,
1007 + k_pe: torch.Tensor,
1008 + kv_cache: torch.Tensor,
1009 + slot_mapping: torch.Tensor,
1010 + kv_cache_dtype: str,
1011 + scale: torch.Tensor,
1012 +) -> None:
1013 + torch.ops._C_cache_ops.concat_and_cache_mla(kv_c, k_pe, kv_cache,
1014 + slot_mapping, kv_cache_dtype,
1015 + scale)
1016 +
1017 +
1018 def copy_blocks(key_caches: List[torch.Tensor],
1019 value_caches: List[torch.Tensor],
1020 block_mapping: torch.Tensor) -> None:
vllm/attention/backends/abstract.py
+16
@@ -276,3 +276,19 @@ class AttentionImpl(ABC, Generic[T]):
276 output: Optional[torch.Tensor] = None,
277 ) -> torch.Tensor:
278 raise NotImplementedError
279 +
280 +
281 +class MLAAttentionImpl(AttentionImpl[T], Generic[T]):
282 +
283 + @abstractmethod
284 + def forward(
285 + self,
286 + layer: AttentionLayer,
287 + hidden_states_or_cq: torch.Tensor,
288 + kv_c_normed: torch.Tensor,
289 + k_pe: torch.Tensor,
290 + kv_cache: torch.Tensor,
291 + attn_metadata: T,
292 + output: Optional[torch.Tensor] = None,
293 + ) -> torch.Tensor:
294 + raise NotImplementedError
vllm/attention/backends/mla/__init__.py
vllm/attention/backends/mla/utils.py new
+365
@@ -0,0 +1,365 @@
1 +from abc import abstractmethod
2 +from dataclasses import dataclass
3 +from typing import Any, Dict, Generic, List, Optional
4 +
5 +import torch
6 +
7 +from vllm import _custom_ops as ops
8 +from vllm import envs
9 +from vllm.attention.backends.abstract import (AttentionLayer,
10 + AttentionMetadata,
11 + MLAAttentionImpl, T)
12 +from vllm.distributed import get_tensor_model_parallel_world_size
13 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 + RowParallelLinear)
15 +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
16 +from vllm.vllm_flash_attn import flash_attn_varlen_func
17 +
18 +
19 +@dataclass
20 +class MLACommonMetadata(AttentionMetadata):
21 + # Input positions for rotrary embeddings since for MLA the rotary
22 + # position embeddings are applied inside the attention backend
23 + input_positions: torch.Tensor
24 +
25 +
26 +class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
27 + """
28 + Common class for implementing repeated parts
29 +
30 + Main reference: DeepseekV2 paper, and FlashInfer Implementation
31 + (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
32 +
33 + Deepseek's MLA attention works the following way:
34 + * Use a single latent vector to represent the entire KV cache.
35 + * The attention "simulates" a multi-head attention, while the compute is
36 + similar to multi-query attention.
37 + * The dataflow is as follows,
38 +
39 + * B: batch/sequence length
40 + * H: hidden size
41 + * N: number of attention heads
42 + * Lq: latent dimension for Q
43 + * Lkv: latent dimension for K/V
44 + * P: nope dimension, P+R is the actual head_dim in common attention.
45 + * R: rope dimension, this slide of the head_dim goes through rope.
46 + * V: V head dim.
47 + * kv_c: latent/compressed KV
48 + * q_c: latent/compressed Q
49 +
50 + #
51 + # Outside the MLA attention backend
52 + #
53 +
54 + 1. The hidden states (B, H) are projected down into cq (B, Lq) and
55 + kv_c_k_pe (B, Lkv+R).
56 + 2. The kv_c_k_pe is split into kv_c (B, Lkv) and k_pe (B, R). cq
57 + and kv_c are normalized.
58 +
59 + #
60 + # Inside the MLA attention backend
61 + #
62 +
63 + * if prefill:
64 +
65 + 3. The q_c is then projected up into the multi-head version.
66 + * q_c goes from (B, Lq) to (B, N, (P+R)), which is split into q_nope
67 + (B, N, P) and q_pe (B, N, R).
68 + 4. q_pe, k_pe are then passed through rotary embeddings.
69 + 5. kv_c and k_pe are concatenated and inserted into the cache
70 + 6. The kv_c is then projected up into the multi-head version.
71 + * kv_c goes from (B, Lkv) to (B, N, (P+V)) which has the nope
72 + dimensions for K and V, which is split into k_nope (B, N, P)
73 + and v (B, N, V).
74 + 7. q (B, N, (P+R)) and k (B, N, (P+R)) matrices are assembled from
75 + q_nope, q_pe, k_nope, k_pe.
76 + 8. Attention is computued with q, k, v.
77 + 9. The attention computation returns (B, N, V), which is projected back
78 + to (B, H) using out projection.
79 +
80 + * if decode:
81 +
82 + 3. Here's the change, we do not perform up the full up projection for
83 + q_c, and there is no up projection at all for kv_c. This is
84 + achieved by the technique of "weight absorption". The paper says
85 + "Fortunately, due to the associative law of matrix multiplication,
86 + we can absorb WUK into WUQ, and WUV into WO"
87 + * The q up projection turns (B, Lq) into (B, N, (P+R)), we split it
88 + into W_UQ (Lq, N, P) and W_QR (Lq, N, R).
89 + * The kv_c up projection turns (B, Lkv) into (B, N, (P+V)), we split
90 + it into W_UK (Lkv, N, P) and W_UV (Lkv, N, V).
91 + * The out projection shape W_O (N*V, H) turns (B, N, V) into (B, H).
92 + * We can precompute the product of W_UQ and W_UK into
93 + W_UQ_UK (Lq, N, Lkv), which is possible due to QK^T operation in
94 + attention.
95 + * We can precompute the product of W_UV and W_O into
96 + W_UV_O (N, Lkv, H), which is possible due to V@O as the
97 + "epilogue" of attention
98 + 4. We still need to compute q_pe (B, N, R) by applying W_QR to q_latent.
99 + 5. q_pe, k_pe are then passed through rotary embeddings.
100 + 6. kv_c and k_pe are concatenated and inserted into the cache
101 + 7. By applying W_UQ_UK to q_latent, we have the new q_nope of shape
102 + (B, N, Lkv).
103 + 8. q (B, N, (Lkv+R)), k (B, (Lkv+R)) are assembled from q_nope, q_pe,
104 + kv_a, k_pe. v (B, Lkv) is exactly the same vector as kv_a.
105 + 9. The attention is computed with q, k, v. Note that we just performed
106 + a MQA attention with (LKv+R) as our head dim.
107 + 10. The KV cache is updated using the new entries k (B, N, (Lkv+R)),
108 + which included the v and rope values.
109 + 11. The attention computation returns (B, N, Lkv), which is projected
110 + back to (B, H) using W_UV_O.
111 +
112 + From @tsu-bin's calculation, we only want to use the absorption technique
113 + for decode. The prefill algorithm should still use the up-projected MHA
114 + for less flops and memory usage.
115 +
116 + """
117 +
118 + def __init__(
119 + self,
120 + num_heads: int,
121 + head_size: int,
122 + scale: float,
123 + num_kv_heads: int,
124 + alibi_slopes: Optional[List[float]],
125 + sliding_window: Optional[int],
126 + kv_cache_dtype: str,
127 + blocksparse_params: Optional[Dict[str, Any]],
128 + logits_soft_cap: Optional[float],
129 + attn_type: str,
130 + # MLA Specific Arguments
131 + q_lora_rank: Optional[int],
132 + kv_lora_rank: int,
133 + qk_nope_head_dim: int,
134 + qk_rope_head_dim: int,
135 + qk_head_dim: int,
136 + v_head_dim: int,
137 + rotary_emb: RotaryEmbedding,
138 + # q_proj should be q_b_proj if q_lora_rank is not None, but from an
139 + # attention backend perspective we rely on the layer to pass in the
140 + # correct matrix
141 + q_proj: ColumnParallelLinear,
142 + kv_b_proj: ColumnParallelLinear,
143 + o_proj: RowParallelLinear,
144 + ) -> None:
145 + self.num_heads = num_heads
146 + self.head_size = head_size
147 + self.scale = float(scale)
148 + self.num_kv_heads = num_kv_heads
149 + self.kv_cache_dtype = kv_cache_dtype
150 +
151 + self.q_lora_rank = q_lora_rank
152 + self.kv_lora_rank = kv_lora_rank
153 + self.qk_nope_head_dim = qk_nope_head_dim
154 + self.qk_rope_head_dim = qk_rope_head_dim
155 + self.qk_head_dim = qk_head_dim
156 + self.v_head_dim = v_head_dim
157 +
158 + self.rotary_emb = rotary_emb
159 + self.q_proj = q_proj
160 + self.kv_b_proj = kv_b_proj
161 + self.o_proj = o_proj
162 +
163 + def _v_up_proj_and_o_proj(self, x):
164 + if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
165 + return self.o_proj_absorbed(
166 + x.reshape(-1, self.num_heads * self.kv_lora_rank))[0]
167 + else:
168 + x = torch.einsum("bnl,lnv->bnv", x, self.W_UV)
169 + return self.o_proj(x.reshape(-1,
170 + self.num_heads * self.v_head_dim))[0]
171 +
172 + def _q_proj_and_k_up_proj(self, x):
173 + if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
174 + return torch.matmul(x, self.W_Q_UK)\
175 + .view(-1, self.num_heads, self.kv_lora_rank)
176 + else:
177 + x = torch.matmul(x, self.W_Q)\
178 + .view(-1, self.num_heads, self.qk_nope_head_dim)
179 + return torch.einsum("bnp,lnp->bnl", x, self.W_UK)\
180 + .view(-1, self.num_heads, self.kv_lora_rank)
181 +
182 + def process_weights_after_loading(self):
183 + kv_b_proj_weight = self.kv_b_proj.weight.T
184 + assert kv_b_proj_weight.shape == (
185 + self.kv_lora_rank,
186 + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim)), (
187 + f"{kv_b_proj_weight.shape=}, "
188 + f"{self.kv_lora_rank=}, "
189 + f"{self.num_heads=}, "
190 + f"{self.qk_nope_head_dim=}, "
191 + f"{self.v_head_dim=}")
192 + kv_b_proj_weight = kv_b_proj_weight.view(
193 + self.kv_lora_rank,
194 + self.num_heads,
195 + self.qk_nope_head_dim + self.v_head_dim,
196 + )
197 +
198 + W_UK, W_UV = kv_b_proj_weight.split(
199 + [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
200 +
201 + q_proj = self.q_proj.weight.T\
202 + .view(-1, self.num_heads, self.qk_head_dim)
203 +
204 + # can be W_Q or W_UQ depending q_lora_rank, the former if
205 + # q_lora_rank is None, the latter otherwise. From the Attention backend
206 + # perspective though we call these both W_Q and rely on the layer
207 + # to pass in the correct matrix
208 + W_Q = q_proj[..., :self.qk_nope_head_dim]
209 + self.W_QR = q_proj[..., self.qk_nope_head_dim:]\
210 + .flatten(start_dim=1).contiguous()
211 +
212 + if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
213 + #
214 + # Perform matrix-absorption following
215 + # https://github.com/flashinfer-ai/flashinfer/pull/551
216 + # for decode, as a result we end up with absorbed weights for decode
217 + # and another copy of raw weights for prefill.
218 + #
219 + self.W_UK, self.W_UV = kv_b_proj_weight.split(
220 + [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
221 + # We absorb `W_UK` into `W_Q` resulting in either W_Q_UK or W_UQ_UK
222 + # depending q_lora_rank, the former if q_lora_rank is None, the
223 + # latter otherwise
224 + # basically if q_lora_rank is none we are absorbing into q_proj
225 + # instead of UQ
226 + self.W_Q_UK = torch.einsum("qnd,lnd -> qnl", W_Q, W_UK)\
227 + .flatten(start_dim=1).contiguous()
228 +
229 + W_O = self.o_proj.weight\
230 + .view(-1, self.num_heads, self.v_head_dim)
231 + self.W_UV_O = torch.einsum("lnd,hnd -> nlh", W_UV, W_O)\
232 + .flatten(start_dim=0, end_dim=1).contiguous()
233 +
234 + tp_size = get_tensor_model_parallel_world_size()
235 + self.o_proj_absorbed = RowParallelLinear(
236 + self.W_UV_O.shape[0] * tp_size,
237 + self.W_UV_O.shape[1],
238 + bias=False,
239 + # TODO(lucas) figure out how to properly forward quant_method
240 + #quant_config=self.o_proj.quant_method,
241 + )
242 +
243 + self.o_proj_absorbed.weight = torch.nn.Parameter(self.W_UV_O.T)
244 + else:
245 + self.W_UV = W_UV
246 + self.W_UK = W_UK
247 + self.W_Q = W_Q.flatten(start_dim=1)
248 +
249 + @abstractmethod
250 + def _forward_prefill(
251 + self,
252 + q: torch.Tensor,
253 + kv_c_normed: torch.Tensor,
254 + k_pe: torch.Tensor,
255 + attn_metadata: T,
256 + ) -> torch.Tensor:
257 + raise NotImplementedError
258 +
259 + @abstractmethod
260 + def _forward_decode(
261 + self,
262 + q_nope: torch.Tensor,
263 + q_pe: torch.Tensor,
264 + kv_cache: torch.Tensor,
265 + attn_metadata: T,
266 + ) -> torch.Tensor:
267 + raise NotImplementedError
268 +
269 + def forward(
270 + self,
271 + layer: AttentionLayer,
272 + hidden_states_or_q_c: torch.Tensor, # query in unified attn
273 + k_c_normed: torch.Tensor, # key in unified attn
274 + k_pe: torch.Tensor, # value in unified attn
275 + kv_cache: torch.Tensor,
276 + attn_metadata: T,
277 + output: Optional[torch.Tensor] = None,
278 + ) -> torch.Tensor:
279 + if output is not None:
280 + raise NotImplementedError(
281 + "output is not yet supported for MLAImplBase")
282 +
283 + is_decode = attn_metadata.decode_metadata is not None
284 + is_prefill = attn_metadata.prefill_metadata is not None
285 +
286 + if (is_decode and is_prefill):
287 + raise NotImplementedError(
288 + "chunked prefill is not supported for MLAImplBase")
289 +
290 + # Restore head dim (for rotary embedding)
291 + k_pe = k_pe.unsqueeze(1)
292 + assert hasattr(attn_metadata, "input_positions")
293 +
294 + if is_decode:
295 + q_nope = self._q_proj_and_k_up_proj(hidden_states_or_q_c)
296 + q_pe = torch.matmul(hidden_states_or_q_c, self.W_QR)\
297 + .view(-1, self.num_heads, self.qk_rope_head_dim)
298 + q_pe, k_pe = \
299 + self.rotary_emb(attn_metadata.input_positions, q_pe, k_pe)
300 + else:
301 + assert is_prefill
302 + q = self.q_proj(hidden_states_or_q_c)[0]\
303 + .view(-1, self.num_heads, self.qk_head_dim)
304 +
305 + # TODO(lucas): there must be a nicer way to write this line
306 + q[..., self.qk_nope_head_dim:], k_pe = \
307 + self.rotary_emb(
308 + attn_metadata.input_positions,
309 + q[..., self.qk_nope_head_dim:], k_pe)
310 +
311 + # write the latent and rope to kv cache
312 + if kv_cache.numel() > 0:
313 + ops.concat_and_cache_mla(
314 + k_c_normed,
315 + k_pe.squeeze(1),
316 + kv_cache,
317 + attn_metadata.slot_mapping.flatten(),
318 + kv_cache_dtype=self.kv_cache_dtype,
319 + scale=layer._k_scale,
320 + )
321 +
322 + if attn_metadata.prefill_metadata is not None:
323 + return self._forward_prefill(q, k_c_normed, k_pe, attn_metadata)
324 +
325 + if attn_metadata.decode_metadata is not None:
326 + return self._forward_decode(q_nope, q_pe, kv_cache, attn_metadata)
327 +
328 + # Optional common flash-attn based prefill
329 + def _forward_prefill_flash(
330 + self,
331 + q: torch.Tensor,
332 + k_c_normed: torch.Tensor,
333 + k_pe: torch.Tensor,
334 + seq_start_loc: torch.Tensor,
335 + max_prefill_seq_len: int,
336 + ) -> torch.Tensor:
337 +
338 + kv_nope = self.kv_b_proj(k_c_normed)[0]\
339 + .view(-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
340 + k_nope, v = kv_nope\
341 + .split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
342 +
343 + k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1)
344 +
345 + # For MLA the v head dim is smaller than qk head dim so we pad out
346 + # v with 0s to match the qk head dim
347 + v_padded = torch.nn.functional.pad(v, [0, q.shape[-1] - v.shape[-1]],
348 + value=0)
349 +
350 + attn_output = flash_attn_varlen_func(
351 + q=q,
352 + k=k,
353 + v=v_padded,
354 + cu_seqlens_q=seq_start_loc,
355 + cu_seqlens_k=seq_start_loc,
356 + max_seqlen_q=max_prefill_seq_len,
357 + max_seqlen_k=max_prefill_seq_len,
358 + softmax_scale=self.scale,
359 + causal=True,
360 + )
361 + attn_output = attn_output\
362 + .view(-1, self.num_heads, q.shape[-1])[..., :v.shape[-1]]\
363 + .reshape(-1, self.num_heads * v.shape[-1])
364 +
365 + return self.o_proj(attn_output)[0]
vllm/attention/backends/triton_mla.py new
+749
@@ -0,0 +1,749 @@
1 +from collections import defaultdict
2 +from contextlib import contextmanager
3 +from dataclasses import dataclass
4 +from itertools import accumulate
5 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
6 +
7 +from vllm.multimodal import MultiModalPlaceholderMap
8 +
9 +try:
10 + from flashinfer import BatchDecodeMlaWithPagedKVCacheWrapper
11 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
12 +except ImportError:
13 + BatchDecodeMlaWithPagedKVCacheWrapper = None
14 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
15 +
16 +import torch
17 +
18 +from vllm import _custom_ops as ops
19 +from vllm.attention.backends.abstract import (AttentionBackend,
20 + AttentionMetadata,
21 + AttentionMetadataBuilder,
22 + AttentionState, AttentionType)
23 +from vllm.attention.backends.mla.utils import MLACommonImpl, MLACommonMetadata
24 +from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
25 + compute_slot_mapping_start_idx,
26 + is_block_tables_empty)
27 +from vllm.attention.ops.paged_attn import PagedAttention
28 +from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
29 +from vllm.utils import async_tensor_h2d, make_tensor_with_pad
30 +
31 +if TYPE_CHECKING:
32 + from vllm.worker.model_runner import (ModelInputForGPUBuilder,
33 + ModelInputForGPUWithSamplingMetadata)
34 +
35 +
36 +class TritonMLABackend(AttentionBackend):
37 +
38 + @staticmethod
39 + def get_name() -> str:
40 + return "TRITON_MLA"
41 +
42 + @staticmethod
43 + def get_impl_cls() -> Type["TritonMLAImpl"]:
44 + return TritonMLAImpl
45 +
46 + @staticmethod
47 + def get_metadata_cls() -> Type["AttentionMetadata"]:
48 + return TritonMLAMetadata
49 +
50 + @staticmethod
51 + def get_builder_cls() -> Type["TritonMLAMetadataBuilder"]:
52 + return TritonMLAMetadataBuilder
53 +
54 + @staticmethod
55 + def get_state_cls() -> Type["TritonMLAState"]:
56 + return TritonMLAState
57 +
58 + @staticmethod
59 + def get_kv_cache_shape(
60 + num_blocks: int,
61 + block_size: int,
62 + num_kv_heads: int, # assumed to be 1 for MLA
63 + kv_lora_rank: int, # passed via head_size
64 + ) -> Tuple[int, ...]:
65 + # TODO(lucas): remove hardcoding k_pe size as 1/8th of kv_lora_rank
66 + k_pe_size = kv_lora_rank // 8
67 + return (num_blocks, block_size, kv_lora_rank + k_pe_size)
68 +
69 + @staticmethod
70 + def swap_blocks(
71 + src_kv_cache: torch.Tensor,
72 + dst_kv_cache: torch.Tensor,
73 + src_to_dst: torch.Tensor,
74 + ) -> None:
75 + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
76 +
77 + @staticmethod
78 + def copy_blocks(
79 + kv_caches: List[torch.Tensor],
80 + src_to_dists: torch.Tensor,
81 + ) -> None:
82 + PagedAttention.copy_blocks(kv_caches, src_to_dists)
83 +
84 + @staticmethod
85 + def get_supported_head_sizes() -> List[int]:
86 + return [512]
87 +
88 +
89 +class TritonMLAState(AttentionState):
90 +
91 + def __init__(self, runner):
92 + self.runner = runner
93 + self._is_graph_capturing = False
94 +
95 + @contextmanager
96 + def graph_capture(self, max_batch_size: int):
97 + self._is_graph_capturing = True
98 +
99 + self._graph_slot_mapping = torch.full((max_batch_size, ),
100 + PAD_SLOT_ID,
101 + dtype=torch.long,
102 + device=self.runner.device)
103 + self._graph_seq_lens = torch.ones(max_batch_size,
104 + dtype=torch.int32,
105 + device=self.runner.device)
106 + self._graph_block_tables = torch.from_numpy(
107 + self.runner.graph_block_tables).to(device=self.runner.device)
108 +
109 + self._positions = torch.zeros((max_batch_size, ),
110 + dtype=torch.long,
111 + device=self.runner.device)
112 +
113 + yield
114 +
115 + self._is_graph_capturing = False
116 + del self._graph_slot_mapping
117 + del self._graph_seq_lens
118 + del self._graph_block_tables
119 + del self._positions
120 +
121 + def graph_clone(self, batch_size: int):
122 + assert self._is_graph_capturing
123 + return self.__class__(self.runner)
124 +
125 + def graph_capture_get_metadata_for_batch(
126 + self, batch_size: int, is_encoder_decoder_model: bool = False):
127 + assert self._is_graph_capturing
128 +
129 + attn_metadata = self.runner.attn_backend.make_metadata(
130 + num_prefills=0,
131 + num_prefill_tokens=0,
132 + num_decode_tokens=batch_size,
133 + slot_mapping=self._graph_slot_mapping[:batch_size],
134 + multi_modal_placeholder_index_maps=None,
135 + enable_kv_scales_calculation=True,
136 + seq_lens=None,
137 + seq_lens_tensor=self._graph_seq_lens[:batch_size],
138 + max_query_len=1,
139 + max_decode_query_len=1,
140 + max_prefill_seq_len=0,
141 + max_decode_seq_len=self.runner.max_seq_len_to_capture,
142 + query_start_loc=None,
143 + seq_start_loc=None,
144 + context_lens_tensor=None,
145 + block_tables=self._graph_block_tables[:batch_size],
146 + use_cuda_graph=True,
147 + input_positions=self._positions[:batch_size],
148 + head_dim=self.runner.model_config.get_head_size())
149 +
150 + if is_encoder_decoder_model:
151 + raise NotImplementedError(
152 + "TritonMLAState does not support encoder/decoder yet")
153 +
154 + return attn_metadata
155 +
156 + def get_graph_input_buffers(self,
157 + attn_metadata,
158 + is_encoder_decoder_model: bool = False):
159 + input_buffers = {
160 + "slot_mapping": attn_metadata.slot_mapping,
161 + "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
162 + "block_tables": attn_metadata.decode_metadata.block_tables,
163 + "input_positions": attn_metadata.decode_metadata.input_positions,
164 + }
165 + if is_encoder_decoder_model:
166 + raise NotImplementedError(
167 + "TritonMLAState does not support encoder/decoder yet")
168 +
169 + return input_buffers
170 +
171 + def prepare_graph_input_buffers(self,
172 + input_buffers,
173 + attn_metadata,
174 + is_encoder_decoder_model: bool = False):
175 + input_positions = attn_metadata.input_positions
176 + num_positions = input_positions.shape[0]
177 + input_buffers["seq_lens_tensor"].copy_(
178 + attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
179 + input_buffers["block_tables"].copy_(
180 + attn_metadata.decode_metadata.block_tables, non_blocking=True)
181 + # CUDA graph buffer is padded so only perform a partial copy based on
182 + # num_positions
183 + input_buffers["input_positions"][:num_positions].copy_(
184 + input_positions, non_blocking=True)
185 + if is_encoder_decoder_model:
186 + raise NotImplementedError(
187 + "TritonMLAState does not support encoder/decoder yet")
188 +
189 + def begin_forward(self, model_input):
190 + return
191 +
192 +
193 +@dataclass
194 +class TritonMLAMetadata(MLACommonMetadata):
195 + """Metadata for TritonMLAMetadata.
196 +
197 + NOTE: Any python object stored here is not updated when it is
198 + cuda-graph replayed. If you have values that need to be changed
199 + dynamically, it should be stored in tensor. The tensor has to be
200 + updated from `CUDAGraphRunner.forward` API.
201 + """
202 + # (batch_size,). The sequence length per sequence. Sequence length means
203 + # the computed tokens + new tokens None if it is a decoding.
204 + seq_lens: Optional[List[int]]
205 + # seq_lens stored as a tensor.
206 + seq_lens_tensor: Optional[torch.Tensor]
207 +
208 + # NOTE(sang): Definition of context_len, query_len, and seq_len.
209 + # |---------- N-1 iteration --------|
210 + # |---------------- N iteration ---------------------|
211 + # |- tokenA -|......................|-- newTokens ---|
212 + # |---------- context_len ----------|
213 + # |-------------------- seq_len ---------------------|
214 + # |-- query_len ---|
215 +
216 + # Maximum sequence length among prefill batch. 0 if there are decoding
217 + # requests only.
218 + max_prefill_seq_len: int
219 + # Maximum sequence length among decode batch. 0 if there are prefill
220 + # requests only.
221 + max_decode_seq_len: int
222 + # (batch_size,) A tensor of context lengths (tokens that are computed
223 + # so far).
224 + context_lens_tensor: Optional[torch.Tensor]
225 +
226 + # (batch_size, max_blocks_per_seq).
227 + # Block addresses per sequence. (Seq id -> list of physical block)
228 + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
229 + # in the kv cache. Each block can contain up to block_size tokens.
230 + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
231 + # captured.
232 + block_tables: Optional[torch.Tensor]
233 +
234 + # Whether or not if cuda graph is enabled.
235 + # Cuda-graph is currently enabled for decoding only.
236 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
237 +
238 + use_cuda_graph: bool
239 +
240 + # Maximum query length in the batch.
241 + max_query_len: Optional[int] = None
242 +
243 + # Max number of query tokens among request in the batch.
244 + max_decode_query_len: Optional[int] = None
245 +
246 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
247 + # the batch, used to index into subquery. E.g., if the subquery length
248 + # is [4, 6], it is [0, 4, 10].
249 + query_start_loc: Optional[torch.Tensor] = None
250 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
251 + # the batch, used to index into sequence. E.g., if the sequence length is
252 + # [4, 6], it is [0, 4, 10].
253 + seq_start_loc: Optional[torch.Tensor] = None
254 +
255 + _cached_prefill_metadata: Optional["TritonMLAMetadata"] = None
256 + _cached_decode_metadata: Optional["TritonMLAMetadata"] = None
257 +
258 + num_prefill_tokens: int
259 +
260 + num_kv_splits: int = 4 # TODO(lucas) add heuristic
261 + attn_logits: Optional[torch.Tensor] = None
262 + req_idx: Optional[torch.Tensor] = None
263 +
264 + # The dimension of the attention heads
265 + head_dim: Optional[int] = None
266 +
267 + def __post_init__(self):
268 + supported_head_sizes = TritonMLABackend.get_supported_head_sizes()
269 + if self.head_dim is not None and self.head_dim \
270 + not in supported_head_sizes:
271 + raise ValueError(
272 + f"Only {supported_head_sizes} are supported for head_dim,",
273 + f"received {self.head_dim}.")
274 +
275 + @property
276 + def prefill_metadata(self) -> Optional["TritonMLAMetadata"]:
277 + if self.num_prefills == 0:
278 + return None
279 +
280 + if self._cached_prefill_metadata is not None:
281 + return self._cached_prefill_metadata
282 +
283 + assert self.seq_lens is not None
284 + assert self.seq_lens_tensor is not None
285 +
286 + # Compute some attn_metadata fields which default to None
287 + query_start_loc = (None if self.query_start_loc is None else
288 + self.query_start_loc[:self.num_prefills + 1])
289 + slot_mapping = (None if self.slot_mapping is None else
290 + self.slot_mapping[:self.num_prefill_tokens])
291 + seq_lens = (None if self.seq_lens is None else
292 + self.seq_lens[:self.num_prefills])
293 + seq_lens_tensor = (None if self.seq_lens_tensor is None else
294 + self.seq_lens_tensor[:self.num_prefills])
295 + seq_start_loc = (None if self.seq_start_loc is None else
296 + self.seq_start_loc[:self.num_prefills + 1])
297 + context_lens_tensor = (None if self.context_lens_tensor is None else
298 + self.context_lens_tensor[:self.num_prefills])
299 + block_tables = (None if self.block_tables is None else
300 + self.block_tables[:self.num_prefills])
301 + input_positions = (None if self.input_positions is None else
302 + self.input_positions[:self.num_prefill_tokens])
303 +
304 + self._cached_prefill_metadata = TritonMLAMetadata(
305 + num_prefills=self.num_prefills,
306 + num_prefill_tokens=self.num_prefill_tokens,
307 + num_decode_tokens=0,
308 + slot_mapping=slot_mapping,
309 + multi_modal_placeholder_index_maps=self.
310 + multi_modal_placeholder_index_maps,
311 + enable_kv_scales_calculation=self.enable_kv_scales_calculation,
312 + input_positions=input_positions,
313 + seq_lens=seq_lens,
314 + seq_lens_tensor=seq_lens_tensor,
315 + max_query_len=self.max_query_len,
316 + max_prefill_seq_len=self.max_prefill_seq_len,
317 + max_decode_query_len=0,
318 + max_decode_seq_len=0,
319 + query_start_loc=query_start_loc,
320 + seq_start_loc=seq_start_loc,
321 + context_lens_tensor=context_lens_tensor,
322 + block_tables=block_tables,
323 + use_cuda_graph=False,
324 + head_dim=self.head_dim)
325 + return self._cached_prefill_metadata
326 +
327 + @property
328 + def decode_metadata(self) -> Optional["TritonMLAMetadata"]:
329 + if self.num_decode_tokens == 0:
330 + return None
331 +
332 + if self._cached_decode_metadata is not None:
333 + return self._cached_decode_metadata
334 + assert self.seq_lens_tensor is not None
335 +
336 + # Compute some attn_metadata fields which default to None
337 + slot_mapping = (None if self.slot_mapping is None else
338 + self.slot_mapping[self.num_prefill_tokens:])
339 + seq_lens_tensor = (None if self.seq_lens_tensor is None else
340 + self.seq_lens_tensor[self.num_prefills:])
341 + block_tables = (None if self.block_tables is None else
342 + self.block_tables[self.num_prefills:])
343 + input_positions = (None if self.input_positions is None else
344 + self.input_positions[self.num_prefill_tokens:])
345 +
346 + self._cached_decode_metadata = TritonMLAMetadata(
347 + num_prefills=0,
348 + num_prefill_tokens=0,
349 + num_decode_tokens=self.num_decode_tokens,
350 + slot_mapping=slot_mapping,
351 + multi_modal_placeholder_index_maps=None,
352 + enable_kv_scales_calculation=True,
353 + seq_lens=None,
354 + seq_lens_tensor=seq_lens_tensor,
355 + max_decode_query_len=self.max_decode_query_len,
356 + max_query_len=self.max_query_len,
357 + max_prefill_seq_len=0,
358 + max_decode_seq_len=self.max_decode_seq_len,
359 + # Batch may be composed of prefill|decodes, adjust query start
360 + # indices to refer to the start of decodes. E.g.
361 + # in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
362 + query_start_loc=(self.query_start_loc[self.num_prefills:] -
363 + self.query_start_loc[self.num_prefills])
364 + if self.query_start_loc is not None else None,
365 + seq_start_loc=self.seq_start_loc[self.num_prefills:]
366 + if self.seq_start_loc is not None else None,
367 + context_lens_tensor=None,
368 + block_tables=block_tables,
369 + use_cuda_graph=self.use_cuda_graph,
370 + input_positions=input_positions,
371 + head_dim=self.head_dim)
372 + return self._cached_decode_metadata
373 +
374 + def advance_step(self,
375 + model_input: "ModelInputForGPUWithSamplingMetadata",
376 + sampled_token_ids: Optional[torch.Tensor],
377 + block_size: int,
378 + num_seqs: int,
379 + num_queries: int,
380 + turn_prefills_into_decodes: bool = False):
381 + """
382 + Update metadata in-place to advance one decode step.
383 + """
384 + # When using cudagraph, the num_seqs is padded to the next captured
385 + # batch sized, but num_queries tracks the actual number of requests in
386 + # the batch. For --enforce-eager mode, num_seqs == num_queries
387 + if num_seqs != num_queries:
388 + assert num_seqs > num_queries
389 + assert self.use_cuda_graph
390 +
391 + if turn_prefills_into_decodes:
392 + # When Mutli-Step is enabled with Chunked-Prefill, prefills and
393 + # decodes are scheduled together. In the first step, all the
394 + # prefills turn into decodes. This update reflects that
395 + # conversion.
396 + assert self.num_decode_tokens + self.num_prefills == num_seqs
397 + self.num_decode_tokens += self.num_prefills
398 + self.num_prefills = 0
399 + self.num_prefill_tokens = 0
400 + self.max_prefill_seq_len = 0
401 + self.max_query_len = 1
402 +
403 + self.slot_mapping = self.slot_mapping[:num_seqs]
404 + else:
405 + assert self.seq_lens is not None
406 + assert self.max_decode_seq_len == max(self.seq_lens)
407 +
408 + assert self.num_prefills == 0
409 + assert self.num_prefill_tokens == 0
410 + assert self.num_decode_tokens == num_seqs
411 + assert self.slot_mapping.shape == (num_seqs, )
412 +
413 + assert self.seq_lens is not None
414 + assert len(self.seq_lens) == num_seqs
415 + assert self.seq_lens_tensor is not None
416 + assert self.seq_lens_tensor.shape == (num_seqs, )
417 + assert self.max_query_len == 1
418 + assert self.max_prefill_seq_len == 0
419 +
420 + assert self.query_start_loc is not None
421 + assert self.query_start_loc.shape == (num_queries + 1, )
422 + assert self.seq_start_loc is not None
423 + assert self.seq_start_loc.shape == (num_seqs + 1, )
424 +
425 + assert self.context_lens_tensor is not None
426 + assert self.context_lens_tensor.shape == (num_queries, )
427 +
428 + assert self.block_tables is not None
429 + assert self.block_tables.shape[0] == num_seqs
430 +
431 + # Update query lengths. Note that we update only queries and not seqs,
432 + # since tensors may be padded due to captured cuda graph batch size
433 + for i in range(num_queries):
434 + self.seq_lens[i] += 1
435 + self.max_decode_seq_len = max(self.seq_lens)
436 +
437 + ops.advance_step_flashattn(num_seqs=num_seqs,
438 + num_queries=num_queries,
439 + block_size=block_size,
440 + input_tokens=model_input.input_tokens,
441 + sampled_token_ids=sampled_token_ids,
442 + input_positions=model_input.input_positions,
443 + seq_lens=self.seq_lens_tensor,
444 + slot_mapping=self.slot_mapping,
445 + block_tables=self.block_tables)
446 +
447 +
448 +class TritonMLAMetadataBuilder(AttentionMetadataBuilder[TritonMLAMetadata]):
449 +
450 + def __init__(self, input_builder: "ModelInputForGPUBuilder"):
451 + self.input_builder = input_builder
452 + self.runner = input_builder.runner
453 + self.sliding_window = input_builder.sliding_window
454 + self.block_size = input_builder.block_size
455 +
456 + def prepare(self):
457 + self.slot_mapping: List[int] = []
458 + self.prefill_seq_lens: List[int] = []
459 + self.context_lens: List[int] = []
460 + self.block_tables: List[List[int]] = []
461 + self.curr_seq_lens: List[int] = []
462 + self.input_positions: List[int] = []
463 + self.multimodal_placeholder_maps: Dict[
464 + str,
465 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
466 + self.num_prefills = 0
467 + self.num_prefill_tokens = 0
468 + self.num_decode_tokens = 0
469 + self.has_prefix_cache_hit = False
470 +
471 + def _add_seq_group(
472 + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
473 + chunked_prefill_enabled: bool, prefix_cache_hit: bool):
474 + """Add a sequence group to the metadata. Specifically update/append
475 + 1. context length.
476 + 2. block table.
477 + 3. slot mapping.
478 + """
479 + is_prompt = inter_data.is_prompt
480 + block_tables = inter_data.block_tables
481 +
482 + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
483 + curr_sliding_window_block, input_positions) in zip(
484 + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
485 + inter_data.orig_seq_lens, inter_data.seq_lens,
486 + inter_data.query_lens, inter_data.context_lens,
487 + inter_data.curr_sliding_window_blocks,
488 + inter_data.input_positions):
489 + self.input_positions.extend(input_positions)
490 + self.context_lens.append(context_len)
491 + if is_prompt:
492 + mm_maps = inter_data.multi_modal_placeholder_maps
493 + if mm_maps:
494 + for modality, placeholders in mm_maps.items():
495 + self.multimodal_placeholder_maps[modality].extend(
496 + placeholders)
497 +
498 + self.num_prefills += 1
499 + self.num_prefill_tokens += token_len
500 + self.prefill_seq_lens.append(seq_len)
501 + else:
502 + self.num_decode_tokens += query_len
503 + self.curr_seq_lens.append(curr_seq_len)
504 +
505 + # Compute block table.
506 + # TODO(sang): Combine chunked prefill and prefix caching by
507 + # only allowing multiple of block_size chunk size.
508 + # NOTE: This only works for oooooooxxx style attention.
509 + block_table = []
510 + if prefix_cache_hit:
511 + # NOTE(woosuk): For flash-attn, the block table should
512 + # include the entries for the incoming prefill tokens.
513 + block_table = block_tables[seq_id]
514 + elif ((chunked_prefill_enabled or not is_prompt)
515 + and block_tables is not None):
516 + if curr_sliding_window_block == 0:
517 + block_table = block_tables[seq_id]
518 + else:
519 + block_table = block_tables[seq_id][
520 + -curr_sliding_window_block:]
521 + self.block_tables.append(block_table)
522 +
523 + # Compute slot mapping.
524 + is_profile_run = is_block_tables_empty(block_tables)
525 + start_idx = compute_slot_mapping_start_idx(is_prompt, query_len,
526 + context_len,
527 + self.sliding_window)
528 + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
529 + seq_len, context_len, start_idx,
530 + self.block_size, inter_data.block_tables)
531 +
532 + def _get_graph_runner_block_tables(
533 + self, num_seqs: int,
534 + block_tables: List[List[int]]) -> torch.Tensor:
535 + # The shape of graph_block_tables is
536 + # [max batch size, max context len // block size].
537 + max_batch_size, max_blocks = self.runner.graph_block_tables.shape
538 + assert max_batch_size >= num_seqs
539 +
540 + graph_block_tables = self.runner.graph_block_tables[:num_seqs]
541 + for i, block_table in enumerate(block_tables):
542 + if block_table:
543 + num_blocks = len(block_table)
544 + if num_blocks <= max_blocks:
545 + graph_block_tables[i, :num_blocks] = block_table
546 + else:
547 + # It may be possible to have more blocks allocated due
548 + # to lookahead slots of multi-step, however, they are
549 + # not used anyway, so can be safely ignored.
550 + graph_block_tables[
551 + i, :max_blocks] = block_table[:max_blocks]
552 +
553 + return torch.from_numpy(graph_block_tables).to(
554 + device=self.runner.device, non_blocking=True)
555 +
556 + def build(self, seq_lens: List[int], query_lens: List[int],
557 + cuda_graph_pad_size: int, batch_size: int):
558 + """Build attention metadata with on-device tensors.
559 +
560 + Args:
561 + seq_lens: The maybe padded sequence lengths of the input sequences.
562 + query_lens: The query lengths of the input sequences.
563 + cuda_graph_pad_size: The padding size for cuda graph.
564 + -1 if cuda graph is not used.
565 + batch_size: The maybe padded batch size.
566 + """
567 + prefix_cache_hit = any([
568 + inter_data.prefix_cache_hit
569 + for inter_data in self.input_builder.inter_data_list
570 + ])
571 + for inter_data in self.input_builder.inter_data_list:
572 + self._add_seq_group(inter_data,
573 + self.input_builder.chunked_prefill_enabled,
574 + prefix_cache_hit)
575 +
576 + device = self.runner.device
577 + use_captured_graph = cuda_graph_pad_size != -1
578 +
579 + max_query_len = max(query_lens)
580 + decode_query_lens = query_lens[self.num_prefills:]
581 + if len(decode_query_lens) > 0:
582 + max_decode_query_len = max(decode_query_lens)
583 + else:
584 + max_decode_query_len = 1
585 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
586 + max_decode_seq_len = max(self.curr_seq_lens, default=0)
587 + num_decode_tokens = self.num_decode_tokens
588 + query_start_loc = list(accumulate(query_lens, initial=0))
589 + seq_start_loc = list(accumulate(seq_lens, initial=0))
590 +
591 + num_seqs = len(seq_lens)
592 + if use_captured_graph:
593 + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
594 + self.block_tables.extend([] * cuda_graph_pad_size)
595 + num_decode_tokens = batch_size - self.num_prefill_tokens
596 + block_tables = self._get_graph_runner_block_tables(
597 + num_seqs, self.block_tables)
598 + else:
599 + block_tables = make_tensor_with_pad(
600 + self.block_tables,
601 + pad=0,
602 + dtype=torch.int,
603 + device=device,
604 + )
605 + assert max_query_len > 0, ("query_lens: {}".format(query_lens))
606 +
607 + assert device is not None
608 + context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int,
609 + device, self.runner.pin_memory)
610 + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
611 + self.runner.pin_memory)
612 + input_positions = async_tensor_h2d(self.input_positions, torch.long,
613 + device, self.runner.pin_memory)
614 + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
615 + device, self.runner.pin_memory)
616 + query_start_loc_tensor = async_tensor_h2d(query_start_loc, torch.int32,
617 + device,
618 + self.runner.pin_memory)
619 + seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
620 + device, self.runner.pin_memory)
621 + placeholder_index_maps = {
622 + modality: placeholder_map.index_map()
623 + for modality, placeholder_map in
624 + self.multimodal_placeholder_maps.items()
625 + }
626 +
627 + num_kv_splits = 8
628 +
629 + return TritonMLAMetadata(
630 + num_prefills=self.num_prefills,
631 + slot_mapping=slot_mapping_tensor,
632 + num_prefill_tokens=self.num_prefill_tokens,
633 + num_decode_tokens=num_decode_tokens,
634 + seq_lens=seq_lens,
635 + multi_modal_placeholder_index_maps=placeholder_index_maps,
636 + enable_kv_scales_calculation=True,
637 + input_positions=input_positions,
638 + seq_lens_tensor=seq_lens_tensor,
639 + max_query_len=max_query_len,
640 + max_decode_query_len=max_decode_query_len,
641 + max_prefill_seq_len=max_prefill_seq_len,
642 + max_decode_seq_len=max_decode_seq_len,
643 + query_start_loc=query_start_loc_tensor,
644 + seq_start_loc=seq_start_loc_tensor,
645 + context_lens_tensor=context_lens_tensor,
646 + block_tables=block_tables,
647 + use_cuda_graph=use_captured_graph,
648 + num_kv_splits=num_kv_splits,
649 + head_dim=self.runner.model_config.get_head_size(),
650 + )
651 +
652 +
653 +class TritonMLAImpl(MLACommonImpl[TritonMLAMetadata]):
654 +
655 + def __init__(
656 + self,
657 + num_heads: int,
658 + head_size: int,
659 + scale: float,
660 + num_kv_heads: int,
661 + alibi_slopes: Optional[List[float]],
662 + sliding_window: Optional[int],
663 + kv_cache_dtype: str,
664 + blocksparse_params: Optional[Dict[str, Any]],
665 + logits_soft_cap: Optional[float],
666 + attn_type: str,
667 + # MLA Specific Arguments
668 + **kwargs) -> None:
669 + super().__init__(num_heads, head_size, scale, num_kv_heads,
670 + alibi_slopes, sliding_window, kv_cache_dtype,
671 + blocksparse_params, logits_soft_cap, attn_type,
672 + **kwargs)
673 +
674 + unsupported_features = [
675 + alibi_slopes, sliding_window, blocksparse_params, logits_soft_cap
676 + ]
677 + if any(unsupported_features):
678 + raise NotImplementedError(
679 + "TritonMLAImpl does not support one of the following: "
680 + "alibi_slopes, sliding_window, blocksparse_params, "
681 + "logits_soft_cap")
682 +
683 + if attn_type != AttentionType.DECODER:
684 + raise NotImplementedError("Encoder self-attention and "
685 + "encoder/decoder cross-attention "
686 + "are not implemented for "
687 + "TritonMLAImpl")
688 +
689 + def _forward_prefill(
690 + self,
691 + q: torch.Tensor,
692 + kv_c_normed: torch.Tensor,
693 + k_pe: torch.Tensor,
694 + attn_metadata: TritonMLAMetadata,
695 + ) -> torch.Tensor:
696 + assert isinstance(attn_metadata, TritonMLAMetadata)
697 + return self._forward_prefill_flash(q, kv_c_normed, k_pe,
698 + attn_metadata.seq_start_loc,
699 + attn_metadata.max_prefill_seq_len)
700 +
701 + def _forward_decode(
702 + self,
703 + q_nope: torch.Tensor,
704 + q_pe: torch.Tensor,
705 + kv_c_and_k_pe_cache: torch.Tensor,
706 + attn_metadata: TritonMLAMetadata,
707 + ) -> torch.Tensor:
708 + assert kv_c_and_k_pe_cache.numel() > 0
709 + if self.kv_cache_dtype.startswith("fp8"):
710 + raise NotImplementedError("FP8 Triton MLA not yet supported")
711 +
712 + decode_meta = attn_metadata.decode_metadata
713 + assert decode_meta is not None
714 + B = q_nope.shape[0]
715 +
716 + q = torch.cat([q_nope, q_pe], dim=-1)
717 + o = torch.zeros(B,
718 + self.num_heads,
719 + self.kv_lora_rank,
720 + dtype=q.dtype,
721 + device=q.device)
722 +
723 + # TODO(lucas) Allocate ahead of time
724 + attn_logits = torch.empty(
725 + (
726 + B,
727 + self.num_heads,
728 + attn_metadata.num_kv_splits,
729 + # NOTE(lucas) idk why the +1 is here but sglang has it so we
730 + # just mirror that
731 + self.kv_lora_rank + 1,
732 + ),
733 + dtype=torch.float32,
734 + device=q.device,
735 + )
736 +
737 + # Add a head dim of 1
738 + kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.unsqueeze(2)
739 + kv_c_cache = kv_c_and_k_pe_cache[..., :self.kv_lora_rank]
740 + PAGE_SIZE = kv_c_and_k_pe_cache.size(1)
741 +
742 + # Run MQA
743 + decode_attention_fwd(q, kv_c_and_k_pe_cache, kv_c_cache, o,
744 + decode_meta.block_tables,
745 + decode_meta.seq_lens_tensor, attn_logits,
746 + attn_metadata.num_kv_splits, self.scale,
747 + PAGE_SIZE)
748 +
749 + return self._v_up_proj_and_o_proj(o)
vllm/attention/backends/utils.py
+4
@@ -289,7 +289,9 @@ class CommonAttentionState(AttentionState):
289
290 @contextmanager
291 def graph_capture(self, max_batch_size: int):
292 +
293 self._is_graph_capturing = True
294 +
295 self._graph_slot_mapping = torch.full((max_batch_size, ),
296 PAD_SLOT_ID,
297 dtype=torch.long,
@@ -299,7 +301,9 @@ class CommonAttentionState(AttentionState):
301 device=self.runner.device)
302 self._graph_block_tables = torch.from_numpy(
303 self.runner.graph_block_tables).to(device=self.runner.device)
304 +
305 yield
306 +
307 self._is_graph_capturing = False
308 del self._graph_slot_mapping
309 del self._graph_seq_lens
vllm/attention/layer.py
+15 -4
@@ -41,8 +41,10 @@ class Attention(nn.Module):
41 blocksparse_params: Optional[Dict[str, Any]] = None,
42 logits_soft_cap: Optional[float] = None,
43 per_layer_sliding_window: Optional[int] = None,
44 + use_mla: bool = False,
45 prefix: str = "",
46 attn_type: str = AttentionType.DECODER,
47 + **extra_impl_args,
48 ) -> None:
49 super().__init__()
50 if per_layer_sliding_window is not None:
@@ -101,13 +103,18 @@ class Attention(nn.Module):
103 # During model initialization, the default dtype is set as the model
104 # weight and activation dtype.
105 dtype = torch.get_default_dtype()
104 - attn_backend = get_attn_backend(head_size, dtype, kv_cache_dtype,
105 - block_size, is_attention_free,
106 - blocksparse_params is not None)
106 + attn_backend = get_attn_backend(head_size,
107 + dtype,
108 + kv_cache_dtype,
109 + block_size,
110 + is_attention_free,
111 + blocksparse_params is not None,
112 + use_mla=use_mla)
113 impl_cls = attn_backend.get_impl_cls()
114 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
115 alibi_slopes, sliding_window, kv_cache_dtype,
110 - blocksparse_params, logits_soft_cap, attn_type)
116 + blocksparse_params, logits_soft_cap, attn_type,
117 + **extra_impl_args)
118 self.num_heads = num_heads
119 self.head_size = head_size
120 self.num_kv_heads = num_kv_heads
@@ -193,6 +200,10 @@ class Attention(nn.Module):
200 s += f", backend={self.impl.__class__.__name__}"
201 return s
202
203 + def process_weights_after_loading(self):
204 + if hasattr(self.impl, "process_weights_after_loading"):
205 + self.impl.process_weights_after_loading()
206 +
207
208 class MultiHeadAttention(nn.Module):
209 """Multi-headed attention without any cache, used for ViT."""
vllm/attention/ops/triton_decode_attention.py new
+667
@@ -0,0 +1,667 @@
1 +# Adapted from
2 +# https://github.com/sgl-project/sglang/blob/9f635ea50de920aa507f486daafba26a5b837574/python/sglang/srt/layers/attention/triton_ops/decode_attention.py
3 +# which was originally adapted from
4 +# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage1.py
5 +# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage2.py
6 +
7 +# Changes:
8 +# - Add support for page size >= 1.
9 +
10 +# Copyright 2025 vLLM Team
11 +# Copyright 2023-2024 SGLang Team
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +# ==============================================================================
24 +"""
25 +Memory-efficient attention for decoding.
26 +It supports page size >= 1.
27 +"""
28 +
29 +import logging
30 +
31 +import triton
32 +import triton.language as tl
33 +
34 +from vllm.platforms import current_platform
35 +
36 +is_hip_ = current_platform.is_rocm()
37 +
38 +logger = logging.getLogger(__name__)
39 +
40 +# TODO: Remove this when triton>=3.2.0. This issue will not affect performance
41 +# and accuracy.
42 +logger.warning(
43 + "The following error message 'operation scheduled before its operands' "
44 + "can be ignored.")
45 +
46 +
47 +@triton.jit
48 +def tanh(x):
49 + # Tanh is just a scaled sigmoid
50 + return 2 * tl.sigmoid(2 * x) - 1
51 +
52 +
53 +@triton.jit
54 +def _fwd_kernel_stage1(
55 + Q,
56 + K_Buffer,
57 + V_Buffer,
58 + sm_scale,
59 + Req_to_tokens,
60 + B_Seqlen,
61 + Att_Out,
62 + stride_req_to_tokens_b,
63 + stride_qbs,
64 + stride_qh,
65 + stride_buf_kbs,
66 + stride_buf_kh,
67 + stride_buf_vbs,
68 + stride_buf_vh,
69 + stride_mid_ob,
70 + stride_mid_oh,
71 + stride_mid_os,
72 + kv_group_num: tl.constexpr,
73 + BLOCK_DMODEL: tl.constexpr,
74 + BLOCK_DV: tl.constexpr,
75 + BLOCK_N: tl.constexpr,
76 + NUM_KV_SPLITS: tl.constexpr,
77 + PAGE_SIZE: tl.constexpr,
78 + logit_cap: tl.constexpr,
79 + Lk: tl.constexpr,
80 + Lv: tl.constexpr,
81 +):
82 + cur_batch = tl.program_id(0)
83 + cur_head = tl.program_id(1)
84 + split_kv_id = tl.program_id(2)
85 +
86 + cur_kv_head = cur_head // kv_group_num
87 +
88 + offs_d = tl.arange(0, BLOCK_DMODEL)
89 + offs_dv = tl.arange(0, BLOCK_DV)
90 + mask_d = offs_d < Lk
91 + mask_dv = offs_dv < Lv
92 + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
93 + cur_batch_req_idx = cur_batch
94 +
95 + off_q = cur_batch * stride_qbs + cur_head * stride_qh + offs_d
96 + q = tl.load(Q + off_q, mask=mask_d, other=0.0)
97 +
98 + kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS)
99 + split_kv_start = kv_len_per_split * split_kv_id
100 + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split,
101 + cur_batch_seq_len)
102 +
103 + e_max = -float("inf")
104 + e_sum = 0.0
105 + acc = tl.zeros([BLOCK_DV], dtype=tl.float32)
106 +
107 + if split_kv_end > split_kv_start:
108 + for start_n in range(split_kv_start, split_kv_end, BLOCK_N):
109 + offs_n = start_n + tl.arange(0, BLOCK_N)
110 + kv_page_number = tl.load(
111 + Req_to_tokens + stride_req_to_tokens_b * cur_batch_req_idx +
112 + offs_n // PAGE_SIZE,
113 + mask=offs_n < split_kv_end,
114 + other=0,
115 + )
116 + kv_loc = kv_page_number * PAGE_SIZE + offs_n % PAGE_SIZE
117 + offs_buf_k = (kv_loc[:, None] * stride_buf_kbs +
118 + cur_kv_head * stride_buf_kh + offs_d[None, :])
119 + k = tl.load(
120 + K_Buffer + offs_buf_k,
121 + mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]),
122 + other=0.0,
123 + )
124 + qk = tl.sum(q[None, :] * k, 1)
125 + qk *= sm_scale
126 +
127 + if logit_cap > 0:
128 + qk = logit_cap * tanh(qk / logit_cap)
129 +
130 + qk = tl.where(offs_n < split_kv_end, qk, float("-inf"))
131 +
132 + offs_buf_v = (kv_loc[:, None] * stride_buf_vbs +
133 + cur_kv_head * stride_buf_vh + offs_dv[None, :])
134 + v = tl.load(
135 + V_Buffer + offs_buf_v,
136 + mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]),
137 + other=0.0,
138 + )
139 +
140 + n_e_max = tl.maximum(tl.max(qk, 0), e_max)
141 + re_scale = tl.exp(e_max - n_e_max)
142 + p = tl.exp(qk - n_e_max)
143 + acc *= re_scale
144 + acc += tl.sum(p[:, None] * v, 0)
145 +
146 + e_sum = e_sum * re_scale + tl.sum(p, 0)
147 + e_max = n_e_max
148 +
149 + offs_mid_o = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh +
150 + split_kv_id * stride_mid_os + offs_dv)
151 +
152 + tl.store(
153 + Att_Out + offs_mid_o,
154 + acc / e_sum,
155 + mask=(mask_dv),
156 + )
157 +
158 + offs_mid_o_1 = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh +
159 + split_kv_id * stride_mid_os + Lv)
160 +
161 + tl.store(
162 + Att_Out + offs_mid_o_1,
163 + e_max + tl.log(e_sum),
164 + )
165 +
166 +
167 +def _decode_att_m_fwd(
168 + q,
169 + k_buffer,
170 + v_buffer,
171 + att_out,
172 + Req_to_tokens,
173 + B_Seqlen,
174 + num_kv_splits,
175 + sm_scale,
176 + page_size,
177 + logit_cap,
178 +):
179 + BLOCK = 64
180 + NUM_KV_SPLITS = num_kv_splits
181 + Lk = k_buffer.shape[-1]
182 + Lv = v_buffer.shape[-1]
183 +
184 + batch, head_num = q.shape[0], q.shape[1]
185 +
186 + grid = (batch, head_num, NUM_KV_SPLITS)
187 + kv_group_num = q.shape[1] // k_buffer.shape[-2]
188 +
189 + num_warps = 4 if kv_group_num == 1 else 2
190 +
191 + BLOCK_DMODEL = triton.next_power_of_2(Lk)
192 + BLOCK_DV = triton.next_power_of_2(Lv)
193 +
194 + _fwd_kernel_stage1[grid](
195 + q,
196 + k_buffer,
197 + v_buffer,
198 + sm_scale,
199 + Req_to_tokens,
200 + B_Seqlen,
201 + att_out,
202 + Req_to_tokens.stride(0),
203 + q.stride(0),
204 + q.stride(1),
205 + k_buffer.stride(-2),
206 + k_buffer.stride(-1),
207 + v_buffer.stride(-2),
208 + v_buffer.stride(-1),
209 + att_out.stride(0),
210 + att_out.stride(1),
211 + att_out.stride(2),
212 + kv_group_num=kv_group_num,
213 + BLOCK_DMODEL=BLOCK_DMODEL,
214 + BLOCK_DV=BLOCK_DV,
215 + BLOCK_N=BLOCK,
216 + NUM_KV_SPLITS=NUM_KV_SPLITS,
217 + PAGE_SIZE=page_size,
218 + logit_cap=logit_cap,
219 + num_warps=num_warps,
220 + num_stages=2,
221 + Lk=Lk,
222 + Lv=Lv,
223 + )
224 +
225 +
226 +@triton.jit
227 +def _fwd_grouped_kernel_stage1(
228 + Q,
229 + K_Buffer,
230 + V_Buffer,
231 + sm_scale,
232 + Req_to_tokens,
233 + B_Seqlen,
234 + Att_Out,
235 + stride_req_to_tokens_b,
236 + stride_qbs,
237 + stride_qh,
238 + stride_buf_kbs,
239 + stride_buf_kh,
240 + stride_buf_vbs,
241 + stride_buf_vh,
242 + stride_mid_ob,
243 + stride_mid_oh,
244 + stride_mid_os,
245 + kv_group_num: tl.constexpr,
246 + q_head_num: tl.constexpr,
247 + BLOCK_DMODEL: tl.constexpr,
248 + BLOCK_DPE: tl.constexpr,
249 + BLOCK_DV: tl.constexpr,
250 + BLOCK_N: tl.constexpr,
251 + BLOCK_H: tl.constexpr,
252 + NUM_KV_SPLITS: tl.constexpr,
253 + PAGE_SIZE: tl.constexpr,
254 + logit_cap: tl.constexpr,
255 + Lk: tl.constexpr,
256 + Lv: tl.constexpr,
257 +):
258 + cur_batch = tl.program_id(0)
259 + cur_head_id = tl.program_id(1)
260 + cur_kv_head = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H)
261 + split_kv_id = tl.program_id(2)
262 +
263 + if kv_group_num > BLOCK_H:
264 + VALID_BLOCK_H: tl.constexpr = BLOCK_H
265 + else:
266 + VALID_BLOCK_H: tl.constexpr = kv_group_num
267 + cur_head = cur_head_id * VALID_BLOCK_H + tl.arange(0, BLOCK_H)
268 + mask_h = cur_head < (cur_head_id + 1) * VALID_BLOCK_H
269 + mask_h = mask_h & (cur_head < q_head_num)
270 +
271 + offs_d = tl.arange(0, BLOCK_DMODEL)
272 + offs_dv = tl.arange(0, BLOCK_DV)
273 + mask_d = offs_d < Lk
274 + mask_dv = offs_dv < Lv
275 + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
276 + cur_batch_req_idx = cur_batch
277 +
278 + offs_q = cur_batch * stride_qbs + cur_head[:, None] * stride_qh + offs_d[
279 + None, :]
280 + q = tl.load(Q + offs_q,
281 + mask=(mask_h[:, None]) & (mask_d[None, :]),
282 + other=0.0)
283 +
284 + if BLOCK_DPE > 0:
285 + offs_dpe = BLOCK_DMODEL + tl.arange(0, BLOCK_DPE)
286 + mask_dpe = offs_dpe < Lk
287 + off_qpe = (cur_batch * stride_qbs + cur_head[:, None] * stride_qh +
288 + offs_dpe[None, :])
289 + qpe = tl.load(Q + off_qpe,
290 + mask=(mask_h[:, None]) & (mask_dpe[None, :]),
291 + other=0.0)
292 +
293 + kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS)
294 + split_kv_start = kv_len_per_split * split_kv_id
295 + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split,
296 + cur_batch_seq_len)
297 +
298 + e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - float("inf")
299 + e_sum = tl.zeros([BLOCK_H], dtype=tl.float32)
300 + acc = tl.zeros([BLOCK_H, BLOCK_DV], dtype=tl.float32)
301 +
302 + if split_kv_end > split_kv_start:
303 + for start_n in range(split_kv_start, split_kv_end, BLOCK_N):
304 + offs_n = start_n + tl.arange(0, BLOCK_N)
305 + kv_page_number = tl.load(
306 + Req_to_tokens + stride_req_to_tokens_b * cur_batch_req_idx +
307 + offs_n // PAGE_SIZE,
308 + mask=offs_n < split_kv_end,
309 + other=0,
310 + )
311 + kv_loc = kv_page_number * PAGE_SIZE + offs_n % PAGE_SIZE
312 + offs_buf_k = (kv_loc[None, :] * stride_buf_kbs +
313 + cur_kv_head * stride_buf_kh + offs_d[:, None])
314 + k = tl.load(
315 + K_Buffer + offs_buf_k,
316 + mask=(offs_n[None, :] < split_kv_end) & (mask_d[:, None]),
317 + other=0.0,
318 + )
319 + qk = tl.dot(q, k.to(q.dtype))
320 + if BLOCK_DPE > 0:
321 + offs_buf_kpe = (kv_loc[None, :] * stride_buf_kbs +
322 + cur_kv_head * stride_buf_kh +
323 + offs_dpe[:, None])
324 + kpe = tl.load(
325 + K_Buffer + offs_buf_kpe,
326 + mask=(offs_n[None, :] < split_kv_end) &
327 + (mask_dpe[:, None]),
328 + other=0.0,
329 + )
330 + qk += tl.dot(qpe, kpe.to(qpe.dtype))
331 + qk *= sm_scale
332 +
333 + if logit_cap > 0:
334 + qk = logit_cap * tanh(qk / logit_cap)
335 +
336 + qk = tl.where(mask_h[:, None] & (offs_n[None, :] < split_kv_end),
337 + qk, float("-inf"))
338 +
339 + offs_buf_v = (kv_loc[:, None] * stride_buf_vbs +
340 + cur_kv_head * stride_buf_vh + offs_dv[None, :])
341 + v = tl.load(
342 + V_Buffer + offs_buf_v,
343 + mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]),
344 + other=0.0,
345 + )
346 +
347 + n_e_max = tl.maximum(tl.max(qk, 1), e_max)
348 + re_scale = tl.exp(e_max - n_e_max)
349 + p = tl.exp(qk - n_e_max[:, None])
350 + acc *= re_scale[:, None]
351 + acc += tl.dot(p.to(v.dtype), v)
352 +
353 + e_sum = e_sum * re_scale + tl.sum(p, 1)
354 + e_max = n_e_max
355 +
356 + offs_mid_o = (cur_batch * stride_mid_ob +
357 + cur_head[:, None] * stride_mid_oh +
358 + split_kv_id * stride_mid_os + offs_dv[None, :])
359 +
360 + tl.store(
361 + Att_Out + offs_mid_o,
362 + acc / e_sum[:, None],
363 + mask=(mask_h[:, None]) & (mask_dv[None, :]),
364 + )
365 +
366 + offs_mid_o_1 = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh +
367 + split_kv_id * stride_mid_os + Lv)
368 +
369 + tl.store(
370 + Att_Out + offs_mid_o_1,
371 + e_max + tl.log(e_sum),
372 + mask=mask_h,
373 + )
374 +
375 +
376 +def _decode_grouped_att_m_fwd(
377 + q,
378 + k_buffer,
379 + v_buffer,
380 + att_out,
381 + Req_to_tokens,
382 + B_Seqlen,
383 + num_kv_splits,
384 + sm_scale,
385 + page_size,
386 + logit_cap,
387 +):
388 + BLOCK = 32
389 + Lk = k_buffer.shape[-1]
390 + Lv = v_buffer.shape[-1]
391 +
392 + # [TODO] work around shmem limit on MI3xx
393 + if is_hip_ and Lk >= 576:
394 + BLOCK = 16
395 +
396 + if Lk == 576:
397 + BLOCK_DMODEL = 512
398 + BLOCK_DPE = 64
399 + elif Lk == 288:
400 + BLOCK_DMODEL = 256
401 + BLOCK_DPE = 32
402 + else:
403 + BLOCK_DMODEL = triton.next_power_of_2(Lk)
404 + BLOCK_DPE = 0
405 + BLOCK_DV = triton.next_power_of_2(Lv)
406 +
407 + batch, head_num = q.shape[0], q.shape[1]
408 + kv_group_num = q.shape[1] // k_buffer.shape[-2]
409 +
410 + BLOCK_H = 16
411 + NUM_KV_SPLITS = num_kv_splits
412 + grid = (
413 + batch,
414 + triton.cdiv(head_num, min(BLOCK_H, kv_group_num)),
415 + NUM_KV_SPLITS,
416 + )
417 +
418 + extra_kargs = {}
419 + if is_hip_:
420 + # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html
421 + # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py
422 + extra_kargs = {
423 + "waves_per_eu": 4,
424 + "matrix_instr_nonkdim": 16,
425 + "kpack": 2
426 + }
427 +
428 + _fwd_grouped_kernel_stage1[grid](
429 + q,
430 + k_buffer,
431 + v_buffer,
432 + sm_scale,
433 + Req_to_tokens,
434 + B_Seqlen,
435 + att_out,
436 + Req_to_tokens.stride(0),
437 + q.stride(0),
438 + q.stride(1),
439 + k_buffer.stride(-2),
440 + k_buffer.stride(-1),
441 + v_buffer.stride(-2),
442 + v_buffer.stride(-1),
443 + att_out.stride(0),
444 + att_out.stride(1),
445 + att_out.stride(2),
446 + kv_group_num=kv_group_num,
447 + q_head_num=head_num,
448 + BLOCK_DMODEL=BLOCK_DMODEL,
449 + BLOCK_DPE=BLOCK_DPE,
450 + BLOCK_DV=BLOCK_DV,
451 + BLOCK_N=BLOCK,
452 + BLOCK_H=BLOCK_H,
453 + NUM_KV_SPLITS=NUM_KV_SPLITS,
454 + PAGE_SIZE=page_size,
455 + logit_cap=logit_cap,
456 + num_warps=4,
457 + num_stages=2,
458 + Lk=Lk,
459 + Lv=Lv,
460 + **extra_kargs,
461 + )
462 +
463 +
464 +@triton.jit
465 +def _fwd_kernel_stage2(
466 + Mid_O,
467 + o,
468 + B_Seqlen,
469 + stride_mid_ob,
470 + stride_mid_oh,
471 + stride_mid_os,
472 + stride_obs,
473 + stride_oh,
474 + NUM_KV_SPLITS: tl.constexpr,
475 + BLOCK_DV: tl.constexpr,
476 + Lv: tl.constexpr,
477 +):
478 + cur_batch = tl.program_id(0)
479 + cur_head = tl.program_id(1)
480 +
481 + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
482 +
483 + offs_d = tl.arange(0, BLOCK_DV)
484 + mask_d = offs_d < Lv
485 +
486 + e_sum = 0.0
487 + e_max = -float("inf")
488 + acc = tl.zeros([BLOCK_DV], dtype=tl.float32)
489 +
490 + offs_v = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + offs_d
491 + offs_logic = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + Lv
492 +
493 + for split_kv_id in range(0, NUM_KV_SPLITS):
494 + kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS)
495 + split_kv_start = kv_len_per_split * split_kv_id
496 + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split,
497 + cur_batch_seq_len)
498 +
499 + if split_kv_end > split_kv_start:
500 + tv = tl.load(Mid_O + offs_v + split_kv_id * stride_mid_os,
501 + mask=mask_d,
502 + other=0.0)
503 + tlogic = tl.load(Mid_O + offs_logic + split_kv_id * stride_mid_os)
504 + n_e_max = tl.maximum(tlogic, e_max)
505 +
506 + old_scale = tl.exp(e_max - n_e_max)
507 + acc *= old_scale
508 + exp_logic = tl.exp(tlogic - n_e_max)
509 + acc += exp_logic * tv
510 +
511 + e_sum = e_sum * old_scale + exp_logic
512 + e_max = n_e_max
513 +
514 + tl.store(
515 + o + cur_batch * stride_obs + cur_head * stride_oh + offs_d,
516 + acc / e_sum,
517 + mask=mask_d,
518 + )
519 +
520 +
521 +def _decode_softmax_reducev_fwd(
522 + logits,
523 + q,
524 + o,
525 + v_buffer,
526 + b_seq_len,
527 + num_kv_splits,
528 +):
529 + batch, head_num = q.shape[0], q.shape[1]
530 + Lv = v_buffer.shape[-1]
531 + BLOCK_DV = triton.next_power_of_2(Lv)
532 +
533 + NUM_KV_SPLITS = num_kv_splits
534 +
535 + extra_kargs = {}
536 + if is_hip_:
537 + # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html
538 + # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py
539 + extra_kargs = {
540 + "waves_per_eu": 4,
541 + "matrix_instr_nonkdim": 16,
542 + "kpack": 2
543 + }
544 +
545 + grid = (batch, head_num)
546 + _fwd_kernel_stage2[grid](
547 + logits,
548 + o,
549 + b_seq_len,
550 + logits.stride(0),
551 + logits.stride(1),
552 + logits.stride(2),
553 + o.stride(0),
554 + o.stride(1),
555 + NUM_KV_SPLITS=NUM_KV_SPLITS,
556 + BLOCK_DV=BLOCK_DV,
557 + Lv=Lv,
558 + num_warps=4,
559 + num_stages=2,
560 + **extra_kargs,
561 + )
562 +
563 +
564 +def decode_attention_fwd_normal(
565 + q,
566 + k_buffer,
567 + v_buffer,
568 + o,
569 + req_to_token,
570 + b_seq_len,
571 + attn_logits,
572 + num_kv_splits,
573 + sm_scale,
574 + page_size,
575 + logit_cap=0.0,
576 +):
577 + _decode_att_m_fwd(
578 + q,
579 + k_buffer,
580 + v_buffer,
581 + attn_logits,
582 + req_to_token,
583 + b_seq_len,
584 + num_kv_splits,
585 + sm_scale,
586 + page_size,
587 + logit_cap,
588 + )
589 + _decode_softmax_reducev_fwd(attn_logits, q, o, v_buffer, b_seq_len,
590 + num_kv_splits)
591 +
592 +
593 +def decode_attention_fwd_grouped(
594 + q,
595 + k_buffer,
596 + v_buffer,
597 + o,
598 + req_to_token,
599 + b_seq_len,
600 + attn_logits,
601 + num_kv_splits,
602 + sm_scale,
603 + page_size,
604 + logit_cap=0.0,
605 +):
606 + _decode_grouped_att_m_fwd(
607 + q,
608 + k_buffer,
609 + v_buffer,
610 + attn_logits,
611 + req_to_token,
612 + b_seq_len,
613 + num_kv_splits,
614 + sm_scale,
615 + page_size,
616 + logit_cap,
617 + )
618 + _decode_softmax_reducev_fwd(attn_logits, q, o, v_buffer, b_seq_len,
619 + num_kv_splits)
620 +
621 +
622 +def decode_attention_fwd(
623 + q,
624 + k_buffer,
625 + v_buffer,
626 + o,
627 + req_to_token,
628 + b_seq_len,
629 + attn_logits,
630 + num_kv_splits,
631 + sm_scale,
632 + page_size=1,
633 + logit_cap=0.0,
634 +):
635 + assert num_kv_splits == attn_logits.shape[2]
636 + kv_group_num = q.shape[1] // v_buffer.shape[-2]
637 +
638 + if kv_group_num == 1:
639 + # MHA
640 + decode_attention_fwd_normal(
641 + q,
642 + k_buffer,
643 + v_buffer,
644 + o,
645 + req_to_token,
646 + b_seq_len,
647 + attn_logits,
648 + num_kv_splits,
649 + sm_scale,
650 + page_size,
651 + logit_cap,
652 + )
653 + else:
654 + # GQA/MQA/MLA
655 + decode_attention_fwd_grouped(
656 + q,
657 + k_buffer,
658 + v_buffer,
659 + o,
660 + req_to_token,
661 + b_seq_len,
662 + attn_logits,
663 + num_kv_splits,
664 + sm_scale,
665 + page_size,
666 + logit_cap,
667 + )
vllm/attention/selector.py
+5 -1
@@ -83,6 +83,7 @@ def get_attn_backend(
83 block_size: int,
84 is_attention_free: bool,
85 is_blocksparse: bool = False,
86 + use_mla: bool = False,
87 ) -> Type[AttentionBackend]:
88 """Selects which attention backend to use and lazily imports it."""
89 # Accessing envs.* behind an @lru_cache decorator can cause the wrong
@@ -97,6 +98,7 @@ def get_attn_backend(
98 is_attention_free=is_attention_free,
99 is_blocksparse=is_blocksparse,
100 use_v1=envs.VLLM_USE_V1,
101 + use_mla=use_mla,
102 )
103
104
@@ -109,6 +111,7 @@ def _cached_get_attn_backend(
111 is_attention_free: bool,
112 is_blocksparse: bool = False,
113 use_v1: bool = False,
114 + use_mla: bool = False,
115 ) -> Type[AttentionBackend]:
116 if is_blocksparse:
117 logger.info("Using BlocksparseFlashAttention backend.")
@@ -141,7 +144,8 @@ def _cached_get_attn_backend(
144
145 # get device-specific attn_backend
146 attention_cls = current_platform.get_attn_backend_cls(
144 - selected_backend, head_size, dtype, kv_cache_dtype, block_size, use_v1)
147 + selected_backend, head_size, dtype, kv_cache_dtype, block_size, use_v1,
148 + use_mla)
149 if not attention_cls:
150 raise ValueError(
151 f"Invalid attention backend for {current_platform.device_name}")
vllm/config.py
+26 -9
@@ -736,17 +736,25 @@ class ModelConfig:
736 def get_hidden_size(self) -> int:
737 return self.hf_text_config.hidden_size
738
739 + @property
740 + def is_deepseek_mla(self) -> bool:
741 + # TODO add deepseek_v3
742 + return hasattr(self.hf_text_config,
743 + "model_type") and (self.hf_text_config.model_type
744 + in ('deepseek_v2'))
745 +
746 def get_head_size(self) -> int:
747 # TODO remove hard code
741 - if hasattr(self.hf_text_config,
742 - "model_type") and (self.hf_text_config.model_type
743 - in ('deepseek_v2', 'deepseek_v3')):
744 - qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
745 - 0)
746 - qk_nope_head_dim = getattr(self.hf_text_config, "qk_nope_head_dim",
747 - 0)
748 - if qk_rope_head_dim and qk_nope_head_dim:
749 - return qk_rope_head_dim + qk_nope_head_dim
748 + if self.is_deepseek_mla:
749 + if self.use_mla:
750 + return self.hf_text_config.kv_lora_rank
751 + else:
752 + qk_rope_head_dim = getattr(self.hf_text_config,
753 + "qk_rope_head_dim", 0)
754 + qk_nope_head_dim = getattr(self.hf_text_config,
755 + "qk_nope_head_dim", 0)
756 + if qk_rope_head_dim and qk_nope_head_dim:
757 + return qk_rope_head_dim + qk_nope_head_dim
758
759 if self.is_attention_free:
760 return 0
@@ -805,6 +813,10 @@ class ModelConfig:
813
814 def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
815 """Returns the number of KV heads per GPU."""
816 + if self.use_mla:
817 + # When using MLA during decode it becomes MQA
818 + return 1
819 +
820 total_num_kv_heads = self.get_total_num_kv_heads()
821 # If tensor parallelism is used, we divide the number of KV heads by
822 # the tensor parallel size. We will replicate the KV heads in the
@@ -955,6 +967,11 @@ class ModelConfig:
967 architectures = getattr(self.hf_config, "architectures", [])
968 return ModelRegistry.is_cross_encoder_model(architectures)
969
970 + @property
971 + def use_mla(self) -> bool:
972 + use_mla = (self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE)
973 + return use_mla
974 +
975 @property
976 def supported_runner_types(self) -> Set[RunnerType]:
977 return {_TASK_RUNNER[task] for task in self.supported_tasks}
vllm/engine/arg_utils.py
-1
@@ -931,7 +931,6 @@ class EngineArgs:
931 type=str,
932 default="auto",
933 help='The worker class to use for distributed execution.')
934 -
934 parser.add_argument(
935 "--generation-config",
936 type=nullable_str,
vllm/envs.py
+14
@@ -77,6 +77,8 @@ if TYPE_CHECKING:
77 V_SCALE_CONSTANT: int = 100
78 VLLM_SERVER_DEV_MODE: bool = False
79 VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
80 + VLLM_MLA_DISABLE: bool = False
81 + VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
82
83
84 def get_default_cache_root():
@@ -506,6 +508,18 @@ environment_variables: Dict[str, Callable[[], Any]] = {
508 # TTFT and overall throughput.
509 "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE":
510 lambda: int(os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128")),
511 +
512 + # If set, vLLM will disable the MLA attention optimizations.
513 + "VLLM_MLA_DISABLE":
514 + lambda: bool(int(os.getenv("VLLM_MLA_DISABLE", "0"))),
515 +
516 + # Flag that can control whether or not we perform matrix-absorption for MLA
517 + # decode, i.e. absorb W_UK into W_Q/W_UK and W_UV into W_O, absorbing the
518 + # matrices reduces the runtime FLOPs needed to compute MLA but requires
519 + # storing more weights, W_Q_UK and W_UV_O, so can increase memory usage,
520 + # the is enabled by default
521 + "VLLM_MLA_PERFORM_MATRIX_ABSORPTION":
522 + lambda: bool(int(os.getenv("VLLM_MLA_PERFORM_MATRIX_ABSORPTION", "1")))
523 }
524
525 # end-env-vars-definition
vllm/model_executor/model_loader/loader.py
+6
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import AutoModelForCausalLM
24 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
25
26 +from vllm.attention import Attention
27 from vllm.config import (LoadConfig, LoadFormat, ModelConfig, ParallelConfig,
28 VllmConfig, set_current_vllm_config)
29 from vllm.distributed import (get_tensor_model_parallel_rank,
@@ -397,6 +398,11 @@ class DefaultModelLoader(BaseModelLoader):
398 # parameters onto device for processing and back off after.
399 with device_loading_context(module, target_device):
400 quant_method.process_weights_after_loading(module)
401 + elif isinstance(module, Attention) and \
402 + hasattr(module, "process_weights_after_loading"):
403 + # When attention modules need to process weights after
404 + # currently only used by MLA
405 + module.process_weights_after_loading()
406 return model.eval()
407
408
vllm/model_executor/models/deepseek_v2.py
+152 -2
@@ -28,7 +28,7 @@ from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, VllmConfig
31 +from vllm.config import CacheConfig, ModelConfig, VllmConfig
32 from vllm.distributed import (get_pp_group,
33 get_tensor_model_parallel_world_size,
34 tensor_model_parallel_all_reduce)
@@ -326,12 +326,156 @@ class DeepseekV2Attention(nn.Module):
326 return output
327
328
329 +class DeepseekV2MLAAttention(nn.Module):
330 + """
331 + Main reference: DeepseekV2 paper, and FlashInfer Implementation
332 + (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
333 +
334 + For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py
335 + """
336 +
337 + def __init__(
338 + self,
339 + config: PretrainedConfig,
340 + hidden_size: int,
341 + num_heads: int,
342 + qk_nope_head_dim: int,
343 + qk_rope_head_dim: int,
344 + v_head_dim: int,
345 + q_lora_rank: Optional[int],
346 + kv_lora_rank: int,
347 + rope_theta: float = 10000,
348 + rope_scaling: Optional[Dict[str, Any]] = None,
349 + max_position_embeddings: int = 8192,
350 + cache_config: Optional[CacheConfig] = None,
351 + quant_config: Optional[QuantizationConfig] = None,
352 + prefix: str = "",
353 + ) -> None:
354 + super().__init__()
355 + self.hidden_size = hidden_size
356 + self.qk_nope_head_dim = qk_nope_head_dim
357 + self.qk_rope_head_dim = qk_rope_head_dim
358 + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
359 + self.v_head_dim = v_head_dim
360 +
361 + self.q_lora_rank = q_lora_rank
362 + self.kv_lora_rank = kv_lora_rank
363 +
364 + self.num_heads = num_heads
365 + tp_size = get_tensor_model_parallel_world_size()
366 + assert num_heads % tp_size == 0
367 + self.num_local_heads = num_heads // tp_size
368 +
369 + self.scaling = self.qk_head_dim**-0.5
370 + self.rope_theta = rope_theta
371 + self.max_position_embeddings = max_position_embeddings
372 +
373 + if self.q_lora_rank is not None:
374 + self.q_a_proj = ReplicatedLinear(self.hidden_size,
375 + self.q_lora_rank,
376 + bias=False,
377 + quant_config=quant_config,
378 + prefix=f"{prefix}.q_a_proj")
379 + self.q_a_layernorm = RMSNorm(self.q_lora_rank,
380 + eps=config.rms_norm_eps)
381 + self.q_b_proj = ColumnParallelLinear(q_lora_rank,
382 + self.num_heads *
383 + self.qk_head_dim,
384 + bias=False,
385 + quant_config=quant_config,
386 + prefix=f"{prefix}.q_b_proj")
387 + else:
388 + self.q_proj = ColumnParallelLinear(self.hidden_size,
389 + self.num_heads *
390 + self.qk_head_dim,
391 + bias=False,
392 + quant_config=quant_config,
393 + prefix=f"{prefix}.q_proj")
394 +
395 + self.kv_a_proj_with_mqa = ReplicatedLinear(
396 + self.hidden_size,
397 + self.kv_lora_rank + self.qk_rope_head_dim,
398 + bias=False,
399 + quant_config=quant_config,
400 + prefix=f"{prefix}.kv_a_proj_with_mqa")
401 + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
402 + eps=config.rms_norm_eps)
403 + self.kv_b_proj = ColumnParallelLinear(
404 + self.kv_lora_rank,
405 + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
406 + bias=False,
407 + quant_config=quant_config,
408 + prefix=f"{prefix}.kv_b_proj")
409 + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
410 + self.hidden_size,
411 + bias=False,
412 + quant_config=quant_config,
413 + prefix=f"{prefix}.o_proj")
414 +
415 + rope_scaling["rope_type"] = 'deepseek_yarn'
416 + self.rotary_emb = get_rope(qk_rope_head_dim,
417 + rotary_dim=qk_rope_head_dim,
418 + max_position=max_position_embeddings,
419 + base=rope_theta,
420 + rope_scaling=rope_scaling,
421 + is_neox_style=False)
422 + if rope_scaling:
423 + mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
424 + scaling_factor = rope_scaling["factor"]
425 + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
426 + self.scaling = self.scaling * mscale * mscale
427 +
428 + self.mla_attn = Attention(
429 + num_heads=self.num_local_heads,
430 + head_size=self.kv_lora_rank,
431 + scale=self.scaling,
432 + num_kv_heads=1,
433 + cache_config=cache_config,
434 + quant_config=quant_config,
435 + prefix=f"{prefix}.attn",
436 + use_mla=True,
437 + # MLA Args
438 + q_lora_rank=self.q_lora_rank,
439 + kv_lora_rank=self.kv_lora_rank,
440 + qk_nope_head_dim=self.qk_nope_head_dim,
441 + qk_rope_head_dim=self.qk_rope_head_dim,
442 + qk_head_dim=self.qk_head_dim,
443 + v_head_dim=self.v_head_dim,
444 + rotary_emb=self.rotary_emb,
445 + q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj,
446 + kv_b_proj=self.kv_b_proj,
447 + o_proj=self.o_proj,
448 + )
449 +
450 + self.prefix = prefix
451 + self.debug_layer_idx = int(self.prefix.split(".")[-2])
452 +
453 + def forward(
454 + self,
455 + positions: torch.Tensor,
456 + hidden_states: torch.Tensor,
457 + kv_cache: torch.Tensor,
458 + attn_metadata: AttentionMetadata,
459 + ) -> torch.Tensor:
460 + if self.q_lora_rank is not None:
461 + ckq = self.q_a_proj(hidden_states)[0]
462 + hidden_states_or_q_c = self.q_a_layernorm(ckq)
463 + else:
464 + hidden_states_or_q_c = hidden_states
465 + kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states)[0].split(
466 + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
467 + kv_c_normed = self.kv_a_layernorm(kv_c.contiguous())
468 + return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, kv_cache,
469 + attn_metadata)
470 +
471 +
472 class DeepseekV2DecoderLayer(nn.Module):
473
474 def __init__(
475 self,
476 config: PretrainedConfig,
477 prefix: str,
478 + model_config: ModelConfig,
479 cache_config: Optional[CacheConfig] = None,
480 quant_config: Optional[QuantizationConfig] = None,
481 ) -> None:
@@ -344,7 +488,11 @@ class DeepseekV2DecoderLayer(nn.Module):
488 # DecoderLayers are created with `make_layers` which passes the prefix
489 # with the layer's index.
490 layer_idx = int(prefix.split(sep='.')[-1])
347 - self.self_attn = DeepseekV2Attention(
491 + if model_config.use_mla:
492 + attn_cls = DeepseekV2MLAAttention
493 + else:
494 + attn_cls = DeepseekV2Attention
495 + self.self_attn = attn_cls(
496 config=config,
497 hidden_size=self.hidden_size,
498 num_heads=config.num_attention_heads,
@@ -421,6 +569,7 @@ class DeepseekV2Model(nn.Module):
569 super().__init__()
570
571 config = vllm_config.model_config.hf_config
572 + model_config = vllm_config.model_config
573 cache_config = vllm_config.cache_config
574 quant_config = vllm_config.quant_config
575
@@ -440,6 +589,7 @@ class DeepseekV2Model(nn.Module):
589 lambda prefix: DeepseekV2DecoderLayer(
590 config,
591 prefix,
592 + model_config=model_config,
593 cache_config=cache_config,
594 quant_config=quant_config,
595 ),
vllm/platforms/cpu.py
+2 -1
@@ -31,7 +31,8 @@ class CpuPlatform(Platform):
31 @classmethod
32 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
33 dtype: torch.dtype, kv_cache_dtype: Optional[str],
34 - block_size: int, use_v1: bool) -> str:
34 + block_size: int, use_v1: bool,
35 + use_mla: bool) -> str:
36 if selected_backend != _Backend.TORCH_SDPA:
37 logger.info("Cannot use %s backend on CPU.", selected_backend)
38 logger.info("Using Torch SDPA backend.")
vllm/platforms/cuda.py
+7 -2
@@ -157,10 +157,14 @@ class CudaPlatformBase(Platform):
157
158 @classmethod
159 def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
160 - kv_cache_dtype, block_size, use_v1) -> str:
160 + kv_cache_dtype, block_size, use_v1,
161 + use_mla) -> str:
162 if use_v1:
163 logger.info("Using Flash Attention backend on V1 engine.")
164 return "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend"
165 + if use_mla:
166 + logger.info("Using Triton MLA backend.")
167 + return "vllm.attention.backends.triton_mla.TritonMLABackend"
168 if selected_backend == _Backend.FLASHINFER:
169 logger.info("Using FlashInfer backend.")
170 return "vllm.attention.backends.flashinfer.FlashInferBackend"
@@ -171,7 +175,8 @@ class CudaPlatformBase(Platform):
175 pass
176 elif selected_backend:
177 raise ValueError(
174 - f"Invalid attention backend for {cls.device_name}")
178 + f"Invalid attention backend for {cls.device_name}, "
179 + f"with use_v1: {use_v1} use_mla: {use_mla}")
180
181 target_backend = _Backend.FLASH_ATTN
182 if not cls.has_device_capability(80):
vllm/platforms/hpu.py
+2 -1
@@ -27,7 +27,8 @@ class HpuPlatform(Platform):
27 @classmethod
28 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
29 dtype: torch.dtype, kv_cache_dtype: Optional[str],
30 - block_size: int, use_v1: bool) -> str:
30 + block_size: int, use_v1: bool,
31 + use_mla: bool) -> str:
32 logger.info("Using HPUAttention backend.")
33 return "vllm.attention.backends.hpu_attn.HPUAttentionBackend"
34
vllm/platforms/interface.py
+3 -1
@@ -30,6 +30,7 @@ class _Backend(enum.Enum):
30 TORCH_SDPA = enum.auto()
31 OPENVINO = enum.auto()
32 FLASHINFER = enum.auto()
33 + TRITON_MLA = enum.auto()
34 HPU_ATTN = enum.auto()
35 PALLAS = enum.auto()
36 IPEX = enum.auto()
@@ -139,7 +140,8 @@ class Platform:
140 @classmethod
141 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
142 dtype: torch.dtype, kv_cache_dtype: Optional[str],
142 - block_size: int, use_v1: bool) -> str:
143 + block_size: int, use_v1: bool,
144 + use_mla: bool) -> str:
145 """Get the attention backend class of a device."""
146 return ""
147
vllm/platforms/openvino.py
+2 -1
@@ -30,7 +30,8 @@ class OpenVinoPlatform(Platform):
30 @classmethod
31 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
32 dtype: torch.dtype, kv_cache_dtype: Optional[str],
33 - block_size: int, use_v1: bool) -> str:
33 + block_size: int, use_v1: bool,
34 + use_mla: bool) -> str:
35 if selected_backend != _Backend.OPENVINO:
36 logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
37 logger.info("Using OpenVINO Attention backend.")
vllm/platforms/rocm.py
+2 -1
@@ -75,7 +75,8 @@ class RocmPlatform(Platform):
75
76 @classmethod
77 def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
78 - kv_cache_dtype, block_size, use_v1) -> str:
78 + kv_cache_dtype, block_size, use_v1,
79 + use_mla) -> str:
80 selected_backend = (_Backend.ROCM_FLASH if selected_backend
81 == _Backend.FLASH_ATTN else selected_backend)
82 if selected_backend == _Backend.ROCM_FLASH:
vllm/platforms/tpu.py
+2 -1
@@ -29,7 +29,8 @@ class TpuPlatform(Platform):
29 @classmethod
30 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
31 dtype: torch.dtype, kv_cache_dtype: Optional[str],
32 - block_size: int, use_v1: bool) -> str:
32 + block_size: int, use_v1: bool,
33 + use_mla: bool) -> str:
34 if selected_backend != _Backend.PALLAS:
35 logger.info("Cannot use %s backend on TPU.", selected_backend)
36 logger.info("Using Pallas backend.")
vllm/platforms/xpu.py
+2 -1
@@ -27,7 +27,8 @@ class XPUPlatform(Platform):
27 @classmethod
28 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
29 dtype: torch.dtype, kv_cache_dtype: Optional[str],
30 - block_size: int, use_v1: bool) -> str:
30 + block_size: int, use_v1: bool,
31 + use_mla: bool) -> str:
32 if selected_backend != _Backend.IPEX:
33 logger.info("Cannot use %s backend on XPU.", selected_backend)
34 logger.info("Using IPEX attention backend.")
vllm/worker/cache_engine.py
+2 -1
@@ -56,7 +56,8 @@ class CacheEngine:
56 model_config.dtype,
57 cache_config.cache_dtype,
58 self.block_size,
59 - model_config.is_attention_free)
59 + model_config.is_attention_free,
60 + use_mla=model_config.use_mla)
61
62 # Initialize the cache.
63 self.gpu_cache = self._allocate_kv_cache(
vllm/worker/model_runner.py
+3 -1
@@ -1066,6 +1066,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1066 self.kv_cache_dtype,
1067 self.block_size,
1068 self.model_config.is_attention_free,
1069 + use_mla=self.model_config.use_mla,
1070 ) if needs_attn_backend else None
1071 if self.attn_backend:
1072 self.attn_state = self.attn_backend.get_state_cls()(
@@ -1973,7 +1974,8 @@ class CUDAGraphRunner(nn.Module):
1974
1975 # Copy the input tensors to the input buffers.
1976 self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
1976 - self.input_buffers["positions"].copy_(positions, non_blocking=True)
1977 + if positions is not None:
1978 + self.input_buffers["positions"].copy_(positions, non_blocking=True)
1979
1980 if self.backend_name != "NO_ATTENTION":
1981 self.input_buffers["slot_mapping"].copy_(