A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Remove hardcoded `head_size=256` for Deepseek v2 and v3 (#12067)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 16, 2025 at 18:11 UTC
dd7c9ad87074b68d201208a196e0a4b2b5ecc27a
4 files changed
+23
-40
tests/kernels/test_attention.py
+3
-3
@@ -31,9 +31,9 @@ NUM_GEN_SEQS = [7] # Arbitrary values for testing
31
NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
32
NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
33
34
-# FlashAttention forward only supports head dimension at most 128
35
-# https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
36
-HEAD_SIZES = [64, 80, 120, 256]
34
+# This should be sync with get_supported_head_sizes() in
35
+# vllm.attention.ops.paged_attn.PagedAttention
36
+HEAD_SIZES = [32, 64, 80, 96, 112, 120, 128, 192, 256]
37
38
BLOCK_SIZES = [16, 32]
39
USE_ALIBI = [False, True]
vllm/config.py
+6
-3
@@ -733,9 +733,12 @@ class ModelConfig:
733
if hasattr(self.hf_text_config,
734
"model_type") and (self.hf_text_config.model_type
735
in ('deepseek_v2', 'deepseek_v3')):
736
- # FlashAttention supports only head_size 32, 64, 128, 256,
737
- # we need to pad head_size 192 to 256
738
- return 256
736
+ qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
737
+ 0)
738
+ qk_nope_head_dim = getattr(self.hf_text_config, "qk_nope_head_dim",
739
+ 0)
740
+ if qk_rope_head_dim and qk_nope_head_dim:
741
+ return qk_rope_head_dim + qk_nope_head_dim
742
743
if self.is_attention_free:
744
return 0
vllm/model_executor/models/deepseek_v2.py
+7
-17
@@ -262,14 +262,8 @@ class DeepseekV2Attention(nn.Module):
262
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
263
self.scaling = self.scaling * mscale * mscale
264
265
- # self.attn = Attention(self.num_heads,
266
- # self.qk_head_dim,
267
- # self.scaling,
268
- # num_kv_heads=self.num_heads)
269
-
270
- # TODO, support head_size 192
265
self.attn = Attention(self.num_local_heads,
272
- 256,
266
+ self.qk_head_dim,
267
self.scaling,
268
num_kv_heads=self.num_local_heads,
269
cache_config=cache_config,
@@ -319,18 +313,14 @@ class DeepseekV2Attention(nn.Module):
313
k = torch.empty_like(q)
314
k[..., :self.qk_nope_head_dim] = k_nope
315
k[..., self.qk_nope_head_dim:] = k_pe
322
- q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim],
323
- value=0).view(-1,
324
- self.num_local_heads * 256)
325
- k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim],
326
- value=0).view(-1,
327
- self.num_local_heads * 256)
328
- v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim],
329
- value=0).view(-1,
330
- self.num_local_heads * 256)
316
+ # padding value to qk_head_dim for alignment
317
+ v = torch.nn.functional.pad(
318
+ v, [0, self.qk_head_dim - self.v_head_dim],
319
+ value=0).view(-1, self.num_local_heads * self.qk_head_dim)
320
attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
321
attn_output = attn_output.view(
333
- -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape(
322
+ -1, self.num_local_heads,
323
+ self.qk_head_dim)[..., :self.v_head_dim].reshape(
324
-1, self.num_local_heads * self.v_head_dim)
325
output, _ = self.o_proj(attn_output)
326
return output
vllm/model_executor/models/deepseek_v3.py
+7
-17
@@ -269,14 +269,8 @@ class DeepseekV3Attention(nn.Module):
269
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
270
self.scaling = self.scaling * mscale * mscale
271
272
- # self.attn = Attention(self.num_heads,
273
- # self.qk_head_dim,
274
- # self.scaling,
275
- # num_kv_heads=self.num_heads)
276
-
277
- # TODO, support head_size 192
272
self.attn = Attention(self.num_local_heads,
279
- 256,
273
+ self.qk_head_dim,
274
self.scaling,
275
num_kv_heads=self.num_local_heads,
276
cache_config=cache_config,
@@ -326,18 +320,14 @@ class DeepseekV3Attention(nn.Module):
320
k = torch.empty_like(q)
321
k[..., :self.qk_nope_head_dim] = k_nope
322
k[..., self.qk_nope_head_dim:] = k_pe
329
- q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim],
330
- value=0).view(-1,
331
- self.num_local_heads * 256)
332
- k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim],
333
- value=0).view(-1,
334
- self.num_local_heads * 256)
335
- v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim],
336
- value=0).view(-1,
337
- self.num_local_heads * 256)
323
+ # padding value to qk_head_dim for alignment
324
+ v = torch.nn.functional.pad(
325
+ v, [0, self.qk_head_dim - self.v_head_dim],
326
+ value=0).view(-1, self.num_local_heads * self.qk_head_dim)
327
attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
328
attn_output = attn_output.view(
340
- -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape(
329
+ -1, self.num_local_heads,
330
+ self.qk_head_dim)[..., :self.v_head_dim].reshape(
331
-1, self.num_local_heads * self.v_head_dim)
332
output, _ = self.o_proj(attn_output)
333
return output