A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[AMD][ROCm] Enable DeepSeek model on ROCm (#12662)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
Hongxia Yang committed
Feb 4, 2025 at 03:24 UTC
c36ac98d0118537ec5f3f405a68311a10f9b59a5
5 files changed
+58
-1
tests/kernels/test_rocm_attention_selector.py
new
+31
@@ -0,0 +1,31 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from unittest.mock import patch
4
+
5
+import pytest
6
+import torch
7
+
8
+from tests.kernels.utils import override_backend_env_variable
9
+from vllm.attention.selector import _cached_get_attn_backend, get_attn_backend
10
+from vllm.platforms.rocm import RocmPlatform
11
+
12
+
13
+@pytest.fixture(autouse=True)
14
+def clear_cache():
15
+ """Clear lru cache to ensure each test case runs without caching.
16
+ """
17
+ _cached_get_attn_backend.cache_clear()
18
+
19
+
20
+def test_selector(monkeypatch):
21
+ """Test that the attention selector for ROCm.
22
+ """
23
+ override_backend_env_variable(monkeypatch, "ROCM_FLASH")
24
+
25
+ with patch("vllm.attention.selector.current_platform", RocmPlatform()):
26
+ backend = get_attn_backend(16, torch.float16, torch.float16, 16, False)
27
+ assert backend.get_name() == "ROCM_FLASH"
28
+ # mla test for deepseek related
29
+ backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False,
30
+ False, True)
31
+ assert backend.get_name() == "TRITON_MLA"
tests/worker/test_model_runner.py
+9
@@ -24,6 +24,15 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
24
return model_runner
25
26
27
+def test_deepseek_mla_attn_backend_module():
28
+ model_runner = _create_model_runner(
29
+ "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
30
+ trust_remote_code=True,
31
+ enable_chunked_prefill=False,
32
+ )
33
+ assert model_runner.attn_backend.__name__ == "TritonMLABackend"
34
+
35
+
36
@pytest.mark.parametrize("batch_size", list(range(1, 257)))
37
def test_prepare_prompt(batch_size):
38
model_runner = _create_model_runner(
vllm/attention/backends/mla/utils.py
+5
-1
@@ -27,7 +27,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
27
from vllm.model_executor.layers.quantization.utils.quant_utils import (
28
scaled_dequantize, scaled_quantize)
29
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
30
-from vllm.vllm_flash_attn import flash_attn_varlen_func
30
+
31
+try:
32
+ from vllm.vllm_flash_attn import flash_attn_varlen_func
33
+except ImportError:
34
+ from flash_attn import flash_attn_varlen_func
35
36
37
@dataclass
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+10
@@ -47,6 +47,16 @@ def apply_w8a8_block_fp8_linear(
47
48
shape_supported_by_cutlass = (weight.shape[0] % 128 == 0
49
and weight.shape[1] % 128 == 0)
50
+ if current_platform.is_rocm():
51
+ scale_a_shape = ((input_2d.shape[-1] // block_size[1], ) +
52
+ input_2d.shape[:-1])[::-1]
53
+ scale_b_shape = (weight_scale.view(-1, 1)
54
+ if weight_scale.dim() <= 1 else weight_scale.T).shape
55
+ ar, ac = scale_a_shape
56
+ br, bc = scale_b_shape
57
+ if (ac > 1 or bc > 1 or ar not in (1, input_2d.shape[0])
58
+ or br not in (1, weight.shape[0])):
59
+ shape_supported_by_cutlass = False
60
if cutlass_block_fp8_supported and shape_supported_by_cutlass:
61
q_input, x_scale = per_token_group_quant_fp8(input_2d,
62
block_size[1],
vllm/platforms/rocm.py
+3
@@ -79,6 +79,9 @@ class RocmPlatform(Platform):
79
def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
80
kv_cache_dtype, block_size, use_v1,
81
use_mla) -> str:
82
+ if use_mla:
83
+ logger.info("Using Triton MLA backend.")
84
+ return "vllm.attention.backends.triton_mla.TritonMLABackend"
85
selected_backend = (_Backend.ROCM_FLASH if selected_backend
86
== _Backend.FLASH_ATTN else selected_backend)
87
if selected_backend == _Backend.ROCM_FLASH: