@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c36ac98d01
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[AMD][ROCm] Enable DeepSeek model on ROCm (#12662)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>

Hongxia Yang committed Feb 4, 2025 at 03:24 UTC c36ac98d0118537ec5f3f405a68311a10f9b59a5
5 files changed +58 -1
tests/kernels/test_rocm_attention_selector.py new
+31
@@ -0,0 +1,31 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from unittest.mock import patch
4 +
5 +import pytest
6 +import torch
7 +
8 +from tests.kernels.utils import override_backend_env_variable
9 +from vllm.attention.selector import _cached_get_attn_backend, get_attn_backend
10 +from vllm.platforms.rocm import RocmPlatform
11 +
12 +
13 +@pytest.fixture(autouse=True)
14 +def clear_cache():
15 + """Clear lru cache to ensure each test case runs without caching.
16 + """
17 + _cached_get_attn_backend.cache_clear()
18 +
19 +
20 +def test_selector(monkeypatch):
21 + """Test that the attention selector for ROCm.
22 + """
23 + override_backend_env_variable(monkeypatch, "ROCM_FLASH")
24 +
25 + with patch("vllm.attention.selector.current_platform", RocmPlatform()):
26 + backend = get_attn_backend(16, torch.float16, torch.float16, 16, False)
27 + assert backend.get_name() == "ROCM_FLASH"
28 + # mla test for deepseek related
29 + backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False,
30 + False, True)
31 + assert backend.get_name() == "TRITON_MLA"
tests/worker/test_model_runner.py
+9
@@ -24,6 +24,15 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
24 return model_runner
25
26
27 +def test_deepseek_mla_attn_backend_module():
28 + model_runner = _create_model_runner(
29 + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
30 + trust_remote_code=True,
31 + enable_chunked_prefill=False,
32 + )
33 + assert model_runner.attn_backend.__name__ == "TritonMLABackend"
34 +
35 +
36 @pytest.mark.parametrize("batch_size", list(range(1, 257)))
37 def test_prepare_prompt(batch_size):
38 model_runner = _create_model_runner(
vllm/attention/backends/mla/utils.py
+5 -1
@@ -27,7 +27,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
27 from vllm.model_executor.layers.quantization.utils.quant_utils import (
28 scaled_dequantize, scaled_quantize)
29 from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
30 -from vllm.vllm_flash_attn import flash_attn_varlen_func
30 +
31 +try:
32 + from vllm.vllm_flash_attn import flash_attn_varlen_func
33 +except ImportError:
34 + from flash_attn import flash_attn_varlen_func
35
36
37 @dataclass
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+10
@@ -47,6 +47,16 @@ def apply_w8a8_block_fp8_linear(
47
48 shape_supported_by_cutlass = (weight.shape[0] % 128 == 0
49 and weight.shape[1] % 128 == 0)
50 + if current_platform.is_rocm():
51 + scale_a_shape = ((input_2d.shape[-1] // block_size[1], ) +
52 + input_2d.shape[:-1])[::-1]
53 + scale_b_shape = (weight_scale.view(-1, 1)
54 + if weight_scale.dim() <= 1 else weight_scale.T).shape
55 + ar, ac = scale_a_shape
56 + br, bc = scale_b_shape
57 + if (ac > 1 or bc > 1 or ar not in (1, input_2d.shape[0])
58 + or br not in (1, weight.shape[0])):
59 + shape_supported_by_cutlass = False
60 if cutlass_block_fp8_supported and shape_supported_by_cutlass:
61 q_input, x_scale = per_token_group_quant_fp8(input_2d,
62 block_size[1],
vllm/platforms/rocm.py
+3
@@ -79,6 +79,9 @@ class RocmPlatform(Platform):
79 def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
80 kv_cache_dtype, block_size, use_v1,
81 use_mla) -> str:
82 + if use_mla:
83 + logger.info("Using Triton MLA backend.")
84 + return "vllm.attention.backends.triton_mla.TritonMLABackend"
85 selected_backend = (_Backend.ROCM_FLASH if selected_backend
86 == _Backend.FLASH_ATTN else selected_backend)
87 if selected_backend == _Backend.ROCM_FLASH: