@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba59b78a9c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[ROCm][V1] Add intial ROCm support to V1 (#12790)

Sage Moore committed Feb 13, 2025 at 22:21 UTC ba59b78a9c5a8e9a535a0be081a438951679347f
5 files changed +236 -18
requirements-rocm-build.txt new
+16
@@ -0,0 +1,16 @@
1 +# Common dependencies
2 +-r requirements-common.txt
3 +
4 +--extra-index-url https://download.pytorch.org/whl/rocm6.2
5 +torch==2.5.1
6 +torchvision==0.20.1
7 +torchaudio==2.5.1
8 +
9 +cmake>=3.26
10 +ninja
11 +packaging
12 +setuptools>=61
13 +setuptools-scm>=8
14 +wheel
15 +jinja2
16 +amdsmi==6.2.4
vllm/attention/ops/prefix_prefill.py
+4 -2
@@ -718,7 +718,8 @@ if triton.__version__ >= "2.1.0":
718 k_scale: torch.Tensor,
719 v_scale: torch.Tensor,
720 alibi_slopes=None,
721 - sliding_window=None):
721 + sliding_window=None,
722 + sm_scale=None):
723
724 q_dtype_is_f32 = q.dtype is torch.float32
725 # need to reduce num. blocks when using fp32
@@ -759,7 +760,8 @@ if triton.__version__ >= "2.1.0":
760 # round up Lk to a power of 2 - this is required for Triton block size
761 Lk_padded = triton.next_power_of_2(Lk)
762
762 - sm_scale = 1.0 / (Lq**0.5)
763 + if sm_scale is None:
764 + sm_scale = 1.0 / (Lq**0.5)
765 batch, head = b_seq_len.shape[0], q.shape[1]
766 num_queries_per_kv = q.shape[1] // k.shape[1]
767
vllm/platforms/rocm.py
+30 -15
@@ -1,6 +1,5 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -import os
3 from functools import lru_cache
4 from typing import TYPE_CHECKING, Dict, List, Optional
5
@@ -29,12 +28,6 @@ try:
28 except ImportError as e:
29 logger.warning("Failed to import from vllm._rocm_C with %r", e)
30
32 -if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
33 - logger.warning("`fork` method is not supported by ROCm. "
34 - "VLLM_WORKER_MULTIPROC_METHOD is overridden to"
35 - " `spawn` instead.")
36 - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
37 -
31 # Models not supported by ROCm.
32 _ROCM_UNSUPPORTED_MODELS: List[str] = []
33
@@ -84,6 +77,9 @@ class RocmPlatform(Platform):
77 return "vllm.attention.backends.triton_mla.TritonMLABackend"
78 selected_backend = (_Backend.ROCM_FLASH if selected_backend
79 == _Backend.FLASH_ATTN else selected_backend)
80 + if envs.VLLM_USE_V1:
81 + logger.info("Using ROCm Attention backend on V1 engine.")
82 + return "vllm.v1.attention.backends.rocm_attn.ROCmAttentionBackend"
83 if selected_backend == _Backend.ROCM_FLASH:
84 if not cls.has_device_capability(90):
85 # not Instinct series GPUs.
@@ -102,7 +98,11 @@ class RocmPlatform(Platform):
98 @classmethod
99 @lru_cache(maxsize=8)
100 def get_device_name(cls, device_id: int = 0) -> str:
105 - return torch.cuda.get_device_name(device_id)
101 + # NOTE: When using V1 this function is called when overriding the
102 + # engine args. Calling torch.cuda.get_device_name(device_id) here
103 + # will result in the ROCm context being initialized before other
104 + # processes can be created.
105 + return "AMD"
106
107 @classmethod
108 def get_device_total_memory(cls, device_id: int = 0) -> int:
@@ -129,15 +129,30 @@ class RocmPlatform(Platform):
129 scheduler_config = vllm_config.scheduler_config
130 if parallel_config.worker_cls == "auto":
131 if scheduler_config.is_multi_step:
132 - parallel_config.worker_cls = \
133 - "vllm.worker.multi_step_worker.MultiStepWorker"
132 + if envs.VLLM_USE_V1:
133 + raise NotImplementedError(
134 + "Multi-step scheduling is not supported (and not "
135 + "needed) on VLLM V1. Please launch without "
136 + "--num-scheduler-steps.")
137 + else:
138 + parallel_config.worker_cls = \
139 + "vllm.worker.multi_step_worker.MultiStepWorker"
140 elif vllm_config.speculative_config:
135 - parallel_config.worker_cls = \
136 - "vllm.spec_decode.spec_decode_worker.create_spec_worker"
137 - parallel_config.sd_worker_cls = \
138 - "vllm.worker.worker.Worker"
141 + if envs.VLLM_USE_V1:
142 + raise NotImplementedError(
143 + "Speculative decoding is not yet supported on VLLM V1."
144 + )
145 + else:
146 + parallel_config.worker_cls = \
147 + "vllm.spec_decode.spec_decode_worker.create_spec_worker"
148 + parallel_config.sd_worker_cls = \
149 + "vllm.worker.worker.Worker"
150 else:
140 - parallel_config.worker_cls = "vllm.worker.worker.Worker"
151 + if envs.VLLM_USE_V1:
152 + parallel_config.worker_cls = \
153 + "vllm.v1.worker.gpu_worker.Worker"
154 + else:
155 + parallel_config.worker_cls = "vllm.worker.worker.Worker"
156
157 @classmethod
158 def verify_model_arch(cls, model_arch: str) -> None:
vllm/v1/attention/backends/flash_attn.py
+4 -1
@@ -12,8 +12,11 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12 AttentionMetadata, AttentionType)
13 from vllm.attention.backends.utils import get_flash_attn_version
14 from vllm.logger import init_logger
15 +from vllm.platforms import current_platform
16 from vllm.utils import cdiv
16 -from vllm.vllm_flash_attn import flash_attn_varlen_func
17 +
18 +if current_platform.is_cuda():
19 + from vllm.vllm_flash_attn import flash_attn_varlen_func
20
21 logger = init_logger(__name__)
22
vllm/v1/attention/backends/rocm_attn.py new
+182
@@ -0,0 +1,182 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""Attention layer with PagedAttention on rocm"""
3 +from typing import Any, Dict, List, Optional, Tuple, Type
4 +
5 +import torch
6 +
7 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 + AttentionMetadata, AttentionType)
9 +from vllm.attention.ops.paged_attn import PagedAttention
10 +from vllm.attention.ops.prefix_prefill import context_attention_fwd
11 +from vllm.logger import init_logger
12 +from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class ROCmAttentionBackend(AttentionBackend):
18 +
19 + accept_output_buffer: bool = True
20 +
21 + @staticmethod
22 + def get_supported_head_sizes() -> List[int]:
23 + return [32, 64, 96, 128, 160, 192, 224, 256]
24 +
25 + @staticmethod
26 + def get_name() -> str:
27 + return "ROCM_ATTN_VLLM_V1"
28 +
29 + @staticmethod
30 + def get_impl_cls() -> Type["ROCmAttentionImpl"]:
31 + return ROCmAttentionImpl
32 +
33 + @staticmethod
34 + def get_metadata_cls() -> Type["AttentionMetadata"]:
35 + return FlashAttentionMetadata
36 +
37 + @staticmethod
38 + def get_kv_cache_shape(
39 + num_blocks: int,
40 + block_size: int,
41 + num_kv_heads: int,
42 + head_size: int,
43 + ) -> Tuple[int, ...]:
44 + if block_size % 16 != 0:
45 + raise ValueError("Block size must be a multiple of 16.")
46 + return (2, num_blocks, block_size, num_kv_heads, head_size)
47 +
48 + @staticmethod
49 + def use_cascade_attention(*args, **kwargs) -> bool:
50 + return False
51 +
52 +
53 +class ROCmAttentionImpl(AttentionImpl):
54 +
55 + def __init__(
56 + self,
57 + num_heads: int,
58 + head_size: int,
59 + scale: float,
60 + num_kv_heads: int,
61 + alibi_slopes: Optional[List[float]],
62 + sliding_window: Optional[int],
63 + kv_cache_dtype: str,
64 + blocksparse_params: Optional[Dict[str, Any]] = None,
65 + logits_soft_cap: Optional[float] = None,
66 + attn_type: AttentionType = AttentionType.DECODER,
67 + ) -> None:
68 + if blocksparse_params is not None:
69 + raise ValueError(
70 + "ROCmAttention does not support block-sparse attention.")
71 + self.num_heads = num_heads
72 + self.head_size = head_size
73 + self.scale = float(scale)
74 + self.num_kv_heads = num_kv_heads
75 + if alibi_slopes is not None:
76 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
77 + self.alibi_slopes = alibi_slopes
78 + if sliding_window is None:
79 + self.sliding_window = (-1, -1)
80 + else:
81 + self.sliding_window = (sliding_window - 1, 0)
82 + self.kv_cache_dtype = kv_cache_dtype
83 +
84 + assert self.num_heads % self.num_kv_heads == 0
85 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
86 +
87 + support_head_sizes = ROCmAttentionBackend.get_supported_head_sizes()
88 + if head_size not in support_head_sizes:
89 + raise ValueError(
90 + f"Head size {head_size} is not supported by ROCmAttention. "
91 + f"Supported head sizes are: {support_head_sizes}.")
92 +
93 + if attn_type != AttentionType.DECODER:
94 + raise NotImplementedError("Encoder self-attention and "
95 + "encoder/decoder cross-attention "
96 + "are not implemented for "
97 + "ROCmAttentionImpl")
98 +
99 + def forward(
100 + self,
101 + layer: torch.nn.Module,
102 + query: torch.Tensor,
103 + key: torch.Tensor,
104 + value: torch.Tensor,
105 + kv_cache: torch.Tensor,
106 + attn_metadata: FlashAttentionMetadata,
107 + output: Optional[torch.Tensor] = None,
108 + ) -> torch.Tensor:
109 + """Forward pass with FlashAttention.
110 +
111 + Args:
112 + query: shape = [num_tokens, num_heads, head_size]
113 + key: shape = [num_tokens, num_kv_heads, head_size]
114 + value: shape = [num_tokens, num_kv_heads, head_size]
115 + kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
116 + attn_metadata: Metadata for attention.
117 + Returns:
118 + shape = [num_tokens, num_heads * head_size]
119 + """
120 + assert output is not None, "Output tensor must be provided."
121 +
122 + if attn_metadata is None:
123 + # Profiling run.
124 + return output
125 +
126 + assert attn_metadata.use_cascade is False
127 +
128 + # IMPORTANT!
129 + # NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in
130 + # eager-mode PyTorch. Thus, we need to be careful about any CPU overhead
131 + # in this method. For example, `view` and `slice` (or `[:n]`) operations
132 + # are surprisingly slow even in the case they do not invoke any GPU ops.
133 + # Minimize the PyTorch ops in this method as much as possible.
134 + # Whenever making a change in this method, please benchmark the
135 + # performance to make sure it does not introduce any overhead.
136 +
137 + num_actual_tokens = attn_metadata.num_actual_tokens
138 + key_cache, value_cache = PagedAttention.split_kv_cache(
139 + kv_cache, self.num_kv_heads, self.head_size)
140 +
141 + # Reshape the input keys and values and store them in the cache.
142 + PagedAttention.write_to_paged_cache(
143 + key,
144 + value,
145 + key_cache,
146 + value_cache,
147 + attn_metadata.slot_mapping,
148 + self.kv_cache_dtype,
149 + layer._k_scale,
150 + layer._v_scale,
151 + )
152 +
153 + # TODO(sage): Refactor the context_attention_fwd kernel so that this
154 + # overhead can be removed
155 + context_lens = torch.empty_like(attn_metadata.seq_lens)
156 + batch_size = len(attn_metadata.query_start_loc) - 1
157 + assert len(context_lens) == batch_size
158 + for i in range(batch_size):
159 + query_start = attn_metadata.query_start_loc[i]
160 + query_end = attn_metadata.query_start_loc[i + 1]
161 + context_lens[i] = attn_metadata.seq_lens[i] - (query_end -
162 + query_start)
163 +
164 + # Compute attention and update output up to `num_actual_tokens`.
165 + context_attention_fwd(q=query[:num_actual_tokens],
166 + k=key[:num_actual_tokens],
167 + v=value[:num_actual_tokens],
168 + o=output[:num_actual_tokens],
169 + kv_cache_dtype=self.kv_cache_dtype,
170 + k_cache=key_cache,
171 + v_cache=value_cache,
172 + b_loc=attn_metadata.block_table,
173 + b_start_loc=attn_metadata.query_start_loc,
174 + b_seq_len=attn_metadata.seq_lens,
175 + b_ctx_len=context_lens,
176 + max_input_len=attn_metadata.max_query_len,
177 + k_scale=layer._k_scale,
178 + v_scale=layer._v_scale,
179 + alibi_slopes=self.alibi_slopes,
180 + sliding_window=self.sliding_window[0],
181 + sm_scale=self.scale)
182 + return output