A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[ROCm][V1] Add intial ROCm support to V1 (#12790)
Sage Moore committed
Feb 13, 2025 at 22:21 UTC
ba59b78a9c5a8e9a535a0be081a438951679347f
5 files changed
+236
-18
requirements-rocm-build.txt
new
+16
@@ -0,0 +1,16 @@
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+--extra-index-url https://download.pytorch.org/whl/rocm6.2
5
+torch==2.5.1
6
+torchvision==0.20.1
7
+torchaudio==2.5.1
8
+
9
+cmake>=3.26
10
+ninja
11
+packaging
12
+setuptools>=61
13
+setuptools-scm>=8
14
+wheel
15
+jinja2
16
+amdsmi==6.2.4
vllm/attention/ops/prefix_prefill.py
+4
-2
@@ -718,7 +718,8 @@ if triton.__version__ >= "2.1.0":
718
k_scale: torch.Tensor,
719
v_scale: torch.Tensor,
720
alibi_slopes=None,
721
- sliding_window=None):
721
+ sliding_window=None,
722
+ sm_scale=None):
723
724
q_dtype_is_f32 = q.dtype is torch.float32
725
# need to reduce num. blocks when using fp32
@@ -759,7 +760,8 @@ if triton.__version__ >= "2.1.0":
760
# round up Lk to a power of 2 - this is required for Triton block size
761
Lk_padded = triton.next_power_of_2(Lk)
762
762
- sm_scale = 1.0 / (Lq**0.5)
763
+ if sm_scale is None:
764
+ sm_scale = 1.0 / (Lq**0.5)
765
batch, head = b_seq_len.shape[0], q.shape[1]
766
num_queries_per_kv = q.shape[1] // k.shape[1]
767
vllm/platforms/rocm.py
+30
-15
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import os
3
from functools import lru_cache
4
from typing import TYPE_CHECKING, Dict, List, Optional
5
@@ -29,12 +28,6 @@ try:
28
except ImportError as e:
29
logger.warning("Failed to import from vllm._rocm_C with %r", e)
30
32
-if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
33
- logger.warning("`fork` method is not supported by ROCm. "
34
- "VLLM_WORKER_MULTIPROC_METHOD is overridden to"
35
- " `spawn` instead.")
36
- os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
37
-
31
# Models not supported by ROCm.
32
_ROCM_UNSUPPORTED_MODELS: List[str] = []
33
@@ -84,6 +77,9 @@ class RocmPlatform(Platform):
77
return "vllm.attention.backends.triton_mla.TritonMLABackend"
78
selected_backend = (_Backend.ROCM_FLASH if selected_backend
79
== _Backend.FLASH_ATTN else selected_backend)
80
+ if envs.VLLM_USE_V1:
81
+ logger.info("Using ROCm Attention backend on V1 engine.")
82
+ return "vllm.v1.attention.backends.rocm_attn.ROCmAttentionBackend"
83
if selected_backend == _Backend.ROCM_FLASH:
84
if not cls.has_device_capability(90):
85
# not Instinct series GPUs.
@@ -102,7 +98,11 @@ class RocmPlatform(Platform):
98
@classmethod
99
@lru_cache(maxsize=8)
100
def get_device_name(cls, device_id: int = 0) -> str:
105
- return torch.cuda.get_device_name(device_id)
101
+ # NOTE: When using V1 this function is called when overriding the
102
+ # engine args. Calling torch.cuda.get_device_name(device_id) here
103
+ # will result in the ROCm context being initialized before other
104
+ # processes can be created.
105
+ return "AMD"
106
107
@classmethod
108
def get_device_total_memory(cls, device_id: int = 0) -> int:
@@ -129,15 +129,30 @@ class RocmPlatform(Platform):
129
scheduler_config = vllm_config.scheduler_config
130
if parallel_config.worker_cls == "auto":
131
if scheduler_config.is_multi_step:
132
- parallel_config.worker_cls = \
133
- "vllm.worker.multi_step_worker.MultiStepWorker"
132
+ if envs.VLLM_USE_V1:
133
+ raise NotImplementedError(
134
+ "Multi-step scheduling is not supported (and not "
135
+ "needed) on VLLM V1. Please launch without "
136
+ "--num-scheduler-steps.")
137
+ else:
138
+ parallel_config.worker_cls = \
139
+ "vllm.worker.multi_step_worker.MultiStepWorker"
140
elif vllm_config.speculative_config:
135
- parallel_config.worker_cls = \
136
- "vllm.spec_decode.spec_decode_worker.create_spec_worker"
137
- parallel_config.sd_worker_cls = \
138
- "vllm.worker.worker.Worker"
141
+ if envs.VLLM_USE_V1:
142
+ raise NotImplementedError(
143
+ "Speculative decoding is not yet supported on VLLM V1."
144
+ )
145
+ else:
146
+ parallel_config.worker_cls = \
147
+ "vllm.spec_decode.spec_decode_worker.create_spec_worker"
148
+ parallel_config.sd_worker_cls = \
149
+ "vllm.worker.worker.Worker"
150
else:
140
- parallel_config.worker_cls = "vllm.worker.worker.Worker"
151
+ if envs.VLLM_USE_V1:
152
+ parallel_config.worker_cls = \
153
+ "vllm.v1.worker.gpu_worker.Worker"
154
+ else:
155
+ parallel_config.worker_cls = "vllm.worker.worker.Worker"
156
157
@classmethod
158
def verify_model_arch(cls, model_arch: str) -> None:
vllm/v1/attention/backends/flash_attn.py
+4
-1
@@ -12,8 +12,11 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
AttentionMetadata, AttentionType)
13
from vllm.attention.backends.utils import get_flash_attn_version
14
from vllm.logger import init_logger
15
+from vllm.platforms import current_platform
16
from vllm.utils import cdiv
16
-from vllm.vllm_flash_attn import flash_attn_varlen_func
17
+
18
+if current_platform.is_cuda():
19
+ from vllm.vllm_flash_attn import flash_attn_varlen_func
20
21
logger = init_logger(__name__)
22
vllm/v1/attention/backends/rocm_attn.py
new
+182
@@ -0,0 +1,182 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""Attention layer with PagedAttention on rocm"""
3
+from typing import Any, Dict, List, Optional, Tuple, Type
4
+
5
+import torch
6
+
7
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8
+ AttentionMetadata, AttentionType)
9
+from vllm.attention.ops.paged_attn import PagedAttention
10
+from vllm.attention.ops.prefix_prefill import context_attention_fwd
11
+from vllm.logger import init_logger
12
+from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class ROCmAttentionBackend(AttentionBackend):
18
+
19
+ accept_output_buffer: bool = True
20
+
21
+ @staticmethod
22
+ def get_supported_head_sizes() -> List[int]:
23
+ return [32, 64, 96, 128, 160, 192, 224, 256]
24
+
25
+ @staticmethod
26
+ def get_name() -> str:
27
+ return "ROCM_ATTN_VLLM_V1"
28
+
29
+ @staticmethod
30
+ def get_impl_cls() -> Type["ROCmAttentionImpl"]:
31
+ return ROCmAttentionImpl
32
+
33
+ @staticmethod
34
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
35
+ return FlashAttentionMetadata
36
+
37
+ @staticmethod
38
+ def get_kv_cache_shape(
39
+ num_blocks: int,
40
+ block_size: int,
41
+ num_kv_heads: int,
42
+ head_size: int,
43
+ ) -> Tuple[int, ...]:
44
+ if block_size % 16 != 0:
45
+ raise ValueError("Block size must be a multiple of 16.")
46
+ return (2, num_blocks, block_size, num_kv_heads, head_size)
47
+
48
+ @staticmethod
49
+ def use_cascade_attention(*args, **kwargs) -> bool:
50
+ return False
51
+
52
+
53
+class ROCmAttentionImpl(AttentionImpl):
54
+
55
+ def __init__(
56
+ self,
57
+ num_heads: int,
58
+ head_size: int,
59
+ scale: float,
60
+ num_kv_heads: int,
61
+ alibi_slopes: Optional[List[float]],
62
+ sliding_window: Optional[int],
63
+ kv_cache_dtype: str,
64
+ blocksparse_params: Optional[Dict[str, Any]] = None,
65
+ logits_soft_cap: Optional[float] = None,
66
+ attn_type: AttentionType = AttentionType.DECODER,
67
+ ) -> None:
68
+ if blocksparse_params is not None:
69
+ raise ValueError(
70
+ "ROCmAttention does not support block-sparse attention.")
71
+ self.num_heads = num_heads
72
+ self.head_size = head_size
73
+ self.scale = float(scale)
74
+ self.num_kv_heads = num_kv_heads
75
+ if alibi_slopes is not None:
76
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
77
+ self.alibi_slopes = alibi_slopes
78
+ if sliding_window is None:
79
+ self.sliding_window = (-1, -1)
80
+ else:
81
+ self.sliding_window = (sliding_window - 1, 0)
82
+ self.kv_cache_dtype = kv_cache_dtype
83
+
84
+ assert self.num_heads % self.num_kv_heads == 0
85
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
86
+
87
+ support_head_sizes = ROCmAttentionBackend.get_supported_head_sizes()
88
+ if head_size not in support_head_sizes:
89
+ raise ValueError(
90
+ f"Head size {head_size} is not supported by ROCmAttention. "
91
+ f"Supported head sizes are: {support_head_sizes}.")
92
+
93
+ if attn_type != AttentionType.DECODER:
94
+ raise NotImplementedError("Encoder self-attention and "
95
+ "encoder/decoder cross-attention "
96
+ "are not implemented for "
97
+ "ROCmAttentionImpl")
98
+
99
+ def forward(
100
+ self,
101
+ layer: torch.nn.Module,
102
+ query: torch.Tensor,
103
+ key: torch.Tensor,
104
+ value: torch.Tensor,
105
+ kv_cache: torch.Tensor,
106
+ attn_metadata: FlashAttentionMetadata,
107
+ output: Optional[torch.Tensor] = None,
108
+ ) -> torch.Tensor:
109
+ """Forward pass with FlashAttention.
110
+
111
+ Args:
112
+ query: shape = [num_tokens, num_heads, head_size]
113
+ key: shape = [num_tokens, num_kv_heads, head_size]
114
+ value: shape = [num_tokens, num_kv_heads, head_size]
115
+ kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
116
+ attn_metadata: Metadata for attention.
117
+ Returns:
118
+ shape = [num_tokens, num_heads * head_size]
119
+ """
120
+ assert output is not None, "Output tensor must be provided."
121
+
122
+ if attn_metadata is None:
123
+ # Profiling run.
124
+ return output
125
+
126
+ assert attn_metadata.use_cascade is False
127
+
128
+ # IMPORTANT!
129
+ # NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in
130
+ # eager-mode PyTorch. Thus, we need to be careful about any CPU overhead
131
+ # in this method. For example, `view` and `slice` (or `[:n]`) operations
132
+ # are surprisingly slow even in the case they do not invoke any GPU ops.
133
+ # Minimize the PyTorch ops in this method as much as possible.
134
+ # Whenever making a change in this method, please benchmark the
135
+ # performance to make sure it does not introduce any overhead.
136
+
137
+ num_actual_tokens = attn_metadata.num_actual_tokens
138
+ key_cache, value_cache = PagedAttention.split_kv_cache(
139
+ kv_cache, self.num_kv_heads, self.head_size)
140
+
141
+ # Reshape the input keys and values and store them in the cache.
142
+ PagedAttention.write_to_paged_cache(
143
+ key,
144
+ value,
145
+ key_cache,
146
+ value_cache,
147
+ attn_metadata.slot_mapping,
148
+ self.kv_cache_dtype,
149
+ layer._k_scale,
150
+ layer._v_scale,
151
+ )
152
+
153
+ # TODO(sage): Refactor the context_attention_fwd kernel so that this
154
+ # overhead can be removed
155
+ context_lens = torch.empty_like(attn_metadata.seq_lens)
156
+ batch_size = len(attn_metadata.query_start_loc) - 1
157
+ assert len(context_lens) == batch_size
158
+ for i in range(batch_size):
159
+ query_start = attn_metadata.query_start_loc[i]
160
+ query_end = attn_metadata.query_start_loc[i + 1]
161
+ context_lens[i] = attn_metadata.seq_lens[i] - (query_end -
162
+ query_start)
163
+
164
+ # Compute attention and update output up to `num_actual_tokens`.
165
+ context_attention_fwd(q=query[:num_actual_tokens],
166
+ k=key[:num_actual_tokens],
167
+ v=value[:num_actual_tokens],
168
+ o=output[:num_actual_tokens],
169
+ kv_cache_dtype=self.kv_cache_dtype,
170
+ k_cache=key_cache,
171
+ v_cache=value_cache,
172
+ b_loc=attn_metadata.block_table,
173
+ b_start_loc=attn_metadata.query_start_loc,
174
+ b_seq_len=attn_metadata.seq_lens,
175
+ b_ctx_len=context_lens,
176
+ max_input_len=attn_metadata.max_query_len,
177
+ k_scale=layer._k_scale,
178
+ v_scale=layer._v_scale,
179
+ alibi_slopes=self.alibi_slopes,
180
+ sliding_window=self.sliding_window[0],
181
+ sm_scale=self.scale)
182
+ return output