A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[bugfix] [AMD] add multi-step advance_step to ROCmFlashAttentionMetadata (#8474)
William Lin committed
Sep 19, 2024 at 20:49 UTC
9e5ec35b1f8239453b1aaab28e7a02307db4ab1f
2 files changed
+58
-2
vllm/attention/backends/rocm_flash_attn.py
+57
-1
@@ -1,6 +1,6 @@
1
"""Attention layer ROCm GPUs."""
2
from dataclasses import dataclass
3
-from typing import Any, Dict, List, Optional, Tuple, Type
3
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
4
5
import torch
6
@@ -15,6 +15,9 @@ from vllm.attention.ops.paged_attn import (PagedAttention,
15
from vllm.logger import init_logger
16
from vllm.platforms import current_platform
17
18
+if TYPE_CHECKING:
19
+ from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
20
+
21
logger = init_logger(__name__)
22
23
_PARTITION_SIZE_ROCM = 512
@@ -180,6 +183,59 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
183
)
184
return self._cached_decode_metadata
185
186
+ def advance_step(self, model_input: "ModelInputForGPUWithSamplingMetadata",
187
+ sampled_token_ids: Optional[torch.Tensor],
188
+ block_size: int, num_seqs: int, num_queries: int):
189
+ """
190
+ Update metadata in-place to advance one decode step.
191
+ """
192
+ # When using cudagraph, the num_seqs is padded to the next captured
193
+ # batch sized, but num_queries tracks the actual number of requests in
194
+ # the batch. For --enforce-eager mode, num_seqs == num_queries
195
+ if num_seqs != num_queries:
196
+ assert num_seqs > num_queries
197
+ assert self.use_cuda_graph
198
+
199
+ assert self.num_prefills == 0
200
+ assert self.num_prefill_tokens == 0
201
+ assert self.num_decode_tokens == num_seqs
202
+ assert self.slot_mapping.shape == (num_seqs, )
203
+
204
+ assert self.seq_lens is not None
205
+ assert len(self.seq_lens) == num_seqs
206
+ assert self.seq_lens_tensor is not None
207
+ assert self.seq_lens_tensor.shape == (num_seqs, )
208
+ assert self.max_query_len == 1
209
+ assert self.max_prefill_seq_len == 0
210
+ assert self.max_decode_seq_len == max(self.seq_lens)
211
+
212
+ assert self.query_start_loc is not None
213
+ assert self.query_start_loc.shape == (num_queries + 1, )
214
+ assert self.seq_start_loc is not None
215
+ assert self.seq_start_loc.shape == (num_seqs + 1, )
216
+
217
+ assert self.context_lens_tensor is not None
218
+ assert self.context_lens_tensor.shape == (num_queries, )
219
+
220
+ assert self.block_tables is not None
221
+ assert self.block_tables.shape[0] == num_seqs
222
+
223
+ # Update query lengths. Note that we update only queries and not seqs,
224
+ # since tensors may be padded due to captured cuda graph batch size
225
+ for i in range(num_queries):
226
+ self.seq_lens[i] += 1
227
+ self.max_decode_seq_len = max(self.seq_lens)
228
+
229
+ ops.advance_step_flashattn(num_seqs=num_seqs,
230
+ num_queries=num_queries,
231
+ block_size=block_size,
232
+ input_tokens=model_input.input_tokens,
233
+ sampled_token_ids=sampled_token_ids,
234
+ input_positions=model_input.input_positions,
235
+ seq_lens=self.seq_lens_tensor,
236
+ slot_mapping=self.slot_mapping,
237
+ block_tables=self.block_tables)
238
+
239
240
class ROCmFlashAttentionMetadataBuilder(
241
CommonMetadataBuilder[ROCmFlashAttentionMetadata]):
vllm/worker/multi_step_model_runner.py
+1
-1
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
29
30
logger = init_logger(__name__)
31
32
-MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "flashinfer"]
32
+MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "rocm-flash-attn", "flashinfer"]
33
34
35
def seq_output_builder():