A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[spec decode] [4/N] Move update_flash_attn_metadata to attn backend (#7571)
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
William Lin committed
Aug 16, 2024 at 11:41 UTC
f366f6339b1dbd9176470489b99dae4fb454f5ce
3 files changed
+49
-33
vllm/attention/backends/abstract.py
+3
@@ -75,6 +75,9 @@ class AttentionBackend(ABC):
75
) -> None:
76
raise NotImplementedError
77
78
+ def advance_step(self, num_seqs: int, num_queries: int):
79
+ raise NotImplementedError
80
+
81
82
@dataclass
83
class AttentionMetadata:
vllm/attention/backends/flash_attn.py
+45
@@ -297,6 +297,51 @@ class FlashAttentionMetadata(AttentionMetadata):
297
)
298
return self._cached_decode_metadata
299
300
+ def advance_step(self, num_seqs: int, num_queries: int):
301
+ """
302
+ Update metadata in-place to advance one decode step.
303
+ """
304
+ # GPU in-place update is currently called separately through
305
+ # custom_ops.advance_step(). See draft_model_runner. TODO(will): Move
306
+ # this logic to the backend.
307
+
308
+ # When using cudagraph, the num_seqs is padded to the next captured
309
+ # batch sized, but num_queries tracks the actual number of requests in
310
+ # the batch. For --enforce-eager mode, num_seqs == num_queries
311
+ if num_seqs != num_queries:
312
+ assert num_seqs > num_queries
313
+ assert self.use_cuda_graph
314
+
315
+ assert self.num_prefills == 0
316
+ assert self.num_prefill_tokens == 0
317
+ assert self.num_decode_tokens == num_seqs
318
+ assert self.slot_mapping.shape == (num_seqs, )
319
+
320
+ assert self.seq_lens is not None
321
+ assert len(self.seq_lens) == num_seqs
322
+ assert self.seq_lens_tensor is not None
323
+ assert self.seq_lens_tensor.shape == (num_seqs, )
324
+ assert self.max_query_len == 1
325
+ assert self.max_prefill_seq_len == 0
326
+ assert self.max_decode_seq_len == max(self.seq_lens)
327
+
328
+ assert self.query_start_loc is not None
329
+ assert self.query_start_loc.shape == (num_queries + 1, )
330
+ assert self.seq_start_loc is not None
331
+ assert self.seq_start_loc.shape == (num_seqs + 1, )
332
+
333
+ assert self.context_lens_tensor is not None
334
+ assert self.context_lens_tensor.shape == (num_queries, )
335
+
336
+ assert self.block_tables is not None
337
+ assert self.block_tables.shape[0] == num_seqs
338
+
339
+ # Update query lengths. Note that we update only queries and not seqs,
340
+ # since tensors may be padded due to captured cuda graph batch size
341
+ for i in range(num_queries):
342
+ self.seq_lens[i] += 1
343
+ self.max_decode_seq_len = max(self.seq_lens)
344
+
345
346
class FlashAttentionMetadataBuilder(
347
AttentionMetadataBuilder[FlashAttentionMetadata]):
vllm/spec_decode/draft_model_runner.py
+1
-33
@@ -97,38 +97,6 @@ class TP1DraftModelRunner(ModelRunner):
97
self.flashinfer_prefill_workspace_buffer = None
98
self.flashinfer_prefill_wrapper = None
99
100
- def _update_flash_attn_metadata(self, attn_metadata, num_seqs,
101
- num_queries):
102
- assert isinstance(attn_metadata, FlashAttentionMetadata)
103
-
104
- if num_seqs != num_queries:
105
- assert num_seqs > num_queries
106
- assert attn_metadata.use_cuda_graph
107
-
108
- assert attn_metadata.num_prefills == 0
109
- assert attn_metadata.num_prefill_tokens == 0
110
- assert attn_metadata.num_decode_tokens == num_seqs
111
- assert attn_metadata.slot_mapping.shape == (num_seqs, )
112
-
113
- assert len(attn_metadata.seq_lens) == num_seqs
114
- assert attn_metadata.seq_lens_tensor.shape == (num_seqs, )
115
- assert attn_metadata.max_query_len == 1
116
- assert attn_metadata.max_prefill_seq_len == 0
117
- assert attn_metadata.max_decode_seq_len == max(attn_metadata.seq_lens)
118
-
119
- assert attn_metadata.query_start_loc.shape == (num_queries + 1, )
120
- assert attn_metadata.seq_start_loc.shape == (num_seqs + 1, )
121
-
122
- assert attn_metadata.context_lens_tensor.shape == (num_queries, )
123
-
124
- assert attn_metadata.block_tables.shape[0] == num_seqs
125
-
126
- # Update query lengths. Note that we update only queries and not seqs,
127
- # since tensors may be padded due to captured cuda graph batch size
128
- for i in range(num_queries):
129
- attn_metadata.seq_lens[i] += 1
130
- attn_metadata.max_decode_seq_len = max(attn_metadata.seq_lens)
131
-
100
def _update_sampling_metadata(self, sampling_metadata, num_seqs,
101
num_queries):
102
@@ -166,7 +134,7 @@ class TP1DraftModelRunner(ModelRunner):
134
# Update attn_metadata
135
attn_metadata = model_input.attn_metadata
136
assert isinstance(attn_metadata, FlashAttentionMetadata)
169
- self._update_flash_attn_metadata(attn_metadata, num_seqs, num_queries)
137
+ attn_metadata.advance_step(num_seqs, num_queries)
138
139
# Update GPU tensors
140
ops.advance_step(num_seqs=num_seqs,