@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f366f6339b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[spec decode] [4/N] Move update_flash_attn_metadata to attn backend (#7571)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

William Lin committed Aug 16, 2024 at 11:41 UTC f366f6339b1dbd9176470489b99dae4fb454f5ce
3 files changed +49 -33
vllm/attention/backends/abstract.py
+3
@@ -75,6 +75,9 @@ class AttentionBackend(ABC):
75 ) -> None:
76 raise NotImplementedError
77
78 + def advance_step(self, num_seqs: int, num_queries: int):
79 + raise NotImplementedError
80 +
81
82 @dataclass
83 class AttentionMetadata:
vllm/attention/backends/flash_attn.py
+45
@@ -297,6 +297,51 @@ class FlashAttentionMetadata(AttentionMetadata):
297 )
298 return self._cached_decode_metadata
299
300 + def advance_step(self, num_seqs: int, num_queries: int):
301 + """
302 + Update metadata in-place to advance one decode step.
303 + """
304 + # GPU in-place update is currently called separately through
305 + # custom_ops.advance_step(). See draft_model_runner. TODO(will): Move
306 + # this logic to the backend.
307 +
308 + # When using cudagraph, the num_seqs is padded to the next captured
309 + # batch sized, but num_queries tracks the actual number of requests in
310 + # the batch. For --enforce-eager mode, num_seqs == num_queries
311 + if num_seqs != num_queries:
312 + assert num_seqs > num_queries
313 + assert self.use_cuda_graph
314 +
315 + assert self.num_prefills == 0
316 + assert self.num_prefill_tokens == 0
317 + assert self.num_decode_tokens == num_seqs
318 + assert self.slot_mapping.shape == (num_seqs, )
319 +
320 + assert self.seq_lens is not None
321 + assert len(self.seq_lens) == num_seqs
322 + assert self.seq_lens_tensor is not None
323 + assert self.seq_lens_tensor.shape == (num_seqs, )
324 + assert self.max_query_len == 1
325 + assert self.max_prefill_seq_len == 0
326 + assert self.max_decode_seq_len == max(self.seq_lens)
327 +
328 + assert self.query_start_loc is not None
329 + assert self.query_start_loc.shape == (num_queries + 1, )
330 + assert self.seq_start_loc is not None
331 + assert self.seq_start_loc.shape == (num_seqs + 1, )
332 +
333 + assert self.context_lens_tensor is not None
334 + assert self.context_lens_tensor.shape == (num_queries, )
335 +
336 + assert self.block_tables is not None
337 + assert self.block_tables.shape[0] == num_seqs
338 +
339 + # Update query lengths. Note that we update only queries and not seqs,
340 + # since tensors may be padded due to captured cuda graph batch size
341 + for i in range(num_queries):
342 + self.seq_lens[i] += 1
343 + self.max_decode_seq_len = max(self.seq_lens)
344 +
345
346 class FlashAttentionMetadataBuilder(
347 AttentionMetadataBuilder[FlashAttentionMetadata]):
vllm/spec_decode/draft_model_runner.py
+1 -33
@@ -97,38 +97,6 @@ class TP1DraftModelRunner(ModelRunner):
97 self.flashinfer_prefill_workspace_buffer = None
98 self.flashinfer_prefill_wrapper = None
99
100 - def _update_flash_attn_metadata(self, attn_metadata, num_seqs,
101 - num_queries):
102 - assert isinstance(attn_metadata, FlashAttentionMetadata)
103 -
104 - if num_seqs != num_queries:
105 - assert num_seqs > num_queries
106 - assert attn_metadata.use_cuda_graph
107 -
108 - assert attn_metadata.num_prefills == 0
109 - assert attn_metadata.num_prefill_tokens == 0
110 - assert attn_metadata.num_decode_tokens == num_seqs
111 - assert attn_metadata.slot_mapping.shape == (num_seqs, )
112 -
113 - assert len(attn_metadata.seq_lens) == num_seqs
114 - assert attn_metadata.seq_lens_tensor.shape == (num_seqs, )
115 - assert attn_metadata.max_query_len == 1
116 - assert attn_metadata.max_prefill_seq_len == 0
117 - assert attn_metadata.max_decode_seq_len == max(attn_metadata.seq_lens)
118 -
119 - assert attn_metadata.query_start_loc.shape == (num_queries + 1, )
120 - assert attn_metadata.seq_start_loc.shape == (num_seqs + 1, )
121 -
122 - assert attn_metadata.context_lens_tensor.shape == (num_queries, )
123 -
124 - assert attn_metadata.block_tables.shape[0] == num_seqs
125 -
126 - # Update query lengths. Note that we update only queries and not seqs,
127 - # since tensors may be padded due to captured cuda graph batch size
128 - for i in range(num_queries):
129 - attn_metadata.seq_lens[i] += 1
130 - attn_metadata.max_decode_seq_len = max(attn_metadata.seq_lens)
131 -
100 def _update_sampling_metadata(self, sampling_metadata, num_seqs,
101 num_queries):
102
@@ -166,7 +134,7 @@ class TP1DraftModelRunner(ModelRunner):
134 # Update attn_metadata
135 attn_metadata = model_input.attn_metadata
136 assert isinstance(attn_metadata, FlashAttentionMetadata)
169 - self._update_flash_attn_metadata(attn_metadata, num_seqs, num_queries)
137 + attn_metadata.advance_step(num_seqs, num_queries)
138
139 # Update GPU tensors
140 ops.advance_step(num_seqs=num_seqs,