A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix decode tokens w. CUDA graph (#6757)
Cody Yu committed
Jul 24, 2024 at 22:33 UTC
309aaef8255fb832bf674c6ed7d9d84211629421
4 files changed
+31
-4
tests/worker/test_model_runner.py
+1
@@ -193,6 +193,7 @@ def test_prepare_decode_cuda_graph(batch_size):
193
for _ in range(expected_bs - len(seq_lens)):
194
seq_lens.append(1)
195
assert attn_metadata.seq_lens == seq_lens
196
+ assert attn_metadata.num_decode_tokens == len(seq_lens)
197
start_idx = 0
198
start_loc = [start_idx]
199
for _ in context_lens:
vllm/attention/backends/flash_attn.py
+10
-2
@@ -272,7 +272,15 @@ class FlashAttentionMetadataBuilder(
272
273
def build(self, seq_lens: List[int], query_lens: List[int],
274
cuda_graph_pad_size: int, batch_size: int):
275
- """Build attention metadata with on-device tensors."""
275
+ """Build attention metadata with on-device tensors.
276
+
277
+ Args:
278
+ seq_lens: The maybe padded sequence lengths of the input sequences.
279
+ query_lens: The query lengths of the input sequences.
280
+ cuda_graph_pad_size: The padding size for cuda graph.
281
+ -1 if cuda graph is not used.
282
+ batch_size: The maybe padded batch size.
283
+ """
284
for inter_data in self.input_builder.inter_data_list:
285
self._add_seq_group(inter_data,
286
self.input_builder.chunked_prefill_enabled)
@@ -297,7 +305,7 @@ class FlashAttentionMetadataBuilder(
305
if use_captured_graph:
306
self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
307
self.block_tables.extend([] * cuda_graph_pad_size)
300
- num_decode_tokens = batch_size + cuda_graph_pad_size
308
+ num_decode_tokens = batch_size
309
310
# The shape of graph_block_tables is
311
# [max batch size, max context len // block size].
vllm/attention/backends/flashinfer.py
+10
-1
@@ -320,6 +320,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
320
321
def build(self, seq_lens: List[int], query_lens: List[int],
322
cuda_graph_pad_size: int, batch_size: int):
323
+ """Build attention metadata with on-device tensors.
324
+
325
+ Args:
326
+ seq_lens: The maybe padded sequence lengths of the input sequences.
327
+ query_lens: The query lengths of the input sequences.
328
+ cuda_graph_pad_size: The padding size for cuda graph.
329
+ -1 if cuda graph is not used.
330
+ batch_size: The maybe padded batch size.
331
+ """
332
for inter_data in self.input_builder.inter_data_list:
333
self._add_seq_group(inter_data,
334
self.input_builder.chunked_prefill_enabled)
@@ -334,7 +343,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
343
if use_captured_graph:
344
self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
345
self.block_tables.extend([] * cuda_graph_pad_size)
337
- num_decode_tokens = batch_size + cuda_graph_pad_size
346
+ num_decode_tokens = batch_size
347
348
# The shape of graph_block_tables is
349
# [max batch size, max context len // block size].
vllm/attention/backends/utils.py
+10
-1
@@ -149,6 +149,15 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
149
150
def build(self, seq_lens: List[int], query_lens: List[int],
151
cuda_graph_pad_size: int, batch_size: int):
152
+ """Build attention metadata with on-device tensors.
153
+
154
+ Args:
155
+ seq_lens: The maybe padded sequence lengths of the input sequences.
156
+ query_lens: The query lengths of the input sequences.
157
+ cuda_graph_pad_size: The padding size for cuda graph.
158
+ -1 if cuda graph is not used.
159
+ batch_size: The maybe padded batch size.
160
+ """
161
for inter_data in self.input_builder.inter_data_list:
162
self._add_seq_group(inter_data,
163
self.input_builder.chunked_prefill_enabled)
@@ -173,7 +182,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
182
if use_captured_graph:
183
self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
184
self.block_tables.extend([] * cuda_graph_pad_size)
176
- num_decode_tokens = batch_size + cuda_graph_pad_size
185
+ num_decode_tokens = batch_size
186
187
# The shape of graph_block_tables is
188
# [max batch size, max context len // block size].