@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 309aaef825
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix decode tokens w. CUDA graph (#6757)

Cody Yu committed Jul 24, 2024 at 22:33 UTC 309aaef8255fb832bf674c6ed7d9d84211629421
4 files changed +31 -4
tests/worker/test_model_runner.py
+1
@@ -193,6 +193,7 @@ def test_prepare_decode_cuda_graph(batch_size):
193 for _ in range(expected_bs - len(seq_lens)):
194 seq_lens.append(1)
195 assert attn_metadata.seq_lens == seq_lens
196 + assert attn_metadata.num_decode_tokens == len(seq_lens)
197 start_idx = 0
198 start_loc = [start_idx]
199 for _ in context_lens:
vllm/attention/backends/flash_attn.py
+10 -2
@@ -272,7 +272,15 @@ class FlashAttentionMetadataBuilder(
272
273 def build(self, seq_lens: List[int], query_lens: List[int],
274 cuda_graph_pad_size: int, batch_size: int):
275 - """Build attention metadata with on-device tensors."""
275 + """Build attention metadata with on-device tensors.
276 +
277 + Args:
278 + seq_lens: The maybe padded sequence lengths of the input sequences.
279 + query_lens: The query lengths of the input sequences.
280 + cuda_graph_pad_size: The padding size for cuda graph.
281 + -1 if cuda graph is not used.
282 + batch_size: The maybe padded batch size.
283 + """
284 for inter_data in self.input_builder.inter_data_list:
285 self._add_seq_group(inter_data,
286 self.input_builder.chunked_prefill_enabled)
@@ -297,7 +305,7 @@ class FlashAttentionMetadataBuilder(
305 if use_captured_graph:
306 self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
307 self.block_tables.extend([] * cuda_graph_pad_size)
300 - num_decode_tokens = batch_size + cuda_graph_pad_size
308 + num_decode_tokens = batch_size
309
310 # The shape of graph_block_tables is
311 # [max batch size, max context len // block size].
vllm/attention/backends/flashinfer.py
+10 -1
@@ -320,6 +320,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
320
321 def build(self, seq_lens: List[int], query_lens: List[int],
322 cuda_graph_pad_size: int, batch_size: int):
323 + """Build attention metadata with on-device tensors.
324 +
325 + Args:
326 + seq_lens: The maybe padded sequence lengths of the input sequences.
327 + query_lens: The query lengths of the input sequences.
328 + cuda_graph_pad_size: The padding size for cuda graph.
329 + -1 if cuda graph is not used.
330 + batch_size: The maybe padded batch size.
331 + """
332 for inter_data in self.input_builder.inter_data_list:
333 self._add_seq_group(inter_data,
334 self.input_builder.chunked_prefill_enabled)
@@ -334,7 +343,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
343 if use_captured_graph:
344 self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
345 self.block_tables.extend([] * cuda_graph_pad_size)
337 - num_decode_tokens = batch_size + cuda_graph_pad_size
346 + num_decode_tokens = batch_size
347
348 # The shape of graph_block_tables is
349 # [max batch size, max context len // block size].
vllm/attention/backends/utils.py
+10 -1
@@ -149,6 +149,15 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
149
150 def build(self, seq_lens: List[int], query_lens: List[int],
151 cuda_graph_pad_size: int, batch_size: int):
152 + """Build attention metadata with on-device tensors.
153 +
154 + Args:
155 + seq_lens: The maybe padded sequence lengths of the input sequences.
156 + query_lens: The query lengths of the input sequences.
157 + cuda_graph_pad_size: The padding size for cuda graph.
158 + -1 if cuda graph is not used.
159 + batch_size: The maybe padded batch size.
160 + """
161 for inter_data in self.input_builder.inter_data_list:
162 self._add_seq_group(inter_data,
163 self.input_builder.chunked_prefill_enabled)
@@ -173,7 +182,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
182 if use_captured_graph:
183 self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
184 self.block_tables.extend([] * cuda_graph_pad_size)
176 - num_decode_tokens = batch_size + cuda_graph_pad_size
185 + num_decode_tokens = batch_size
186
187 # The shape of graph_block_tables is
188 # [max batch size, max context len // block size].