A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] CUDA Graphs for Multi-Step + Chunked-Prefill (#8645)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Oct 2, 2024 at 15:44 UTC
afb050b29d0cac27c32c19c8206a9ac2a4662de2
3 files changed
+97
-34
csrc/prepare_inputs/advance_step.cu
+11
@@ -17,6 +17,17 @@ __global__ void advance_step_flashattn_kernel(
17
long const* sampled_token_ids_ptr, long* input_positions_ptr,
18
int* seq_lens_ptr, long* slot_mapping_ptr, int const* block_tables_ptr,
19
int64_t const block_tables_stride) {
20
+ int const n_pad = num_seqs - num_queries;
21
+ if (n_pad && blockIdx.x == 0) {
22
+ // Handle cuda graph padding
23
+ int const offset = num_queries;
24
+ for (int i = threadIdx.x; i < n_pad; i += blockDim.x) {
25
+ input_tokens_ptr[offset + i] = 0;
26
+ input_positions_ptr[offset + i] = 0;
27
+ slot_mapping_ptr[offset + i] = -1;
28
+ }
29
+ }
30
+
31
int num_query_blocks = div_ceil(num_queries, num_threads);
32
33
if (blockIdx.x >= num_query_blocks) {
vllm/attention/backends/flash_attn.py
+28
-20
@@ -500,6 +500,30 @@ class FlashAttentionMetadataBuilder(
500
seq_len, context_len, start_idx,
501
self.block_size, inter_data.block_tables)
502
503
+ def _get_graph_runner_block_tables(
504
+ self, num_seqs: int,
505
+ block_tables: List[List[int]]) -> torch.Tensor:
506
+ # The shape of graph_block_tables is
507
+ # [max batch size, max context len // block size].
508
+ max_batch_size, max_blocks = self.runner.graph_block_tables.shape
509
+ assert max_batch_size >= num_seqs
510
+
511
+ graph_block_tables = self.runner.graph_block_tables[:num_seqs]
512
+ for i, block_table in enumerate(block_tables):
513
+ if block_table:
514
+ num_blocks = len(block_table)
515
+ if num_blocks <= max_blocks:
516
+ graph_block_tables[i, :num_blocks] = block_table
517
+ else:
518
+ # It may be possible to have more blocks allocated due
519
+ # to lookahead slots of multi-step, however, they are
520
+ # not used anyway, so can be safely ignored.
521
+ graph_block_tables[
522
+ i, :max_blocks] = block_table[:max_blocks]
523
+
524
+ return torch.from_numpy(graph_block_tables).to(
525
+ device=self.runner.device, non_blocking=True)
526
+
527
def build(self, seq_lens: List[int], query_lens: List[int],
528
cuda_graph_pad_size: int, batch_size: int):
529
"""Build attention metadata with on-device tensors.
@@ -533,29 +557,13 @@ class FlashAttentionMetadataBuilder(
557
max_decode_seq_len = max(self.curr_seq_lens, default=0)
558
num_decode_tokens = self.num_decode_tokens
559
560
+ num_seqs = len(seq_lens)
561
if use_captured_graph:
562
self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
563
self.block_tables.extend([] * cuda_graph_pad_size)
539
- num_decode_tokens = batch_size
540
-
541
- # The shape of graph_block_tables is
542
- # [max batch size, max context len // block size].
543
- input_block_tables = self.runner.graph_block_tables[:batch_size]
544
- max_blocks = input_block_tables.shape[1]
545
- for i, block_table in enumerate(self.block_tables):
546
- if block_table:
547
- num_blocks = len(block_table)
548
- if num_blocks <= max_blocks:
549
- input_block_tables[i, :num_blocks] = block_table
550
- else:
551
- # It may be possible to have more blocks allocated due
552
- # to lookahead slots of multi-step, however, they are
553
- # not used anyway, so can be safely ignored.
554
- input_block_tables[
555
- i, :max_blocks] = block_table[:max_blocks]
556
-
557
- block_tables = torch.from_numpy(input_block_tables).to(
558
- device=device, non_blocking=True)
564
+ num_decode_tokens = batch_size - self.num_prefill_tokens
565
+ block_tables = self._get_graph_runner_block_tables(
566
+ num_seqs, self.block_tables)
567
else:
568
block_tables = make_tensor_with_pad(
569
self.block_tables,
vllm/worker/model_runner.py
+58
-14
@@ -712,14 +712,62 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
712
713
def _use_captured_graph(self,
714
batch_size: int,
715
+ decode_only: bool,
716
max_decode_seq_len: int,
717
max_encoder_seq_len: int = 0) -> bool:
717
- return (self.decode_only and not self.runner.model_config.enforce_eager
718
+ return (decode_only and not self.runner.model_config.enforce_eager
719
and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
720
and max_decode_seq_len <= self.runner.max_seq_len_to_capture
721
and max_encoder_seq_len <= self.runner.max_seq_len_to_capture
722
and batch_size <= self.runner.max_batchsize_to_capture)
723
724
+ def _get_cuda_graph_pad_size(self,
725
+ num_seqs: int,
726
+ max_decode_seq_len: int,
727
+ max_encoder_seq_len: int = 0) -> int:
728
+ """
729
+ Determine the number of padding sequences required for running in
730
+ CUDA graph mode. Returns -1 if CUDA graphs cannot be used.
731
+
732
+ In the multi-step + chunked-prefill case, only the first step
733
+ has Prefills (if any). The rest of the steps are guaranteed to be all
734
+ decodes. In this case, we set up the padding as if all the sequences
735
+ are decodes so we may run all steps except the first step in CUDA graph
736
+ mode. The padding is accounted for in the multi-step `advance_step`
737
+ family of functions.
738
+
739
+ Args:
740
+ num_seqs (int): Number of sequences scheduled to run.
741
+ max_decode_seq_len (int): Greatest of all the decode sequence
742
+ lengths. Used only in checking the viablility of using
743
+ CUDA graphs.
744
+ max_encoder_seq_len (int, optional): Greatest of all the encode
745
+ sequence lengths. Defaults to 0. Used only in checking the
746
+ viability of using CUDA graphs.
747
+ Returns:
748
+ int: Returns the determined number of padding sequences. If
749
+ CUDA graphs is not viable, returns -1.
750
+ """
751
+ is_mscp: bool = self.runner.scheduler_config.is_multi_step and \
752
+ self.runner.scheduler_config.chunked_prefill_enabled
753
+ decode_only = self.decode_only or is_mscp
754
+ if not decode_only:
755
+ # Early exit so we can treat num_seqs as the batch_size below.
756
+ return -1
757
+
758
+ # batch_size out of this function refers to the number of input
759
+ # tokens being scheduled. This conflation of num_seqs as batch_size
760
+ # is valid as this is a decode-only case.
761
+ batch_size = num_seqs
762
+ if not self._use_captured_graph(batch_size, decode_only,
763
+ max_decode_seq_len,
764
+ max_encoder_seq_len):
765
+ return -1
766
+
767
+ graph_batch_size = _get_graph_batch_size(batch_size)
768
+ assert graph_batch_size >= batch_size
769
+ return graph_batch_size - batch_size
770
+
771
def build(self) -> ModelInputForGPU:
772
"""Finalize the builder intermediate data and
773
create on-device tensors.
@@ -778,21 +826,17 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
826
for data in self.inter_data_list
827
}
828
781
- batch_size = len(input_tokens)
782
- use_captured_graph = self._use_captured_graph(
783
- batch_size,
784
- max_decode_seq_len,
829
+ cuda_graph_pad_size = self._get_cuda_graph_pad_size(
830
+ num_seqs=len(seq_lens),
831
+ max_decode_seq_len=max_encoder_seq_len,
832
max_encoder_seq_len=max_encoder_seq_len)
833
787
- # If cuda graph can be used, pad tensors accordingly.
788
- # See `capture_model` API for more details.
789
- # vLLM uses cuda graph only for decoding requests.
790
- cuda_graph_pad_size = -1
791
- if use_captured_graph:
792
- graph_batch_size = _get_graph_batch_size(batch_size)
793
- assert graph_batch_size >= batch_size
794
- cuda_graph_pad_size = graph_batch_size - batch_size
795
- batch_size = graph_batch_size
834
+ batch_size = len(input_tokens)
835
+ if cuda_graph_pad_size != -1:
836
+ # If cuda graph can be used, pad tensors accordingly.
837
+ # See `capture_model` API for more details.
838
+ # vLLM uses cuda graph only for decoding requests.
839
+ batch_size += cuda_graph_pad_size
840
841
# Tokens and positions.
842
if cuda_graph_pad_size: