@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e76466dde2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] draft_model_runner: Implement prepare_inputs on GPU for advance_step (#6338)

Alexander Matveev committed Jul 17, 2024 at 17:30 UTC e76466dde2bc9525d55165ceaa600d298c7bf773
12 files changed +568 -130
CMakeLists.txt
+1
@@ -151,6 +151,7 @@ set(VLLM_EXT_SRC
151 "csrc/quantization/fp8/common.cu"
152 "csrc/cuda_utils_kernels.cu"
153 "csrc/moe_align_block_size_kernels.cu"
154 + "csrc/prepare_inputs/advance_step.cu"
155 "csrc/torch_bindings.cpp")
156
157 if(VLLM_GPU_LANG STREQUAL "CUDA")
csrc/ops.h
+5
@@ -52,6 +52,11 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input);
52
53 void gelu_quick(torch::Tensor& out, torch::Tensor& input);
54
55 +void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
56 + torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
57 + torch::Tensor& input_positions, torch::Tensor& seq_lens,
58 + torch::Tensor& slot_mapping, torch::Tensor& block_tables);
59 +
60 #ifndef USE_ROCM
61 torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
62 const torch::Tensor& codebooks,
csrc/prepare_inputs/advance_step.cu new
+131
@@ -0,0 +1,131 @@
1 +/*
2 + * The goal of this GPU kernel is to advance input tensors on the GPU directly
3 + * PR: https://github.com/vllm-project/vllm/pull/6338
4 + * Current restrictions:
5 + * 1. Specialized for DraftModelRunner
6 + * 2. Supports flash_attn only
7 + */
8 +
9 +#include "advance_step.cuh"
10 +
11 +namespace prepare_inputs {
12 +
13 +//
14 +template <int const num_threads>
15 +__global__ void advance_step_kernel(int num_seqs, int num_queries,
16 + int block_size, long* input_tokens_ptr,
17 + long const* sampled_token_ids_ptr,
18 + long* input_positions_ptr,
19 + int* seq_lens_ptr, long* slot_mapping_ptr,
20 + int const* block_tables_ptr,
21 + int64_t const block_tables_stride) {
22 + int num_query_blocks = div_ceil(num_queries, num_threads);
23 +
24 + if (blockIdx.x >= num_query_blocks) {
25 + return;
26 + }
27 +
28 + int cur_query_id = blockIdx.x * num_threads + threadIdx.x;
29 +
30 + if (cur_query_id >= num_queries) {
31 + return;
32 + }
33 +
34 + // Update input_tokens
35 + input_tokens_ptr[cur_query_id] = sampled_token_ids_ptr[cur_query_id];
36 +
37 + int seq_len = seq_lens_ptr[cur_query_id];
38 + int next_seq_len = seq_len + 1;
39 + int next_input_pos = next_seq_len - 1;
40 +
41 + // Update seq_lens
42 + seq_lens_ptr[cur_query_id] = next_seq_len;
43 + // Update input_positions
44 + input_positions_ptr[cur_query_id] = next_input_pos;
45 +
46 + int const* seq_block_tables_ptr =
47 + block_tables_ptr + block_tables_stride * cur_query_id;
48 +
49 + int block_index = next_input_pos / block_size;
50 + int block_offset = next_input_pos % block_size;
51 +
52 + int slot_num = seq_block_tables_ptr[block_index] * block_size + block_offset;
53 + // Update slot_mapping
54 + slot_mapping_ptr[cur_query_id] = slot_num;
55 +}
56 +
57 +inline void verify_tensor(std::string const& name, torch::Tensor& t,
58 + int64_t const size_0, int64_t const size_1,
59 + c10::ScalarType const type) {
60 + bool size_0_cond = true;
61 + if (size_0 != -1) {
62 + size_0_cond = t.size(0) == size_0;
63 + }
64 +
65 + bool size_1_cond = true;
66 + if (size_1 != -1) {
67 + size_1_cond = t.size(1) == size_1;
68 + }
69 +
70 + bool is_contiguous = t.is_contiguous();
71 + bool same_type = t.dtype() == type;
72 +
73 + bool pass = size_0_cond && size_1_cond && is_contiguous && same_type;
74 + if (!pass) {
75 + TORCH_CHECK(false, "tensor: name = ", name, ", shape = ", t.sizes(),
76 + " is_cont = ", t.is_contiguous(), ", type = ", t.dtype(),
77 + " is not as expected: shape = [", size_0, ", ", size_1,
78 + "], type = ", type);
79 + }
80 +}
81 +
82 +void advance_step(int num_seqs, int num_queries, int block_size,
83 + torch::Tensor& input_tokens, // type: long
84 + torch::Tensor& sampled_token_ids, // type: long
85 + torch::Tensor& input_positions, // type: long
86 + torch::Tensor& seq_lens, // type: int
87 + torch::Tensor& slot_mapping, // type: long
88 + torch::Tensor& block_tables) { // type: int
89 +
90 + if (logging) {
91 + printf("advance_step:\n");
92 + printf(" num_seqs = %d\n", num_seqs);
93 + printf(" num_queries = %d\n", num_queries);
94 + printf(" block_size = %d\n", block_size);
95 + }
96 + // Verify all tensors
97 + verify_tensor("input_tokens", input_tokens, num_seqs, -1, at::kLong);
98 + verify_tensor("sampled_token_ids", sampled_token_ids, num_queries, 1,
99 + at::kLong);
100 + verify_tensor("input_positions", input_positions, num_seqs, -1, at::kLong);
101 + verify_tensor("seq_lens", seq_lens, num_seqs, -1, at::kInt);
102 + verify_tensor("slot_mapping", slot_mapping, num_seqs, -1, at::kLong);
103 + verify_tensor("block_tables", block_tables, num_seqs, -1, at::kInt);
104 +
105 + int dev = sampled_token_ids.get_device();
106 + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
107 +
108 + int blocks;
109 + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
110 +
111 + advance_step_kernel<max_threads><<<blocks, max_threads, 0, stream>>>(
112 + num_seqs, num_queries, block_size,
113 + reinterpret_cast<long*>(input_tokens.data_ptr()),
114 + reinterpret_cast<long const*>(sampled_token_ids.data_ptr()),
115 + reinterpret_cast<long*>(input_positions.data_ptr()),
116 + reinterpret_cast<int*>(seq_lens.data_ptr()),
117 + reinterpret_cast<long*>(slot_mapping.data_ptr()),
118 + reinterpret_cast<int const*>(block_tables.data_ptr()),
119 + block_tables.stride(0));
120 +}
121 +
122 +} // namespace prepare_inputs
123 +
124 +void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
125 + torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
126 + torch::Tensor& input_positions, torch::Tensor& seq_lens,
127 + torch::Tensor& slot_mapping, torch::Tensor& block_tables) {
128 + prepare_inputs::advance_step(num_seqs, num_queries, block_size, input_tokens,
129 + sampled_token_ids, input_positions, seq_lens,
130 + slot_mapping, block_tables);
131 +}
\ No newline at end of file
csrc/prepare_inputs/advance_step.cuh new
+19
@@ -0,0 +1,19 @@
1 +#pragma once
2 +
3 +#include <torch/all.h>
4 +
5 +#include <ATen/cuda/CUDAContext.h>
6 +#include <c10/cuda/CUDAGuard.h>
7 +#include <cuda.h>
8 +#include <cuda_fp16.h>
9 +#include <cuda_runtime.h>
10 +#include <iostream>
11 +
12 +namespace prepare_inputs {
13 +
14 +static constexpr int max_threads = 256;
15 +static constexpr bool logging = false;
16 +
17 +constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
18 +
19 +} // namespace prepare_inputs
csrc/torch_bindings.cpp
+4
@@ -72,6 +72,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
72 ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
73 ops.impl("gelu_quick", torch::kCUDA, &gelu_quick);
74
75 + // prepare_inputs advance_step
76 + ops.def("advance_step", &advance_step);
77 + ops.impl("advance_step", torch::kCUDA, &advance_step);
78 +
79 // Layernorm
80 // Apply Root Mean Square (RMS) Normalization to the input tensor.
81 ops.def(
tests/spec_decode/e2e/conftest.py
+1
@@ -227,6 +227,7 @@ def get_output_from_llm_generator(
227 maybe_assert_ngram_worker(llm)
228
229 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
230 +
231 token_ids = [output.outputs[0].token_ids for output in outputs]
232 tokens = [output.outputs[0].text for output in outputs]
233
tests/spec_decode/test_multi_step_worker.py
+48
@@ -642,3 +642,51 @@ def test_draft_proposals_mixed_k():
642 assert proposals.proposal_lens.tolist() == [
643 k for _ in range(expected_num_proposal_seqs - 1)
644 ] + [0 for _ in range(expected_num_no_proposal_seqs)] + [k]
645 +
646 +
647 +@torch.inference_mode()
648 +def test_use_draft_model_runner_advance_step():
649 + """Verify that draft model runner triggers advance step
650 + when applicable.
651 + """
652 + seed = 100
653 + model_name = 'JackFram/llama-68m'
654 +
655 + k = 5
656 + batch_size = 32
657 + block_size = 32
658 + num_gpu_blocks = 2048 // block_size
659 + worker = create_worker(
660 + MultiStepWorker,
661 + model_name,
662 + block_size,
663 + num_gpu_blocks,
664 + seed,
665 + model_runner_cls=TP1DraftModelRunner,
666 + )
667 +
668 + # Mock "_gpu_advance_step" to raise an exception when called.
669 + exception_secret = "artificial stop"
670 + worker.model_runner._gpu_advance_step = MagicMock()
671 + worker.model_runner._gpu_advance_step.side_effect = ValueError(
672 + exception_secret)
673 +
674 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
675 +
676 + # Fallback (should not call) when num_steps=1.
677 + execute_model_req = ExecuteModelRequest(
678 + seq_group_metadata_list=seq_group_metadata_list,
679 + num_lookahead_slots=k,
680 + num_steps=1)
681 + worker.execute_model(execute_model_req=execute_model_req)
682 +
683 + # Expect exception if _gpu_advance_step is called.
684 + execute_model_req = ExecuteModelRequest(
685 + seq_group_metadata_list=seq_group_metadata_list,
686 + num_lookahead_slots=k,
687 + num_steps=k)
688 +
689 + with pytest.raises(ValueError, match=exception_secret):
690 + worker.execute_model(execute_model_req=execute_model_req)
691 + call_args_list = worker.model_runner._gpu_advance_step.call_args_list
692 + assert len(call_args_list) == 1
vllm/_custom_ops.py
+12
@@ -166,6 +166,18 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
166 torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
167
168
169 +def advance_step(num_seqs: int, num_queries: int, block_size: int,
170 + input_tokens: torch.Tensor, sampled_token_ids: torch.Tensor,
171 + input_positions: torch.Tensor, seq_lens: torch.Tensor,
172 + slot_mapping: torch.Tensor,
173 + block_tables: torch.Tensor) -> None:
174 + """Advance a step on GPU for existing inputs for a multi-step runner"""
175 + return torch.ops._C.advance_step(num_seqs, num_queries, block_size,
176 + input_tokens, sampled_token_ids,
177 + input_positions, seq_lens, slot_mapping,
178 + block_tables)
179 +
180 +
181 # quantization ops
182 # awq
183 def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
vllm/model_executor/layers/sampler.py
+100 -47
@@ -47,6 +47,32 @@ class Sampler(nn.Module):
47 # speculative decoding.
48 self.include_gpu_probs_tensor = False
49
50 + def _init_sampling_tensors(
51 + self,
52 + logits: torch.Tensor,
53 + sampling_metadata: SamplingMetadata,
54 + ):
55 + """The goal here is to reuse sampling tensors between similar decode
56 + runs. This is possible because sampling logic does not change between
57 + decodes of the same sequences.
58 + """
59 + _, vocab_size = logits.shape
60 +
61 + # First free any existing stored sampling tensors.
62 + # This is necessary because some sampling tensors may
63 + # have pinned memory.
64 + self._sampling_tensors = None
65 +
66 + # Initialize new sampling tensors
67 + (sampling_tensors, do_penalties, do_top_p_top_k,
68 + do_min_p) = SamplingTensors.from_sampling_metadata(
69 + sampling_metadata, vocab_size, logits.device, logits.dtype)
70 +
71 + self._sampling_tensors = sampling_tensors
72 + self._do_penalties = do_penalties
73 + self._do_top_p_top_k = do_top_p_top_k
74 + self._do_min_p = do_min_p
75 +
76 def forward(
77 self,
78 logits: torch.Tensor,
@@ -60,12 +86,23 @@ class Sampler(nn.Module):
86 assert logits is not None
87 _, vocab_size = logits.shape
88
63 - logits = _apply_min_tokens_penalty(logits, sampling_metadata)
64 -
89 # Prepare sampling tensors with pinned memory to avoid blocking.
66 - (sampling_tensors, do_penalties, do_top_p_top_k,
67 - do_min_p) = SamplingTensors.from_sampling_metadata(
68 - sampling_metadata, vocab_size, logits.device, logits.dtype)
90 + if not sampling_metadata.reuse_sampling_tensors:
91 + self._init_sampling_tensors(logits, sampling_metadata)
92 + elif self._do_penalties:
93 + # In this case, the sampling tensors logic depends on
94 + # "output_tokens" of a sequence. As a result, we cannot
95 + # reuse sampling tensors, since "output_tokens" changes
96 + # between decode runs.
97 + self._init_sampling_tensors(logits, sampling_metadata)
98 +
99 + assert self._sampling_tensors is not None
100 + sampling_tensors = self._sampling_tensors
101 + do_penalties = self._do_penalties
102 + do_top_p_top_k = self._do_top_p_top_k
103 + do_min_p = self._do_min_p
104 +
105 + logits = _apply_min_tokens_penalty(logits, sampling_metadata)
106
107 # Apply presence and frequency penalties.
108 if do_penalties:
@@ -77,7 +114,7 @@ class Sampler(nn.Module):
114
115 # Apply temperature scaling.
116 # Use in-place division to avoid creating a new tensor.
80 - logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1))
117 + logits.div_(sampling_tensors.temperatures.unsqueeze(dim=1))
118
119 if do_top_p_top_k:
120 logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps,
@@ -109,13 +146,19 @@ class Sampler(nn.Module):
146 on_device_tensors = None
147
148 # Get the logprobs query results.
112 - prompt_logprobs, sample_logprobs = _get_logprobs(
113 - logprobs, sampling_metadata, sample_results)
114 - return _build_sampler_output(sample_results,
115 - sampling_metadata,
116 - prompt_logprobs,
117 - sample_logprobs,
118 - on_device_tensors=on_device_tensors)
149 + prompt_logprobs = None
150 + sample_logprobs = None
151 + if not sampling_metadata.skip_sampler_cpu_output:
152 + prompt_logprobs, sample_logprobs = _get_logprobs(
153 + logprobs, sampling_metadata, sample_results)
154 +
155 + return _build_sampler_output(
156 + sample_results,
157 + sampling_metadata,
158 + prompt_logprobs,
159 + sample_logprobs,
160 + on_device_tensors=on_device_tensors,
161 + skip_sampler_cpu_output=sampling_metadata.skip_sampler_cpu_output)
162
163 @property
164 def _should_modify_greedy_probs_inplace(self) -> bool:
@@ -535,24 +578,29 @@ def _sample_with_torch(
578
579 # GPU<->CPU sync happens in the loop below.
580 # This also converts the sample output to Python objects.
538 - for sampling_type in SamplingType:
539 - if sampling_type not in sample_metadata:
540 - continue
541 - (seq_group_id, seq_groups) = sample_metadata[sampling_type]
542 - if sampling_type == SamplingType.GREEDY:
543 - sample_results = _greedy_sample(seq_groups, greedy_samples)
544 - elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
545 - sample_results = _random_sample(seq_groups,
546 - multinomial_samples[sampling_type])
547 - elif sampling_type == SamplingType.BEAM:
548 - sample_results = _beam_search_sample(seq_groups,
549 - beam_search_logprobs)
550 - sample_results_dict.update(zip(seq_group_id, sample_results))
581 + if not sampling_metadata.skip_sampler_cpu_output:
582 + for sampling_type in SamplingType:
583 + if sampling_type not in sample_metadata:
584 + continue
585 + (seq_group_id, seq_groups) = sample_metadata[sampling_type]
586 + if sampling_type == SamplingType.GREEDY:
587 + sample_results = _greedy_sample(seq_groups, greedy_samples)
588 + elif sampling_type in (SamplingType.RANDOM,
589 + SamplingType.RANDOM_SEED):
590 + sample_results = _random_sample(
591 + seq_groups, multinomial_samples[sampling_type])
592 + elif sampling_type == SamplingType.BEAM:
593 + sample_results = _beam_search_sample(seq_groups,
594 + beam_search_logprobs)
595 + sample_results_dict.update(zip(seq_group_id, sample_results))
596 +
597 + sample_results = [
598 + sample_results_dict.get(i, ([], []))
599 + for i in range(len(sampling_metadata.seq_groups))
600 + ]
601 + else:
602 + sample_results = []
603
552 - sample_results = [
553 - sample_results_dict.get(i, ([], []))
554 - for i in range(len(sampling_metadata.seq_groups))
555 - ]
604 return sample_results, sampled_token_ids_tensor
605
606
@@ -997,10 +1045,11 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
1045 def _build_sampler_output(
1046 sample_results: SampleResultType,
1047 sampling_metadata: SamplingMetadata,
1000 - prompt_logprobs: List[Optional[PromptLogprobs]],
1001 - sample_logprobs: List[SampleLogprobs],
1048 + prompt_logprobs: Optional[List[Optional[PromptLogprobs]]],
1049 + sample_logprobs: Optional[List[SampleLogprobs]],
1050 on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor,
1051 torch.Tensor]],
1052 + skip_sampler_cpu_output: bool = False,
1053 ) -> SamplerOutput:
1054 """Construct Python objects with the output of sampling.
1055
@@ -1010,22 +1059,26 @@ def _build_sampler_output(
1059 allows post-processing without copies to CPU/serialization, e.g. in
1060 speculative decoding rejection sampling.
1061 """
1013 -
1062 sampler_output: List[CompletionSequenceGroupOutput] = []
1015 - for (seq_group, sample_result, group_prompt_logprobs,
1016 - group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1017 - sample_results, prompt_logprobs,
1018 - sample_logprobs):
1019 - seq_ids = seq_group.seq_ids
1020 - next_token_ids, parent_ids = sample_result
1021 - seq_outputs: List[SequenceOutput] = []
1022 - for parent_id, next_token_id, logprobs in zip(parent_ids,
1023 - next_token_ids,
1024 - group_sample_logprobs):
1025 - seq_outputs.append(
1026 - SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
1027 - sampler_output.append(
1028 - CompletionSequenceGroupOutput(seq_outputs, group_prompt_logprobs))
1063 + if not skip_sampler_cpu_output:
1064 + assert prompt_logprobs is not None
1065 + assert sample_logprobs is not None
1066 +
1067 + for (seq_group, sample_result, group_prompt_logprobs,
1068 + group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1069 + sample_results, prompt_logprobs,
1070 + sample_logprobs):
1071 + seq_ids = seq_group.seq_ids
1072 + next_token_ids, parent_ids = sample_result
1073 + seq_outputs: List[SequenceOutput] = []
1074 + for parent_id, next_token_id, logprobs in zip(
1075 + parent_ids, next_token_ids, group_sample_logprobs):
1076 + seq_outputs.append(
1077 + SequenceOutput(seq_ids[parent_id], next_token_id,
1078 + logprobs))
1079 + sampler_output.append(
1080 + CompletionSequenceGroupOutput(seq_outputs,
1081 + group_prompt_logprobs))
1082
1083 # If not specified, store None values in SamplerOutput.
1084 if on_device_tensors is not None:
vllm/model_executor/sampling_metadata.py
+10
@@ -87,6 +87,12 @@ class SamplingMetadata:
87 The first tuple is [1, 2] (sampled index within original logit),
88 and the second tuple is [0, 1] (sampled index within pruned logit).
89 num_prompts: Number of prompt sequence groups in seq_groups.
90 + skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU
91 + serialization of token outputs.
92 + reuse_sampling_tensors: Indicates if we want to reuse sampling
93 + tensors that are part of the sampler forward pass. Currently,
94 + it is mainly used for multi-step decode.
95 +
96 """
97
98 def __init__(
@@ -95,11 +101,15 @@ class SamplingMetadata:
101 selected_token_indices: torch.Tensor,
102 categorized_sample_indices: Dict[SamplingType, torch.Tensor],
103 num_prompts: int,
104 + skip_sampler_cpu_output: bool = False,
105 + reuse_sampling_tensors: bool = False,
106 ) -> None:
107 self.seq_groups = seq_groups
108 self.selected_token_indices = selected_token_indices
109 self.categorized_sample_indices = categorized_sample_indices
110 self.num_prompts = num_prompts
111 + self.skip_sampler_cpu_output = skip_sampler_cpu_output
112 + self.reuse_sampling_tensors = reuse_sampling_tensors
113
114 @staticmethod
115 def prepare(
vllm/spec_decode/draft_model_runner.py
+225 -80
@@ -2,17 +2,22 @@ from typing import List, Optional
2
3 import torch
4
5 +from vllm import _custom_ops as ops
6 +from vllm.attention.backends.flash_attn import FlashAttentionMetadata
7 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8 ModelConfig, MultiModalConfig, ParallelConfig,
9 PromptAdapterConfig, SchedulerConfig)
10 from vllm.logger import init_logger
9 -from vllm.sequence import (IntermediateTensors, SamplerOutput,
10 - SequenceGroupMetadata)
11 +from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
12 + SamplerOutput)
13 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
14 ModelRunner)
15
16 logger = init_logger(__name__)
17
18 +debug_advance_input = False
19 +enable_gpu_advance_step = True
20 +
21
22 class TP1DraftModelRunner(ModelRunner):
23 """Specialized model runner for speculative decoding draft model.
@@ -21,18 +26,9 @@ class TP1DraftModelRunner(ModelRunner):
26 we could get rid of most CPU-GPU synchronization and data transfer
27 overheads by keeping model input and output tensors on GPU all the time.
28
24 - This runner is still under development so there's no performance gain
25 - at this moment. Currently we adopt a temporary solution that caches the
26 - seq_group_metadata_list for multi-step execution, so that we can
27 - leverage existing prepare_model_input to be compatible with the current
28 - execution flow, but we plan to remove this cache and avoid calling
29 - prepare_model_input in execute_model at all.
30 -
31 - The detail development plan includes:
32 - 1. Use "update_model_input" to update existing model_input without
33 - creating a new one.
34 - 2. Improve the performance of "update_model_input" with a GPU kernel.
35 - 3. Support TP > 1 (this requires some designs because we do not expect
29 + TODOs:
30 + 1. Currently supports only flash-attn, add support for other attn_backends.
31 + 2. Support TP > 1 (this requires some designs because we do not expect
32 any broadcasting inside execute_model).
33 """
34
@@ -71,51 +67,156 @@ class TP1DraftModelRunner(ModelRunner):
67 return_hidden_states=return_hidden_states,
68 )
69
74 - # TODO: Remove this cache when we are able to update model_input
75 - # directly in advance_step.
76 - self.cached_seq_group_metadata_list: Optional[
77 - List[SequenceGroupMetadata]] = None
70 + def _update_flash_attn_metadata(self, attn_metadata, num_seqs,
71 + num_queries):
72 + assert isinstance(attn_metadata, FlashAttentionMetadata)
73
79 - def prepare_model_input(
80 - self,
81 - seq_group_metadata_list: List[SequenceGroupMetadata],
82 - virtual_engine: int = 0,
83 - finished_requests_ids: Optional[List[str]] = None
84 - ) -> ModelInputForGPUWithSamplingMetadata:
85 - """A temporary solution that caches the seq_group_metadata_list
86 - for multi-step execution.
87 - TODO: In-place update model_input and remove this function.
88 - """
89 - self.cached_seq_group_metadata_list = seq_group_metadata_list
90 - return super().prepare_model_input(
91 - seq_group_metadata_list,
92 - finished_requests_ids=finished_requests_ids)
74 + if num_seqs != num_queries:
75 + assert num_seqs > num_queries
76 + assert attn_metadata.use_cuda_graph
77 +
78 + assert attn_metadata.num_prefills == 0
79 + assert attn_metadata.num_prefill_tokens == 0
80 + assert attn_metadata.num_decode_tokens == num_seqs
81 + assert attn_metadata.slot_mapping.shape == (num_seqs, )
82 +
83 + assert len(attn_metadata.seq_lens) == num_seqs
84 + assert attn_metadata.seq_lens_tensor.shape == (num_seqs, )
85 + assert attn_metadata.max_query_len == 1
86 + assert attn_metadata.max_prefill_seq_len == 0
87 + assert attn_metadata.max_decode_seq_len == max(attn_metadata.seq_lens)
88 +
89 + assert attn_metadata.query_start_loc.shape == (num_queries + 1, )
90 + assert attn_metadata.seq_start_loc.shape == (num_seqs + 1, )
91 +
92 + assert attn_metadata.context_lens_tensor.shape == (num_queries, )
93 +
94 + assert attn_metadata.block_tables.shape[0] == num_seqs
95 +
96 + # Update query lengths. Note that we update only queries and not seqs,
97 + # since tensors may be padded due to captured cuda graph batch size
98 + for i in range(num_queries):
99 + attn_metadata.seq_lens[i] += 1
100 + attn_metadata.max_decode_seq_len = max(attn_metadata.seq_lens)
101
94 - def update_model_input(
102 + def _update_sampling_metadata(self, sampling_metadata, num_seqs,
103 + num_queries):
104 +
105 + assert sampling_metadata.num_prompts == 0
106 + assert len(sampling_metadata.seq_groups) == num_queries
107 + assert sampling_metadata.selected_token_indices.shape == (
108 + num_queries, )
109 + # assert sampling_metadata.categorized_sample_indices == TODO: Add if needed # noqa: E501
110 +
111 + # Verify that all sequences are decodes
112 + for i in range(num_queries):
113 + seq_group = sampling_metadata.seq_groups[i]
114 +
115 + assert seq_group.is_prompt is False # No prompt
116 + assert seq_group.prompt_logprob_indices == [] # No prompt
117 + assert seq_group.sample_indices == [i] # Simple
118 + assert seq_group.seq_len is None # Decode
119 + assert seq_group.query_len is None # Decode
120 +
121 + def _gpu_advance_step(
122 self, model_input: ModelInputForGPUWithSamplingMetadata,
123 last_output: SamplerOutput
124 ) -> ModelInputForGPUWithSamplingMetadata:
98 - """Prepare the model inputs for the next step.
99 - TODO: In-place update model_input instead of calling
100 - prepare_model_input.
125 + # Currently, we expect "decode mode" only
126 + assert not model_input.is_prompt
127 +
128 + # Get num_seqs
129 + num_seqs = len(model_input.seq_lens)
130 + num_queries = len(model_input.query_lens)
131 +
132 + # Get output tokens GPU tensor
133 + sampled_token_ids = last_output.sampled_token_ids
134 + assert sampled_token_ids is not None
135 +
136 + # Update attn_metadata
137 + attn_metadata = model_input.attn_metadata
138 + assert isinstance(attn_metadata, FlashAttentionMetadata)
139 + self._update_flash_attn_metadata(attn_metadata, num_seqs, num_queries)
140 +
141 + # Update GPU tensors
142 + ops.advance_step(num_seqs=num_seqs,
143 + num_queries=num_queries,
144 + block_size=self.block_size,
145 + input_tokens=model_input.input_tokens,
146 + sampled_token_ids=sampled_token_ids,
147 + input_positions=model_input.input_positions,
148 + seq_lens=attn_metadata.seq_lens_tensor,
149 + slot_mapping=attn_metadata.slot_mapping,
150 + block_tables=attn_metadata.block_tables)
151 +
152 + # Update sampling_metadata
153 + sampling_metadata = model_input.sampling_metadata
154 + self._update_sampling_metadata(sampling_metadata, num_seqs,
155 + num_queries)
156 +
157 + # Create new input
158 + new_model_input = self._model_input_cls(
159 + input_tokens=model_input.input_tokens,
160 + input_positions=model_input.input_positions,
161 + attn_metadata=attn_metadata,
162 + seq_lens=attn_metadata.seq_lens,
163 + query_lens=model_input.query_lens,
164 + lora_mapping=model_input.lora_mapping,
165 + lora_requests=model_input.lora_requests,
166 + multi_modal_kwargs=model_input.multi_modal_kwargs,
167 + sampling_metadata=model_input.sampling_metadata,
168 + is_prompt=False,
169 + )
170 +
171 + # Ensure we skip CPU samples
172 + assert new_model_input.sampling_metadata.skip_sampler_cpu_output is True
173 + # We can reuse sampling tensors since every decode iteration is the same
174 + new_model_input.sampling_metadata.reuse_sampling_tensors = True
175 +
176 + if debug_advance_input:
177 + logger.debug("NEW INPUT: ")
178 + logger.debug(" input_tokens = %s", new_model_input.input_tokens)
179 + logger.debug(" input_positions = %s",
180 + new_model_input.input_positions)
181 + logger.debug(" seq_lens = %d", new_model_input.seq_lens)
182 + logger.debug(" query_lens = %d", new_model_input.query_lens)
183 + logger.debug(" attn_metadata:")
184 + logger.debug(" seq_lens_tensor: %s",
185 + attn_metadata.seq_lens_tensor)
186 + logger.debug(" slot_mapping: %s", attn_metadata.slot_mapping)
187 + logger.debug(" block_tables: %s", attn_metadata.block_tables)
188 +
189 + return new_model_input
190 +
191 + def supports_gpu_multi_step(self, execute_model_req: ExecuteModelRequest):
192 + """Determines if draft_model_runner GPU multi-step can be used.
193 + Currently required conditions are:
194 + 1. Only decodes
195 + 2. Only flash-attn
196 + 3. No LORA
197 + 4. No prompt_adapter_config
198 """
199 + if not enable_gpu_advance_step:
200 + return False
201
103 - # Append the output token to the sequence data.
104 - assert self.cached_seq_group_metadata_list is not None
105 - for seq_group_metadata, sequence_group_outputs in zip(
106 - self.cached_seq_group_metadata_list, last_output.outputs):
107 - seq_group_metadata.is_prompt = False
202 + # We allow multi-step GPU only in decode mode
203 + for seq_group in execute_model_req.seq_group_metadata_list:
204 + if seq_group.is_prompt:
205 + return False
206
109 - for seq_output in sequence_group_outputs.samples:
110 - seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
207 + # TODO: Add support for other attn backends
208 + if self.attn_backend.get_name() != "flash-attn":
209 + return False
210
112 - token_id = seq_output.output_token
113 - token_logprob = seq_output.logprobs[token_id]
211 + # TODO: Add support for LORA
212 + if self.lora_config:
213 + return False
214
115 - seq.append_token_id(token_id, token_logprob.logprob)
116 - seq.update_num_computed_tokens(1)
215 + # TODO: Add soft-tuning prompt adapter support
216 + if self.prompt_adapter_config:
217 + return False
218
118 - return self.prepare_model_input(self.cached_seq_group_metadata_list)
219 + return True
220
221 @torch.inference_mode()
222 def execute_model(
@@ -125,42 +226,86 @@ class TP1DraftModelRunner(ModelRunner):
226 intermediate_tensors: Optional[IntermediateTensors] = None,
227 num_steps: int = 1,
228 ) -> Optional[List[SamplerOutput]]:
128 - # Since we do not broadcast data inside execute_model anymore,
129 - # we need to figure out the best way to support TP > 1 in this
130 - # case, because we will at least need to broadcast the sampled
131 - # tokens to all workers.
132 - if not self.is_driver_worker:
133 - raise ValueError("TP1DraftModelRunner only supports TP=1.")
229 + """Executes num_steps forward passes with advacement of input tensors
230 + on the GPU. Look at supports_gpu_multi_step(..) for pre-conditions.
231
135 - if self.lora_config:
136 - assert model_input.lora_requests is not None
137 - assert model_input.lora_mapping is not None
138 - self.set_active_loras(model_input.lora_requests,
139 - model_input.lora_mapping)
232 + Optimizations used:
233 + 1. Input tensors are updated on the GPU directly
234 + 2. Skips GPU=>CPU serialization of sampler outputs (we don't need
235 + them since we do batch expansion later that uses GPU outputs)
236 + 3. Reuses sampling tensors (since we run only decodes and they have
237 + a repeating sampling logic)
238 + """
239
141 - if self.prompt_adapter_config:
142 - assert model_input.prompt_adapter_requests is not None
143 - assert model_input.prompt_adapter_mapping is not None
144 - self.set_active_prompt_adapters(
145 - model_input.prompt_adapter_requests,
146 - model_input.prompt_adapter_mapping)
240 + # When num_steps == 1, we execute the fallback here for the GPU
241 + # advance_step, which runs prepare_inputs on CPU and for each spec
242 + # iteration invokes this function only once
243 + # (Look at multi-step-worker code)
244 + is_fallback = num_steps == 1
245 + if not is_fallback:
246 + # Since we do not broadcast data inside execute_model anymore,
247 + # we need to figure out the best way to support TP > 1 in this
248 + # case, because we will at least need to broadcast the sampled
249 + # tokens to all workers.
250 + if not self.is_driver_worker:
251 + raise ValueError("TP1DraftModelRunner only supports TP=1.")
252 +
253 + # Sanity
254 + if self.lora_config is not None:
255 + raise ValueError("TP1DraftModelRunner has no support for LORA")
256 + if self.prompt_adapter_config is not None:
257 + raise ValueError("TP1DraftModelRunner has no support for "
258 + "prompt_adapter_config")
259 + if model_input.multi_modal_kwargs:
260 + raise ValueError(
261 + "TP1DraftModelRunner has no support for multi_modal_kwargs"
262 + )
263 + else:
264 + if self.lora_config:
265 + assert model_input.lora_requests is not None
266 + assert model_input.lora_mapping is not None
267 + self.set_active_loras(model_input.lora_requests,
268 + model_input.lora_mapping)
269 +
270 + if self.prompt_adapter_config:
271 + assert model_input.prompt_adapter_requests is not None
272 + assert model_input.prompt_adapter_mapping is not None
273 + self.set_active_prompt_adapters(
274 + model_input.prompt_adapter_requests,
275 + model_input.prompt_adapter_mapping)
276 +
277 + # Detect exec mode
278 + assert model_input.attn_metadata is not None
279 + use_cuda_graph = False
280 + if model_input.attn_metadata.num_prefills > 0:
281 + # In this case, execute_model(..) was called directly
282 + if num_steps > 1:
283 + raise ValueError(
284 + "execute_model(..) of draft_model_runner can be called "
285 + "directly only with a single-step prefill")
286 + else:
287 + # We can skip CPU samples for spec token generation.
288 + # (We do allow CPU samples for num_steps == 1 to support the
289 + # fallback case, where supports_gpu_multi_step(..) does not pass)
290 + model_input.sampling_metadata.skip_sampler_cpu_output = (
291 + not is_fallback)
292 +
293 + # Attn attr defines if we use cuda graphs
294 + use_cuda_graph = model_input.attn_metadata.use_cuda_graph
295 +
296 + # Get model
297 + if use_cuda_graph:
298 + graph_batch_size = model_input.input_tokens.shape[0]
299 + model_executable = (self.graph_runners[model_input.virtual_engine]
300 + [graph_batch_size])
301 + else:
302 + model_executable = self.model
303
148 - virtual_engine = model_input.virtual_engine
304 outputs: List[SamplerOutput] = []
305 for step in range(num_steps):
151 - # Currently cuda graph is only supported by the decode phase.
152 - assert model_input.attn_metadata is not None
153 - prefill_meta = model_input.attn_metadata.prefill_metadata
154 - decode_meta = model_input.attn_metadata.decode_metadata
155 - if prefill_meta is None and decode_meta.use_cuda_graph:
156 - assert model_input.input_tokens is not None
157 - graph_batch_size = model_input.input_tokens.shape[0]
158 - model_executable = (
159 - self.graph_runners[virtual_engine][graph_batch_size])
160 - else:
161 - model_executable = self.model
162 -
306 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
307 +
308 + # Run model
309 hidden_states = model_executable(
310 input_ids=model_input.input_tokens,
311 positions=model_input.input_positions,
@@ -181,8 +326,8 @@ class TP1DraftModelRunner(ModelRunner):
326 sampling_metadata=model_input.sampling_metadata,
327 ))
328
184 - # Prepare the inputs for the next step.
329 + # Prepare inputs for the next step
330 if step != num_steps - 1:
186 - model_input = self.update_model_input(model_input, outputs[-1])
331 + model_input = self._gpu_advance_step(model_input, outputs[-1])
332
333 return outputs
vllm/spec_decode/multi_step_worker.py
+12 -3
@@ -67,14 +67,23 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
67 expanded_request, indices_of_seq_with_bonus_tokens =\
68 self._expand_execute_model_request(
69 execute_model_req, seq_ids_with_bonus_token_in_last_step)
70 +
71 # Run model sample_len times.
72 model_outputs: List[SamplerOutput] = []
72 - if isinstance(self.model_runner, TP1DraftModelRunner):
73 + if isinstance(
74 + self.model_runner, TP1DraftModelRunner
75 + ) and self.model_runner.supports_gpu_multi_step(expanded_request):
76 + # Here we run the draft_model_runner with multi-step prepare
77 + # on the GPU directly
78 expanded_request.num_steps = sample_len
79 model_outputs = self.execute_model(
80 execute_model_req=expanded_request)
81 else:
77 - # TODO: Remove this branch once DraftModelRunner supports TP>1.
82 + # Here we run multi-step directly, with every step prepared
83 + # on the CPU.
84 + # TODO: Remove this branch once DraftModelRunner supports TP>1
85 + # and other restrictions that are part of DraftModelRunner's
86 + # supports_gpu_multi_step(..)
87 for _ in range(sample_len):
88 model_output: List[SamplerOutput] = super().execute_model(
89 execute_model_req=expanded_request)
@@ -171,7 +180,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
180 outputs=[
181 expanded_batch_output.outputs[i]
182 for i in output_indices_to_retain
174 - ],
183 + ] if len(expanded_batch_output.outputs) > 0 else [],
184 sampled_token_probs=(
185 expanded_batch_output.
186 sampled_token_probs[output_indices_to_retain]