A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] draft_model_runner: Implement prepare_inputs on GPU for advance_step (#6338)
Alexander Matveev committed
Jul 17, 2024 at 17:30 UTC
e76466dde2bc9525d55165ceaa600d298c7bf773
12 files changed
+568
-130
CMakeLists.txt
+1
@@ -151,6 +151,7 @@ set(VLLM_EXT_SRC
151
"csrc/quantization/fp8/common.cu"
152
"csrc/cuda_utils_kernels.cu"
153
"csrc/moe_align_block_size_kernels.cu"
154
+ "csrc/prepare_inputs/advance_step.cu"
155
"csrc/torch_bindings.cpp")
156
157
if(VLLM_GPU_LANG STREQUAL "CUDA")
csrc/ops.h
+5
@@ -52,6 +52,11 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input);
52
53
void gelu_quick(torch::Tensor& out, torch::Tensor& input);
54
55
+void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
56
+ torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
57
+ torch::Tensor& input_positions, torch::Tensor& seq_lens,
58
+ torch::Tensor& slot_mapping, torch::Tensor& block_tables);
59
+
60
#ifndef USE_ROCM
61
torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
62
const torch::Tensor& codebooks,
csrc/prepare_inputs/advance_step.cu
new
+131
@@ -0,0 +1,131 @@
1
+/*
2
+ * The goal of this GPU kernel is to advance input tensors on the GPU directly
3
+ * PR: https://github.com/vllm-project/vllm/pull/6338
4
+ * Current restrictions:
5
+ * 1. Specialized for DraftModelRunner
6
+ * 2. Supports flash_attn only
7
+ */
8
+
9
+#include "advance_step.cuh"
10
+
11
+namespace prepare_inputs {
12
+
13
+//
14
+template <int const num_threads>
15
+__global__ void advance_step_kernel(int num_seqs, int num_queries,
16
+ int block_size, long* input_tokens_ptr,
17
+ long const* sampled_token_ids_ptr,
18
+ long* input_positions_ptr,
19
+ int* seq_lens_ptr, long* slot_mapping_ptr,
20
+ int const* block_tables_ptr,
21
+ int64_t const block_tables_stride) {
22
+ int num_query_blocks = div_ceil(num_queries, num_threads);
23
+
24
+ if (blockIdx.x >= num_query_blocks) {
25
+ return;
26
+ }
27
+
28
+ int cur_query_id = blockIdx.x * num_threads + threadIdx.x;
29
+
30
+ if (cur_query_id >= num_queries) {
31
+ return;
32
+ }
33
+
34
+ // Update input_tokens
35
+ input_tokens_ptr[cur_query_id] = sampled_token_ids_ptr[cur_query_id];
36
+
37
+ int seq_len = seq_lens_ptr[cur_query_id];
38
+ int next_seq_len = seq_len + 1;
39
+ int next_input_pos = next_seq_len - 1;
40
+
41
+ // Update seq_lens
42
+ seq_lens_ptr[cur_query_id] = next_seq_len;
43
+ // Update input_positions
44
+ input_positions_ptr[cur_query_id] = next_input_pos;
45
+
46
+ int const* seq_block_tables_ptr =
47
+ block_tables_ptr + block_tables_stride * cur_query_id;
48
+
49
+ int block_index = next_input_pos / block_size;
50
+ int block_offset = next_input_pos % block_size;
51
+
52
+ int slot_num = seq_block_tables_ptr[block_index] * block_size + block_offset;
53
+ // Update slot_mapping
54
+ slot_mapping_ptr[cur_query_id] = slot_num;
55
+}
56
+
57
+inline void verify_tensor(std::string const& name, torch::Tensor& t,
58
+ int64_t const size_0, int64_t const size_1,
59
+ c10::ScalarType const type) {
60
+ bool size_0_cond = true;
61
+ if (size_0 != -1) {
62
+ size_0_cond = t.size(0) == size_0;
63
+ }
64
+
65
+ bool size_1_cond = true;
66
+ if (size_1 != -1) {
67
+ size_1_cond = t.size(1) == size_1;
68
+ }
69
+
70
+ bool is_contiguous = t.is_contiguous();
71
+ bool same_type = t.dtype() == type;
72
+
73
+ bool pass = size_0_cond && size_1_cond && is_contiguous && same_type;
74
+ if (!pass) {
75
+ TORCH_CHECK(false, "tensor: name = ", name, ", shape = ", t.sizes(),
76
+ " is_cont = ", t.is_contiguous(), ", type = ", t.dtype(),
77
+ " is not as expected: shape = [", size_0, ", ", size_1,
78
+ "], type = ", type);
79
+ }
80
+}
81
+
82
+void advance_step(int num_seqs, int num_queries, int block_size,
83
+ torch::Tensor& input_tokens, // type: long
84
+ torch::Tensor& sampled_token_ids, // type: long
85
+ torch::Tensor& input_positions, // type: long
86
+ torch::Tensor& seq_lens, // type: int
87
+ torch::Tensor& slot_mapping, // type: long
88
+ torch::Tensor& block_tables) { // type: int
89
+
90
+ if (logging) {
91
+ printf("advance_step:\n");
92
+ printf(" num_seqs = %d\n", num_seqs);
93
+ printf(" num_queries = %d\n", num_queries);
94
+ printf(" block_size = %d\n", block_size);
95
+ }
96
+ // Verify all tensors
97
+ verify_tensor("input_tokens", input_tokens, num_seqs, -1, at::kLong);
98
+ verify_tensor("sampled_token_ids", sampled_token_ids, num_queries, 1,
99
+ at::kLong);
100
+ verify_tensor("input_positions", input_positions, num_seqs, -1, at::kLong);
101
+ verify_tensor("seq_lens", seq_lens, num_seqs, -1, at::kInt);
102
+ verify_tensor("slot_mapping", slot_mapping, num_seqs, -1, at::kLong);
103
+ verify_tensor("block_tables", block_tables, num_seqs, -1, at::kInt);
104
+
105
+ int dev = sampled_token_ids.get_device();
106
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
107
+
108
+ int blocks;
109
+ cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
110
+
111
+ advance_step_kernel<max_threads><<<blocks, max_threads, 0, stream>>>(
112
+ num_seqs, num_queries, block_size,
113
+ reinterpret_cast<long*>(input_tokens.data_ptr()),
114
+ reinterpret_cast<long const*>(sampled_token_ids.data_ptr()),
115
+ reinterpret_cast<long*>(input_positions.data_ptr()),
116
+ reinterpret_cast<int*>(seq_lens.data_ptr()),
117
+ reinterpret_cast<long*>(slot_mapping.data_ptr()),
118
+ reinterpret_cast<int const*>(block_tables.data_ptr()),
119
+ block_tables.stride(0));
120
+}
121
+
122
+} // namespace prepare_inputs
123
+
124
+void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
125
+ torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
126
+ torch::Tensor& input_positions, torch::Tensor& seq_lens,
127
+ torch::Tensor& slot_mapping, torch::Tensor& block_tables) {
128
+ prepare_inputs::advance_step(num_seqs, num_queries, block_size, input_tokens,
129
+ sampled_token_ids, input_positions, seq_lens,
130
+ slot_mapping, block_tables);
131
+}
\ No newline at end of file
csrc/prepare_inputs/advance_step.cuh
new
+19
@@ -0,0 +1,19 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+
5
+#include <ATen/cuda/CUDAContext.h>
6
+#include <c10/cuda/CUDAGuard.h>
7
+#include <cuda.h>
8
+#include <cuda_fp16.h>
9
+#include <cuda_runtime.h>
10
+#include <iostream>
11
+
12
+namespace prepare_inputs {
13
+
14
+static constexpr int max_threads = 256;
15
+static constexpr bool logging = false;
16
+
17
+constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
18
+
19
+} // namespace prepare_inputs
csrc/torch_bindings.cpp
+4
@@ -72,6 +72,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
72
ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
73
ops.impl("gelu_quick", torch::kCUDA, &gelu_quick);
74
75
+ // prepare_inputs advance_step
76
+ ops.def("advance_step", &advance_step);
77
+ ops.impl("advance_step", torch::kCUDA, &advance_step);
78
+
79
// Layernorm
80
// Apply Root Mean Square (RMS) Normalization to the input tensor.
81
ops.def(
tests/spec_decode/e2e/conftest.py
+1
@@ -227,6 +227,7 @@ def get_output_from_llm_generator(
227
maybe_assert_ngram_worker(llm)
228
229
outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
230
+
231
token_ids = [output.outputs[0].token_ids for output in outputs]
232
tokens = [output.outputs[0].text for output in outputs]
233
tests/spec_decode/test_multi_step_worker.py
+48
@@ -642,3 +642,51 @@ def test_draft_proposals_mixed_k():
642
assert proposals.proposal_lens.tolist() == [
643
k for _ in range(expected_num_proposal_seqs - 1)
644
] + [0 for _ in range(expected_num_no_proposal_seqs)] + [k]
645
+
646
+
647
+@torch.inference_mode()
648
+def test_use_draft_model_runner_advance_step():
649
+ """Verify that draft model runner triggers advance step
650
+ when applicable.
651
+ """
652
+ seed = 100
653
+ model_name = 'JackFram/llama-68m'
654
+
655
+ k = 5
656
+ batch_size = 32
657
+ block_size = 32
658
+ num_gpu_blocks = 2048 // block_size
659
+ worker = create_worker(
660
+ MultiStepWorker,
661
+ model_name,
662
+ block_size,
663
+ num_gpu_blocks,
664
+ seed,
665
+ model_runner_cls=TP1DraftModelRunner,
666
+ )
667
+
668
+ # Mock "_gpu_advance_step" to raise an exception when called.
669
+ exception_secret = "artificial stop"
670
+ worker.model_runner._gpu_advance_step = MagicMock()
671
+ worker.model_runner._gpu_advance_step.side_effect = ValueError(
672
+ exception_secret)
673
+
674
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
675
+
676
+ # Fallback (should not call) when num_steps=1.
677
+ execute_model_req = ExecuteModelRequest(
678
+ seq_group_metadata_list=seq_group_metadata_list,
679
+ num_lookahead_slots=k,
680
+ num_steps=1)
681
+ worker.execute_model(execute_model_req=execute_model_req)
682
+
683
+ # Expect exception if _gpu_advance_step is called.
684
+ execute_model_req = ExecuteModelRequest(
685
+ seq_group_metadata_list=seq_group_metadata_list,
686
+ num_lookahead_slots=k,
687
+ num_steps=k)
688
+
689
+ with pytest.raises(ValueError, match=exception_secret):
690
+ worker.execute_model(execute_model_req=execute_model_req)
691
+ call_args_list = worker.model_runner._gpu_advance_step.call_args_list
692
+ assert len(call_args_list) == 1
vllm/_custom_ops.py
+12
@@ -166,6 +166,18 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
166
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
167
168
169
+def advance_step(num_seqs: int, num_queries: int, block_size: int,
170
+ input_tokens: torch.Tensor, sampled_token_ids: torch.Tensor,
171
+ input_positions: torch.Tensor, seq_lens: torch.Tensor,
172
+ slot_mapping: torch.Tensor,
173
+ block_tables: torch.Tensor) -> None:
174
+ """Advance a step on GPU for existing inputs for a multi-step runner"""
175
+ return torch.ops._C.advance_step(num_seqs, num_queries, block_size,
176
+ input_tokens, sampled_token_ids,
177
+ input_positions, seq_lens, slot_mapping,
178
+ block_tables)
179
+
180
+
181
# quantization ops
182
# awq
183
def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
vllm/model_executor/layers/sampler.py
+100
-47
@@ -47,6 +47,32 @@ class Sampler(nn.Module):
47
# speculative decoding.
48
self.include_gpu_probs_tensor = False
49
50
+ def _init_sampling_tensors(
51
+ self,
52
+ logits: torch.Tensor,
53
+ sampling_metadata: SamplingMetadata,
54
+ ):
55
+ """The goal here is to reuse sampling tensors between similar decode
56
+ runs. This is possible because sampling logic does not change between
57
+ decodes of the same sequences.
58
+ """
59
+ _, vocab_size = logits.shape
60
+
61
+ # First free any existing stored sampling tensors.
62
+ # This is necessary because some sampling tensors may
63
+ # have pinned memory.
64
+ self._sampling_tensors = None
65
+
66
+ # Initialize new sampling tensors
67
+ (sampling_tensors, do_penalties, do_top_p_top_k,
68
+ do_min_p) = SamplingTensors.from_sampling_metadata(
69
+ sampling_metadata, vocab_size, logits.device, logits.dtype)
70
+
71
+ self._sampling_tensors = sampling_tensors
72
+ self._do_penalties = do_penalties
73
+ self._do_top_p_top_k = do_top_p_top_k
74
+ self._do_min_p = do_min_p
75
+
76
def forward(
77
self,
78
logits: torch.Tensor,
@@ -60,12 +86,23 @@ class Sampler(nn.Module):
86
assert logits is not None
87
_, vocab_size = logits.shape
88
63
- logits = _apply_min_tokens_penalty(logits, sampling_metadata)
64
-
89
# Prepare sampling tensors with pinned memory to avoid blocking.
66
- (sampling_tensors, do_penalties, do_top_p_top_k,
67
- do_min_p) = SamplingTensors.from_sampling_metadata(
68
- sampling_metadata, vocab_size, logits.device, logits.dtype)
90
+ if not sampling_metadata.reuse_sampling_tensors:
91
+ self._init_sampling_tensors(logits, sampling_metadata)
92
+ elif self._do_penalties:
93
+ # In this case, the sampling tensors logic depends on
94
+ # "output_tokens" of a sequence. As a result, we cannot
95
+ # reuse sampling tensors, since "output_tokens" changes
96
+ # between decode runs.
97
+ self._init_sampling_tensors(logits, sampling_metadata)
98
+
99
+ assert self._sampling_tensors is not None
100
+ sampling_tensors = self._sampling_tensors
101
+ do_penalties = self._do_penalties
102
+ do_top_p_top_k = self._do_top_p_top_k
103
+ do_min_p = self._do_min_p
104
+
105
+ logits = _apply_min_tokens_penalty(logits, sampling_metadata)
106
107
# Apply presence and frequency penalties.
108
if do_penalties:
@@ -77,7 +114,7 @@ class Sampler(nn.Module):
114
115
# Apply temperature scaling.
116
# Use in-place division to avoid creating a new tensor.
80
- logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1))
117
+ logits.div_(sampling_tensors.temperatures.unsqueeze(dim=1))
118
119
if do_top_p_top_k:
120
logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps,
@@ -109,13 +146,19 @@ class Sampler(nn.Module):
146
on_device_tensors = None
147
148
# Get the logprobs query results.
112
- prompt_logprobs, sample_logprobs = _get_logprobs(
113
- logprobs, sampling_metadata, sample_results)
114
- return _build_sampler_output(sample_results,
115
- sampling_metadata,
116
- prompt_logprobs,
117
- sample_logprobs,
118
- on_device_tensors=on_device_tensors)
149
+ prompt_logprobs = None
150
+ sample_logprobs = None
151
+ if not sampling_metadata.skip_sampler_cpu_output:
152
+ prompt_logprobs, sample_logprobs = _get_logprobs(
153
+ logprobs, sampling_metadata, sample_results)
154
+
155
+ return _build_sampler_output(
156
+ sample_results,
157
+ sampling_metadata,
158
+ prompt_logprobs,
159
+ sample_logprobs,
160
+ on_device_tensors=on_device_tensors,
161
+ skip_sampler_cpu_output=sampling_metadata.skip_sampler_cpu_output)
162
163
@property
164
def _should_modify_greedy_probs_inplace(self) -> bool:
@@ -535,24 +578,29 @@ def _sample_with_torch(
578
579
# GPU<->CPU sync happens in the loop below.
580
# This also converts the sample output to Python objects.
538
- for sampling_type in SamplingType:
539
- if sampling_type not in sample_metadata:
540
- continue
541
- (seq_group_id, seq_groups) = sample_metadata[sampling_type]
542
- if sampling_type == SamplingType.GREEDY:
543
- sample_results = _greedy_sample(seq_groups, greedy_samples)
544
- elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
545
- sample_results = _random_sample(seq_groups,
546
- multinomial_samples[sampling_type])
547
- elif sampling_type == SamplingType.BEAM:
548
- sample_results = _beam_search_sample(seq_groups,
549
- beam_search_logprobs)
550
- sample_results_dict.update(zip(seq_group_id, sample_results))
581
+ if not sampling_metadata.skip_sampler_cpu_output:
582
+ for sampling_type in SamplingType:
583
+ if sampling_type not in sample_metadata:
584
+ continue
585
+ (seq_group_id, seq_groups) = sample_metadata[sampling_type]
586
+ if sampling_type == SamplingType.GREEDY:
587
+ sample_results = _greedy_sample(seq_groups, greedy_samples)
588
+ elif sampling_type in (SamplingType.RANDOM,
589
+ SamplingType.RANDOM_SEED):
590
+ sample_results = _random_sample(
591
+ seq_groups, multinomial_samples[sampling_type])
592
+ elif sampling_type == SamplingType.BEAM:
593
+ sample_results = _beam_search_sample(seq_groups,
594
+ beam_search_logprobs)
595
+ sample_results_dict.update(zip(seq_group_id, sample_results))
596
+
597
+ sample_results = [
598
+ sample_results_dict.get(i, ([], []))
599
+ for i in range(len(sampling_metadata.seq_groups))
600
+ ]
601
+ else:
602
+ sample_results = []
603
552
- sample_results = [
553
- sample_results_dict.get(i, ([], []))
554
- for i in range(len(sampling_metadata.seq_groups))
555
- ]
604
return sample_results, sampled_token_ids_tensor
605
606
@@ -997,10 +1045,11 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
1045
def _build_sampler_output(
1046
sample_results: SampleResultType,
1047
sampling_metadata: SamplingMetadata,
1000
- prompt_logprobs: List[Optional[PromptLogprobs]],
1001
- sample_logprobs: List[SampleLogprobs],
1048
+ prompt_logprobs: Optional[List[Optional[PromptLogprobs]]],
1049
+ sample_logprobs: Optional[List[SampleLogprobs]],
1050
on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor,
1051
torch.Tensor]],
1052
+ skip_sampler_cpu_output: bool = False,
1053
) -> SamplerOutput:
1054
"""Construct Python objects with the output of sampling.
1055
@@ -1010,22 +1059,26 @@ def _build_sampler_output(
1059
allows post-processing without copies to CPU/serialization, e.g. in
1060
speculative decoding rejection sampling.
1061
"""
1013
-
1062
sampler_output: List[CompletionSequenceGroupOutput] = []
1015
- for (seq_group, sample_result, group_prompt_logprobs,
1016
- group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1017
- sample_results, prompt_logprobs,
1018
- sample_logprobs):
1019
- seq_ids = seq_group.seq_ids
1020
- next_token_ids, parent_ids = sample_result
1021
- seq_outputs: List[SequenceOutput] = []
1022
- for parent_id, next_token_id, logprobs in zip(parent_ids,
1023
- next_token_ids,
1024
- group_sample_logprobs):
1025
- seq_outputs.append(
1026
- SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
1027
- sampler_output.append(
1028
- CompletionSequenceGroupOutput(seq_outputs, group_prompt_logprobs))
1063
+ if not skip_sampler_cpu_output:
1064
+ assert prompt_logprobs is not None
1065
+ assert sample_logprobs is not None
1066
+
1067
+ for (seq_group, sample_result, group_prompt_logprobs,
1068
+ group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1069
+ sample_results, prompt_logprobs,
1070
+ sample_logprobs):
1071
+ seq_ids = seq_group.seq_ids
1072
+ next_token_ids, parent_ids = sample_result
1073
+ seq_outputs: List[SequenceOutput] = []
1074
+ for parent_id, next_token_id, logprobs in zip(
1075
+ parent_ids, next_token_ids, group_sample_logprobs):
1076
+ seq_outputs.append(
1077
+ SequenceOutput(seq_ids[parent_id], next_token_id,
1078
+ logprobs))
1079
+ sampler_output.append(
1080
+ CompletionSequenceGroupOutput(seq_outputs,
1081
+ group_prompt_logprobs))
1082
1083
# If not specified, store None values in SamplerOutput.
1084
if on_device_tensors is not None:
vllm/model_executor/sampling_metadata.py
+10
@@ -87,6 +87,12 @@ class SamplingMetadata:
87
The first tuple is [1, 2] (sampled index within original logit),
88
and the second tuple is [0, 1] (sampled index within pruned logit).
89
num_prompts: Number of prompt sequence groups in seq_groups.
90
+ skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU
91
+ serialization of token outputs.
92
+ reuse_sampling_tensors: Indicates if we want to reuse sampling
93
+ tensors that are part of the sampler forward pass. Currently,
94
+ it is mainly used for multi-step decode.
95
+
96
"""
97
98
def __init__(
@@ -95,11 +101,15 @@ class SamplingMetadata:
101
selected_token_indices: torch.Tensor,
102
categorized_sample_indices: Dict[SamplingType, torch.Tensor],
103
num_prompts: int,
104
+ skip_sampler_cpu_output: bool = False,
105
+ reuse_sampling_tensors: bool = False,
106
) -> None:
107
self.seq_groups = seq_groups
108
self.selected_token_indices = selected_token_indices
109
self.categorized_sample_indices = categorized_sample_indices
110
self.num_prompts = num_prompts
111
+ self.skip_sampler_cpu_output = skip_sampler_cpu_output
112
+ self.reuse_sampling_tensors = reuse_sampling_tensors
113
114
@staticmethod
115
def prepare(
vllm/spec_decode/draft_model_runner.py
+225
-80
@@ -2,17 +2,22 @@ from typing import List, Optional
2
3
import torch
4
5
+from vllm import _custom_ops as ops
6
+from vllm.attention.backends.flash_attn import FlashAttentionMetadata
7
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8
ModelConfig, MultiModalConfig, ParallelConfig,
9
PromptAdapterConfig, SchedulerConfig)
10
from vllm.logger import init_logger
9
-from vllm.sequence import (IntermediateTensors, SamplerOutput,
10
- SequenceGroupMetadata)
11
+from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
12
+ SamplerOutput)
13
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
14
ModelRunner)
15
16
logger = init_logger(__name__)
17
18
+debug_advance_input = False
19
+enable_gpu_advance_step = True
20
+
21
22
class TP1DraftModelRunner(ModelRunner):
23
"""Specialized model runner for speculative decoding draft model.
@@ -21,18 +26,9 @@ class TP1DraftModelRunner(ModelRunner):
26
we could get rid of most CPU-GPU synchronization and data transfer
27
overheads by keeping model input and output tensors on GPU all the time.
28
24
- This runner is still under development so there's no performance gain
25
- at this moment. Currently we adopt a temporary solution that caches the
26
- seq_group_metadata_list for multi-step execution, so that we can
27
- leverage existing prepare_model_input to be compatible with the current
28
- execution flow, but we plan to remove this cache and avoid calling
29
- prepare_model_input in execute_model at all.
30
-
31
- The detail development plan includes:
32
- 1. Use "update_model_input" to update existing model_input without
33
- creating a new one.
34
- 2. Improve the performance of "update_model_input" with a GPU kernel.
35
- 3. Support TP > 1 (this requires some designs because we do not expect
29
+ TODOs:
30
+ 1. Currently supports only flash-attn, add support for other attn_backends.
31
+ 2. Support TP > 1 (this requires some designs because we do not expect
32
any broadcasting inside execute_model).
33
"""
34
@@ -71,51 +67,156 @@ class TP1DraftModelRunner(ModelRunner):
67
return_hidden_states=return_hidden_states,
68
)
69
74
- # TODO: Remove this cache when we are able to update model_input
75
- # directly in advance_step.
76
- self.cached_seq_group_metadata_list: Optional[
77
- List[SequenceGroupMetadata]] = None
70
+ def _update_flash_attn_metadata(self, attn_metadata, num_seqs,
71
+ num_queries):
72
+ assert isinstance(attn_metadata, FlashAttentionMetadata)
73
79
- def prepare_model_input(
80
- self,
81
- seq_group_metadata_list: List[SequenceGroupMetadata],
82
- virtual_engine: int = 0,
83
- finished_requests_ids: Optional[List[str]] = None
84
- ) -> ModelInputForGPUWithSamplingMetadata:
85
- """A temporary solution that caches the seq_group_metadata_list
86
- for multi-step execution.
87
- TODO: In-place update model_input and remove this function.
88
- """
89
- self.cached_seq_group_metadata_list = seq_group_metadata_list
90
- return super().prepare_model_input(
91
- seq_group_metadata_list,
92
- finished_requests_ids=finished_requests_ids)
74
+ if num_seqs != num_queries:
75
+ assert num_seqs > num_queries
76
+ assert attn_metadata.use_cuda_graph
77
+
78
+ assert attn_metadata.num_prefills == 0
79
+ assert attn_metadata.num_prefill_tokens == 0
80
+ assert attn_metadata.num_decode_tokens == num_seqs
81
+ assert attn_metadata.slot_mapping.shape == (num_seqs, )
82
+
83
+ assert len(attn_metadata.seq_lens) == num_seqs
84
+ assert attn_metadata.seq_lens_tensor.shape == (num_seqs, )
85
+ assert attn_metadata.max_query_len == 1
86
+ assert attn_metadata.max_prefill_seq_len == 0
87
+ assert attn_metadata.max_decode_seq_len == max(attn_metadata.seq_lens)
88
+
89
+ assert attn_metadata.query_start_loc.shape == (num_queries + 1, )
90
+ assert attn_metadata.seq_start_loc.shape == (num_seqs + 1, )
91
+
92
+ assert attn_metadata.context_lens_tensor.shape == (num_queries, )
93
+
94
+ assert attn_metadata.block_tables.shape[0] == num_seqs
95
+
96
+ # Update query lengths. Note that we update only queries and not seqs,
97
+ # since tensors may be padded due to captured cuda graph batch size
98
+ for i in range(num_queries):
99
+ attn_metadata.seq_lens[i] += 1
100
+ attn_metadata.max_decode_seq_len = max(attn_metadata.seq_lens)
101
94
- def update_model_input(
102
+ def _update_sampling_metadata(self, sampling_metadata, num_seqs,
103
+ num_queries):
104
+
105
+ assert sampling_metadata.num_prompts == 0
106
+ assert len(sampling_metadata.seq_groups) == num_queries
107
+ assert sampling_metadata.selected_token_indices.shape == (
108
+ num_queries, )
109
+ # assert sampling_metadata.categorized_sample_indices == TODO: Add if needed # noqa: E501
110
+
111
+ # Verify that all sequences are decodes
112
+ for i in range(num_queries):
113
+ seq_group = sampling_metadata.seq_groups[i]
114
+
115
+ assert seq_group.is_prompt is False # No prompt
116
+ assert seq_group.prompt_logprob_indices == [] # No prompt
117
+ assert seq_group.sample_indices == [i] # Simple
118
+ assert seq_group.seq_len is None # Decode
119
+ assert seq_group.query_len is None # Decode
120
+
121
+ def _gpu_advance_step(
122
self, model_input: ModelInputForGPUWithSamplingMetadata,
123
last_output: SamplerOutput
124
) -> ModelInputForGPUWithSamplingMetadata:
98
- """Prepare the model inputs for the next step.
99
- TODO: In-place update model_input instead of calling
100
- prepare_model_input.
125
+ # Currently, we expect "decode mode" only
126
+ assert not model_input.is_prompt
127
+
128
+ # Get num_seqs
129
+ num_seqs = len(model_input.seq_lens)
130
+ num_queries = len(model_input.query_lens)
131
+
132
+ # Get output tokens GPU tensor
133
+ sampled_token_ids = last_output.sampled_token_ids
134
+ assert sampled_token_ids is not None
135
+
136
+ # Update attn_metadata
137
+ attn_metadata = model_input.attn_metadata
138
+ assert isinstance(attn_metadata, FlashAttentionMetadata)
139
+ self._update_flash_attn_metadata(attn_metadata, num_seqs, num_queries)
140
+
141
+ # Update GPU tensors
142
+ ops.advance_step(num_seqs=num_seqs,
143
+ num_queries=num_queries,
144
+ block_size=self.block_size,
145
+ input_tokens=model_input.input_tokens,
146
+ sampled_token_ids=sampled_token_ids,
147
+ input_positions=model_input.input_positions,
148
+ seq_lens=attn_metadata.seq_lens_tensor,
149
+ slot_mapping=attn_metadata.slot_mapping,
150
+ block_tables=attn_metadata.block_tables)
151
+
152
+ # Update sampling_metadata
153
+ sampling_metadata = model_input.sampling_metadata
154
+ self._update_sampling_metadata(sampling_metadata, num_seqs,
155
+ num_queries)
156
+
157
+ # Create new input
158
+ new_model_input = self._model_input_cls(
159
+ input_tokens=model_input.input_tokens,
160
+ input_positions=model_input.input_positions,
161
+ attn_metadata=attn_metadata,
162
+ seq_lens=attn_metadata.seq_lens,
163
+ query_lens=model_input.query_lens,
164
+ lora_mapping=model_input.lora_mapping,
165
+ lora_requests=model_input.lora_requests,
166
+ multi_modal_kwargs=model_input.multi_modal_kwargs,
167
+ sampling_metadata=model_input.sampling_metadata,
168
+ is_prompt=False,
169
+ )
170
+
171
+ # Ensure we skip CPU samples
172
+ assert new_model_input.sampling_metadata.skip_sampler_cpu_output is True
173
+ # We can reuse sampling tensors since every decode iteration is the same
174
+ new_model_input.sampling_metadata.reuse_sampling_tensors = True
175
+
176
+ if debug_advance_input:
177
+ logger.debug("NEW INPUT: ")
178
+ logger.debug(" input_tokens = %s", new_model_input.input_tokens)
179
+ logger.debug(" input_positions = %s",
180
+ new_model_input.input_positions)
181
+ logger.debug(" seq_lens = %d", new_model_input.seq_lens)
182
+ logger.debug(" query_lens = %d", new_model_input.query_lens)
183
+ logger.debug(" attn_metadata:")
184
+ logger.debug(" seq_lens_tensor: %s",
185
+ attn_metadata.seq_lens_tensor)
186
+ logger.debug(" slot_mapping: %s", attn_metadata.slot_mapping)
187
+ logger.debug(" block_tables: %s", attn_metadata.block_tables)
188
+
189
+ return new_model_input
190
+
191
+ def supports_gpu_multi_step(self, execute_model_req: ExecuteModelRequest):
192
+ """Determines if draft_model_runner GPU multi-step can be used.
193
+ Currently required conditions are:
194
+ 1. Only decodes
195
+ 2. Only flash-attn
196
+ 3. No LORA
197
+ 4. No prompt_adapter_config
198
"""
199
+ if not enable_gpu_advance_step:
200
+ return False
201
103
- # Append the output token to the sequence data.
104
- assert self.cached_seq_group_metadata_list is not None
105
- for seq_group_metadata, sequence_group_outputs in zip(
106
- self.cached_seq_group_metadata_list, last_output.outputs):
107
- seq_group_metadata.is_prompt = False
202
+ # We allow multi-step GPU only in decode mode
203
+ for seq_group in execute_model_req.seq_group_metadata_list:
204
+ if seq_group.is_prompt:
205
+ return False
206
109
- for seq_output in sequence_group_outputs.samples:
110
- seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
207
+ # TODO: Add support for other attn backends
208
+ if self.attn_backend.get_name() != "flash-attn":
209
+ return False
210
112
- token_id = seq_output.output_token
113
- token_logprob = seq_output.logprobs[token_id]
211
+ # TODO: Add support for LORA
212
+ if self.lora_config:
213
+ return False
214
115
- seq.append_token_id(token_id, token_logprob.logprob)
116
- seq.update_num_computed_tokens(1)
215
+ # TODO: Add soft-tuning prompt adapter support
216
+ if self.prompt_adapter_config:
217
+ return False
218
118
- return self.prepare_model_input(self.cached_seq_group_metadata_list)
219
+ return True
220
221
@torch.inference_mode()
222
def execute_model(
@@ -125,42 +226,86 @@ class TP1DraftModelRunner(ModelRunner):
226
intermediate_tensors: Optional[IntermediateTensors] = None,
227
num_steps: int = 1,
228
) -> Optional[List[SamplerOutput]]:
128
- # Since we do not broadcast data inside execute_model anymore,
129
- # we need to figure out the best way to support TP > 1 in this
130
- # case, because we will at least need to broadcast the sampled
131
- # tokens to all workers.
132
- if not self.is_driver_worker:
133
- raise ValueError("TP1DraftModelRunner only supports TP=1.")
229
+ """Executes num_steps forward passes with advacement of input tensors
230
+ on the GPU. Look at supports_gpu_multi_step(..) for pre-conditions.
231
135
- if self.lora_config:
136
- assert model_input.lora_requests is not None
137
- assert model_input.lora_mapping is not None
138
- self.set_active_loras(model_input.lora_requests,
139
- model_input.lora_mapping)
232
+ Optimizations used:
233
+ 1. Input tensors are updated on the GPU directly
234
+ 2. Skips GPU=>CPU serialization of sampler outputs (we don't need
235
+ them since we do batch expansion later that uses GPU outputs)
236
+ 3. Reuses sampling tensors (since we run only decodes and they have
237
+ a repeating sampling logic)
238
+ """
239
141
- if self.prompt_adapter_config:
142
- assert model_input.prompt_adapter_requests is not None
143
- assert model_input.prompt_adapter_mapping is not None
144
- self.set_active_prompt_adapters(
145
- model_input.prompt_adapter_requests,
146
- model_input.prompt_adapter_mapping)
240
+ # When num_steps == 1, we execute the fallback here for the GPU
241
+ # advance_step, which runs prepare_inputs on CPU and for each spec
242
+ # iteration invokes this function only once
243
+ # (Look at multi-step-worker code)
244
+ is_fallback = num_steps == 1
245
+ if not is_fallback:
246
+ # Since we do not broadcast data inside execute_model anymore,
247
+ # we need to figure out the best way to support TP > 1 in this
248
+ # case, because we will at least need to broadcast the sampled
249
+ # tokens to all workers.
250
+ if not self.is_driver_worker:
251
+ raise ValueError("TP1DraftModelRunner only supports TP=1.")
252
+
253
+ # Sanity
254
+ if self.lora_config is not None:
255
+ raise ValueError("TP1DraftModelRunner has no support for LORA")
256
+ if self.prompt_adapter_config is not None:
257
+ raise ValueError("TP1DraftModelRunner has no support for "
258
+ "prompt_adapter_config")
259
+ if model_input.multi_modal_kwargs:
260
+ raise ValueError(
261
+ "TP1DraftModelRunner has no support for multi_modal_kwargs"
262
+ )
263
+ else:
264
+ if self.lora_config:
265
+ assert model_input.lora_requests is not None
266
+ assert model_input.lora_mapping is not None
267
+ self.set_active_loras(model_input.lora_requests,
268
+ model_input.lora_mapping)
269
+
270
+ if self.prompt_adapter_config:
271
+ assert model_input.prompt_adapter_requests is not None
272
+ assert model_input.prompt_adapter_mapping is not None
273
+ self.set_active_prompt_adapters(
274
+ model_input.prompt_adapter_requests,
275
+ model_input.prompt_adapter_mapping)
276
+
277
+ # Detect exec mode
278
+ assert model_input.attn_metadata is not None
279
+ use_cuda_graph = False
280
+ if model_input.attn_metadata.num_prefills > 0:
281
+ # In this case, execute_model(..) was called directly
282
+ if num_steps > 1:
283
+ raise ValueError(
284
+ "execute_model(..) of draft_model_runner can be called "
285
+ "directly only with a single-step prefill")
286
+ else:
287
+ # We can skip CPU samples for spec token generation.
288
+ # (We do allow CPU samples for num_steps == 1 to support the
289
+ # fallback case, where supports_gpu_multi_step(..) does not pass)
290
+ model_input.sampling_metadata.skip_sampler_cpu_output = (
291
+ not is_fallback)
292
+
293
+ # Attn attr defines if we use cuda graphs
294
+ use_cuda_graph = model_input.attn_metadata.use_cuda_graph
295
+
296
+ # Get model
297
+ if use_cuda_graph:
298
+ graph_batch_size = model_input.input_tokens.shape[0]
299
+ model_executable = (self.graph_runners[model_input.virtual_engine]
300
+ [graph_batch_size])
301
+ else:
302
+ model_executable = self.model
303
148
- virtual_engine = model_input.virtual_engine
304
outputs: List[SamplerOutput] = []
305
for step in range(num_steps):
151
- # Currently cuda graph is only supported by the decode phase.
152
- assert model_input.attn_metadata is not None
153
- prefill_meta = model_input.attn_metadata.prefill_metadata
154
- decode_meta = model_input.attn_metadata.decode_metadata
155
- if prefill_meta is None and decode_meta.use_cuda_graph:
156
- assert model_input.input_tokens is not None
157
- graph_batch_size = model_input.input_tokens.shape[0]
158
- model_executable = (
159
- self.graph_runners[virtual_engine][graph_batch_size])
160
- else:
161
- model_executable = self.model
162
-
306
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
307
+
308
+ # Run model
309
hidden_states = model_executable(
310
input_ids=model_input.input_tokens,
311
positions=model_input.input_positions,
@@ -181,8 +326,8 @@ class TP1DraftModelRunner(ModelRunner):
326
sampling_metadata=model_input.sampling_metadata,
327
))
328
184
- # Prepare the inputs for the next step.
329
+ # Prepare inputs for the next step
330
if step != num_steps - 1:
186
- model_input = self.update_model_input(model_input, outputs[-1])
331
+ model_input = self._gpu_advance_step(model_input, outputs[-1])
332
333
return outputs
vllm/spec_decode/multi_step_worker.py
+12
-3
@@ -67,14 +67,23 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
67
expanded_request, indices_of_seq_with_bonus_tokens =\
68
self._expand_execute_model_request(
69
execute_model_req, seq_ids_with_bonus_token_in_last_step)
70
+
71
# Run model sample_len times.
72
model_outputs: List[SamplerOutput] = []
72
- if isinstance(self.model_runner, TP1DraftModelRunner):
73
+ if isinstance(
74
+ self.model_runner, TP1DraftModelRunner
75
+ ) and self.model_runner.supports_gpu_multi_step(expanded_request):
76
+ # Here we run the draft_model_runner with multi-step prepare
77
+ # on the GPU directly
78
expanded_request.num_steps = sample_len
79
model_outputs = self.execute_model(
80
execute_model_req=expanded_request)
81
else:
77
- # TODO: Remove this branch once DraftModelRunner supports TP>1.
82
+ # Here we run multi-step directly, with every step prepared
83
+ # on the CPU.
84
+ # TODO: Remove this branch once DraftModelRunner supports TP>1
85
+ # and other restrictions that are part of DraftModelRunner's
86
+ # supports_gpu_multi_step(..)
87
for _ in range(sample_len):
88
model_output: List[SamplerOutput] = super().execute_model(
89
execute_model_req=expanded_request)
@@ -171,7 +180,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
180
outputs=[
181
expanded_batch_output.outputs[i]
182
for i in output_indices_to_retain
174
- ],
183
+ ] if len(expanded_batch_output.outputs) > 0 else [],
184
sampled_token_probs=(
185
expanded_batch_output.
186
sampled_token_probs[output_indices_to_retain]