A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[multi-step] add flashinfer backend (#7928)
William Lin committed
Sep 12, 2024 at 11:16 UTC
a6c0f3658da4f2f23460e3e15bfa7d70ac7e60c1
9 files changed
+371
-84
csrc/ops.h
+15
-4
@@ -54,10 +54,21 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input);
54
55
void gelu_quick(torch::Tensor& out, torch::Tensor& input);
56
57
-void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
58
- torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
59
- torch::Tensor& input_positions, torch::Tensor& seq_lens,
60
- torch::Tensor& slot_mapping, torch::Tensor& block_tables);
57
+void advance_step_flashattn(int64_t num_seqs, int64_t num_queries,
58
+ int64_t block_size, torch::Tensor& input_tokens,
59
+ torch::Tensor& sampled_token_ids,
60
+ torch::Tensor& input_positions,
61
+ torch::Tensor& seq_lens,
62
+ torch::Tensor& slot_mapping,
63
+ torch::Tensor& block_tables);
64
+
65
+void advance_step_flashinfer(
66
+ int64_t num_seqs, int64_t num_queries, int64_t block_size,
67
+ torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
68
+ torch::Tensor& input_positions, torch::Tensor& seq_lens,
69
+ torch::Tensor& slot_mapping, torch::Tensor& block_tables,
70
+ torch::Tensor& paged_kv_indices, torch::Tensor& paged_kv_indptr,
71
+ torch::Tensor& paged_kv_last_page_len, torch::Tensor& block_table_bounds);
72
73
#ifndef USE_ROCM
74
torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
csrc/prepare_inputs/advance_step.cu
+200
-25
@@ -12,13 +12,11 @@ namespace prepare_inputs {
12
13
//
14
template <int const num_threads>
15
-__global__ void advance_step_kernel(int num_seqs, int num_queries,
16
- int block_size, long* input_tokens_ptr,
17
- long const* sampled_token_ids_ptr,
18
- long* input_positions_ptr,
19
- int* seq_lens_ptr, long* slot_mapping_ptr,
20
- int const* block_tables_ptr,
21
- int64_t const block_tables_stride) {
15
+__global__ void advance_step_flashattn_kernel(
16
+ int num_seqs, int num_queries, int block_size, long* input_tokens_ptr,
17
+ long const* sampled_token_ids_ptr, long* input_positions_ptr,
18
+ int* seq_lens_ptr, long* slot_mapping_ptr, int const* block_tables_ptr,
19
+ int64_t const block_tables_stride) {
20
int num_query_blocks = div_ceil(num_queries, num_threads);
21
22
if (blockIdx.x >= num_query_blocks) {
@@ -79,16 +77,91 @@ inline void verify_tensor(std::string const& name, torch::Tensor& t,
77
}
78
}
79
82
-void advance_step(int num_seqs, int num_queries, int block_size,
83
- torch::Tensor& input_tokens, // type: long
84
- torch::Tensor& sampled_token_ids, // type: long
85
- torch::Tensor& input_positions, // type: long
86
- torch::Tensor& seq_lens, // type: int
87
- torch::Tensor& slot_mapping, // type: long
88
- torch::Tensor& block_tables) { // type: int
80
+__global__ void advance_step_flashinfer_kernel(
81
+ int num_threads, int num_seqs, int num_queries, int block_size,
82
+ long* input_tokens_ptr, long const* sampled_token_ids_ptr,
83
+ long* input_positions_ptr, int* seq_lens_ptr, long* slot_mapping_ptr,
84
+ int const* block_tables_ptr, int64_t const block_tables_stride,
85
+ int* paged_kv_last_page_len_ptr, int* block_table_bound_ptr) {
86
+ int num_query_blocks = div_ceil(num_queries, num_threads);
87
+
88
+ if (blockIdx.x < num_query_blocks) {
89
+ int cur_query_id = blockIdx.x * num_threads + threadIdx.x;
90
+
91
+ if (cur_query_id < num_queries) {
92
+ // Update input_tokens
93
+ input_tokens_ptr[cur_query_id] = sampled_token_ids_ptr[cur_query_id];
94
+
95
+ int seq_len = seq_lens_ptr[cur_query_id];
96
+ int next_seq_len = seq_len + 1;
97
+ int next_input_pos = next_seq_len - 1;
98
+
99
+ // Update seq_lens
100
+ seq_lens_ptr[cur_query_id] = next_seq_len;
101
+ // Update input_positions
102
+ input_positions_ptr[cur_query_id] = next_input_pos;
103
+
104
+ int const* seq_block_tables_ptr =
105
+ block_tables_ptr + block_tables_stride * cur_query_id;
106
+
107
+ int block_index = next_input_pos / block_size;
108
+ int block_offset = next_input_pos % block_size;
109
+
110
+ // Update paged_kv_last_page_len
111
+ paged_kv_last_page_len_ptr[cur_query_id] = block_offset + 1;
112
+
113
+ int slot_num =
114
+ seq_block_tables_ptr[block_index] * block_size + block_offset;
115
+ // Update slot_mapping
116
+ slot_mapping_ptr[cur_query_id] = slot_num;
117
+ block_table_bound_ptr[cur_query_id] = div_ceil(next_seq_len, block_size);
118
+ }
119
+ }
120
+}
121
+
122
+__global__ void advance_step_flashinfer_indptr_kernel(
123
+ int num_threads, int num_seqs, int num_queries, int* paged_kv_indptr_ptr,
124
+ int* block_table_bound_ptr) {
125
+ int idx = blockIdx.x * num_threads + threadIdx.x;
126
+
127
+ // Update paged_kv_indptr
128
+ if (idx < num_queries) {
129
+ int sum = 0;
130
+ for (int i = 0; i <= idx; ++i) {
131
+ sum += block_table_bound_ptr[i];
132
+ }
133
+ paged_kv_indptr_ptr[idx + 1] = sum;
134
+ }
135
+}
136
+
137
+__global__ void advance_step_flashinfer_indices_kernel(
138
+ int num_threads, int num_seqs, int num_queries, int const* block_tables_ptr,
139
+ int64_t const block_tables_stride, int* paged_kv_indices_ptr,
140
+ int* paged_kv_indptr_ptr, int* block_table_bound_ptr) {
141
+ int idx = blockIdx.x * num_threads + threadIdx.x;
142
+ int row = idx / block_tables_stride;
143
+ int col = idx % block_tables_stride;
144
+
145
+ if (row < num_queries && col < block_table_bound_ptr[row]) {
146
+ paged_kv_indices_ptr[paged_kv_indptr_ptr[row] + col] =
147
+ block_tables_ptr[row * block_tables_stride + col];
148
+ }
149
+ // if cudagraph, fill padded seqs with the last valid seq's indptr
150
+ if (num_queries < row && row <= num_seqs) {
151
+ paged_kv_indptr_ptr[row] = paged_kv_indptr_ptr[num_queries];
152
+ }
153
+}
154
+
155
+void advance_step_flashattn(int num_seqs, int num_queries, int block_size,
156
+ torch::Tensor& input_tokens, // type: long
157
+ torch::Tensor& sampled_token_ids, // type: long
158
+ torch::Tensor& input_positions, // type: long
159
+ torch::Tensor& seq_lens, // type: int
160
+ torch::Tensor& slot_mapping, // type: long
161
+ torch::Tensor& block_tables) { // type: int
162
163
if (logging) {
91
- printf("advance_step:\n");
164
+ printf("advance_step_flashattn:\n");
165
printf(" num_seqs = %d\n", num_seqs);
166
printf(" num_queries = %d\n", num_queries);
167
printf(" block_size = %d\n", block_size);
@@ -108,24 +181,126 @@ void advance_step(int num_seqs, int num_queries, int block_size,
181
int blocks;
182
cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
183
111
- advance_step_kernel<max_threads><<<blocks, max_threads, 0, stream>>>(
112
- num_seqs, num_queries, block_size,
184
+ advance_step_flashattn_kernel<max_threads>
185
+ <<<blocks, max_threads, 0, stream>>>(
186
+ num_seqs, num_queries, block_size,
187
+ reinterpret_cast<long*>(input_tokens.data_ptr()),
188
+ reinterpret_cast<long const*>(sampled_token_ids.data_ptr()),
189
+ reinterpret_cast<long*>(input_positions.data_ptr()),
190
+ reinterpret_cast<int*>(seq_lens.data_ptr()),
191
+ reinterpret_cast<long*>(slot_mapping.data_ptr()),
192
+ reinterpret_cast<int const*>(block_tables.data_ptr()),
193
+ block_tables.stride(0));
194
+}
195
+
196
+void advance_step_flashinfer(
197
+ int num_seqs, int num_queries, int block_size,
198
+ torch::Tensor& input_tokens, // type: long
199
+ torch::Tensor& sampled_token_ids, // type: long
200
+ torch::Tensor& input_positions, // type: long
201
+ torch::Tensor& seq_lens, // type: int
202
+ torch::Tensor& slot_mapping, // type: long
203
+ torch::Tensor& block_tables, // type: int
204
+ torch::Tensor& paged_kv_indices, // type: int
205
+ torch::Tensor& paged_kv_indptr, // type: int
206
+ torch::Tensor& paged_kv_last_page_len, // type: int
207
+ torch::Tensor& block_table_bound) { // type: int
208
+
209
+ if (logging) {
210
+ printf("advance_step_flashinfer:\n");
211
+ printf(" num_seqs = %d\n", num_seqs);
212
+ printf(" num_queries = %d\n", num_queries);
213
+ printf(" block_size = %d\n", block_size);
214
+ printf(" block_tables.stride(0) = %d\n", block_tables.stride(0));
215
+ }
216
+ // Verify all tensors
217
+ verify_tensor("input_tokens", input_tokens, num_seqs, -1, at::kLong);
218
+ // verify_tensor("sampled_token_ids", sampled_token_ids, num_queries, 1,
219
+ // at::kLong);
220
+ verify_tensor("input_positions", input_positions, num_seqs, -1, at::kLong);
221
+ verify_tensor("seq_lens", seq_lens, num_seqs, -1, at::kInt);
222
+ verify_tensor("slot_mapping", slot_mapping, num_seqs, -1, at::kLong);
223
+ verify_tensor("block_tables", block_tables, num_seqs, -1, at::kInt);
224
+
225
+ verify_tensor("paged_kv_indices", paged_kv_indices, -1, -1, at::kInt);
226
+ verify_tensor("paged_kv_indptr", paged_kv_indptr, num_seqs + 1, -1, at::kInt);
227
+ verify_tensor("paged_kv_last_page_len", paged_kv_last_page_len, num_seqs, -1,
228
+ at::kInt);
229
+
230
+ verify_tensor("block_table_bound", block_table_bound, num_seqs, -1, at::kInt);
231
+
232
+ int dev = sampled_token_ids.get_device();
233
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
234
+
235
+ int blocks;
236
+ int threads;
237
+ cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
238
+ cudaDeviceGetAttribute(&threads, cudaDevAttrMaxThreadsPerBlock, dev);
239
+ if (logging) {
240
+ printf("launching kernel with %d blocks\n", blocks);
241
+ }
242
+
243
+ // TODO(will): support arbitrary block_tables stride
244
+ if ((blocks * threads) / block_tables.stride(0) < num_queries) {
245
+ TORCH_CHECK(false,
246
+ "multi-step: not enough threads to map block_table to"
247
+ "FlashInfer's paged_kv_indices on GPU. Try reducing the number "
248
+ "of seqs,",
249
+ " increasing the block size or take smaller steps.",
250
+ " num_queries = ", num_queries,
251
+ " block_tables.stride(0) = ", block_tables.stride(0),
252
+ " blocks = ", blocks, " max_threads = ", threads);
253
+ }
254
+
255
+ advance_step_flashinfer_kernel<<<blocks, threads, 0, stream>>>(
256
+ threads, num_seqs, num_queries, block_size,
257
reinterpret_cast<long*>(input_tokens.data_ptr()),
258
reinterpret_cast<long const*>(sampled_token_ids.data_ptr()),
259
reinterpret_cast<long*>(input_positions.data_ptr()),
260
reinterpret_cast<int*>(seq_lens.data_ptr()),
261
reinterpret_cast<long*>(slot_mapping.data_ptr()),
262
reinterpret_cast<int const*>(block_tables.data_ptr()),
119
- block_tables.stride(0));
263
+ block_tables.stride(0),
264
+ reinterpret_cast<int*>(paged_kv_last_page_len.data_ptr()),
265
+ reinterpret_cast<int*>(block_table_bound.data_ptr()));
266
+
267
+ advance_step_flashinfer_indptr_kernel<<<blocks, threads, 0, stream>>>(
268
+ threads, num_seqs, num_queries,
269
+ reinterpret_cast<int*>(paged_kv_indptr.data_ptr()),
270
+ reinterpret_cast<int*>(block_table_bound.data_ptr()));
271
+
272
+ advance_step_flashinfer_indices_kernel<<<blocks, threads, 0, stream>>>(
273
+ threads, num_seqs, num_queries,
274
+ reinterpret_cast<int const*>(block_tables.data_ptr()),
275
+ block_tables.stride(0),
276
+ reinterpret_cast<int*>(paged_kv_indices.data_ptr()),
277
+ reinterpret_cast<int*>(paged_kv_indptr.data_ptr()),
278
+ reinterpret_cast<int*>(block_table_bound.data_ptr()));
279
}
280
281
} // namespace prepare_inputs
282
124
-void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
125
- torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
126
- torch::Tensor& input_positions, torch::Tensor& seq_lens,
127
- torch::Tensor& slot_mapping, torch::Tensor& block_tables) {
128
- prepare_inputs::advance_step(num_seqs, num_queries, block_size, input_tokens,
129
- sampled_token_ids, input_positions, seq_lens,
130
- slot_mapping, block_tables);
283
+void advance_step_flashattn(int64_t num_seqs, int64_t num_queries,
284
+ int64_t block_size, torch::Tensor& input_tokens,
285
+ torch::Tensor& sampled_token_ids,
286
+ torch::Tensor& input_positions,
287
+ torch::Tensor& seq_lens,
288
+ torch::Tensor& slot_mapping,
289
+ torch::Tensor& block_tables) {
290
+ prepare_inputs::advance_step_flashattn(
291
+ num_seqs, num_queries, block_size, input_tokens, sampled_token_ids,
292
+ input_positions, seq_lens, slot_mapping, block_tables);
293
+}
294
+
295
+void advance_step_flashinfer(
296
+ int64_t num_seqs, int64_t num_queries, int64_t block_size,
297
+ torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
298
+ torch::Tensor& input_positions, torch::Tensor& seq_lens,
299
+ torch::Tensor& slot_mapping, torch::Tensor& block_tables,
300
+ torch::Tensor& paged_kv_indices, torch::Tensor& paged_kv_indptr,
301
+ torch::Tensor& paged_kv_last_page_len, torch::Tensor& block_table_bound) {
302
+ prepare_inputs::advance_step_flashinfer(
303
+ num_seqs, num_queries, block_size, input_tokens, sampled_token_ids,
304
+ input_positions, seq_lens, slot_mapping, block_tables, paged_kv_indices,
305
+ paged_kv_indptr, paged_kv_last_page_len, block_table_bound);
306
}
\ No newline at end of file
csrc/torch_bindings.cpp
+13
-2
@@ -74,11 +74,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
74
75
// prepare_inputs advance_step
76
ops.def(
77
- "advance_step(int num_seqs, int num_queries, int block_size, "
77
+ "advance_step_flashattn(int num_seqs, int num_queries, int block_size, "
78
"Tensor! input_tokens, Tensor sampled_token_ids, "
79
"Tensor! input_positions, Tensor! seq_lens, Tensor! slot_mapping, "
80
"Tensor block_tables) -> ()");
81
- ops.impl("advance_step", torch::kCUDA, &advance_step);
81
+ ops.impl("advance_step_flashattn", torch::kCUDA, &advance_step_flashattn);
82
+
83
+ ops.def(
84
+ "advance_step_flashinfer("
85
+ " int num_seqs, int num_queries, int block_size,"
86
+ " Tensor! input_tokens, Tensor sampled_token_ids,"
87
+ " Tensor! input_positions, Tensor! seq_lens, Tensor! slot_mapping,"
88
+ " Tensor block_tables, Tensor! paged_kv_indices,"
89
+ " Tensor! paged_kv_indptr, Tensor! paged_kv_last_page_len,"
90
+ " Tensor! block_table_bounds"
91
+ ") -> ()");
92
+ ops.impl("advance_step_flashinfer", torch::kCUDA, &advance_step_flashinfer);
93
94
// Layernorm
95
// Apply Root Mean Square (RMS) Normalization to the input tensor.
tests/multi_step/test_correctness_async_llm.py
+9
-3
@@ -1,9 +1,10 @@
1
# Test the AsyncLLMEngine with multi-step-decoding
2
-
2
from typing import List, Optional
3
4
import pytest
5
6
+from tests.kernels.utils import override_backend_env_variable
7
+
8
from ..models.utils import check_logprobs_close
9
from ..utils import (completions_with_server_args, get_client_text_generations,
10
get_client_text_logprob_generations)
@@ -33,8 +34,9 @@ DEFAULT_SERVER_ARGS: List[str] = [
34
@pytest.mark.parametrize("eager_mode", [False, True])
35
@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
36
@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
36
-@pytest.mark.parametrize("num_logprobs", [None, 5])
37
-@pytest.mark.parametrize("is_async", [False, True])
37
+@pytest.mark.parametrize("num_logprobs", [5])
38
+@pytest.mark.parametrize("is_async", [True])
39
+@pytest.mark.parametrize("attention_backend", ["FLASHINFER", "FLASH_ATTN"])
40
@pytest.mark.asyncio
41
async def test_multi_step(
42
example_prompts,
@@ -46,6 +48,8 @@ async def test_multi_step(
48
num_prompts: int,
49
is_async: bool,
50
num_logprobs: Optional[int],
51
+ attention_backend: str,
52
+ monkeypatch,
53
) -> None:
54
"""Test vLLM engine with multi-step scheduling in an OpenAI-protocol
55
client/server environment.
@@ -71,6 +75,8 @@ async def test_multi_step(
75
completions endpoint; `None` -> no logprobs
76
"""
77
78
+ override_backend_env_variable(monkeypatch, attention_backend)
79
+
80
prompts = example_prompts
81
if len(prompts) < num_prompts:
82
prompts = prompts * ((num_prompts // len(prompts)) + 1)
vllm/_custom_ops.py
+29
-9
@@ -161,16 +161,36 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
161
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
162
163
164
-def advance_step(num_seqs: int, num_queries: int, block_size: int,
165
- input_tokens: torch.Tensor, sampled_token_ids: torch.Tensor,
166
- input_positions: torch.Tensor, seq_lens: torch.Tensor,
167
- slot_mapping: torch.Tensor,
168
- block_tables: torch.Tensor) -> None:
164
+def advance_step_flashattn(num_seqs: int, num_queries: int, block_size: int,
165
+ input_tokens: torch.Tensor,
166
+ sampled_token_ids: torch.Tensor,
167
+ input_positions: torch.Tensor,
168
+ seq_lens: torch.Tensor, slot_mapping: torch.Tensor,
169
+ block_tables: torch.Tensor) -> None:
170
"""Advance a step on GPU for existing inputs for a multi-step runner"""
170
- return torch.ops._C.advance_step(num_seqs, num_queries, block_size,
171
- input_tokens, sampled_token_ids,
172
- input_positions, seq_lens, slot_mapping,
173
- block_tables)
171
+ return torch.ops._C.advance_step_flashattn(num_seqs, num_queries,
172
+ block_size, input_tokens,
173
+ sampled_token_ids,
174
+ input_positions, seq_lens,
175
+ slot_mapping, block_tables)
176
+
177
+
178
+def advance_step_flashinfer(num_seqs: int, num_queries: int, block_size: int,
179
+ input_tokens: torch.Tensor,
180
+ sampled_token_ids: torch.Tensor,
181
+ input_positions: torch.Tensor,
182
+ seq_lens: torch.Tensor, slot_mapping: torch.Tensor,
183
+ block_tables: torch.Tensor,
184
+ paged_kv_indices: torch.Tensor,
185
+ paged_kv_indptr: torch.Tensor,
186
+ paged_kv_last_page_len: torch.Tensor,
187
+ block_table_bound: torch.Tensor) -> None:
188
+
189
+ return torch.ops._C.advance_step_flashinfer(
190
+ num_seqs, num_queries, block_size, input_tokens, sampled_token_ids,
191
+ input_positions, seq_lens, slot_mapping, block_tables,
192
+ paged_kv_indices, paged_kv_indptr, paged_kv_last_page_len,
193
+ block_table_bound)
194
195
196
# quantization ops
vllm/attention/backends/abstract.py
+3
-1
@@ -83,7 +83,9 @@ class AttentionBackend(ABC):
83
) -> None:
84
raise NotImplementedError
85
86
- def advance_step(self, num_seqs: int, num_queries: int):
86
+ def advance_step(self, model_input: "ModelRunnerInputBase",
87
+ sampled_token_ids: Optional[torch.Tensor],
88
+ block_size: int, num_seqs: int, num_queries: int) -> None:
89
raise NotImplementedError
90
91
vllm/attention/backends/flash_attn.py
+9
-9
@@ -380,15 +380,15 @@ class FlashAttentionMetadata(AttentionMetadata):
380
self.seq_lens[i] += 1
381
self.max_decode_seq_len = max(self.seq_lens)
382
383
- ops.advance_step(num_seqs=num_seqs,
384
- num_queries=num_queries,
385
- block_size=block_size,
386
- input_tokens=model_input.input_tokens,
387
- sampled_token_ids=sampled_token_ids,
388
- input_positions=model_input.input_positions,
389
- seq_lens=self.seq_lens_tensor,
390
- slot_mapping=self.slot_mapping,
391
- block_tables=self.block_tables)
383
+ ops.advance_step_flashattn(num_seqs=num_seqs,
384
+ num_queries=num_queries,
385
+ block_size=block_size,
386
+ input_tokens=model_input.input_tokens,
387
+ sampled_token_ids=sampled_token_ids,
388
+ input_positions=model_input.input_positions,
389
+ seq_lens=self.seq_lens_tensor,
390
+ slot_mapping=self.slot_mapping,
391
+ block_tables=self.block_tables)
392
393
394
class FlashAttentionMetadataBuilder(
vllm/attention/backends/flashinfer.py
+77
-10
@@ -30,7 +30,8 @@ from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
30
make_tensor_with_pad)
31
32
if TYPE_CHECKING:
33
- from vllm.worker.model_runner import ModelInputForGPUBuilder
33
+ from vllm.worker.model_runner import (ModelInputForGPUBuilder,
34
+ ModelInputForGPUWithSamplingMetadata)
35
36
37
class FlashInferBackend(AttentionBackend):
@@ -268,6 +269,10 @@ class FlashInferMetadata(AttentionMetadata):
269
query_start_loc: Optional[torch.Tensor] = None
270
block_tables: Optional[torch.Tensor] = None
271
272
+ # used for GPU in-place advance_step
273
+ seq_lens_tensor: Optional[torch.Tensor] = None
274
+ block_table_bound: Optional[torch.Tensor] = None
275
+
276
# An example for paged_kv_indices, paged_kv_indptr:
277
# request 1, page indices [0, 5, 8]
278
# request 2, page indices [1, 6, 7]
@@ -318,6 +323,8 @@ class FlashInferMetadata(AttentionMetadata):
323
assert self.paged_kv_indices is not None
324
assert self.paged_kv_indptr is not None
325
assert self.paged_kv_last_page_len is not None
326
+ assert self.block_table_bound is not None
327
+ assert self.seq_lens_tensor is not None
328
batch_size = self.query_start_loc.shape[0] - 1
329
assert batch_size >= 0
330
# We will use flash attention for profiling to
@@ -327,6 +334,8 @@ class FlashInferMetadata(AttentionMetadata):
334
self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
335
self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
336
self.device)
337
+ self.block_table_bound = self.block_table_bound.to(self.device)
338
+ self.seq_lens_tensor = self.seq_lens_tensor.to(self.device)
339
self.paged_kv_indices = self.paged_kv_indices.to(self.device)
340
self.prefill_wrapper.end_forward()
341
self.prefill_wrapper.begin_forward(
@@ -335,14 +344,18 @@ class FlashInferMetadata(AttentionMetadata):
344
self.num_qo_heads, self.num_kv_heads, self.head_dim,
345
self.page_size)
346
else:
338
- if not self.use_cuda_graph:
339
- assert self.paged_kv_indices is not None
340
- assert self.paged_kv_indptr is not None
341
- assert self.paged_kv_last_page_len is not None
342
- self.paged_kv_indices = self.paged_kv_indices.to(self.device)
343
- self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
344
- self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
345
- self.device)
347
+ assert self.paged_kv_indices is not None
348
+ assert self.paged_kv_indptr is not None
349
+ assert self.paged_kv_last_page_len is not None
350
+ self.paged_kv_indices = self.paged_kv_indices.to(self.device)
351
+ self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
352
+ self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
353
+ self.device)
354
+ # handle model warmup path
355
+ if self.block_table_bound is not None:
356
+ self.block_table_bound = self.block_table_bound.to(self.device)
357
+ if self.seq_lens_tensor is not None:
358
+ self.seq_lens_tensor = self.seq_lens_tensor.to(self.device)
359
360
assert self.decode_wrapper is not None
361
self.decode_wrapper.end_forward()
@@ -391,6 +404,48 @@ class FlashInferMetadata(AttentionMetadata):
404
405
return self
406
407
+ def advance_step(
408
+ self,
409
+ model_input: "ModelInputForGPUWithSamplingMetadata",
410
+ sampled_token_ids: Optional[torch.Tensor],
411
+ block_size: int,
412
+ num_seqs: int,
413
+ num_queries: int,
414
+ ):
415
+ """
416
+ Update metadata in-place to advance one decode step.
417
+ """
418
+
419
+ assert num_seqs > 0
420
+ assert num_queries > 0
421
+ assert model_input.attn_metadata is not None
422
+ assert sampled_token_ids is not None
423
+
424
+ # When using cudagraph, the num_seqs is padded to the next captured
425
+ # batch sized, but num_queries tracks the actual number of requests in
426
+ # the batch. For --enforce-eager mode, num_seqs == num_queries
427
+ if num_seqs != num_queries:
428
+ assert num_seqs > num_queries
429
+ assert self.use_cuda_graph
430
+
431
+ model_input.input_tokens[:num_queries] = sampled_token_ids.flatten()
432
+
433
+ # Update GPU tensors
434
+ ops.advance_step_flashinfer(
435
+ num_seqs=num_seqs,
436
+ num_queries=num_queries,
437
+ block_size=block_size,
438
+ input_tokens=model_input.input_tokens,
439
+ sampled_token_ids=model_input.input_tokens,
440
+ input_positions=model_input.input_positions,
441
+ seq_lens=self.seq_lens_tensor,
442
+ slot_mapping=self.slot_mapping,
443
+ block_tables=self.block_tables,
444
+ paged_kv_indices=self.paged_kv_indices,
445
+ paged_kv_indptr=self.paged_kv_indptr,
446
+ paged_kv_last_page_len=self.paged_kv_last_page_len,
447
+ block_table_bound=self.block_table_bound)
448
+
449
450
class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
451
@@ -428,7 +483,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
483
self.paged_kv_indptr: List[int] = [0]
484
# paged_kv_last_page_len is the length of the last page of each request
485
self.paged_kv_last_page_len: List[int] = []
431
-
486
+ self.total_blocks = 0
487
self.is_profile_run: bool = False
488
489
def _add_seq_group(
@@ -499,6 +554,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
554
# block_table_bound is 1 with 1 valid block.
555
# If seq_len = 15, block_size = 16,
556
# block_table_bound is 0 + 1 with 1 valid block.
557
+ self.total_blocks += len(block_table)
558
block_table_bound = seq_len // self.block_size + 1 \
559
if seq_len % self.block_size != 0 \
560
else seq_len // self.block_size
@@ -583,6 +639,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
639
out=query_start_loc[1:])
640
641
if len(self.paged_kv_indptr) > 0:
642
+ # extend to the maximum number of blocks as returned by the
643
+ # scheduler
644
+ self.paged_kv_indices.extend(
645
+ [0] * (self.total_blocks - len(self.paged_kv_indices)))
646
paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
647
device="cpu",
648
dtype=torch.int)
@@ -591,10 +651,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
651
dtype=torch.int)
652
paged_kv_last_page_len_tensor = torch.tensor(
653
self.paged_kv_last_page_len, device="cpu", dtype=torch.int)
654
+ block_table_bound_tensor = torch.zeros(len(self.paged_kv_indptr) -
655
+ 1,
656
+ device="cpu",
657
+ dtype=torch.int)
658
else:
659
paged_kv_indices_tensor = None
660
paged_kv_indptr_tensor = None
661
paged_kv_last_page_len_tensor = None
662
+ block_table_bound_tensor = None
663
664
if self.runner.kv_cache_dtype.startswith("fp8"):
665
kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
@@ -613,6 +678,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
678
paged_kv_indptr=paged_kv_indptr_tensor,
679
paged_kv_indices=paged_kv_indices_tensor,
680
paged_kv_last_page_len=paged_kv_last_page_len_tensor,
681
+ block_table_bound=block_table_bound_tensor,
682
+ seq_lens_tensor=seq_lens_tensor,
683
num_qo_heads=self.runner.model_config.get_num_attention_heads(
684
self.runner.parallel_config),
685
num_kv_heads=self.runner.model_config.get_num_kv_heads(
vllm/worker/multi_step_model_runner.py
+16
-21
@@ -4,13 +4,6 @@ from dataclasses import dataclass, field
4
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
5
Union)
6
7
-try:
8
- from vllm.attention.backends.flash_attn import FlashAttentionMetadata
9
-except ModuleNotFoundError:
10
- # vllm_flash_attn is not installed, use the identical ROCm FA metadata
11
- from vllm.attention.backends.rocm_flash_attn import (
12
- ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13
-
7
import torch
8
9
from vllm.distributed import get_pp_group
@@ -36,6 +29,8 @@ if TYPE_CHECKING:
29
30
logger = init_logger(__name__)
31
32
+MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "flashinfer"]
33
+
34
35
def seq_output_builder():
36
return SequenceOutput(
@@ -489,27 +484,27 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
484
485
def _advance_step(self, model_input: StatefulModelInput,
486
out: SamplerOutput) -> StatefulModelInput:
492
- frozen_model_input = model_input.frozen_model_input
493
- assert frozen_model_input is not None
494
- assert frozen_model_input.attn_metadata is not None
487
+ if self.attn_backend.get_name() not in MULTI_STEP_ATTENTION_BACKENDS:
488
+ raise ValueError(
489
+ f"Multi-step not supported for attention backend: "
490
+ f"{self.attn_backend.get_name()}. Set VLLM_ATTENTION_BACKEND "
491
+ f"to a value from {MULTI_STEP_ATTENTION_BACKENDS}.")
492
493
+ sampled_token_ids = model_input.cached_outputs[-1].sampled_token_ids
494
num_seqs = model_input.num_seqs
495
num_queries = model_input.num_queries
498
- assert num_seqs > 0
499
- assert num_queries > 0
500
- assert num_seqs >= num_queries
501
-
496
+ frozen_model_input = model_input.frozen_model_input
497
+ assert frozen_model_input is not None
498
attn_metadata = frozen_model_input.attn_metadata
503
- assert isinstance(attn_metadata, FlashAttentionMetadata)
499
+ assert attn_metadata is not None
500
501
attn_metadata.advance_step(
502
frozen_model_input,
507
- model_input.cached_outputs[-1].sampled_token_ids, self.block_size,
508
- num_seqs, num_queries)
509
-
510
- if frozen_model_input.seq_lens is not None:
511
- for i in range(num_queries):
512
- frozen_model_input.seq_lens[i] = attn_metadata.seq_lens[i]
503
+ sampled_token_ids,
504
+ self.block_size,
505
+ num_seqs,
506
+ num_queries,
507
+ )
508
509
return model_input
510