@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a6c0f3658d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[multi-step] add flashinfer backend (#7928)

William Lin committed Sep 12, 2024 at 11:16 UTC a6c0f3658da4f2f23460e3e15bfa7d70ac7e60c1
9 files changed +371 -84
csrc/ops.h
+15 -4
@@ -54,10 +54,21 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input);
54
55 void gelu_quick(torch::Tensor& out, torch::Tensor& input);
56
57 -void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
58 - torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
59 - torch::Tensor& input_positions, torch::Tensor& seq_lens,
60 - torch::Tensor& slot_mapping, torch::Tensor& block_tables);
57 +void advance_step_flashattn(int64_t num_seqs, int64_t num_queries,
58 + int64_t block_size, torch::Tensor& input_tokens,
59 + torch::Tensor& sampled_token_ids,
60 + torch::Tensor& input_positions,
61 + torch::Tensor& seq_lens,
62 + torch::Tensor& slot_mapping,
63 + torch::Tensor& block_tables);
64 +
65 +void advance_step_flashinfer(
66 + int64_t num_seqs, int64_t num_queries, int64_t block_size,
67 + torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
68 + torch::Tensor& input_positions, torch::Tensor& seq_lens,
69 + torch::Tensor& slot_mapping, torch::Tensor& block_tables,
70 + torch::Tensor& paged_kv_indices, torch::Tensor& paged_kv_indptr,
71 + torch::Tensor& paged_kv_last_page_len, torch::Tensor& block_table_bounds);
72
73 #ifndef USE_ROCM
74 torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
csrc/prepare_inputs/advance_step.cu
+200 -25
@@ -12,13 +12,11 @@ namespace prepare_inputs {
12
13 //
14 template <int const num_threads>
15 -__global__ void advance_step_kernel(int num_seqs, int num_queries,
16 - int block_size, long* input_tokens_ptr,
17 - long const* sampled_token_ids_ptr,
18 - long* input_positions_ptr,
19 - int* seq_lens_ptr, long* slot_mapping_ptr,
20 - int const* block_tables_ptr,
21 - int64_t const block_tables_stride) {
15 +__global__ void advance_step_flashattn_kernel(
16 + int num_seqs, int num_queries, int block_size, long* input_tokens_ptr,
17 + long const* sampled_token_ids_ptr, long* input_positions_ptr,
18 + int* seq_lens_ptr, long* slot_mapping_ptr, int const* block_tables_ptr,
19 + int64_t const block_tables_stride) {
20 int num_query_blocks = div_ceil(num_queries, num_threads);
21
22 if (blockIdx.x >= num_query_blocks) {
@@ -79,16 +77,91 @@ inline void verify_tensor(std::string const& name, torch::Tensor& t,
77 }
78 }
79
82 -void advance_step(int num_seqs, int num_queries, int block_size,
83 - torch::Tensor& input_tokens, // type: long
84 - torch::Tensor& sampled_token_ids, // type: long
85 - torch::Tensor& input_positions, // type: long
86 - torch::Tensor& seq_lens, // type: int
87 - torch::Tensor& slot_mapping, // type: long
88 - torch::Tensor& block_tables) { // type: int
80 +__global__ void advance_step_flashinfer_kernel(
81 + int num_threads, int num_seqs, int num_queries, int block_size,
82 + long* input_tokens_ptr, long const* sampled_token_ids_ptr,
83 + long* input_positions_ptr, int* seq_lens_ptr, long* slot_mapping_ptr,
84 + int const* block_tables_ptr, int64_t const block_tables_stride,
85 + int* paged_kv_last_page_len_ptr, int* block_table_bound_ptr) {
86 + int num_query_blocks = div_ceil(num_queries, num_threads);
87 +
88 + if (blockIdx.x < num_query_blocks) {
89 + int cur_query_id = blockIdx.x * num_threads + threadIdx.x;
90 +
91 + if (cur_query_id < num_queries) {
92 + // Update input_tokens
93 + input_tokens_ptr[cur_query_id] = sampled_token_ids_ptr[cur_query_id];
94 +
95 + int seq_len = seq_lens_ptr[cur_query_id];
96 + int next_seq_len = seq_len + 1;
97 + int next_input_pos = next_seq_len - 1;
98 +
99 + // Update seq_lens
100 + seq_lens_ptr[cur_query_id] = next_seq_len;
101 + // Update input_positions
102 + input_positions_ptr[cur_query_id] = next_input_pos;
103 +
104 + int const* seq_block_tables_ptr =
105 + block_tables_ptr + block_tables_stride * cur_query_id;
106 +
107 + int block_index = next_input_pos / block_size;
108 + int block_offset = next_input_pos % block_size;
109 +
110 + // Update paged_kv_last_page_len
111 + paged_kv_last_page_len_ptr[cur_query_id] = block_offset + 1;
112 +
113 + int slot_num =
114 + seq_block_tables_ptr[block_index] * block_size + block_offset;
115 + // Update slot_mapping
116 + slot_mapping_ptr[cur_query_id] = slot_num;
117 + block_table_bound_ptr[cur_query_id] = div_ceil(next_seq_len, block_size);
118 + }
119 + }
120 +}
121 +
122 +__global__ void advance_step_flashinfer_indptr_kernel(
123 + int num_threads, int num_seqs, int num_queries, int* paged_kv_indptr_ptr,
124 + int* block_table_bound_ptr) {
125 + int idx = blockIdx.x * num_threads + threadIdx.x;
126 +
127 + // Update paged_kv_indptr
128 + if (idx < num_queries) {
129 + int sum = 0;
130 + for (int i = 0; i <= idx; ++i) {
131 + sum += block_table_bound_ptr[i];
132 + }
133 + paged_kv_indptr_ptr[idx + 1] = sum;
134 + }
135 +}
136 +
137 +__global__ void advance_step_flashinfer_indices_kernel(
138 + int num_threads, int num_seqs, int num_queries, int const* block_tables_ptr,
139 + int64_t const block_tables_stride, int* paged_kv_indices_ptr,
140 + int* paged_kv_indptr_ptr, int* block_table_bound_ptr) {
141 + int idx = blockIdx.x * num_threads + threadIdx.x;
142 + int row = idx / block_tables_stride;
143 + int col = idx % block_tables_stride;
144 +
145 + if (row < num_queries && col < block_table_bound_ptr[row]) {
146 + paged_kv_indices_ptr[paged_kv_indptr_ptr[row] + col] =
147 + block_tables_ptr[row * block_tables_stride + col];
148 + }
149 + // if cudagraph, fill padded seqs with the last valid seq's indptr
150 + if (num_queries < row && row <= num_seqs) {
151 + paged_kv_indptr_ptr[row] = paged_kv_indptr_ptr[num_queries];
152 + }
153 +}
154 +
155 +void advance_step_flashattn(int num_seqs, int num_queries, int block_size,
156 + torch::Tensor& input_tokens, // type: long
157 + torch::Tensor& sampled_token_ids, // type: long
158 + torch::Tensor& input_positions, // type: long
159 + torch::Tensor& seq_lens, // type: int
160 + torch::Tensor& slot_mapping, // type: long
161 + torch::Tensor& block_tables) { // type: int
162
163 if (logging) {
91 - printf("advance_step:\n");
164 + printf("advance_step_flashattn:\n");
165 printf(" num_seqs = %d\n", num_seqs);
166 printf(" num_queries = %d\n", num_queries);
167 printf(" block_size = %d\n", block_size);
@@ -108,24 +181,126 @@ void advance_step(int num_seqs, int num_queries, int block_size,
181 int blocks;
182 cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
183
111 - advance_step_kernel<max_threads><<<blocks, max_threads, 0, stream>>>(
112 - num_seqs, num_queries, block_size,
184 + advance_step_flashattn_kernel<max_threads>
185 + <<<blocks, max_threads, 0, stream>>>(
186 + num_seqs, num_queries, block_size,
187 + reinterpret_cast<long*>(input_tokens.data_ptr()),
188 + reinterpret_cast<long const*>(sampled_token_ids.data_ptr()),
189 + reinterpret_cast<long*>(input_positions.data_ptr()),
190 + reinterpret_cast<int*>(seq_lens.data_ptr()),
191 + reinterpret_cast<long*>(slot_mapping.data_ptr()),
192 + reinterpret_cast<int const*>(block_tables.data_ptr()),
193 + block_tables.stride(0));
194 +}
195 +
196 +void advance_step_flashinfer(
197 + int num_seqs, int num_queries, int block_size,
198 + torch::Tensor& input_tokens, // type: long
199 + torch::Tensor& sampled_token_ids, // type: long
200 + torch::Tensor& input_positions, // type: long
201 + torch::Tensor& seq_lens, // type: int
202 + torch::Tensor& slot_mapping, // type: long
203 + torch::Tensor& block_tables, // type: int
204 + torch::Tensor& paged_kv_indices, // type: int
205 + torch::Tensor& paged_kv_indptr, // type: int
206 + torch::Tensor& paged_kv_last_page_len, // type: int
207 + torch::Tensor& block_table_bound) { // type: int
208 +
209 + if (logging) {
210 + printf("advance_step_flashinfer:\n");
211 + printf(" num_seqs = %d\n", num_seqs);
212 + printf(" num_queries = %d\n", num_queries);
213 + printf(" block_size = %d\n", block_size);
214 + printf(" block_tables.stride(0) = %d\n", block_tables.stride(0));
215 + }
216 + // Verify all tensors
217 + verify_tensor("input_tokens", input_tokens, num_seqs, -1, at::kLong);
218 + // verify_tensor("sampled_token_ids", sampled_token_ids, num_queries, 1,
219 + // at::kLong);
220 + verify_tensor("input_positions", input_positions, num_seqs, -1, at::kLong);
221 + verify_tensor("seq_lens", seq_lens, num_seqs, -1, at::kInt);
222 + verify_tensor("slot_mapping", slot_mapping, num_seqs, -1, at::kLong);
223 + verify_tensor("block_tables", block_tables, num_seqs, -1, at::kInt);
224 +
225 + verify_tensor("paged_kv_indices", paged_kv_indices, -1, -1, at::kInt);
226 + verify_tensor("paged_kv_indptr", paged_kv_indptr, num_seqs + 1, -1, at::kInt);
227 + verify_tensor("paged_kv_last_page_len", paged_kv_last_page_len, num_seqs, -1,
228 + at::kInt);
229 +
230 + verify_tensor("block_table_bound", block_table_bound, num_seqs, -1, at::kInt);
231 +
232 + int dev = sampled_token_ids.get_device();
233 + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
234 +
235 + int blocks;
236 + int threads;
237 + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
238 + cudaDeviceGetAttribute(&threads, cudaDevAttrMaxThreadsPerBlock, dev);
239 + if (logging) {
240 + printf("launching kernel with %d blocks\n", blocks);
241 + }
242 +
243 + // TODO(will): support arbitrary block_tables stride
244 + if ((blocks * threads) / block_tables.stride(0) < num_queries) {
245 + TORCH_CHECK(false,
246 + "multi-step: not enough threads to map block_table to"
247 + "FlashInfer's paged_kv_indices on GPU. Try reducing the number "
248 + "of seqs,",
249 + " increasing the block size or take smaller steps.",
250 + " num_queries = ", num_queries,
251 + " block_tables.stride(0) = ", block_tables.stride(0),
252 + " blocks = ", blocks, " max_threads = ", threads);
253 + }
254 +
255 + advance_step_flashinfer_kernel<<<blocks, threads, 0, stream>>>(
256 + threads, num_seqs, num_queries, block_size,
257 reinterpret_cast<long*>(input_tokens.data_ptr()),
258 reinterpret_cast<long const*>(sampled_token_ids.data_ptr()),
259 reinterpret_cast<long*>(input_positions.data_ptr()),
260 reinterpret_cast<int*>(seq_lens.data_ptr()),
261 reinterpret_cast<long*>(slot_mapping.data_ptr()),
262 reinterpret_cast<int const*>(block_tables.data_ptr()),
119 - block_tables.stride(0));
263 + block_tables.stride(0),
264 + reinterpret_cast<int*>(paged_kv_last_page_len.data_ptr()),
265 + reinterpret_cast<int*>(block_table_bound.data_ptr()));
266 +
267 + advance_step_flashinfer_indptr_kernel<<<blocks, threads, 0, stream>>>(
268 + threads, num_seqs, num_queries,
269 + reinterpret_cast<int*>(paged_kv_indptr.data_ptr()),
270 + reinterpret_cast<int*>(block_table_bound.data_ptr()));
271 +
272 + advance_step_flashinfer_indices_kernel<<<blocks, threads, 0, stream>>>(
273 + threads, num_seqs, num_queries,
274 + reinterpret_cast<int const*>(block_tables.data_ptr()),
275 + block_tables.stride(0),
276 + reinterpret_cast<int*>(paged_kv_indices.data_ptr()),
277 + reinterpret_cast<int*>(paged_kv_indptr.data_ptr()),
278 + reinterpret_cast<int*>(block_table_bound.data_ptr()));
279 }
280
281 } // namespace prepare_inputs
282
124 -void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
125 - torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
126 - torch::Tensor& input_positions, torch::Tensor& seq_lens,
127 - torch::Tensor& slot_mapping, torch::Tensor& block_tables) {
128 - prepare_inputs::advance_step(num_seqs, num_queries, block_size, input_tokens,
129 - sampled_token_ids, input_positions, seq_lens,
130 - slot_mapping, block_tables);
283 +void advance_step_flashattn(int64_t num_seqs, int64_t num_queries,
284 + int64_t block_size, torch::Tensor& input_tokens,
285 + torch::Tensor& sampled_token_ids,
286 + torch::Tensor& input_positions,
287 + torch::Tensor& seq_lens,
288 + torch::Tensor& slot_mapping,
289 + torch::Tensor& block_tables) {
290 + prepare_inputs::advance_step_flashattn(
291 + num_seqs, num_queries, block_size, input_tokens, sampled_token_ids,
292 + input_positions, seq_lens, slot_mapping, block_tables);
293 +}
294 +
295 +void advance_step_flashinfer(
296 + int64_t num_seqs, int64_t num_queries, int64_t block_size,
297 + torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
298 + torch::Tensor& input_positions, torch::Tensor& seq_lens,
299 + torch::Tensor& slot_mapping, torch::Tensor& block_tables,
300 + torch::Tensor& paged_kv_indices, torch::Tensor& paged_kv_indptr,
301 + torch::Tensor& paged_kv_last_page_len, torch::Tensor& block_table_bound) {
302 + prepare_inputs::advance_step_flashinfer(
303 + num_seqs, num_queries, block_size, input_tokens, sampled_token_ids,
304 + input_positions, seq_lens, slot_mapping, block_tables, paged_kv_indices,
305 + paged_kv_indptr, paged_kv_last_page_len, block_table_bound);
306 }
\ No newline at end of file
csrc/torch_bindings.cpp
+13 -2
@@ -74,11 +74,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
74
75 // prepare_inputs advance_step
76 ops.def(
77 - "advance_step(int num_seqs, int num_queries, int block_size, "
77 + "advance_step_flashattn(int num_seqs, int num_queries, int block_size, "
78 "Tensor! input_tokens, Tensor sampled_token_ids, "
79 "Tensor! input_positions, Tensor! seq_lens, Tensor! slot_mapping, "
80 "Tensor block_tables) -> ()");
81 - ops.impl("advance_step", torch::kCUDA, &advance_step);
81 + ops.impl("advance_step_flashattn", torch::kCUDA, &advance_step_flashattn);
82 +
83 + ops.def(
84 + "advance_step_flashinfer("
85 + " int num_seqs, int num_queries, int block_size,"
86 + " Tensor! input_tokens, Tensor sampled_token_ids,"
87 + " Tensor! input_positions, Tensor! seq_lens, Tensor! slot_mapping,"
88 + " Tensor block_tables, Tensor! paged_kv_indices,"
89 + " Tensor! paged_kv_indptr, Tensor! paged_kv_last_page_len,"
90 + " Tensor! block_table_bounds"
91 + ") -> ()");
92 + ops.impl("advance_step_flashinfer", torch::kCUDA, &advance_step_flashinfer);
93
94 // Layernorm
95 // Apply Root Mean Square (RMS) Normalization to the input tensor.
tests/multi_step/test_correctness_async_llm.py
+9 -3
@@ -1,9 +1,10 @@
1 # Test the AsyncLLMEngine with multi-step-decoding
2 -
2 from typing import List, Optional
3
4 import pytest
5
6 +from tests.kernels.utils import override_backend_env_variable
7 +
8 from ..models.utils import check_logprobs_close
9 from ..utils import (completions_with_server_args, get_client_text_generations,
10 get_client_text_logprob_generations)
@@ -33,8 +34,9 @@ DEFAULT_SERVER_ARGS: List[str] = [
34 @pytest.mark.parametrize("eager_mode", [False, True])
35 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
36 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
36 -@pytest.mark.parametrize("num_logprobs", [None, 5])
37 -@pytest.mark.parametrize("is_async", [False, True])
37 +@pytest.mark.parametrize("num_logprobs", [5])
38 +@pytest.mark.parametrize("is_async", [True])
39 +@pytest.mark.parametrize("attention_backend", ["FLASHINFER", "FLASH_ATTN"])
40 @pytest.mark.asyncio
41 async def test_multi_step(
42 example_prompts,
@@ -46,6 +48,8 @@ async def test_multi_step(
48 num_prompts: int,
49 is_async: bool,
50 num_logprobs: Optional[int],
51 + attention_backend: str,
52 + monkeypatch,
53 ) -> None:
54 """Test vLLM engine with multi-step scheduling in an OpenAI-protocol
55 client/server environment.
@@ -71,6 +75,8 @@ async def test_multi_step(
75 completions endpoint; `None` -> no logprobs
76 """
77
78 + override_backend_env_variable(monkeypatch, attention_backend)
79 +
80 prompts = example_prompts
81 if len(prompts) < num_prompts:
82 prompts = prompts * ((num_prompts // len(prompts)) + 1)
vllm/_custom_ops.py
+29 -9
@@ -161,16 +161,36 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
161 torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
162
163
164 -def advance_step(num_seqs: int, num_queries: int, block_size: int,
165 - input_tokens: torch.Tensor, sampled_token_ids: torch.Tensor,
166 - input_positions: torch.Tensor, seq_lens: torch.Tensor,
167 - slot_mapping: torch.Tensor,
168 - block_tables: torch.Tensor) -> None:
164 +def advance_step_flashattn(num_seqs: int, num_queries: int, block_size: int,
165 + input_tokens: torch.Tensor,
166 + sampled_token_ids: torch.Tensor,
167 + input_positions: torch.Tensor,
168 + seq_lens: torch.Tensor, slot_mapping: torch.Tensor,
169 + block_tables: torch.Tensor) -> None:
170 """Advance a step on GPU for existing inputs for a multi-step runner"""
170 - return torch.ops._C.advance_step(num_seqs, num_queries, block_size,
171 - input_tokens, sampled_token_ids,
172 - input_positions, seq_lens, slot_mapping,
173 - block_tables)
171 + return torch.ops._C.advance_step_flashattn(num_seqs, num_queries,
172 + block_size, input_tokens,
173 + sampled_token_ids,
174 + input_positions, seq_lens,
175 + slot_mapping, block_tables)
176 +
177 +
178 +def advance_step_flashinfer(num_seqs: int, num_queries: int, block_size: int,
179 + input_tokens: torch.Tensor,
180 + sampled_token_ids: torch.Tensor,
181 + input_positions: torch.Tensor,
182 + seq_lens: torch.Tensor, slot_mapping: torch.Tensor,
183 + block_tables: torch.Tensor,
184 + paged_kv_indices: torch.Tensor,
185 + paged_kv_indptr: torch.Tensor,
186 + paged_kv_last_page_len: torch.Tensor,
187 + block_table_bound: torch.Tensor) -> None:
188 +
189 + return torch.ops._C.advance_step_flashinfer(
190 + num_seqs, num_queries, block_size, input_tokens, sampled_token_ids,
191 + input_positions, seq_lens, slot_mapping, block_tables,
192 + paged_kv_indices, paged_kv_indptr, paged_kv_last_page_len,
193 + block_table_bound)
194
195
196 # quantization ops
vllm/attention/backends/abstract.py
+3 -1
@@ -83,7 +83,9 @@ class AttentionBackend(ABC):
83 ) -> None:
84 raise NotImplementedError
85
86 - def advance_step(self, num_seqs: int, num_queries: int):
86 + def advance_step(self, model_input: "ModelRunnerInputBase",
87 + sampled_token_ids: Optional[torch.Tensor],
88 + block_size: int, num_seqs: int, num_queries: int) -> None:
89 raise NotImplementedError
90
91
vllm/attention/backends/flash_attn.py
+9 -9
@@ -380,15 +380,15 @@ class FlashAttentionMetadata(AttentionMetadata):
380 self.seq_lens[i] += 1
381 self.max_decode_seq_len = max(self.seq_lens)
382
383 - ops.advance_step(num_seqs=num_seqs,
384 - num_queries=num_queries,
385 - block_size=block_size,
386 - input_tokens=model_input.input_tokens,
387 - sampled_token_ids=sampled_token_ids,
388 - input_positions=model_input.input_positions,
389 - seq_lens=self.seq_lens_tensor,
390 - slot_mapping=self.slot_mapping,
391 - block_tables=self.block_tables)
383 + ops.advance_step_flashattn(num_seqs=num_seqs,
384 + num_queries=num_queries,
385 + block_size=block_size,
386 + input_tokens=model_input.input_tokens,
387 + sampled_token_ids=sampled_token_ids,
388 + input_positions=model_input.input_positions,
389 + seq_lens=self.seq_lens_tensor,
390 + slot_mapping=self.slot_mapping,
391 + block_tables=self.block_tables)
392
393
394 class FlashAttentionMetadataBuilder(
vllm/attention/backends/flashinfer.py
+77 -10
@@ -30,7 +30,8 @@ from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
30 make_tensor_with_pad)
31
32 if TYPE_CHECKING:
33 - from vllm.worker.model_runner import ModelInputForGPUBuilder
33 + from vllm.worker.model_runner import (ModelInputForGPUBuilder,
34 + ModelInputForGPUWithSamplingMetadata)
35
36
37 class FlashInferBackend(AttentionBackend):
@@ -268,6 +269,10 @@ class FlashInferMetadata(AttentionMetadata):
269 query_start_loc: Optional[torch.Tensor] = None
270 block_tables: Optional[torch.Tensor] = None
271
272 + # used for GPU in-place advance_step
273 + seq_lens_tensor: Optional[torch.Tensor] = None
274 + block_table_bound: Optional[torch.Tensor] = None
275 +
276 # An example for paged_kv_indices, paged_kv_indptr:
277 # request 1, page indices [0, 5, 8]
278 # request 2, page indices [1, 6, 7]
@@ -318,6 +323,8 @@ class FlashInferMetadata(AttentionMetadata):
323 assert self.paged_kv_indices is not None
324 assert self.paged_kv_indptr is not None
325 assert self.paged_kv_last_page_len is not None
326 + assert self.block_table_bound is not None
327 + assert self.seq_lens_tensor is not None
328 batch_size = self.query_start_loc.shape[0] - 1
329 assert batch_size >= 0
330 # We will use flash attention for profiling to
@@ -327,6 +334,8 @@ class FlashInferMetadata(AttentionMetadata):
334 self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
335 self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
336 self.device)
337 + self.block_table_bound = self.block_table_bound.to(self.device)
338 + self.seq_lens_tensor = self.seq_lens_tensor.to(self.device)
339 self.paged_kv_indices = self.paged_kv_indices.to(self.device)
340 self.prefill_wrapper.end_forward()
341 self.prefill_wrapper.begin_forward(
@@ -335,14 +344,18 @@ class FlashInferMetadata(AttentionMetadata):
344 self.num_qo_heads, self.num_kv_heads, self.head_dim,
345 self.page_size)
346 else:
338 - if not self.use_cuda_graph:
339 - assert self.paged_kv_indices is not None
340 - assert self.paged_kv_indptr is not None
341 - assert self.paged_kv_last_page_len is not None
342 - self.paged_kv_indices = self.paged_kv_indices.to(self.device)
343 - self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
344 - self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
345 - self.device)
347 + assert self.paged_kv_indices is not None
348 + assert self.paged_kv_indptr is not None
349 + assert self.paged_kv_last_page_len is not None
350 + self.paged_kv_indices = self.paged_kv_indices.to(self.device)
351 + self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
352 + self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
353 + self.device)
354 + # handle model warmup path
355 + if self.block_table_bound is not None:
356 + self.block_table_bound = self.block_table_bound.to(self.device)
357 + if self.seq_lens_tensor is not None:
358 + self.seq_lens_tensor = self.seq_lens_tensor.to(self.device)
359
360 assert self.decode_wrapper is not None
361 self.decode_wrapper.end_forward()
@@ -391,6 +404,48 @@ class FlashInferMetadata(AttentionMetadata):
404
405 return self
406
407 + def advance_step(
408 + self,
409 + model_input: "ModelInputForGPUWithSamplingMetadata",
410 + sampled_token_ids: Optional[torch.Tensor],
411 + block_size: int,
412 + num_seqs: int,
413 + num_queries: int,
414 + ):
415 + """
416 + Update metadata in-place to advance one decode step.
417 + """
418 +
419 + assert num_seqs > 0
420 + assert num_queries > 0
421 + assert model_input.attn_metadata is not None
422 + assert sampled_token_ids is not None
423 +
424 + # When using cudagraph, the num_seqs is padded to the next captured
425 + # batch sized, but num_queries tracks the actual number of requests in
426 + # the batch. For --enforce-eager mode, num_seqs == num_queries
427 + if num_seqs != num_queries:
428 + assert num_seqs > num_queries
429 + assert self.use_cuda_graph
430 +
431 + model_input.input_tokens[:num_queries] = sampled_token_ids.flatten()
432 +
433 + # Update GPU tensors
434 + ops.advance_step_flashinfer(
435 + num_seqs=num_seqs,
436 + num_queries=num_queries,
437 + block_size=block_size,
438 + input_tokens=model_input.input_tokens,
439 + sampled_token_ids=model_input.input_tokens,
440 + input_positions=model_input.input_positions,
441 + seq_lens=self.seq_lens_tensor,
442 + slot_mapping=self.slot_mapping,
443 + block_tables=self.block_tables,
444 + paged_kv_indices=self.paged_kv_indices,
445 + paged_kv_indptr=self.paged_kv_indptr,
446 + paged_kv_last_page_len=self.paged_kv_last_page_len,
447 + block_table_bound=self.block_table_bound)
448 +
449
450 class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
451
@@ -428,7 +483,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
483 self.paged_kv_indptr: List[int] = [0]
484 # paged_kv_last_page_len is the length of the last page of each request
485 self.paged_kv_last_page_len: List[int] = []
431 -
486 + self.total_blocks = 0
487 self.is_profile_run: bool = False
488
489 def _add_seq_group(
@@ -499,6 +554,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
554 # block_table_bound is 1 with 1 valid block.
555 # If seq_len = 15, block_size = 16,
556 # block_table_bound is 0 + 1 with 1 valid block.
557 + self.total_blocks += len(block_table)
558 block_table_bound = seq_len // self.block_size + 1 \
559 if seq_len % self.block_size != 0 \
560 else seq_len // self.block_size
@@ -583,6 +639,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
639 out=query_start_loc[1:])
640
641 if len(self.paged_kv_indptr) > 0:
642 + # extend to the maximum number of blocks as returned by the
643 + # scheduler
644 + self.paged_kv_indices.extend(
645 + [0] * (self.total_blocks - len(self.paged_kv_indices)))
646 paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
647 device="cpu",
648 dtype=torch.int)
@@ -591,10 +651,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
651 dtype=torch.int)
652 paged_kv_last_page_len_tensor = torch.tensor(
653 self.paged_kv_last_page_len, device="cpu", dtype=torch.int)
654 + block_table_bound_tensor = torch.zeros(len(self.paged_kv_indptr) -
655 + 1,
656 + device="cpu",
657 + dtype=torch.int)
658 else:
659 paged_kv_indices_tensor = None
660 paged_kv_indptr_tensor = None
661 paged_kv_last_page_len_tensor = None
662 + block_table_bound_tensor = None
663
664 if self.runner.kv_cache_dtype.startswith("fp8"):
665 kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
@@ -613,6 +678,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
678 paged_kv_indptr=paged_kv_indptr_tensor,
679 paged_kv_indices=paged_kv_indices_tensor,
680 paged_kv_last_page_len=paged_kv_last_page_len_tensor,
681 + block_table_bound=block_table_bound_tensor,
682 + seq_lens_tensor=seq_lens_tensor,
683 num_qo_heads=self.runner.model_config.get_num_attention_heads(
684 self.runner.parallel_config),
685 num_kv_heads=self.runner.model_config.get_num_kv_heads(
vllm/worker/multi_step_model_runner.py
+16 -21
@@ -4,13 +4,6 @@ from dataclasses import dataclass, field
4 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
5 Union)
6
7 -try:
8 - from vllm.attention.backends.flash_attn import FlashAttentionMetadata
9 -except ModuleNotFoundError:
10 - # vllm_flash_attn is not installed, use the identical ROCm FA metadata
11 - from vllm.attention.backends.rocm_flash_attn import (
12 - ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13 -
7 import torch
8
9 from vllm.distributed import get_pp_group
@@ -36,6 +29,8 @@ if TYPE_CHECKING:
29
30 logger = init_logger(__name__)
31
32 +MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "flashinfer"]
33 +
34
35 def seq_output_builder():
36 return SequenceOutput(
@@ -489,27 +484,27 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
484
485 def _advance_step(self, model_input: StatefulModelInput,
486 out: SamplerOutput) -> StatefulModelInput:
492 - frozen_model_input = model_input.frozen_model_input
493 - assert frozen_model_input is not None
494 - assert frozen_model_input.attn_metadata is not None
487 + if self.attn_backend.get_name() not in MULTI_STEP_ATTENTION_BACKENDS:
488 + raise ValueError(
489 + f"Multi-step not supported for attention backend: "
490 + f"{self.attn_backend.get_name()}. Set VLLM_ATTENTION_BACKEND "
491 + f"to a value from {MULTI_STEP_ATTENTION_BACKENDS}.")
492
493 + sampled_token_ids = model_input.cached_outputs[-1].sampled_token_ids
494 num_seqs = model_input.num_seqs
495 num_queries = model_input.num_queries
498 - assert num_seqs > 0
499 - assert num_queries > 0
500 - assert num_seqs >= num_queries
501 -
496 + frozen_model_input = model_input.frozen_model_input
497 + assert frozen_model_input is not None
498 attn_metadata = frozen_model_input.attn_metadata
503 - assert isinstance(attn_metadata, FlashAttentionMetadata)
499 + assert attn_metadata is not None
500
501 attn_metadata.advance_step(
502 frozen_model_input,
507 - model_input.cached_outputs[-1].sampled_token_ids, self.block_size,
508 - num_seqs, num_queries)
509 -
510 - if frozen_model_input.seq_lens is not None:
511 - for i in range(num_queries):
512 - frozen_model_input.seq_lens[i] = attn_metadata.seq_lens[i]
503 + sampled_token_ids,
504 + self.block_size,
505 + num_seqs,
506 + num_queries,
507 + )
508
509 return model_input
510