@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2fa4623d9e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Refactor _prepare_model_input_tensors - take 2 (#6164)

Cody Yu committed Jul 17, 2024 at 09:37 UTC 2fa4623d9e673faddb05018ac3f3d65780bb083d
12 files changed +1050 -470
tests/worker/test_model_input.py
+5 -1
@@ -3,7 +3,7 @@ from typing import List, Tuple, Type
3
4 import torch
5
6 -from vllm.attention import AttentionMetadata
6 +from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7 from vllm.attention.backends.abstract import AttentionBackend
8 from vllm.model_executor import SamplingMetadata
9 from vllm.model_executor.pooling_metadata import PoolingMetadata
@@ -26,6 +26,10 @@ class MockAttentionBackend(AttentionBackend):
26 def get_metadata_cls() -> Type["AttentionMetadata"]:
27 return AttentionMetadata
28
29 + @staticmethod
30 + def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
31 + raise AttentionMetadataBuilder
32 +
33 @staticmethod
34 def get_kv_cache_shape(
35 num_blocks: int,
vllm/attention/__init__.py
+3 -1
@@ -1,5 +1,6 @@
1 from vllm.attention.backends.abstract import (AttentionBackend,
2 - AttentionMetadata)
2 + AttentionMetadata,
3 + AttentionMetadataBuilder)
4 from vllm.attention.layer import Attention
5 from vllm.attention.selector import get_attn_backend
6
@@ -7,6 +8,7 @@ __all__ = [
8 "Attention",
9 "AttentionBackend",
10 "AttentionMetadata",
11 + "AttentionMetadataBuilder",
12 "Attention",
13 "get_attn_backend",
14 ]
vllm/attention/backends/abstract.py
+43 -2
@@ -1,11 +1,15 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass, fields
3 from enum import Enum, auto
4 -from typing import (Any, Dict, Generic, List, Optional, Set, Tuple, Type,
5 - TypeVar)
4 +from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
5 + Tuple, Type, TypeVar)
6
7 import torch
8
9 +if TYPE_CHECKING:
10 + from vllm.sequence import SequenceGroupMetadata
11 + from vllm.worker.model_runner_base import ModelRunnerInputBuilderBase
12 +
13
14 class AttentionType(Enum):
15 DECODER = auto() # Decoder attention between previous layer Q/K/V
@@ -35,6 +39,16 @@ class AttentionBackend(ABC):
39 def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":
40 return cls.get_metadata_cls()(*args, **kwargs)
41
42 + @staticmethod
43 + @abstractmethod
44 + def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
45 + raise NotImplementedError
46 +
47 + @classmethod
48 + def make_metadata_builder(cls, *args,
49 + **kwargs) -> "AttentionMetadataBuilder":
50 + return cls.get_builder_cls()(*args, **kwargs)
51 +
52 @staticmethod
53 @abstractmethod
54 def get_kv_cache_shape(
@@ -110,6 +124,33 @@ class AttentionMetadata:
124 T = TypeVar("T", bound=AttentionMetadata)
125
126
127 +class AttentionMetadataBuilder(ABC, Generic[T]):
128 + """Abstract class for attention metadata builders."""
129 +
130 + @abstractmethod
131 + def __init__(self, input_builder) -> None:
132 + raise NotImplementedError
133 +
134 + @abstractmethod
135 + def add_seq_group(self, seq_group_metadata: "SequenceGroupMetadata",
136 + token_lens: List[int], seq_lens: List[int],
137 + curr_seq_lens: List[int], query_lens: List[int],
138 + context_lens: List[int],
139 + curr_sliding_window_blocks: List[int],
140 + prefix_cache_hit: bool, chunked_prefill_enabled: bool):
141 + """Add a sequence group to the metadata and update
142 + corresponding fields (in Python objects).
143 + """
144 + raise NotImplementedError
145 +
146 + @abstractmethod
147 + def build(self, runner: "ModelRunnerInputBuilderBase", seq_lens: List[int],
148 + query_lens: List[int], cuda_graph_pad_size: int,
149 + batch_size: int) -> T:
150 + """Build attention metadata with on-device tensors."""
151 + raise NotImplementedError
152 +
153 +
154 class AttentionImpl(ABC, Generic[T]):
155
156 @abstractmethod
vllm/attention/backends/blocksparse_attn.py
+11
@@ -5,6 +5,7 @@ import torch
5
6 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7 AttentionMetadata, AttentionType)
8 +from vllm.attention.backends.utils import CommonMetadataBuilder
9 from vllm.attention.ops.blocksparse_attention.interface import (
10 LocalStridedBlockSparseAttn, get_head_sliding_step)
11 from vllm.attention.ops.paged_attn import PagedAttention
@@ -93,6 +94,10 @@ class BlocksparseFlashAttentionBackend(AttentionBackend):
94 def get_metadata_cls() -> Type["AttentionMetadata"]:
95 return BlocksparseFlashAttentionMetadata
96
97 + @staticmethod
98 + def get_builder_cls() -> Type["BlocksparseFlashAttentionMetadataBuilder"]:
99 + return BlocksparseFlashAttentionMetadataBuilder
100 +
101 @staticmethod
102 def get_kv_cache_shape(
103 num_blocks: int,
@@ -244,6 +249,12 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
249 return self._cached_decode_metadata
250
251
252 +class BlocksparseFlashAttentionMetadataBuilder(
253 + CommonMetadataBuilder[BlocksparseFlashAttentionMetadata]):
254 +
255 + _metadata_cls = BlocksparseFlashAttentionMetadata
256 +
257 +
258 class BlocksparseFlashAttentionImpl(AttentionImpl):
259 """
260 If the input tensors contain prompt tokens, the layout is as follows:
vllm/attention/backends/flash_attn.py
+181 -2
@@ -1,13 +1,24 @@
1 """Attention layer with FlashAttention."""
2 from dataclasses import dataclass
3 -from typing import Any, Dict, List, Optional, Tuple, Type
3 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
4
5 import torch
6 from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
7
8 from vllm import _custom_ops as ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata, AttentionType)
10 + AttentionMetadata,
11 + AttentionMetadataBuilder,
12 + AttentionType)
13 +from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
14 + compute_slot_mapping_start_idx,
15 + is_block_tables_empty)
16 +from vllm.sequence import SequenceGroupMetadata
17 +from vllm.utils import make_tensor_with_pad
18 +
19 +if TYPE_CHECKING:
20 + from vllm.worker.model_runner import (GPUModelRunnerBase,
21 + ModelInputForGPUBuilder)
22
23
24 class FlashAttentionBackend(AttentionBackend):
@@ -28,6 +39,10 @@ class FlashAttentionBackend(AttentionBackend):
39 def get_metadata_cls() -> Type["AttentionMetadata"]:
40 return FlashAttentionMetadata
41
42 + @staticmethod
43 + def get_builder_cls() -> Type["FlashAttentionMetadataBuilder"]:
44 + return FlashAttentionMetadataBuilder
45 +
46 @staticmethod
47 def get_kv_cache_shape(
48 num_blocks: int,
@@ -184,6 +199,170 @@ class FlashAttentionMetadata(AttentionMetadata):
199 return self._cached_decode_metadata
200
201
202 +class FlashAttentionMetadataBuilder(
203 + AttentionMetadataBuilder[FlashAttentionMetadata]):
204 +
205 + def __init__(self, input_builder: "ModelInputForGPUBuilder"):
206 + self.slot_mapping: List[int] = []
207 + self.prefill_seq_lens: List[int] = []
208 + self.context_lens: List[int] = []
209 + self.block_tables: List[List[int]] = []
210 + self.curr_seq_lens: List[int] = []
211 + self.num_prefills = 0
212 + self.num_prefill_tokens = 0
213 + self.num_decode_tokens = 0
214 +
215 + self.sliding_window = input_builder.sliding_window
216 + self.block_size = input_builder.block_size
217 + self.use_v2_block_manager = (
218 + input_builder.scheduler_config.use_v2_block_manager)
219 +
220 + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
221 + token_lens: List[int], seq_lens: List[int],
222 + curr_seq_lens: List[int], query_lens: List[int],
223 + context_lens: List[int],
224 + curr_sliding_window_blocks: List[int],
225 + prefix_cache_hit: bool, chunked_prefill_enabled: bool):
226 + """Add a sequence group to the metadata. Specifically update/append
227 + 1. context length.
228 + 2. block table.
229 + 3. slot mapping.
230 + """
231 + is_prompt = seq_group_metadata.is_prompt
232 + block_tables = seq_group_metadata.block_tables
233 +
234 + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
235 + curr_sliding_window_block) in zip(
236 + seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
237 + curr_seq_lens, query_lens, context_lens,
238 + curr_sliding_window_blocks):
239 + self.context_lens.append(context_len)
240 +
241 + if is_prompt:
242 + self.num_prefills += 1
243 + self.num_prefill_tokens += token_len
244 + self.prefill_seq_lens.append(seq_len)
245 + else:
246 + assert query_len == 1, (
247 + "seq_len: {}, context_len: {}, query_len: {}".format(
248 + seq_len, context_len, query_len))
249 + self.num_decode_tokens += query_len
250 + self.curr_seq_lens.append(curr_seq_len)
251 +
252 + # Compute block table.
253 + # TODO(sang): Combine chunked prefill and prefix caching by
254 + # only allowing multiple of block_size chunk size.
255 + # NOTE: This only works for oooooooxxx style attention.
256 + block_table = []
257 + if prefix_cache_hit:
258 + # NOTE(woosuk): For flash-attn, the block table should
259 + # include the entries for the incoming prefill tokens.
260 + block_table = block_tables[seq_id]
261 + elif ((chunked_prefill_enabled or not is_prompt)
262 + and block_tables is not None):
263 + block_table = block_tables[seq_id][-curr_sliding_window_block:]
264 + self.block_tables.append(block_table)
265 +
266 + # Compute slot mapping.
267 + is_profile_run = is_block_tables_empty(block_tables)
268 + start_idx = compute_slot_mapping_start_idx(
269 + is_prompt, query_len, context_len, self.sliding_window,
270 + self.use_v2_block_manager)
271 + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
272 + seq_len, context_len, start_idx,
273 + self.block_size,
274 + seq_group_metadata.block_tables)
275 +
276 + def build(self, runner: "GPUModelRunnerBase", seq_lens, query_lens,
277 + cuda_graph_pad_size: int, batch_size: int):
278 + """Build attention metadata with on-device tensors."""
279 + device = runner.device
280 + use_captured_graph = cuda_graph_pad_size != -1
281 +
282 + logits_soft_cap = getattr(runner.model_config.hf_config,
283 + "attn_logit_softcapping", None)
284 + if logits_soft_cap is not None:
285 + raise ValueError(
286 + "Please use Flashinfer backend for models with logits_soft_cap"
287 + " (i.e., Gemma-2). Otherwise, the output might be wrong."
288 + " Set Flashinfer backend by "
289 + "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
290 +
291 + max_query_len = max(query_lens)
292 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
293 + max_decode_seq_len = max(self.curr_seq_lens, default=0)
294 + num_decode_tokens = self.num_decode_tokens
295 +
296 + if use_captured_graph:
297 + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
298 + self.block_tables.extend([] * cuda_graph_pad_size)
299 + num_decode_tokens = batch_size + cuda_graph_pad_size
300 +
301 + # The shape of graph_block_tables is
302 + # [max batch size, max context len // block size].
303 + input_block_tables = runner.graph_block_tables[:batch_size]
304 + for i, block_table in enumerate(self.block_tables):
305 + if block_table:
306 + input_block_tables[i, :len(block_table)] = block_table
307 + block_tables = torch.tensor(input_block_tables, device=device)
308 + else:
309 + max_block_table_len = max(
310 + len(block_table) for block_table in self.block_tables)
311 + block_tables = make_tensor_with_pad(
312 + self.block_tables,
313 + max_len=max_block_table_len,
314 + pad=0,
315 + dtype=torch.int,
316 + device=device,
317 + )
318 + assert max_query_len > 0, ("query_lens: {}".format(query_lens))
319 +
320 + context_lens_tensor = torch.tensor(self.context_lens,
321 + dtype=torch.int,
322 + device=device)
323 + seq_lens_tensor = torch.tensor(seq_lens,
324 + dtype=torch.int,
325 + device=device)
326 + query_lens_tensor = torch.tensor(query_lens,
327 + dtype=torch.long,
328 + device=device)
329 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
330 + dtype=torch.int32,
331 + device=device)
332 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
333 + dtype=torch.int32,
334 + device=device)
335 + torch.cumsum(seq_lens_tensor,
336 + dim=0,
337 + dtype=seq_start_loc.dtype,
338 + out=seq_start_loc[1:])
339 + torch.cumsum(query_lens_tensor,
340 + dim=0,
341 + dtype=query_start_loc.dtype,
342 + out=query_start_loc[1:])
343 +
344 + slot_mapping_tensor = torch.tensor(self.slot_mapping,
345 + dtype=torch.long,
346 + device=device)
347 +
348 + return FlashAttentionMetadata(
349 + num_prefills=self.num_prefills,
350 + slot_mapping=slot_mapping_tensor,
351 + num_prefill_tokens=self.num_prefill_tokens,
352 + num_decode_tokens=num_decode_tokens,
353 + seq_lens=seq_lens,
354 + seq_lens_tensor=seq_lens_tensor,
355 + max_query_len=max_query_len,
356 + max_prefill_seq_len=max_prefill_seq_len,
357 + max_decode_seq_len=max_decode_seq_len,
358 + query_start_loc=query_start_loc,
359 + seq_start_loc=seq_start_loc,
360 + context_lens_tensor=context_lens_tensor,
361 + block_tables=block_tables,
362 + use_cuda_graph=use_captured_graph,
363 + )
364 +
365 +
366 class FlashAttentionImpl(AttentionImpl):
367 """
368 If the input tensors contain prompt tokens, the layout is as follows:
vllm/attention/backends/flashinfer.py
+236 -2
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import Any, Dict, List, Optional, Set, Tuple, Type
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type
3
4 try:
5 from flashinfer import BatchDecodeWithPagedKVCacheWrapper
@@ -14,7 +14,18 @@ import torch
14
15 from vllm import _custom_ops as ops
16 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
17 - AttentionMetadata, AttentionType)
17 + AttentionMetadata,
18 + AttentionMetadataBuilder,
19 + AttentionType)
20 +from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21 + compute_slot_mapping_start_idx,
22 + is_block_tables_empty)
23 +from vllm.sequence import SequenceGroupMetadata
24 +from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
25 +
26 +if TYPE_CHECKING:
27 + from vllm.worker.model_runner import (GPUModelRunnerBase,
28 + ModelInputForGPUBuilder)
29
30
31 class FlashInferBackend(AttentionBackend):
@@ -31,6 +42,10 @@ class FlashInferBackend(AttentionBackend):
42 def get_metadata_cls() -> Type["AttentionMetadata"]:
43 return FlashInferMetadata
44
45 + @staticmethod
46 + def get_builder_cls() -> Type["FlashInferMetadataBuilder"]:
47 + return FlashInferMetadataBuilder
48 +
49 @staticmethod
50 def get_kv_cache_shape(
51 num_blocks: int,
@@ -188,6 +203,225 @@ class FlashInferMetadata(AttentionMetadata):
203 return self
204
205
206 +class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
207 +
208 + def __init__(self, input_builder: "ModelInputForGPUBuilder"):
209 + self.slot_mapping: List[int] = []
210 + self.prefill_seq_lens: List[int] = []
211 + self.context_lens: List[int] = []
212 + self.block_tables: List[List[int]] = []
213 + self.curr_seq_lens: List[int] = []
214 + self.num_prefills = 0
215 + self.num_prefill_tokens = 0
216 + self.num_decode_tokens = 0
217 +
218 + self.sliding_window = input_builder.sliding_window
219 + self.block_size = input_builder.block_size
220 + self.use_v2_block_manager = (
221 + input_builder.scheduler_config.use_v2_block_manager)
222 +
223 + # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
224 + # for the precise definition of the following fields.
225 + # An example:
226 + # request 1, page indices [0, 5, 8]
227 + # request 2, page indices [1, 6, 7]
228 + # request 3, page indices [3, 4]
229 + # paged_kv_indices is a concatenation of page indices of all requests:
230 + # [0, 5, 8, 1, 6, 7, 3, 4]
231 + # paged_kv_indptr is used to index into paged_kv_indices:
232 + # [0, 3, 6, 8]
233 + self.paged_kv_indices: List[int] = []
234 + # 0 at the beginning of paged_kv_indptr indicates the start of the
235 + # first request’s page indices in the paged_kv_indices list.
236 + self.paged_kv_indptr: List[int] = [0]
237 + # paged_kv_last_page_len is the length of the last page of each request
238 + self.paged_kv_last_page_len: List[int] = []
239 +
240 + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
241 + token_lens: List[int], seq_lens: List[int],
242 + curr_seq_lens: List[int], query_lens: List[int],
243 + context_lens: List[int],
244 + curr_sliding_window_blocks: List[int],
245 + prefix_cache_hit: bool, chunked_prefill_enabled: bool):
246 + """Add a sequence group to the metadata. Specifically update/append
247 + 1. context length.
248 + 2. block table.
249 + 3. slot mapping.
250 + """
251 + is_prompt = seq_group_metadata.is_prompt
252 + block_tables = seq_group_metadata.block_tables
253 + computed_block_nums = seq_group_metadata.computed_block_nums
254 +
255 + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
256 + curr_sliding_window_block) in zip(
257 + seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
258 + curr_seq_lens, query_lens, context_lens,
259 + curr_sliding_window_blocks):
260 + self.context_lens.append(context_len)
261 + if is_prompt:
262 + self.num_prefills += 1
263 + self.num_prefill_tokens += token_len
264 + self.prefill_seq_lens.append(seq_len)
265 + else:
266 + assert query_len == 1, (
267 + "seq_len: {}, context_len: {}, query_len: {}".format(
268 + seq_len, context_len, query_len))
269 + self.num_decode_tokens += query_len
270 + self.curr_seq_lens.append(curr_seq_len)
271 +
272 + # Compute block table.
273 + # TODO(sang): Combine chunked prefill and prefix caching by
274 + # only allowing multiple of block_size chunk size.
275 + # NOTE: This only works for oooooooxxx style attention.
276 + block_table = []
277 + if prefix_cache_hit:
278 + block_table = computed_block_nums
279 + elif ((chunked_prefill_enabled or not is_prompt)
280 + and block_tables is not None):
281 + block_table = block_tables[seq_id][-curr_sliding_window_block:]
282 + self.block_tables.append(block_table)
283 +
284 + is_profile_run = is_block_tables_empty(block_tables)
285 +
286 + # Compute slot mapping.
287 + start_idx = compute_slot_mapping_start_idx(
288 + is_prompt, query_len, context_len, self.sliding_window,
289 + self.use_v2_block_manager)
290 + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
291 + seq_len, context_len, start_idx,
292 + self.block_size,
293 + seq_group_metadata.block_tables)
294 +
295 + # It is not necessary to add paged_kv_indices, paged_kv_indptr,
296 + # and paged_kv_last_page_len for profile run because we will
297 + # create dummy inputs.
298 + if is_profile_run:
299 + return
300 +
301 + # Get the number of valid blocks based on sequence length.
302 + # If seq_len = 16, block_size = 16,
303 + # block_table_bound is 1 with 1 valid block.
304 + # If seq_len = 15, block_size = 16,
305 + # block_table_bound is 0 + 1 with 1 valid block.
306 + block_table_bound = seq_len // self.block_size + 1 \
307 + if seq_len % self.block_size != 0 \
308 + else seq_len // self.block_size
309 + block_table = block_tables[seq_id]
310 + self.paged_kv_indices.extend(block_table[:block_table_bound])
311 + self.paged_kv_indptr.append(self.paged_kv_indptr[-1] +
312 + block_table_bound)
313 +
314 + last_page_len = seq_len % self.block_size
315 + if last_page_len == 0:
316 + last_page_len = self.block_size
317 + self.paged_kv_last_page_len.append(last_page_len)
318 +
319 + def build(self, runner: "GPUModelRunnerBase", seq_lens, query_lens,
320 + cuda_graph_pad_size: int, batch_size: int):
321 + device = runner.device
322 + use_captured_graph = cuda_graph_pad_size != -1
323 +
324 + max_query_len = max(query_lens)
325 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
326 + num_decode_tokens = self.num_decode_tokens
327 +
328 + if use_captured_graph:
329 + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
330 + self.block_tables.extend([] * cuda_graph_pad_size)
331 + num_decode_tokens = batch_size + cuda_graph_pad_size
332 +
333 + # The shape of graph_block_tables is
334 + # [max batch size, max context len // block size].
335 + input_block_tables = runner.graph_block_tables[:batch_size]
336 + for i, block_table in enumerate(self.block_tables):
337 + if block_table:
338 + input_block_tables[i, :len(block_table)] = block_table
339 + block_tables = torch.tensor(input_block_tables, device=device)
340 +
341 + last_paged_kv_indptr = self.paged_kv_indptr[-1]
342 + self.paged_kv_indptr.extend([last_paged_kv_indptr] *
343 + cuda_graph_pad_size)
344 + self.paged_kv_last_page_len.extend([0] * cuda_graph_pad_size)
345 + else:
346 + max_block_table_len = max(
347 + len(block_table) for block_table in self.block_tables)
348 + block_tables = make_tensor_with_pad(
349 + self.block_tables,
350 + max_len=max_block_table_len,
351 + pad=0,
352 + dtype=torch.int,
353 + device=device,
354 + )
355 + assert max_query_len > 0, ("query_lens: {}".format(query_lens))
356 +
357 + seq_lens_tensor = torch.tensor(seq_lens,
358 + dtype=torch.int,
359 + device=device)
360 + query_lens_tensor = torch.tensor(query_lens,
361 + dtype=torch.long,
362 + device=device)
363 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
364 + dtype=torch.int32,
365 + device=device)
366 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
367 + dtype=torch.int32,
368 + device=device)
369 + torch.cumsum(seq_lens_tensor,
370 + dim=0,
371 + dtype=seq_start_loc.dtype,
372 + out=seq_start_loc[1:])
373 + torch.cumsum(query_lens_tensor,
374 + dim=0,
375 + dtype=query_start_loc.dtype,
376 + out=query_start_loc[1:])
377 +
378 + slot_mapping_tensor = torch.tensor(self.slot_mapping,
379 + dtype=torch.long,
380 + device=device)
381 +
382 + logits_soft_cap = getattr(runner.model_config.hf_config,
383 + "attn_logit_softcapping", None)
384 +
385 + if len(self.paged_kv_indptr) > 0:
386 + paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
387 + device="cpu",
388 + dtype=torch.int)
389 + paged_kv_indptr_tensor = torch.tensor(self.paged_kv_indptr,
390 + device="cpu",
391 + dtype=torch.int)
392 + paged_kv_last_page_len_tensor = torch.tensor(
393 + self.paged_kv_last_page_len, device="cpu", dtype=torch.int)
394 + else:
395 + paged_kv_indices_tensor = None
396 + paged_kv_indptr_tensor = None
397 + paged_kv_last_page_len_tensor = None
398 +
399 + kv_cache_dtype = get_kv_cache_torch_dtype(runner.kv_cache_dtype,
400 + runner.model_config.dtype)
401 + return FlashInferMetadata(
402 + num_prefills=self.num_prefills,
403 + slot_mapping=slot_mapping_tensor,
404 + num_prefill_tokens=self.num_prefill_tokens,
405 + num_decode_tokens=num_decode_tokens,
406 + max_prefill_seq_len=max_prefill_seq_len,
407 + block_tables=block_tables,
408 + paged_kv_indptr=paged_kv_indptr_tensor,
409 + paged_kv_indices=paged_kv_indices_tensor,
410 + paged_kv_last_page_len=paged_kv_last_page_len_tensor,
411 + num_qo_heads=runner.model_config.get_num_attention_heads(
412 + runner.parallel_config),
413 + num_kv_heads=runner.model_config.get_num_kv_heads(
414 + runner.parallel_config),
415 + head_dim=runner.model_config.get_head_size(),
416 + page_size=self.block_size,
417 + seq_start_loc=seq_start_loc,
418 + query_start_loc=query_start_loc,
419 + device=device,
420 + data_type=kv_cache_dtype,
421 + use_cuda_graph=use_captured_graph,
422 + logits_soft_cap=logits_soft_cap)
423 +
424 +
425 class FlashInferImpl(AttentionImpl):
426
427 def __init__(
vllm/attention/backends/rocm_flash_attn.py
+11
@@ -7,6 +7,7 @@ import torch
7 import vllm.envs as envs
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 AttentionMetadata, AttentionType)
10 +from vllm.attention.backends.utils import CommonMetadataBuilder
11 from vllm.attention.ops.paged_attn import (PagedAttention,
12 PagedAttentionMetadata)
13 from vllm.logger import init_logger
@@ -28,6 +29,10 @@ class ROCmFlashAttentionBackend(AttentionBackend):
29 def get_metadata_cls() -> Type["AttentionMetadata"]:
30 return ROCmFlashAttentionMetadata
31
32 + @staticmethod
33 + def get_builder_cls() -> Type["ROCmFlashAttentionMetadataBuilder"]:
34 + return ROCmFlashAttentionMetadataBuilder
35 +
36 @staticmethod
37 def get_kv_cache_shape(
38 num_blocks: int,
@@ -166,6 +171,12 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
171 return self._cached_decode_metadata
172
173
174 +class ROCmFlashAttentionMetadataBuilder(
175 + CommonMetadataBuilder[ROCmFlashAttentionMetadata]):
176 +
177 + _metadata_cls = ROCmFlashAttentionMetadata
178 +
179 +
180 def _make_alibi_bias(alibi_slopes: torch.Tensor,
181 dtype: torch.dtype,
182 seq_lens: Optional[List[int]],
vllm/attention/backends/utils.py
+233 -1
@@ -1,7 +1,239 @@
1 """Attention backend utils"""
2 +from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
3 +
4 +import torch
5 +
6 +from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7 +from vllm.sequence import SequenceGroupMetadata
8 +from vllm.utils import make_tensor_with_pad
9
10 # Error string(s) for encoder/decoder
11 # unsupported attention scenarios
5 -
12 STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
13 "with encoder/decoder models.")
14 +
15 +PAD_SLOT_ID = -1
16 +
17 +if TYPE_CHECKING:
18 + from vllm.worker.model_runner import (GPUModelRunnerBase,
19 + ModelInputForGPUBuilder)
20 +
21 +
22 +def is_block_tables_empty(block_tables: Union[None, Dict]):
23 + """
24 + Check if block_tables is None or a dictionary with all None values.
25 + """
26 + if block_tables is None:
27 + return True
28 + if isinstance(block_tables, dict) and all(
29 + value is None for value in block_tables.values()):
30 + return True
31 + return False
32 +
33 +
34 +def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
35 + context_len: int, sliding_window: int,
36 + use_v2_block_manager: bool):
37 + """
38 + Compute the start index of slot mapping.
39 + """
40 + start_idx = 0
41 + if is_prompt and sliding_window is not None:
42 + assert use_v2_block_manager or context_len == 0, (
43 + "Prefix caching is currently not supported with "
44 + "sliding window attention in V1 block manager")
45 + # When prefill, we use it to not write slots to kv cache
46 + # to save memory.
47 + start_idx = max(0, query_len - sliding_window)
48 + return start_idx
49 +
50 +
51 +def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
52 + seq_id: int, seq_len: int, context_len: int,
53 + start_idx: int, block_size: int,
54 + block_tables: Dict[int, List[int]]):
55 + """
56 + Compute slot mapping.
57 + """
58 + if is_profile_run:
59 + # During memory profiling, the block tables are not
60 + # initialized yet. In this case, we just use a dummy
61 + # slot mapping.
62 + # In embeddings, the block tables are {seq_id: None}.
63 + slot_mapping.extend([PAD_SLOT_ID] * seq_len)
64 + return
65 +
66 + # Mask the [0, start_idx) tokens of the prompt with
67 + # PAD_SLOT_ID, where start_idx is max(0, seq_len -
68 + # sliding_window). For example, if the prompt len is 10,
69 + # sliding window is 8, and block size is 4, the first two
70 + # tokens are masked and the slot mapping will be
71 + # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
72 + block_table = block_tables[seq_id]
73 + slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
74 + for i in range(max(start_idx, context_len), seq_len):
75 + block_number = block_table[i // block_size]
76 + block_offset = i % block_size
77 + slot = block_number * block_size + block_offset
78 + slot_mapping.append(slot)
79 +
80 +
81 +TAttentionMetadata = TypeVar("TAttentionMetadata", bound='AttentionMetadata')
82 +
83 +
84 +class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
85 +
86 + _metadata_cls: Type[TAttentionMetadata]
87 +
88 + def __init__(self, input_builder: "ModelInputForGPUBuilder"):
89 + self.slot_mapping: List[int] = []
90 + self.prefill_seq_lens: List[int] = []
91 + self.context_lens: List[int] = []
92 + self.block_tables: List[List[int]] = []
93 + self.curr_seq_lens: List[int] = []
94 + self.num_prefills = 0
95 + self.num_prefill_tokens = 0
96 + self.num_decode_tokens = 0
97 +
98 + self.sliding_window = input_builder.sliding_window
99 + self.block_size = input_builder.block_size
100 + self.use_v2_block_manager = (
101 + input_builder.scheduler_config.use_v2_block_manager)
102 +
103 + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
104 + token_lens: List[int], seq_lens: List[int],
105 + curr_seq_lens: List[int], query_lens: List[int],
106 + context_lens: List[int],
107 + curr_sliding_window_blocks: List[int], prefix_cache_hit,
108 + chunked_prefill_enabled):
109 + is_prompt = seq_group_metadata.is_prompt
110 + block_tables = seq_group_metadata.block_tables
111 + computed_block_nums = seq_group_metadata.computed_block_nums
112 +
113 + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
114 + curr_sliding_window_block) in zip(
115 + seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
116 + curr_seq_lens, query_lens, context_lens,
117 + curr_sliding_window_blocks):
118 + self.context_lens.append(context_len)
119 + if is_prompt:
120 + self.num_prefills += 1
121 + self.num_prefill_tokens += token_len
122 + self.prefill_seq_lens.append(seq_len)
123 + else:
124 + assert query_len == 1, (
125 + "seq_len: {}, context_len: {}, query_len: {}".format(
126 + seq_len, context_len, query_len))
127 + self.num_decode_tokens += query_len
128 + self.curr_seq_lens.append(curr_seq_len)
129 +
130 + # Compute block table.
131 + # TODO(sang): Combine chunked prefill and prefix caching by
132 + # only allowing multiple of block_size chunk size.
133 + # NOTE: This only works for oooooooxxx style attention.
134 + block_table = []
135 + if prefix_cache_hit:
136 + block_table = computed_block_nums
137 + elif ((chunked_prefill_enabled or not is_prompt)
138 + and block_tables is not None):
139 + block_table = block_tables[seq_id][-curr_sliding_window_block:]
140 + self.block_tables.append(block_table)
141 +
142 + # Compute slot mapping.
143 + is_profile_run = is_block_tables_empty(block_tables)
144 + start_idx = compute_slot_mapping_start_idx(
145 + is_prompt, query_len, context_len, self.sliding_window,
146 + self.use_v2_block_manager)
147 + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
148 + seq_len, context_len, start_idx,
149 + self.block_size,
150 + seq_group_metadata.block_tables)
151 +
152 + def build(self, runner: "GPUModelRunnerBase", seq_lens: List[int],
153 + query_lens: List[int], cuda_graph_pad_size: int,
154 + batch_size: int):
155 + device = runner.device
156 + use_captured_graph = cuda_graph_pad_size != -1
157 +
158 + logits_soft_cap = getattr(runner.model_config.hf_config,
159 + "attn_logit_softcapping", None)
160 + if logits_soft_cap is not None:
161 + raise ValueError(
162 + "Please use Flashinfer backend for models with logits_soft_cap "
163 + "(i.e., Gemma-2). Otherwise, the output might be wrong. "
164 + "Set Flashinfer backend by "
165 + "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
166 +
167 + max_query_len = max(query_lens)
168 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
169 + max_decode_seq_len = max(self.curr_seq_lens, default=0)
170 + num_decode_tokens = self.num_decode_tokens
171 +
172 + if use_captured_graph:
173 + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
174 + self.block_tables.extend([] * cuda_graph_pad_size)
175 + num_decode_tokens = batch_size + cuda_graph_pad_size
176 +
177 + # The shape of graph_block_tables is
178 + # [max batch size, max context len // block size].
179 + input_block_tables = runner.graph_block_tables[:batch_size]
180 + for i, block_table in enumerate(self.block_tables):
181 + if block_table:
182 + input_block_tables[i, :len(block_table)] = block_table
183 + block_tables = torch.tensor(input_block_tables, device=device)
184 + else:
185 + max_block_table_len = max(
186 + len(block_table) for block_table in self.block_tables)
187 + block_tables = make_tensor_with_pad(
188 + self.block_tables,
189 + max_len=max_block_table_len,
190 + pad=0,
191 + dtype=torch.int,
192 + device=device,
193 + )
194 + assert max_query_len > 0, "query_lens: {}".format(query_lens)
195 +
196 + context_lens_tensor = torch.tensor(self.context_lens,
197 + dtype=torch.int,
198 + device=device)
199 + seq_lens_tensor = torch.tensor(seq_lens,
200 + dtype=torch.int,
201 + device=device)
202 + query_lens_tensor = torch.tensor(query_lens,
203 + dtype=torch.long,
204 + device=device)
205 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
206 + dtype=torch.int32,
207 + device=device)
208 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
209 + dtype=torch.int32,
210 + device=device)
211 + torch.cumsum(seq_lens_tensor,
212 + dim=0,
213 + dtype=seq_start_loc.dtype,
214 + out=seq_start_loc[1:])
215 + torch.cumsum(query_lens_tensor,
216 + dim=0,
217 + dtype=query_start_loc.dtype,
218 + out=query_start_loc[1:])
219 +
220 + slot_mapping_tensor = torch.tensor(self.slot_mapping,
221 + dtype=torch.long,
222 + device=device)
223 +
224 + return self._metadata_cls( # type: ignore
225 + num_prefills=self.num_prefills,
226 + slot_mapping=slot_mapping_tensor,
227 + num_prefill_tokens=self.num_prefill_tokens,
228 + num_decode_tokens=num_decode_tokens,
229 + seq_lens=seq_lens,
230 + seq_lens_tensor=seq_lens_tensor,
231 + max_query_len=max_query_len,
232 + max_prefill_seq_len=max_prefill_seq_len,
233 + max_decode_seq_len=max_decode_seq_len,
234 + query_start_loc=query_start_loc,
235 + seq_start_loc=seq_start_loc,
236 + context_lens_tensor=context_lens_tensor,
237 + block_tables=block_tables,
238 + use_cuda_graph=use_captured_graph,
239 + )
vllm/attention/backends/xformers.py
+10
@@ -11,6 +11,7 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 AttentionMetadata, AttentionType)
14 +from vllm.attention.backends.utils import CommonMetadataBuilder
15 from vllm.attention.ops.paged_attn import (PagedAttention,
16 PagedAttentionMetadata)
17 from vllm.logger import init_logger
@@ -32,6 +33,10 @@ class XFormersBackend(AttentionBackend):
33 def get_metadata_cls() -> Type["AttentionMetadata"]:
34 return XFormersMetadata
35
36 + @staticmethod
37 + def get_builder_cls() -> Type["XFormersMetadataBuilder"]:
38 + return XFormersMetadataBuilder
39 +
40 @staticmethod
41 def get_kv_cache_shape(
42 num_blocks: int,
@@ -362,6 +367,11 @@ def _get_seq_len_block_table_args(
367 raise AttributeError(f"Invalid attention type {str(attn_type)}")
368
369
370 +class XFormersMetadataBuilder(CommonMetadataBuilder[XFormersMetadata]):
371 +
372 + _metadata_cls = XFormersMetadata
373 +
374 +
375 class XFormersImpl(AttentionImpl[XFormersMetadata]):
376 """
377 If the input tensors contain prompt tokens, the layout is as follows:
vllm/attention/selector.py
+3 -2
@@ -7,6 +7,7 @@ import torch
7 import vllm.envs as envs
8 from vllm.attention.backends.abstract import AttentionBackend
9 from vllm.logger import init_logger
10 +from vllm.platforms import current_platform
11 from vllm.utils import is_cpu, is_hip, is_openvino, is_tpu, is_xpu
12
13 logger = init_logger(__name__)
@@ -136,7 +137,7 @@ def which_attn_to_use(
137 selected_backend = (_Backend.ROCM_FLASH if selected_backend
138 == _Backend.FLASH_ATTN else selected_backend)
139 if selected_backend == _Backend.ROCM_FLASH:
139 - if torch.cuda.get_device_capability()[0] != 9:
140 + if current_platform.get_device_capability()[0] != 9:
141 # not Instinct series GPUs.
142 logger.info("flash_attn is not supported on NAVI GPUs.")
143 else:
@@ -145,7 +146,7 @@ def which_attn_to_use(
146
147 # FlashAttn in NVIDIA GPUs.
148 if selected_backend == _Backend.FLASH_ATTN:
148 - if torch.cuda.get_device_capability()[0] < 8:
149 + if current_platform.get_device_capability()[0] < 8:
150 # Volta and Turing NVIDIA GPUs.
151 logger.info(
152 "Cannot use FlashAttention-2 backend for Volta and Turing "
vllm/worker/model_runner.py
+299 -459
@@ -2,6 +2,7 @@ import dataclasses
2 import gc
3 import time
4 import warnings
5 +import weakref
6 from collections import defaultdict
7 from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8 Tuple, Type, TypeVar, Union)
@@ -48,9 +49,9 @@ from vllm.sampling_params import SamplingParams
49 from vllm.sequence import (IntermediateTensors, SamplerOutput,
50 SequenceGroupMetadata)
51 from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
51 - is_pin_memory_available, make_tensor_with_pad)
52 + is_pin_memory_available)
53 from vllm.worker.model_runner_base import (
53 - ModelRunnerBase, ModelRunnerInputBase,
54 + ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
55 _add_attn_metadata_broadcastable_dict,
56 _add_sampling_metadata_broadcastable_dict,
57 _init_attn_metadata_from_tensor_dict,
@@ -165,6 +166,298 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
166 return cls(**tensor_dict)
167
168
169 +class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
170 + """TBA"""
171 +
172 + def __init__(self,
173 + runner: "GPUModelRunnerBase",
174 + finished_requests_ids: Optional[List[str]] = None):
175 + super().__init__()
176 + self.runner = runner
177 + self.model_input_cls = self.runner._model_input_cls
178 + self.attn_backend = self.runner.attn_backend
179 + self.scheduler_config = self.runner.scheduler_config
180 + self.sliding_window = self.runner.sliding_window
181 + self.block_size = self.runner.block_size
182 + self.enable_lora = self.runner.lora_config is not None
183 + self.enable_prompt_adapter = (self.runner.prompt_adapter_config
184 + is not None)
185 + self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
186 + self.finished_requests_ids = finished_requests_ids
187 + self.decode_only = True
188 +
189 + # Common inputs.
190 + self.input_tokens: List[int] = []
191 + self.input_positions: List[int] = []
192 + self.seq_lens: List[int] = []
193 + self.query_lens: List[int] = []
194 + self.max_decode_seq_len: int = 0
195 + self.request_ids_to_seq_ids: Dict[str, List[int]] = defaultdict(list)
196 +
197 + # LoRA inputs.
198 + self.lora_index_mapping: List[int] = []
199 + self.lora_prompt_mapping: List[int] = []
200 + self.lora_requests: Set[LoRARequest] = set()
201 +
202 + # Prompt adapter inputs.
203 + self.prompt_adapter_index_mapping: List[int] = []
204 + self.prompt_adapter_prompt_mapping: List[int] = []
205 + self.prompt_adapter_requests: Set[PromptAdapterRequest] = set()
206 +
207 + # Multi-modal inputs.
208 + self.multi_modal_inputs_list: List[MultiModalInputs] = []
209 +
210 + # Attention metadata inputs.
211 + self.attn_metadata_builder = self.attn_backend.make_metadata_builder(
212 + self)
213 +
214 + # Engine/Model configurations.
215 + self.chunked_prefill_enabled = (
216 + self.scheduler_config is not None
217 + and self.scheduler_config.chunked_prefill_enabled)
218 + if self.sliding_window is not None:
219 + self.sliding_window_blocks = (
220 + self.sliding_window + self.block_size - 1) // self.block_size
221 + self.block_aligned_sliding_window = \
222 + self.sliding_window_blocks * self.block_size
223 +
224 + def _compute_len_for_sliding_window(self, seq_len: int):
225 + curr_sliding_window_blocks = 0
226 + sliding_seq_len = seq_len
227 +
228 + # TODO(sang): This is a hack to make sliding window work with
229 + # paged attn. We can remove it if we make paged attn kernel
230 + # to properly handle slinding window attn.
231 + if self.sliding_window is not None:
232 + curr_sliding_window_blocks = self.sliding_window_blocks
233 + if self.scheduler_config.use_v2_block_manager:
234 + # number of elements in last block
235 + suff_len = seq_len % self.block_size
236 + sliding_seq_len = min(
237 + seq_len, self.block_aligned_sliding_window + suff_len)
238 + if suff_len > 0:
239 + curr_sliding_window_blocks += 1
240 + else:
241 + sliding_seq_len = min(seq_len, self.sliding_window)
242 + return curr_sliding_window_blocks, sliding_seq_len
243 +
244 + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
245 + seq_ids = list(seq_group_metadata.seq_data.keys())
246 + n_seqs = len(seq_ids)
247 + is_prompt = seq_group_metadata.is_prompt
248 + token_chunk_size = seq_group_metadata.token_chunk_size
249 +
250 + if is_prompt:
251 + assert n_seqs == 1
252 + self.decode_only = False
253 +
254 + # Mapping from request IDs to sequence IDs. Used for Jamba models
255 + # that manages the cache by itself.
256 + self.request_ids_to_seq_ids[seq_group_metadata.request_id] = []
257 + # The number of input tokens in each sequence.
258 + token_lens: List[int] = []
259 + # The number of tokens that are already computed.
260 + context_lens: List[int] = []
261 + # The current sliding window block for each sequence.
262 + curr_sliding_window_blocks: List[int] = []
263 + # The original sequence length (before applying sliding window)
264 + # for each sequence.
265 + orig_seq_lens: List[int] = []
266 + # The sequence length (may be capped to the sliding window).
267 + curr_seq_lens: List[int] = []
268 + for seq_id in seq_ids:
269 + seq_data = seq_group_metadata.seq_data[seq_id]
270 + self.request_ids_to_seq_ids[seq_group_metadata.request_id].append(
271 + seq_id)
272 + computed_block_nums = seq_group_metadata.computed_block_nums
273 +
274 + # Check if hit prefix cache (i.e., some blocks are already computed)
275 + # Note that prefix caching does not support sliding window.
276 + prefix_cache_hit = (computed_block_nums is not None
277 + and len(computed_block_nums) > 0
278 + and self.sliding_window is None and is_prompt)
279 + if self.chunked_prefill_enabled and prefix_cache_hit:
280 + raise RuntimeError(
281 + "chunked prefill cannot be used with prefix caching now.")
282 +
283 + # Compute context length (the number of tokens that are
284 + # already computed) and sequence length (total number of tokens).
285 + seq_len = seq_data.get_len()
286 + if is_prompt:
287 + context_len = seq_data.get_num_computed_tokens()
288 + else:
289 + # get_num_computed_tokens is incorrect for spec decoding.
290 + # So, we should have a special logic here.
291 + # TODO(sang): Fix it.
292 + context_len = seq_len - 1
293 + seq_len = min(seq_len, context_len + token_chunk_size)
294 +
295 + # Compute tokens.
296 + if is_prompt:
297 + tokens = seq_data.get_token_ids()[context_len:seq_len]
298 + else:
299 + # Optimization. get_token_ids requires the entire copy of
300 + # tokens.
301 + tokens = [seq_data.get_last_token_id()]
302 + if prefix_cache_hit:
303 + assert computed_block_nums is not None
304 + context_len = len(computed_block_nums) * self.block_size
305 + tokens = tokens[context_len:]
306 +
307 + # These are seq_len/context_len capped to the sliding window.
308 + # They are passed to decode kernel.
309 + # We still need original seq_len/context_len to compute slot
310 + # mapping (and input position) below.
311 + if is_prompt:
312 + curr_sliding_window_block = 0
313 + sliding_seq_len = seq_len
314 + query_len = seq_len - context_len
315 + else:
316 + curr_sliding_window_block, sliding_seq_len = (
317 + self._compute_len_for_sliding_window(seq_len))
318 + query_len = 1
319 +
320 + self.seq_lens.append(sliding_seq_len)
321 + if not is_prompt:
322 + self.max_decode_seq_len = max(self.max_decode_seq_len,
323 + sliding_seq_len)
324 + self.query_lens.append(query_len)
325 + self.input_tokens.extend(tokens)
326 + self.input_positions.extend(list(range(context_len, seq_len)))
327 +
328 + # Intermediate data of the current sequence group for
329 + # the attention metadata.
330 + token_lens.append(len(tokens))
331 + context_lens.append(context_len)
332 + curr_seq_lens.append(sliding_seq_len)
333 + curr_sliding_window_blocks.append(curr_sliding_window_block)
334 + orig_seq_lens.append(seq_len)
335 +
336 + # Update attention metadata. Note that input builder attributes
337 + # (self.xxx) include all added sequences, so we need to slice
338 + # the last n_seqs sequences.
339 + self.attn_metadata_builder.add_seq_group(
340 + seq_group_metadata, token_lens, orig_seq_lens, curr_seq_lens,
341 + self.query_lens[-n_seqs:], context_lens,
342 + curr_sliding_window_blocks, prefix_cache_hit,
343 + self.chunked_prefill_enabled)
344 +
345 + # LoRA data.
346 + if self.enable_lora:
347 + lora_id = seq_group_metadata.lora_int_id
348 + for query_len in self.query_lens[-n_seqs:]:
349 + if lora_id > 0:
350 + self.lora_requests.add(seq_group_metadata.lora_request)
351 + self.lora_index_mapping += [lora_id] * query_len
352 + self.lora_prompt_mapping.extend(
353 + [lora_id] *
354 + (query_len if seq_group_metadata.sampling_params
355 + and seq_group_metadata.sampling_params.prompt_logprobs
356 + is not None else 1))
357 +
358 + # Prompt adapter data. Note that when is_prompt=True,
359 + # we expect only one sequence in the group.
360 + if self.enable_prompt_adapter:
361 + prompt_adapter_id = seq_group_metadata.prompt_adapter_id
362 + if prompt_adapter_id > 0 and is_prompt:
363 + query_len = self.query_lens[-1]
364 + self.prompt_adapter_requests.add(
365 + seq_group_metadata.prompt_adapter_request)
366 +
367 + num_tokens = seq_group_metadata.\
368 + prompt_adapter_num_virtual_tokens
369 + pm = [prompt_adapter_id
370 + ] * num_tokens + [0] * (query_len - num_tokens)
371 + self.prompt_adapter_index_mapping += pm
372 + self.prompt_adapter_prompt_mapping.extend(
373 + [prompt_adapter_id] *
374 + (query_len if seq_group_metadata.sampling_params
375 + and seq_group_metadata.sampling_params.prompt_logprobs
376 + else 1))
377 +
378 + # Multi-modal data.
379 + mm_data = seq_group_metadata.multi_modal_data
380 + if mm_data:
381 + mm_kwargs = self.multi_modal_input_mapper(mm_data)
382 + self.multi_modal_inputs_list.append(mm_kwargs)
383 +
384 + def build(self) -> ModelInputForGPU:
385 + if not self.input_tokens:
386 + return self.model_input_cls()
387 +
388 + batch_size = len(self.input_tokens)
389 + use_captured_graph = (
390 + self.decode_only and not self.runner.model_config.enforce_eager
391 + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
392 + and self.max_decode_seq_len <= self.runner.max_seq_len_to_capture)
393 +
394 + # If cuda graph can be used, pad tensors accordingly.
395 + # See `capture_model` API for more details.
396 + # vLLM uses cuda graph only for decoding requests.
397 + cuda_graph_pad_size = -1
398 + if use_captured_graph:
399 + graph_batch_size = _get_graph_batch_size(batch_size)
400 + assert graph_batch_size >= batch_size
401 + cuda_graph_pad_size = graph_batch_size - batch_size
402 + batch_size = graph_batch_size
403 +
404 + # Tokens and positions.
405 + self.input_tokens.extend([0] * cuda_graph_pad_size)
406 + self.input_positions.extend([0] * cuda_graph_pad_size)
407 + input_tokens_tensor = torch.tensor(self.input_tokens,
408 + dtype=torch.long,
409 + device=self.runner.device)
410 + input_positions_tensor = torch.tensor(self.input_positions,
411 + dtype=torch.long,
412 + device=self.runner.device)
413 +
414 + # Sequence and query lengths.
415 + self.seq_lens.extend([1] * cuda_graph_pad_size)
416 +
417 + # Attention metadata.
418 + attn_metadata = self.attn_metadata_builder.build(
419 + self.runner, self.seq_lens, self.query_lens, cuda_graph_pad_size,
420 + batch_size)
421 +
422 + # LoRA data.
423 + if self.enable_lora:
424 + self.lora_index_mapping.extend([0] * cuda_graph_pad_size)
425 + lora_mapping = LoRAMapping(
426 + self.lora_index_mapping,
427 + self.lora_prompt_mapping,
428 + )
429 + else:
430 + lora_mapping = None
431 +
432 + # Prompt adapter data.
433 + if self.enable_prompt_adapter:
434 + self.prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
435 + prompt_adapter_mapping = PromptAdapterMapping(
436 + self.prompt_adapter_index_mapping,
437 + self.prompt_adapter_prompt_mapping,
438 + )
439 + else:
440 + prompt_adapter_mapping = None
441 +
442 + # Multi-modal data.
443 + multi_modal_kwargs = MultiModalInputs.batch(
444 + self.multi_modal_inputs_list, device=self.runner.device)
445 +
446 + return self.model_input_cls(
447 + input_tokens=input_tokens_tensor,
448 + input_positions=input_positions_tensor,
449 + attn_metadata=attn_metadata,
450 + seq_lens=self.seq_lens,
451 + query_lens=self.query_lens,
452 + lora_mapping=lora_mapping,
453 + lora_requests=self.lora_requests,
454 + multi_modal_kwargs=multi_modal_kwargs,
455 + request_ids_to_seq_ids=self.request_ids_to_seq_ids,
456 + finished_requests_ids=self.finished_requests_ids,
457 + prompt_adapter_mapping=prompt_adapter_mapping,
458 + prompt_adapter_requests=self.prompt_adapter_requests)
459 +
460 +
461 class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
462 """
463 Helper class for shared methods between GPU model runners.
@@ -368,464 +661,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
661
662 If cuda graph is required, this API automatically pads inputs.
663 """
371 - input_tokens: List[int] = []
372 - input_positions: List[int] = []
373 - slot_mapping: List[int] = []
374 - lora_index_mapping: List[int] = []
375 - lora_prompt_mapping: List[int] = []
376 - lora_requests: Set[LoRARequest] = set()
377 - prompt_adapter_index_mapping: List[int] = []
378 - prompt_adapter_prompt_mapping: List[int] = []
379 - prompt_adapter_requests: Set[PromptAdapterRequest] = set()
380 -
381 - seq_lens: List[int] = []
382 - prefill_seq_lens: List[int] = []
383 - decode_seq_lens: List[int] = []
384 - context_lens: List[int] = []
385 - query_lens: List[int] = []
386 - block_tables: List[List[int]] = []
387 - multi_modal_inputs_list: List[MultiModalInputs] = []
388 - request_ids_to_seq_ids: Dict[str, List[int]] = defaultdict(list)
389 - decode_only = True
390 - num_prefills = 0
391 - num_prefill_tokens = 0
392 - num_decode_tokens = 0
393 -
394 - # The following fields are only for flashinfer
395 - # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
396 - # for the precise definition of the following fields.
397 - # An example:
398 - # request 1, page indices [0, 5, 8]
399 - # request 2, page indices [1, 6, 7]
400 - # request 3, page indices [3, 4]
401 - # paged_kv_indices is a concatenation of page indices of all requests:
402 - # [0, 5, 8, 1, 6, 7, 3, 4]
403 - # paged_kv_indptr is used to index into paged_kv_indices:
404 - # [0, 3, 6, 8]
405 - paged_kv_indices: List[int] = []
406 - # 0 at the beginning of paged_kv_indptr indicates the start of the
407 - # first request’s page indices in the paged_kv_indices list.
408 - paged_kv_indptr: List[int] = [0]
409 - # paged_kv_last_page_len is the length of the last page of each request
410 - paged_kv_last_page_len: List[int] = []
411 -
412 - if len(seq_group_metadata_list) == 0:
413 - return self._model_input_cls()
414 -
415 - if self.sliding_window is not None:
416 - sliding_window_blocks = (self.sliding_window + self.block_size -
417 - 1) // self.block_size
418 - block_aligned_sliding_window = \
419 - sliding_window_blocks * self.block_size
420 -
664 + builder = ModelInputForGPUBuilder(weakref.proxy(self),
665 + finished_requests_ids)
666 for seq_group_metadata in seq_group_metadata_list:
422 - seq_ids = list(seq_group_metadata.seq_data.keys())
423 - is_prompt = seq_group_metadata.is_prompt
424 -
425 - for seq_id in seq_ids:
426 - computed_block_nums = seq_group_metadata.computed_block_nums
427 - if (self.scheduler_config is not None
428 - and self.scheduler_config.chunked_prefill_enabled
429 - and not (computed_block_nums is None
430 - or computed_block_nums == [])):
431 - raise RuntimeError(
432 - "chunked prefill cannot be used with prefix caching "
433 - "now.")
434 -
435 - seq_data = seq_group_metadata.seq_data[seq_id]
436 - if is_prompt:
437 - context_len = seq_data.get_num_computed_tokens()
438 - else:
439 - # get_num_computed_tokens is incorrect for spec decoding.
440 - # So, we should have a special logic here.
441 - # TODO(sang): Fix it.
442 - context_len = seq_data.get_len() - 1
443 -
444 - seq_len = min(
445 - seq_data.get_len(),
446 - context_len + seq_group_metadata.token_chunk_size)
447 - if is_prompt:
448 - tokens = seq_data.get_token_ids()[context_len:seq_len]
449 - else:
450 - # Optimization. get_token_ids requires the entire copy of
451 - # tokens.
452 - tokens = [seq_data.get_last_token_id()]
453 -
454 - # Prefix cache was hit.
455 - # Prefix is not supported with sliding_window
456 - prefix_cache_hit = (computed_block_nums is not None
457 - and len(computed_block_nums) > 0
458 - and self.sliding_window is None
459 - and is_prompt)
460 -
461 - # These are seq_len/context_len capped to the sliding window.
462 - # They are passed to decode kernel.
463 - # We still need original seq_len/context_len to compute slot
464 - # mapping (and input position) below.
465 - curr_sliding_window_blocks = None
466 - sliding_seq_len = seq_len
467 - sliding_context_len = context_len
468 -
469 - # TODO(sang): This is a hack to make sliding window work with
470 - # paged attn. We can remove it if we make paged attn kernel
471 - # to properly handle slinding window attn.
472 - if (self.sliding_window is not None and not is_prompt):
473 - curr_sliding_window_blocks = sliding_window_blocks
474 - if self.scheduler_config.use_v2_block_manager:
475 - # number of elements in last block
476 - suff_len = seq_len % self.block_size
477 - sliding_seq_len = min(
478 - seq_len, block_aligned_sliding_window + suff_len)
479 - if suff_len > 0:
480 - curr_sliding_window_blocks += 1
481 - else:
482 - sliding_seq_len = min(seq_len, self.sliding_window)
483 - sliding_context_len = sliding_seq_len - 1
484 -
485 - # TODO(sang): Combine chunked prefill and prefix caching by
486 - # only allowing multiple of block_size chunk size.
487 - # NOTE: This only works for oooooooxxx style attention.
488 - if prefix_cache_hit:
489 - assert computed_block_nums is not None
490 - context_len = len(computed_block_nums) * self.block_size
491 - tokens = tokens[context_len:]
492 -
493 - # need to think what to set it to when we have both sliding
494 - # window and prefix caching...
495 - assert self.sliding_window is None, \
496 - "Prefix caching is not supported with sliding window"
497 - sliding_context_len = context_len
498 -
499 - if self.attn_backend.get_name() == "flash-attn":
500 - # NOTE(woosuk): For flash-attn, the block table should
501 - # include the entries for the incoming prefill tokens.
502 - # TODO(woosuk): This is a temporary fix. We should
503 - # provide a unified interface for different backends.
504 - block_table = seq_group_metadata.block_tables[seq_id]
505 - else:
506 - block_table = computed_block_nums
507 - elif (self.scheduler_config.chunked_prefill_enabled
508 - or not is_prompt):
509 - if seq_group_metadata.block_tables is not None:
510 - # chunked prefill or decode
511 - block_table = seq_group_metadata.block_tables[seq_id]
512 - if curr_sliding_window_blocks is not None:
513 - block_table = block_table[
514 - -curr_sliding_window_blocks:]
515 - else:
516 - # Only happens when memory profiling runs.
517 - block_table = []
518 - else:
519 - # Prefill without chunked prefill or memory profiling.
520 - block_table = []
521 - block_tables.append(block_table)
522 -
523 - seq_lens.append(sliding_seq_len)
524 - context_lens.append(sliding_context_len)
525 - query_len = sliding_seq_len - sliding_context_len
526 - query_lens.append(query_len)
527 - input_tokens.extend(tokens)
528 - input_positions.extend(list(range(context_len, seq_len)))
529 - lora_id = seq_group_metadata.lora_int_id
530 - prompt_adapter_id = seq_group_metadata.prompt_adapter_id
531 -
532 - if is_prompt:
533 - assert len(seq_ids) == 1
534 - num_prefills += 1
535 - num_prefill_tokens += len(tokens)
536 - decode_only = False
537 - prefill_seq_lens.append(seq_len)
538 - else:
539 - assert query_len == 1, (
540 - "seq_len: {}, context_len: {}, query_len: {}".format(
541 - seq_len, context_len, query_len))
542 - num_decode_tokens += query_len
543 - decode_seq_lens.append(sliding_seq_len)
544 -
545 - if lora_id > 0:
546 - lora_requests.add(seq_group_metadata.lora_request)
547 -
548 - lora_index_mapping += [lora_id] * query_len
549 - lora_prompt_mapping.extend(
550 - [lora_id] *
551 - (query_len if seq_group_metadata.sampling_params
552 - and seq_group_metadata.sampling_params.prompt_logprobs
553 - is not None else 1))
554 -
555 - mm_data = seq_group_metadata.multi_modal_data
556 - if mm_data:
557 - # Process multi-modal data
558 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
559 - multi_modal_inputs_list.append(mm_kwargs)
560 -
561 - if prompt_adapter_id > 0 and is_prompt:
562 - prompt_adapter_requests.add(
563 - seq_group_metadata.prompt_adapter_request)
564 -
565 - num_tokens = seq_group_metadata.\
566 - prompt_adapter_num_virtual_tokens
567 - pm = [prompt_adapter_id
568 - ] * num_tokens + [0] * (query_len - num_tokens)
569 - prompt_adapter_index_mapping += pm
570 - prompt_adapter_prompt_mapping.extend(
571 - [prompt_adapter_id] *
572 - (query_len if seq_group_metadata.sampling_params
573 - and seq_group_metadata.sampling_params.prompt_logprobs
574 - else 1))
575 -
576 - is_profile_run = _is_block_tables_empty(
577 - seq_group_metadata.block_tables)
578 - if is_profile_run:
579 - # During memory profiling, the block tables are not
580 - # initialized yet. In this case, we just use a dummy
581 - # slot mapping.
582 - # In embeddings, the block tables are {seq_id: None}.
583 - slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
584 - continue
585 -
586 - # Compute the slot mapping.
587 - block_table = seq_group_metadata.block_tables[seq_id]
588 -
589 - # Mask the [0, start_idx) tokens of the prompt with
590 - # _PAD_SLOT_ID, where start_idx is max(0, seq_len -
591 - # sliding_window). For example, if the prompt len is 10,
592 - # sliding window is 8, and block size is 4, the first two
593 - # tokens are masked and the slot mapping will be
594 - # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
595 - start_idx = 0
596 - if self.sliding_window is not None:
597 - if is_prompt:
598 - assert self.scheduler_config.use_v2_block_manager \
599 - or context_len == 0, (
600 - "Prefix caching is currently not supported with "
601 - "sliding window attention in V1 block manager")
602 - # It is an optimization. When it is decoding, it is always
603 - # 0. When prefill, we use it to not write slots to kv cache
604 - # to save memory.
605 - start_idx = max(0, query_len - self.sliding_window)
606 -
607 - for i in range(context_len, seq_len):
608 - if i < start_idx:
609 - slot_mapping.append(_PAD_SLOT_ID)
610 - continue
611 -
612 - block_number = block_table[i // self.block_size]
613 - block_offset = i % self.block_size
614 - slot = block_number * self.block_size + block_offset
615 - slot_mapping.append(slot)
616 -
617 - # Prepare input tensors for flashinfer
618 - if self.attn_backend.get_name() == "flashinfer":
619 - seq_len = seq_data.get_len()
620 - # Get the number of valid blocks based on sequence length.
621 - # If seq_len = 16, block_size = 16,
622 - # block_table_bound is 1 with 1 valid block.
623 - # If seq_len = 15, block_size = 16,
624 - # block_table_bound is 0 + 1 with 1 valid block.
625 - block_table_bound = seq_len // self.block_size + 1 \
626 - if seq_len % self.block_size != 0 \
627 - else seq_len // self.block_size
628 -
629 - paged_kv_indices.extend(block_table[:block_table_bound])
630 - paged_kv_indptr.append(paged_kv_indptr[-1] +
631 - block_table_bound)
632 -
633 - last_page_len = seq_len % self.block_size
634 - if last_page_len == 0:
635 - last_page_len = self.block_size
636 - paged_kv_last_page_len.append(last_page_len)
637 -
638 - batch_size = len(input_tokens)
639 - max_query_len = max(query_lens)
640 - max_prefill_seq_len = max(prefill_seq_lens, default=0)
641 - max_decode_seq_len = max(decode_seq_lens, default=0)
642 -
643 - # If cuda graph can be used, pad tensors accordingly.
644 - # See `capture_model` API for more details.
645 - # vLLM uses cuda graph only for decoding requests.
646 - use_captured_graph = (
647 - decode_only and not self.model_config.enforce_eager
648 - and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
649 - and max_decode_seq_len <= self.max_seq_len_to_capture)
650 - if use_captured_graph:
651 - graph_batch_size = _get_graph_batch_size(batch_size)
652 - assert graph_batch_size >= batch_size
653 - for _ in range(graph_batch_size - batch_size):
654 - input_tokens.append(0)
655 - input_positions.append(0)
656 - slot_mapping.append(_PAD_SLOT_ID)
657 - seq_lens.append(1)
658 - block_tables.append([])
659 - lora_index_mapping.append(0)
660 - prompt_adapter_index_mapping.append(0)
661 - if self.attn_backend.get_name() == "flashinfer":
662 - last_paged_kv_indptr = paged_kv_indptr[-1]
663 - paged_kv_indptr.append(last_paged_kv_indptr)
664 - paged_kv_last_page_len.append(0)
665 - batch_size = graph_batch_size
666 - num_decode_tokens = batch_size
667 -
668 - if use_captured_graph:
669 - # The shape of graph_block_tables is
670 - # [max batch size, max context len // block size].
671 - input_block_tables = self.graph_block_tables[:batch_size]
672 - for i, block_table in enumerate(block_tables):
673 - if block_table:
674 - input_block_tables[i, :len(block_table)] = block_table
675 - block_tables = torch.tensor(input_block_tables, device=self.device)
676 - else:
677 - max_block_table_len = max(
678 - len(block_table) for block_table in block_tables)
679 - block_tables = make_tensor_with_pad(
680 - block_tables,
681 - max_len=max_block_table_len,
682 - pad=0,
683 - dtype=torch.int,
684 - device=self.device,
685 - )
686 - assert max_query_len > 0, ("query_lens: {}".format(query_lens))
687 -
688 - context_lens_tensor = torch.tensor(context_lens,
689 - dtype=torch.int,
690 - device=self.device)
691 -
692 - seq_lens_tensor = torch.tensor(seq_lens,
693 - dtype=torch.int,
694 - device=self.device)
695 - query_lens_tensor = torch.tensor(query_lens,
696 - dtype=torch.long,
697 - device=self.device)
698 - query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
699 - dtype=torch.int32,
700 - device=self.device)
701 - seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
702 - dtype=torch.int32,
703 - device=self.device)
704 -
705 - torch.cumsum(seq_lens_tensor,
706 - dim=0,
707 - dtype=seq_start_loc.dtype,
708 - out=seq_start_loc[1:])
709 - torch.cumsum(query_lens_tensor,
710 - dim=0,
711 - dtype=query_start_loc.dtype,
712 - out=query_start_loc[1:])
713 -
714 - input_tokens_tensor = torch.tensor(input_tokens,
715 - dtype=torch.long,
716 - device=self.device)
717 - input_positions_tensor = torch.tensor(input_positions,
718 - dtype=torch.long,
719 - device=self.device)
720 - slot_mapping_tensor = torch.tensor(slot_mapping,
721 - dtype=torch.long,
722 - device=self.device)
723 -
724 - logits_soft_cap = getattr(self.model_config.hf_config,
725 - 'attn_logit_softcapping', None)
726 - if logits_soft_cap is not None and self.attn_backend.get_name(
727 - ) != "flashinfer":
728 - raise ValueError("Please use Flashinfer backend for models with"
729 - "logits_soft_cap (i.e., Gemma-2)."
730 - " Otherwise, the output might be wrong."
731 - " Set Flashinfer backend by "
732 - "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
733 -
734 - if self.attn_backend.get_name() == "flashinfer":
735 - if len(paged_kv_indptr) > 0:
736 - paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
737 - device='cpu',
738 - dtype=torch.int)
739 - paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
740 - device='cpu',
741 - dtype=torch.int)
742 - paged_kv_last_page_len_tensor = torch.tensor(
743 - paged_kv_last_page_len, device='cpu', dtype=torch.int)
744 - else:
745 - paged_kv_indices_tensor = None
746 - paged_kv_indptr_tensor = None
747 - paged_kv_last_page_len_tensor = None
748 -
749 - kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
750 - self.model_config.dtype)
751 - attn_metadata = self.attn_backend.make_metadata(
752 - num_prefills=num_prefills,
753 - slot_mapping=slot_mapping_tensor,
754 - num_prefill_tokens=num_prefill_tokens,
755 - num_decode_tokens=num_decode_tokens,
756 - max_prefill_seq_len=max_prefill_seq_len,
757 - block_tables=block_tables,
758 - paged_kv_indptr=paged_kv_indptr_tensor,
759 - paged_kv_indices=paged_kv_indices_tensor,
760 - paged_kv_last_page_len=paged_kv_last_page_len_tensor,
761 - num_qo_heads=self.model_config.get_num_attention_heads(
762 - self.parallel_config),
763 - num_kv_heads=self.model_config.get_num_kv_heads(
764 - self.parallel_config),
765 - head_dim=self.model_config.get_head_size(),
766 - page_size=self.block_size,
767 - seq_start_loc=seq_start_loc,
768 - query_start_loc=query_start_loc,
769 - device=self.device,
770 - data_type=kv_cache_dtype,
771 - use_cuda_graph=use_captured_graph,
772 - logits_soft_cap=logits_soft_cap)
773 -
774 - else:
775 - attn_metadata = self.attn_backend.make_metadata(
776 - num_prefills=num_prefills,
777 - slot_mapping=slot_mapping_tensor,
778 - num_prefill_tokens=num_prefill_tokens,
779 - num_decode_tokens=num_decode_tokens,
780 - seq_lens=seq_lens,
781 - seq_lens_tensor=seq_lens_tensor,
782 - max_query_len=max_query_len,
783 - max_prefill_seq_len=max_prefill_seq_len,
784 - max_decode_seq_len=max_decode_seq_len,
785 - query_start_loc=query_start_loc,
786 - seq_start_loc=seq_start_loc,
787 - context_lens_tensor=context_lens_tensor,
788 - block_tables=block_tables,
789 - use_cuda_graph=use_captured_graph,
790 - )
791 -
792 - if self.lora_config:
793 - lora_mapping = LoRAMapping(
794 - lora_index_mapping,
795 - lora_prompt_mapping,
796 - )
797 - else:
798 - lora_mapping = None
799 -
800 - if self.prompt_adapter_config:
801 - prompt_adapter_mapping = PromptAdapterMapping(
802 - prompt_adapter_index_mapping,
803 - prompt_adapter_prompt_mapping,
804 - )
805 - else:
806 - prompt_adapter_mapping = None
807 -
808 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
809 - device=self.device)
810 - request_ids_to_seq_ids = {
811 - seq_group_metadata.request_id:
812 - list(seq_group_metadata.seq_data.keys())
813 - for seq_group_metadata in seq_group_metadata_list
814 - }
815 - return self._model_input_cls(
816 - input_tokens=input_tokens_tensor,
817 - input_positions=input_positions_tensor,
818 - attn_metadata=attn_metadata,
819 - seq_lens=seq_lens,
820 - query_lens=query_lens,
821 - lora_mapping=lora_mapping,
822 - lora_requests=lora_requests,
823 - multi_modal_kwargs=multi_modal_kwargs,
824 - request_ids_to_seq_ids=request_ids_to_seq_ids,
825 - finished_requests_ids=finished_requests_ids,
826 - prompt_adapter_mapping=prompt_adapter_mapping,
827 - prompt_adapter_requests=prompt_adapter_requests,
828 - )
667 + builder.add_seq_group(seq_group_metadata)
668 + return builder.build() # type: ignore
669
670 @torch.inference_mode()
671 def profile_run(self) -> None:
vllm/worker/model_runner_base.py
+15
@@ -113,6 +113,21 @@ class ModelRunnerInputBase(ABC):
113 raise NotImplementedError
114
115
116 +class ModelRunnerInputBuilderBase(ABC, Generic[T]):
117 + """A builder to create ModelRunnerInputBase objects.
118 + """
119 +
120 + @abstractmethod
121 + def add_seq_group(self, seq_group_metadata):
122 + """TBA"""
123 + raise NotImplementedError
124 +
125 + @abstractmethod
126 + def build(self, *args, **kwargs) -> T:
127 + """Build metadata with on-device tensors."""
128 + raise NotImplementedError
129 +
130 +
131 class ModelRunnerBase(ABC, Generic[T]):
132 """
133 Model runner interface that abstracts a particular hardware and/or type of