A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Modulize prepare input and attention metadata builder (#6596)
Cody Yu committed
Jul 22, 2024 at 17:45 UTC
e0c15758b85827dcac78379e60ea975ebc0ec795
6 files changed
+409
-298
vllm/attention/backends/abstract.py
+3
-17
@@ -7,7 +7,6 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
7
import torch
8
9
if TYPE_CHECKING:
10
- from vllm.sequence import SequenceGroupMetadata
10
from vllm.worker.model_runner_base import ModelRunnerInputBuilderBase
11
12
@@ -128,25 +127,12 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
127
"""Abstract class for attention metadata builders."""
128
129
@abstractmethod
131
- def __init__(self, input_builder) -> None:
130
+ def __init__(self, input_builder: "ModelRunnerInputBuilderBase") -> None:
131
raise NotImplementedError
132
133
@abstractmethod
135
- def add_seq_group(self, seq_group_metadata: "SequenceGroupMetadata",
136
- token_lens: List[int], seq_lens: List[int],
137
- curr_seq_lens: List[int], query_lens: List[int],
138
- context_lens: List[int],
139
- curr_sliding_window_blocks: List[int],
140
- prefix_cache_hit: bool, chunked_prefill_enabled: bool):
141
- """Add a sequence group to the metadata and update
142
- corresponding fields (in Python objects).
143
- """
144
- raise NotImplementedError
145
-
146
- @abstractmethod
147
- def build(self, runner: "ModelRunnerInputBuilderBase", seq_lens: List[int],
148
- query_lens: List[int], cuda_graph_pad_size: int,
149
- batch_size: int) -> T:
134
+ def build(self, seq_lens: List[int], query_lens: List[int],
135
+ cuda_graph_pad_size: int, batch_size: int) -> T:
136
"""Build attention metadata with on-device tensors."""
137
raise NotImplementedError
138
vllm/attention/backends/flash_attn.py
+22
-21
@@ -13,12 +13,10 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
14
compute_slot_mapping_start_idx,
15
is_block_tables_empty)
16
-from vllm.sequence import SequenceGroupMetadata
16
from vllm.utils import make_tensor_with_pad
17
18
if TYPE_CHECKING:
20
- from vllm.worker.model_runner import (GPUModelRunnerBase,
21
- ModelInputForGPUBuilder)
19
+ from vllm.worker.model_runner import ModelInputForGPUBuilder
20
21
22
class FlashAttentionBackend(AttentionBackend):
@@ -212,30 +210,30 @@ class FlashAttentionMetadataBuilder(
210
self.num_prefill_tokens = 0
211
self.num_decode_tokens = 0
212
213
+ self.input_builder = input_builder
214
+ self.runner = input_builder.runner
215
self.sliding_window = input_builder.sliding_window
216
self.block_size = input_builder.block_size
217
self.use_v2_block_manager = (
218
input_builder.scheduler_config.use_v2_block_manager)
219
220
- def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
221
- token_lens: List[int], seq_lens: List[int],
222
- curr_seq_lens: List[int], query_lens: List[int],
223
- context_lens: List[int],
224
- curr_sliding_window_blocks: List[int],
225
- prefix_cache_hit: bool, chunked_prefill_enabled: bool):
220
+ def _add_seq_group(
221
+ self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
222
+ chunked_prefill_enabled: bool):
223
"""Add a sequence group to the metadata. Specifically update/append
224
1. context length.
225
2. block table.
226
3. slot mapping.
227
"""
231
- is_prompt = seq_group_metadata.is_prompt
232
- block_tables = seq_group_metadata.block_tables
228
+ is_prompt = inter_data.is_prompt
229
+ block_tables = inter_data.block_tables
230
231
for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
232
curr_sliding_window_block) in zip(
236
- seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
237
- curr_seq_lens, query_lens, context_lens,
238
- curr_sliding_window_blocks):
233
+ inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
234
+ inter_data.orig_seq_lens, inter_data.seq_lens,
235
+ inter_data.query_lens, inter_data.context_lens,
236
+ inter_data.curr_sliding_window_blocks):
237
self.context_lens.append(context_len)
238
239
if is_prompt:
@@ -254,7 +252,7 @@ class FlashAttentionMetadataBuilder(
252
# only allowing multiple of block_size chunk size.
253
# NOTE: This only works for oooooooxxx style attention.
254
block_table = []
257
- if prefix_cache_hit:
255
+ if inter_data.prefix_cache_hit:
256
# NOTE(woosuk): For flash-attn, the block table should
257
# include the entries for the incoming prefill tokens.
258
block_table = block_tables[seq_id]
@@ -270,16 +268,19 @@ class FlashAttentionMetadataBuilder(
268
self.use_v2_block_manager)
269
compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
270
seq_len, context_len, start_idx,
273
- self.block_size,
274
- seq_group_metadata.block_tables)
271
+ self.block_size, inter_data.block_tables)
272
276
- def build(self, runner: "GPUModelRunnerBase", seq_lens, query_lens,
273
+ def build(self, seq_lens: List[int], query_lens: List[int],
274
cuda_graph_pad_size: int, batch_size: int):
275
"""Build attention metadata with on-device tensors."""
279
- device = runner.device
276
+ for inter_data in self.input_builder.inter_data_list:
277
+ self._add_seq_group(inter_data,
278
+ self.input_builder.chunked_prefill_enabled)
279
+
280
+ device = self.runner.device
281
use_captured_graph = cuda_graph_pad_size != -1
282
282
- logits_soft_cap = getattr(runner.model_config.hf_config,
283
+ logits_soft_cap = getattr(self.runner.model_config.hf_config,
284
"attn_logit_softcapping", None)
285
if logits_soft_cap is not None:
286
raise ValueError(
@@ -300,7 +301,7 @@ class FlashAttentionMetadataBuilder(
301
302
# The shape of graph_block_tables is
303
# [max batch size, max context len // block size].
303
- input_block_tables = runner.graph_block_tables[:batch_size]
304
+ input_block_tables = self.runner.graph_block_tables[:batch_size]
305
for i, block_table in enumerate(self.block_tables):
306
if block_table:
307
input_block_tables[i, :len(block_table)] = block_table
vllm/attention/backends/flashinfer.py
+31
-29
@@ -21,12 +21,10 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21
compute_slot_mapping_start_idx,
22
is_block_tables_empty)
23
from vllm.attention.ops.paged_attn import PagedAttention
24
-from vllm.sequence import SequenceGroupMetadata
24
from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
25
26
if TYPE_CHECKING:
28
- from vllm.worker.model_runner import (GPUModelRunnerBase,
29
- ModelInputForGPUBuilder)
27
+ from vllm.worker.model_runner import ModelInputForGPUBuilder
28
29
30
class FlashInferBackend(AttentionBackend):
@@ -216,6 +214,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
214
self.num_prefill_tokens = 0
215
self.num_decode_tokens = 0
216
217
+ self.input_builder = input_builder
218
+ self.runner = input_builder.runner
219
+
220
self.sliding_window = input_builder.sliding_window
221
self.block_size = input_builder.block_size
222
self.use_v2_block_manager = (
@@ -238,26 +239,24 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
239
# paged_kv_last_page_len is the length of the last page of each request
240
self.paged_kv_last_page_len: List[int] = []
241
241
- def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
242
- token_lens: List[int], seq_lens: List[int],
243
- curr_seq_lens: List[int], query_lens: List[int],
244
- context_lens: List[int],
245
- curr_sliding_window_blocks: List[int],
246
- prefix_cache_hit: bool, chunked_prefill_enabled: bool):
242
+ def _add_seq_group(
243
+ self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
244
+ chunked_prefill_enabled: bool):
245
"""Add a sequence group to the metadata. Specifically update/append
246
1. context length.
247
2. block table.
248
3. slot mapping.
249
"""
252
- is_prompt = seq_group_metadata.is_prompt
253
- block_tables = seq_group_metadata.block_tables
254
- computed_block_nums = seq_group_metadata.computed_block_nums
250
+ is_prompt = inter_data.is_prompt
251
+ block_tables = inter_data.block_tables
252
+ computed_block_nums = inter_data.computed_block_nums
253
254
for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
255
curr_sliding_window_block) in zip(
258
- seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
259
- curr_seq_lens, query_lens, context_lens,
260
- curr_sliding_window_blocks):
256
+ inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
257
+ inter_data.orig_seq_lens, inter_data.seq_lens,
258
+ inter_data.query_lens, inter_data.context_lens,
259
+ inter_data.curr_sliding_window_blocks):
260
self.context_lens.append(context_len)
261
if is_prompt:
262
self.num_prefills += 1
@@ -275,7 +274,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
274
# only allowing multiple of block_size chunk size.
275
# NOTE: This only works for oooooooxxx style attention.
276
block_table = []
278
- if prefix_cache_hit:
277
+ if inter_data.prefix_cache_hit:
278
block_table = computed_block_nums
279
elif ((chunked_prefill_enabled or not is_prompt)
280
and block_tables is not None):
@@ -290,8 +289,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
289
self.use_v2_block_manager)
290
compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
291
seq_len, context_len, start_idx,
293
- self.block_size,
294
- seq_group_metadata.block_tables)
292
+ self.block_size, inter_data.block_tables)
293
294
# It is not necessary to add paged_kv_indices, paged_kv_indptr,
295
# and paged_kv_last_page_len for profile run because we will
@@ -317,9 +315,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
315
last_page_len = self.block_size
316
self.paged_kv_last_page_len.append(last_page_len)
317
320
- def build(self, runner: "GPUModelRunnerBase", seq_lens, query_lens,
318
+ def build(self, seq_lens: List[int], query_lens: List[int],
319
cuda_graph_pad_size: int, batch_size: int):
322
- device = runner.device
320
+ for inter_data in self.input_builder.inter_data_list:
321
+ self._add_seq_group(inter_data,
322
+ self.input_builder.chunked_prefill_enabled)
323
+
324
+ device = self.runner.device
325
use_captured_graph = cuda_graph_pad_size != -1
326
327
max_query_len = max(query_lens)
@@ -333,7 +335,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
335
336
# The shape of graph_block_tables is
337
# [max batch size, max context len // block size].
336
- input_block_tables = runner.graph_block_tables[:batch_size]
338
+ input_block_tables = self.runner.graph_block_tables[:batch_size]
339
for i, block_table in enumerate(self.block_tables):
340
if block_table:
341
input_block_tables[i, :len(block_table)] = block_table
@@ -377,7 +379,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
379
dtype=torch.long,
380
device=device)
381
380
- logits_soft_cap = getattr(runner.model_config.hf_config,
382
+ logits_soft_cap = getattr(self.runner.model_config.hf_config,
383
"attn_logit_softcapping", None)
384
385
if len(self.paged_kv_indptr) > 0:
@@ -394,8 +396,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
396
paged_kv_indptr_tensor = None
397
paged_kv_last_page_len_tensor = None
398
397
- kv_cache_dtype = get_kv_cache_torch_dtype(runner.kv_cache_dtype,
398
- runner.model_config.dtype)
399
+ kv_cache_dtype = get_kv_cache_torch_dtype(
400
+ self.runner.kv_cache_dtype, self.runner.model_config.dtype)
401
return FlashInferMetadata(
402
num_prefills=self.num_prefills,
403
slot_mapping=slot_mapping_tensor,
@@ -406,11 +408,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
408
paged_kv_indptr=paged_kv_indptr_tensor,
409
paged_kv_indices=paged_kv_indices_tensor,
410
paged_kv_last_page_len=paged_kv_last_page_len_tensor,
409
- num_qo_heads=runner.model_config.get_num_attention_heads(
410
- runner.parallel_config),
411
- num_kv_heads=runner.model_config.get_num_kv_heads(
412
- runner.parallel_config),
413
- head_dim=runner.model_config.get_head_size(),
411
+ num_qo_heads=self.runner.model_config.get_num_attention_heads(
412
+ self.runner.parallel_config),
413
+ num_kv_heads=self.runner.model_config.get_num_kv_heads(
414
+ self.runner.parallel_config),
415
+ head_dim=self.runner.model_config.get_head_size(),
416
page_size=self.block_size,
417
seq_start_loc=seq_start_loc,
418
query_start_loc=query_start_loc,
vllm/attention/backends/utils.py
+25
-24
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
4
import torch
5
6
from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7
-from vllm.sequence import SequenceGroupMetadata
7
from vllm.utils import make_tensor_with_pad
8
9
# Error string(s) for encoder/decoder
@@ -15,8 +14,7 @@ STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
14
PAD_SLOT_ID = -1
15
16
if TYPE_CHECKING:
18
- from vllm.worker.model_runner import (GPUModelRunnerBase,
19
- ModelInputForGPUBuilder)
17
+ from vllm.worker.model_runner import ModelInputForGPUBuilder
18
19
20
def is_block_tables_empty(block_tables: Union[None, Dict]):
@@ -95,26 +93,27 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
93
self.num_prefill_tokens = 0
94
self.num_decode_tokens = 0
95
96
+ self.input_builder = input_builder
97
+ self.runner = input_builder.runner
98
+
99
self.sliding_window = input_builder.sliding_window
100
self.block_size = input_builder.block_size
101
self.use_v2_block_manager = (
102
input_builder.scheduler_config.use_v2_block_manager)
103
103
- def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
104
- token_lens: List[int], seq_lens: List[int],
105
- curr_seq_lens: List[int], query_lens: List[int],
106
- context_lens: List[int],
107
- curr_sliding_window_blocks: List[int], prefix_cache_hit,
108
- chunked_prefill_enabled):
109
- is_prompt = seq_group_metadata.is_prompt
110
- block_tables = seq_group_metadata.block_tables
111
- computed_block_nums = seq_group_metadata.computed_block_nums
104
+ def _add_seq_group(
105
+ self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
106
+ chunked_prefill_enabled: bool):
107
+ is_prompt = inter_data.is_prompt
108
+ block_tables = inter_data.block_tables
109
+ computed_block_nums = inter_data.computed_block_nums
110
111
for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
112
curr_sliding_window_block) in zip(
115
- seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
116
- curr_seq_lens, query_lens, context_lens,
117
- curr_sliding_window_blocks):
113
+ inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
114
+ inter_data.orig_seq_lens, inter_data.seq_lens,
115
+ inter_data.query_lens, inter_data.context_lens,
116
+ inter_data.curr_sliding_window_blocks):
117
self.context_lens.append(context_len)
118
if is_prompt:
119
self.num_prefills += 1
@@ -132,7 +131,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
131
# only allowing multiple of block_size chunk size.
132
# NOTE: This only works for oooooooxxx style attention.
133
block_table = []
135
- if prefix_cache_hit:
134
+ if inter_data.prefix_cache_hit:
135
block_table = computed_block_nums
136
elif ((chunked_prefill_enabled or not is_prompt)
137
and block_tables is not None):
@@ -146,16 +145,18 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
145
self.use_v2_block_manager)
146
compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
147
seq_len, context_len, start_idx,
149
- self.block_size,
150
- seq_group_metadata.block_tables)
148
+ self.block_size, inter_data.block_tables)
149
+
150
+ def build(self, seq_lens: List[int], query_lens: List[int],
151
+ cuda_graph_pad_size: int, batch_size: int):
152
+ for inter_data in self.input_builder.inter_data_list:
153
+ self._add_seq_group(inter_data,
154
+ self.input_builder.chunked_prefill_enabled)
155
152
- def build(self, runner: "GPUModelRunnerBase", seq_lens: List[int],
153
- query_lens: List[int], cuda_graph_pad_size: int,
154
- batch_size: int):
155
- device = runner.device
156
+ device = self.runner.device
157
use_captured_graph = cuda_graph_pad_size != -1
158
158
- logits_soft_cap = getattr(runner.model_config.hf_config,
159
+ logits_soft_cap = getattr(self.runner.model_config.hf_config,
160
"attn_logit_softcapping", None)
161
if logits_soft_cap is not None:
162
raise ValueError(
@@ -176,7 +177,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
177
178
# The shape of graph_block_tables is
179
# [max batch size, max context len // block size].
179
- input_block_tables = runner.graph_block_tables[:batch_size]
180
+ input_block_tables = self.runner.graph_block_tables[:batch_size]
181
for i, block_table in enumerate(self.block_tables):
182
if block_table:
183
input_block_tables[i, :len(block_table)] = block_table
vllm/utils.py
+5
@@ -719,6 +719,11 @@ def merge_dicts(dict1: Dict[K, List[T]],
719
return dict(merged_dict)
720
721
722
+def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
723
+ """Flatten a list of lists to a single list."""
724
+ return [item for sublist in lists for item in sublist]
725
+
726
+
727
def init_cached_hf_modules() -> None:
728
"""
729
Lazy initialization of the Hugging Face modules.
vllm/worker/model_runner.py
+323
-207
@@ -3,7 +3,7 @@ import gc
3
import time
4
import warnings
5
import weakref
6
-from collections import defaultdict
6
+from dataclasses import dataclass, field
7
from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8
Tuple, Type, TypeVar, Union)
9
@@ -49,7 +49,8 @@ from vllm.prompt_adapter.worker_manager import (
49
from vllm.sampling_params import SamplingParams
50
from vllm.sequence import (IntermediateTensors, SamplerOutput,
51
SequenceGroupMetadata)
52
-from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
52
+from vllm.utils import (CudaMemoryProfiler, flatten_2d_lists,
53
+ get_kv_cache_torch_dtype, is_hip,
54
is_pin_memory_available)
55
from vllm.worker.model_runner_base import (
56
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -76,7 +77,7 @@ _NUM_WARMUP_ITERS = 2
77
TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
78
79
79
-@dataclasses.dataclass(frozen=True)
80
+@dataclass(frozen=True)
81
class ModelInputForGPU(ModelRunnerInputBase):
82
"""
83
This base class contains metadata needed for the base model forward pass
@@ -126,7 +127,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
127
return cls(**tensor_dict)
128
129
129
-@dataclasses.dataclass(frozen=True)
130
+@dataclass(frozen=True)
131
class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
132
"""
133
Used by the ModelRunner.
@@ -168,12 +169,84 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
169
170
171
class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
171
- """TBA"""
172
+ """Build ModelInputForGPU from SequenceGroupMetadata."""
173
+
174
+ @dataclass
175
+ class InterDataForSeqGroup:
176
+ """Intermediate data for the current sequence group."""
177
+ # From sequence group metadata.
178
+ request_id: str
179
+ seq_ids: List[int]
180
+ is_prompt: bool
181
+ block_tables: Optional[Dict[int, List[int]]]
182
+ computed_block_nums: List[int]
183
+ n_seqs: int = 0
184
+
185
+ # Input tokens and positions.
186
+ input_tokens: List[List[int]] = field(default_factory=list)
187
+ input_positions: List[List[int]] = field(default_factory=list)
188
+
189
+ # The sequence length (may be capped to the sliding window).
190
+ seq_lens: List[int] = field(default_factory=list)
191
+ # The original sequence length (before applying sliding window).
192
+ # This is used to compute slot mapping.
193
+ orig_seq_lens: List[int] = field(default_factory=list)
194
+ # The query length.
195
+ query_lens: List[int] = field(default_factory=list)
196
+ # The number of tokens that are already computed.
197
+ context_lens: List[int] = field(default_factory=list)
198
+ # The current sliding window block.
199
+ curr_sliding_window_blocks: List[int] = field(default_factory=list)
200
+
201
+ # LoRA inputs.
202
+ lora_index_mapping: List[List[int]] = field(default_factory=list)
203
+ lora_prompt_mapping: List[List[int]] = field(default_factory=list)
204
+ lora_requests: Set[LoRARequest] = field(default_factory=set)
205
+
206
+ # Prompt adapter inputs.
207
+ prompt_adapter_index_mapping: List[int] = field(default_factory=list)
208
+ prompt_adapter_prompt_mapping: List[int] = field(default_factory=list)
209
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
210
+
211
+ # Multi-modal inputs.
212
+ multi_modal_inputs: Optional[MultiModalInputs] = None
213
+
214
+ # Whether the prefix cache is hit (prefill only).
215
+ prefix_cache_hit: bool = False
216
+
217
+ def __post_init__(self):
218
+ self.n_seqs = len(self.seq_ids)
219
+
220
+ self.input_tokens = [[] for _ in range(self.n_seqs)]
221
+ self.input_positions = [[] for _ in range(self.n_seqs)]
222
+ self.seq_lens = [0] * self.n_seqs
223
+ self.orig_seq_lens = [0] * self.n_seqs
224
+ self.query_lens = [0] * self.n_seqs
225
+ self.context_lens = [0] * self.n_seqs
226
+ self.curr_sliding_window_blocks = [0] * self.n_seqs
227
+
228
+ self.lora_index_mapping = [[] for _ in range(self.n_seqs)]
229
+ self.lora_prompt_mapping = [[] for _ in range(self.n_seqs)]
230
231
def __init__(self,
232
runner: "GPUModelRunnerBase",
233
finished_requests_ids: Optional[List[str]] = None):
234
super().__init__()
235
+ # Compute functions for each sequence in a sequence group.
236
+ # WARNING: The order of the functions matters!
237
+ self.per_seq_compute_fns = [
238
+ self._compute_lens,
239
+ self._compute_for_prefix_cache_hit,
240
+ self._compute_for_sliding_window,
241
+ self._compute_lora_input,
242
+ ]
243
+ # Compute functions for each sequence group.
244
+ # WARNING: The order of the functions matters!
245
+ self.per_seq_group_compute_fns = [
246
+ self._compute_prompt_adapter_input,
247
+ self._compute_multi_modal_input,
248
+ ]
249
+
250
self.runner = runner
251
self.model_input_cls = self.runner._model_input_cls
252
self.attn_backend = self.runner.attn_backend
@@ -187,30 +260,14 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
260
self.finished_requests_ids = finished_requests_ids
261
self.decode_only = True
262
190
- # Common inputs.
191
- self.input_tokens: List[int] = []
192
- self.input_positions: List[int] = []
193
- self.seq_lens: List[int] = []
194
- self.query_lens: List[int] = []
195
- self.max_decode_seq_len: int = 0
196
- self.request_ids_to_seq_ids: Dict[str, List[int]] = defaultdict(list)
197
-
198
- # LoRA inputs.
199
- self.lora_index_mapping: List[int] = []
200
- self.lora_prompt_mapping: List[int] = []
201
- self.lora_requests: Set[LoRARequest] = set()
202
-
203
- # Prompt adapter inputs.
204
- self.prompt_adapter_index_mapping: List[int] = []
205
- self.prompt_adapter_prompt_mapping: List[int] = []
206
- self.prompt_adapter_requests: Set[PromptAdapterRequest] = set()
207
-
208
- # Multi-modal inputs.
209
- self.multi_modal_inputs_list: List[MultiModalInputs] = []
263
+ # Intermediate data (data in CPU before going to GPU) for
264
+ # the current sequence group.
265
+ self.inter_data_list: List[
266
+ ModelInputForGPUBuilder.InterDataForSeqGroup] = []
267
268
# Attention metadata inputs.
269
self.attn_metadata_builder = self.attn_backend.make_metadata_builder(
213
- self)
270
+ weakref.proxy(self))
271
272
# Engine/Model configurations.
273
self.chunked_prefill_enabled = (
@@ -222,175 +279,222 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
279
self.block_aligned_sliding_window = \
280
self.sliding_window_blocks * self.block_size
281
225
- def _compute_len_for_sliding_window(self, seq_len: int):
226
- curr_sliding_window_blocks = 0
227
- sliding_seq_len = seq_len
282
+ def _compute_lens(self, inter_data: InterDataForSeqGroup, seq_idx: int,
283
+ seq_group_metadata: SequenceGroupMetadata):
284
+ """Compute context length, sequence length and tokens
285
+ for the given sequence data.
286
+ """
287
+ seq_data = seq_group_metadata.seq_data[inter_data.seq_ids[seq_idx]]
288
+ token_chunk_size = seq_group_metadata.token_chunk_size
289
229
- # TODO(sang): This is a hack to make sliding window work with
230
- # paged attn. We can remove it if we make paged attn kernel
231
- # to properly handle slinding window attn.
232
- if self.sliding_window is not None:
233
- curr_sliding_window_blocks = self.sliding_window_blocks
290
+ # Compute context length (the number of tokens that are
291
+ # already computed) and sequence length (total number of tokens).
292
+ seq_len = seq_data.get_len()
293
+ if inter_data.is_prompt:
294
+ context_len = seq_data.get_num_computed_tokens()
295
+ else:
296
+ # get_num_computed_tokens is incorrect for spec decoding.
297
+ # So, we should have a special logic here.
298
+ # TODO(sang): Fix it.
299
+ context_len = seq_len - 1
300
+ seq_len = min(seq_len, context_len + token_chunk_size)
301
+
302
+ # Compute tokens.
303
+ if inter_data.is_prompt:
304
+ tokens = seq_data.get_token_ids()[context_len:seq_len]
305
+ else:
306
+ # Optimization. get_token_ids requires the entire copy of
307
+ # tokens.
308
+ tokens = [seq_data.get_last_token_id()]
309
+
310
+ inter_data.seq_lens[seq_idx] = seq_len
311
+ inter_data.orig_seq_lens[seq_idx] = seq_len
312
+ inter_data.context_lens[seq_idx] = context_len
313
+ inter_data.input_tokens[seq_idx] = tokens
314
+ inter_data.input_positions[seq_idx] = list(range(context_len, seq_len))
315
+ inter_data.query_lens[
316
+ seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
317
+
318
+ def _compute_for_prefix_cache_hit(
319
+ self, inter_data: InterDataForSeqGroup, seq_idx: int,
320
+ seq_group_metadata: SequenceGroupMetadata):
321
+ """Check if hit prefix cache (i.e., some blocks are already computed).
322
+ If hit, update input tokens and positions to only compute the
323
+ remaining blocks.
324
+ """
325
+ computed_block_nums = inter_data.computed_block_nums
326
+
327
+ # Note that prefix caching does not support sliding window.
328
+ prefix_cache_hit = (computed_block_nums is not None
329
+ and len(computed_block_nums) > 0
330
+ and self.sliding_window is None
331
+ and inter_data.is_prompt)
332
+ inter_data.prefix_cache_hit = prefix_cache_hit
333
+ if self.chunked_prefill_enabled and prefix_cache_hit:
334
+ raise RuntimeError(
335
+ "chunked prefill cannot be used with prefix caching now.")
336
+
337
+ # If prefix cache is hit, advance context length to bypass
338
+ # hit blocks. Accordingly, input tokens, position and query length
339
+ # have to be updated.
340
+ if prefix_cache_hit:
341
+ assert computed_block_nums is not None
342
+ context_len = len(computed_block_nums) * self.block_size
343
+ inter_data.input_tokens[seq_idx] = inter_data.input_tokens[
344
+ seq_idx][context_len:]
345
+ inter_data.input_positions[seq_idx] = inter_data.input_positions[
346
+ seq_idx][context_len:]
347
+ inter_data.context_lens[seq_idx] = context_len
348
+ inter_data.query_lens[
349
+ seq_idx] = inter_data.seq_lens[seq_idx] - context_len
350
+
351
+ def _compute_for_sliding_window(self, inter_data: InterDataForSeqGroup,
352
+ seq_idx: int,
353
+ seq_group_metadata: SequenceGroupMetadata):
354
+ """Update seq_len and curr_sliding_window_block for the given
355
+ sequence data (only required by decoding) if sliding window is enabled.
356
+ """
357
+ curr_sliding_window_block = 0
358
+ sliding_seq_len = inter_data.seq_lens[seq_idx]
359
+ if not inter_data.is_prompt and self.sliding_window is not None:
360
+ # TODO(sang): This is a hack to make sliding window work with
361
+ # paged attn. We can remove it if we make paged attn kernel
362
+ # to properly handle slinding window attn.
363
+ curr_sliding_window_block = self.sliding_window_blocks
364
if self.scheduler_config.use_v2_block_manager:
365
# number of elements in last block
236
- suff_len = seq_len % self.block_size
366
+ suff_len = inter_data.seq_lens[seq_idx] % self.block_size
367
sliding_seq_len = min(
238
- seq_len, self.block_aligned_sliding_window + suff_len)
368
+ inter_data.seq_lens[seq_idx],
369
+ self.block_aligned_sliding_window + suff_len)
370
if suff_len > 0:
240
- curr_sliding_window_blocks += 1
371
+ curr_sliding_window_block += 1
372
else:
242
- sliding_seq_len = min(seq_len, self.sliding_window)
243
- return curr_sliding_window_blocks, sliding_seq_len
373
+ sliding_seq_len = min(inter_data.seq_lens[seq_idx],
374
+ self.sliding_window)
375
+
376
+ inter_data.curr_sliding_window_blocks[
377
+ seq_idx] = curr_sliding_window_block
378
+ inter_data.seq_lens[seq_idx] = sliding_seq_len
379
+
380
+ def _compute_lora_input(self, inter_data: InterDataForSeqGroup,
381
+ seq_idx: int,
382
+ seq_group_metadata: SequenceGroupMetadata):
383
+ """If LoRA is enabled, compute LoRA index and prompt mapping."""
384
+ if not self.enable_lora:
385
+ return
386
+
387
+ lora_id = seq_group_metadata.lora_int_id
388
+ if lora_id > 0:
389
+ inter_data.lora_requests.add(seq_group_metadata.lora_request)
390
+ query_len = inter_data.query_lens[seq_idx]
391
+ inter_data.lora_index_mapping.append([lora_id] * query_len)
392
+ inter_data.lora_prompt_mapping.append(
393
+ [lora_id] *
394
+ (query_len if seq_group_metadata.sampling_params
395
+ and seq_group_metadata.sampling_params.prompt_logprobs is not None
396
+ else 1))
397
+
398
+ def _compute_prompt_adapter_input(
399
+ self, inter_data: InterDataForSeqGroup,
400
+ seq_group_metadata: SequenceGroupMetadata):
401
+ """If prompt adapter is enabled, compute index and prompt mapping.
402
+ """
403
+ # Note that when is_prompt=True, we expect only one sequence
404
+ # in the group.
405
+ if not self.enable_prompt_adapter:
406
+ return
407
+
408
+ prompt_adapter_id = seq_group_metadata.prompt_adapter_id
409
+ if prompt_adapter_id <= 0 or not inter_data.is_prompt:
410
+ return
411
+
412
+ # We expect only one sequence in the group when is_prompt=True.
413
+ assert inter_data.n_seqs == 1
414
+ query_len = inter_data.query_lens[0]
415
+ inter_data.prompt_adapter_request = (
416
+ seq_group_metadata.prompt_adapter_request)
417
+
418
+ num_tokens = seq_group_metadata.prompt_adapter_num_virtual_tokens
419
+ inter_data.prompt_adapter_index_mapping = [
420
+ prompt_adapter_id
421
+ ] * num_tokens + [0] * (query_len - num_tokens)
422
+ inter_data.prompt_adapter_prompt_mapping = [prompt_adapter_id] * (
423
+ query_len if seq_group_metadata.sampling_params
424
+ and seq_group_metadata.sampling_params.prompt_logprobs else 1)
425
+
426
+ def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
427
+ seq_group_metadata: SequenceGroupMetadata):
428
+ """If multi-modal data is given, add it to the input."""
429
+ mm_data = seq_group_metadata.multi_modal_data
430
+ if not mm_data:
431
+ return
432
+
433
+ mm_kwargs = self.multi_modal_input_mapper(mm_data)
434
+ inter_data.multi_modal_inputs = mm_kwargs
435
436
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
437
+ """Add a sequence group to the builder."""
438
seq_ids = list(seq_group_metadata.seq_data.keys())
439
n_seqs = len(seq_ids)
440
is_prompt = seq_group_metadata.is_prompt
249
- token_chunk_size = seq_group_metadata.token_chunk_size
441
442
if is_prompt:
443
assert n_seqs == 1
444
self.decode_only = False
445
255
- # Mapping from request IDs to sequence IDs. Used for Jamba models
256
- # that manages the cache by itself.
257
- self.request_ids_to_seq_ids[seq_group_metadata.request_id] = []
258
- # The number of input tokens in each sequence.
259
- token_lens: List[int] = []
260
- # The number of tokens that are already computed.
261
- context_lens: List[int] = []
262
- # The current sliding window block for each sequence.
263
- curr_sliding_window_blocks: List[int] = []
264
- # The original sequence length (before applying sliding window)
265
- # for each sequence.
266
- orig_seq_lens: List[int] = []
267
- # The sequence length (may be capped to the sliding window).
268
- curr_seq_lens: List[int] = []
269
- for seq_id in seq_ids:
270
- seq_data = seq_group_metadata.seq_data[seq_id]
271
- self.request_ids_to_seq_ids[seq_group_metadata.request_id].append(
272
- seq_id)
273
- computed_block_nums = seq_group_metadata.computed_block_nums
274
-
275
- # Check if hit prefix cache (i.e., some blocks are already computed)
276
- # Note that prefix caching does not support sliding window.
277
- prefix_cache_hit = (computed_block_nums is not None
278
- and len(computed_block_nums) > 0
279
- and self.sliding_window is None and is_prompt)
280
- if self.chunked_prefill_enabled and prefix_cache_hit:
281
- raise RuntimeError(
282
- "chunked prefill cannot be used with prefix caching now.")
283
-
284
- # Compute context length (the number of tokens that are
285
- # already computed) and sequence length (total number of tokens).
286
- seq_len = seq_data.get_len()
287
- if is_prompt:
288
- context_len = seq_data.get_num_computed_tokens()
289
- else:
290
- # get_num_computed_tokens is incorrect for spec decoding.
291
- # So, we should have a special logic here.
292
- # TODO(sang): Fix it.
293
- context_len = seq_len - 1
294
- seq_len = min(seq_len, context_len + token_chunk_size)
295
-
296
- # Compute tokens.
297
- if is_prompt:
298
- tokens = seq_data.get_token_ids()[context_len:seq_len]
299
- else:
300
- # Optimization. get_token_ids requires the entire copy of
301
- # tokens.
302
- tokens = [seq_data.get_last_token_id()]
303
- if prefix_cache_hit:
304
- assert computed_block_nums is not None
305
- context_len = len(computed_block_nums) * self.block_size
306
- tokens = tokens[context_len:]
307
-
308
- # These are seq_len/context_len capped to the sliding window.
309
- # They are passed to decode kernel.
310
- # We still need original seq_len/context_len to compute slot
311
- # mapping (and input position) below.
312
- if is_prompt:
313
- curr_sliding_window_block = 0
314
- sliding_seq_len = seq_len
315
- query_len = seq_len - context_len
316
- else:
317
- curr_sliding_window_block, sliding_seq_len = (
318
- self._compute_len_for_sliding_window(seq_len))
319
- query_len = 1
320
-
321
- self.seq_lens.append(sliding_seq_len)
322
- if not is_prompt:
323
- self.max_decode_seq_len = max(self.max_decode_seq_len,
324
- sliding_seq_len)
325
- self.query_lens.append(query_len)
326
- self.input_tokens.extend(tokens)
327
- self.input_positions.extend(list(range(context_len, seq_len)))
328
-
329
- # Intermediate data of the current sequence group for
330
- # the attention metadata.
331
- token_lens.append(len(tokens))
332
- context_lens.append(context_len)
333
- curr_seq_lens.append(sliding_seq_len)
334
- curr_sliding_window_blocks.append(curr_sliding_window_block)
335
- orig_seq_lens.append(seq_len)
336
-
337
- # Update attention metadata. Note that input builder attributes
338
- # (self.xxx) include all added sequences, so we need to slice
339
- # the last n_seqs sequences.
340
- self.attn_metadata_builder.add_seq_group(
341
- seq_group_metadata, token_lens, orig_seq_lens, curr_seq_lens,
342
- self.query_lens[-n_seqs:], context_lens,
343
- curr_sliding_window_blocks, prefix_cache_hit,
344
- self.chunked_prefill_enabled)
345
-
346
- # LoRA data.
347
- if self.enable_lora:
348
- lora_id = seq_group_metadata.lora_int_id
349
- for query_len in self.query_lens[-n_seqs:]:
350
- if lora_id > 0:
351
- self.lora_requests.add(seq_group_metadata.lora_request)
352
- self.lora_index_mapping += [lora_id] * query_len
353
- self.lora_prompt_mapping.extend(
354
- [lora_id] *
355
- (query_len if seq_group_metadata.sampling_params
356
- and seq_group_metadata.sampling_params.prompt_logprobs
357
- is not None else 1))
358
-
359
- # Prompt adapter data. Note that when is_prompt=True,
360
- # we expect only one sequence in the group.
361
- if self.enable_prompt_adapter:
362
- prompt_adapter_id = seq_group_metadata.prompt_adapter_id
363
- if prompt_adapter_id > 0 and is_prompt:
364
- query_len = self.query_lens[-1]
365
- self.prompt_adapter_requests.add(
366
- seq_group_metadata.prompt_adapter_request)
367
-
368
- num_tokens = seq_group_metadata.\
369
- prompt_adapter_num_virtual_tokens
370
- pm = [prompt_adapter_id
371
- ] * num_tokens + [0] * (query_len - num_tokens)
372
- self.prompt_adapter_index_mapping += pm
373
- self.prompt_adapter_prompt_mapping.extend(
374
- [prompt_adapter_id] *
375
- (query_len if seq_group_metadata.sampling_params
376
- and seq_group_metadata.sampling_params.prompt_logprobs
377
- else 1))
446
+ inter_data = self.InterDataForSeqGroup(
447
+ request_id=seq_group_metadata.request_id,
448
+ seq_ids=seq_ids,
449
+ is_prompt=is_prompt,
450
+ block_tables=seq_group_metadata.block_tables,
451
+ computed_block_nums=seq_group_metadata.computed_block_nums)
452
+ self.inter_data_list.append(inter_data)
453
379
- # Multi-modal data.
380
- mm_data = seq_group_metadata.multi_modal_data
381
- if mm_data:
382
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
383
- self.multi_modal_inputs_list.append(mm_kwargs)
454
+ for seq_idx in range(n_seqs):
455
+ for per_seq_fn in self.per_seq_compute_fns:
456
+ per_seq_fn(inter_data, seq_idx, seq_group_metadata)
457
+ for per_seq_group_fn in self.per_seq_group_compute_fns:
458
+ per_seq_group_fn(inter_data, seq_group_metadata)
459
460
def build(self) -> ModelInputForGPU:
386
- if not self.input_tokens:
461
+ """Finalize the builder intermediate data and
462
+ create on-device tensors.
463
+ """
464
+ # Combine and flatten intermediate data.
465
+ input_tokens = flatten_2d_lists([
466
+ flatten_2d_lists(inter_data.input_tokens)
467
+ for inter_data in self.inter_data_list
468
+ ])
469
+ if not input_tokens:
470
+ # This may happen when all prefill requests hit
471
+ # prefix caching and there is no decode request.
472
return self.model_input_cls()
473
+ input_positions = flatten_2d_lists([
474
+ flatten_2d_lists(inter_data.input_positions)
475
+ for inter_data in self.inter_data_list
476
+ ])
477
+ seq_lens = []
478
+ max_decode_seq_len = 0
479
+ for inter_data in self.inter_data_list:
480
+ seq_lens.extend(inter_data.seq_lens)
481
+ if not inter_data.is_prompt:
482
+ max_decode_seq_len = max(max_decode_seq_len,
483
+ max(inter_data.seq_lens))
484
+ query_lens = flatten_2d_lists(
485
+ [inter_data.query_lens for inter_data in self.inter_data_list])
486
+ # Mapping from request IDs to sequence IDs. Used for Jamba models
487
+ # that manages the cache by itself.
488
+ request_ids_to_seq_ids = {
489
+ data.request_id: data.seq_ids
490
+ for data in self.inter_data_list
491
+ }
492
389
- batch_size = len(self.input_tokens)
493
+ batch_size = len(input_tokens)
494
use_captured_graph = (
495
self.decode_only and not self.runner.model_config.enforce_eager
496
and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
393
- and self.max_decode_seq_len <= self.runner.max_seq_len_to_capture)
497
+ and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
498
499
# If cuda graph can be used, pad tensors accordingly.
500
# See `capture_model` API for more details.
@@ -403,60 +507,84 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
507
batch_size = graph_batch_size
508
509
# Tokens and positions.
406
- self.input_tokens.extend([0] * cuda_graph_pad_size)
407
- self.input_positions.extend([0] * cuda_graph_pad_size)
408
- input_tokens_tensor = torch.tensor(self.input_tokens,
510
+ input_tokens.extend([0] * cuda_graph_pad_size)
511
+ input_positions.extend([0] * cuda_graph_pad_size)
512
+ input_tokens_tensor = torch.tensor(input_tokens,
513
dtype=torch.long,
514
device=self.runner.device)
411
- input_positions_tensor = torch.tensor(self.input_positions,
515
+ input_positions_tensor = torch.tensor(input_positions,
516
dtype=torch.long,
517
device=self.runner.device)
518
519
# Sequence and query lengths.
416
- self.seq_lens.extend([1] * cuda_graph_pad_size)
520
+ seq_lens.extend([1] * cuda_graph_pad_size)
521
522
# Attention metadata.
523
attn_metadata = self.attn_metadata_builder.build(
420
- self.runner, self.seq_lens, self.query_lens, cuda_graph_pad_size,
421
- batch_size)
524
+ seq_lens, query_lens, cuda_graph_pad_size, batch_size)
525
526
# LoRA data.
527
+ lora_requests = set()
528
+ lora_mapping = None
529
if self.enable_lora:
425
- self.lora_index_mapping.extend([0] * cuda_graph_pad_size)
530
+ lora_requests = set(r for data in self.inter_data_list
531
+ for r in data.lora_requests)
532
+ lora_index_mapping = flatten_2d_lists([
533
+ flatten_2d_lists(inter_data.lora_index_mapping)
534
+ for inter_data in self.inter_data_list
535
+ ])
536
+ lora_index_mapping.extend([0] * cuda_graph_pad_size)
537
+ lora_prompt_mapping = flatten_2d_lists([
538
+ flatten_2d_lists(inter_data.lora_prompt_mapping)
539
+ for inter_data in self.inter_data_list
540
+ ])
541
lora_mapping = LoRAMapping(
427
- self.lora_index_mapping,
428
- self.lora_prompt_mapping,
542
+ lora_index_mapping,
543
+ lora_prompt_mapping,
544
)
430
- else:
431
- lora_mapping = None
545
546
# Prompt adapter data.
547
+ prompt_adapter_requests: Set[PromptAdapterRequest] = set()
548
+ prompt_adapter_mapping = None
549
if self.enable_prompt_adapter:
435
- self.prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
550
+ prompt_adapter_requests = set(
551
+ data.prompt_adapter_request for data in self.inter_data_list
552
+ if data.prompt_adapter_request is not None)
553
+ prompt_adapter_index_mapping = flatten_2d_lists([
554
+ inter_data.prompt_adapter_index_mapping
555
+ for inter_data in self.inter_data_list
556
+ ])
557
+ prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
558
+ prompt_adapter_prompt_mapping = flatten_2d_lists([
559
+ inter_data.prompt_adapter_prompt_mapping
560
+ for inter_data in self.inter_data_list
561
+ ])
562
prompt_adapter_mapping = PromptAdapterMapping(
437
- self.prompt_adapter_index_mapping,
438
- self.prompt_adapter_prompt_mapping,
563
+ prompt_adapter_index_mapping,
564
+ prompt_adapter_prompt_mapping,
565
)
440
- else:
441
- prompt_adapter_mapping = None
566
567
# Multi-modal data.
444
- multi_modal_kwargs = MultiModalInputs.batch(
445
- self.multi_modal_inputs_list, device=self.runner.device)
568
+ multi_modal_inputs_list = [
569
+ data.multi_modal_inputs for data in self.inter_data_list
570
+ if data.multi_modal_inputs is not None
571
+ ]
572
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
573
+ device=self.runner.device)
574
575
return self.model_input_cls(
576
input_tokens=input_tokens_tensor,
577
input_positions=input_positions_tensor,
578
attn_metadata=attn_metadata,
451
- seq_lens=self.seq_lens,
452
- query_lens=self.query_lens,
579
+ seq_lens=seq_lens,
580
+ query_lens=query_lens,
581
lora_mapping=lora_mapping,
454
- lora_requests=self.lora_requests,
582
+ lora_requests=lora_requests,
583
multi_modal_kwargs=multi_modal_kwargs,
456
- request_ids_to_seq_ids=self.request_ids_to_seq_ids,
584
+ request_ids_to_seq_ids=request_ids_to_seq_ids,
585
finished_requests_ids=self.finished_requests_ids,
586
prompt_adapter_mapping=prompt_adapter_mapping,
459
- prompt_adapter_requests=self.prompt_adapter_requests)
587
+ prompt_adapter_requests=prompt_adapter_requests)
588
589
590
class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
@@ -1393,15 +1521,3 @@ def _get_graph_batch_size(batch_size: int) -> int:
1521
else:
1522
return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
1523
_BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
1396
-
1397
-
1398
-def _is_block_tables_empty(block_tables: Union[None, Dict]):
1399
- """
1400
- Check if block_tables is None or a dictionary with all None values.
1401
- """
1402
- if block_tables is None:
1403
- return True
1404
- if isinstance(block_tables, dict) and all(
1405
- value is None for value in block_tables.values()):
1406
- return True
1407
- return False