@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e0c15758b8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Modulize prepare input and attention metadata builder (#6596)

Cody Yu committed Jul 22, 2024 at 17:45 UTC e0c15758b85827dcac78379e60ea975ebc0ec795
6 files changed +409 -298
vllm/attention/backends/abstract.py
+3 -17
@@ -7,7 +7,6 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
7 import torch
8
9 if TYPE_CHECKING:
10 - from vllm.sequence import SequenceGroupMetadata
10 from vllm.worker.model_runner_base import ModelRunnerInputBuilderBase
11
12
@@ -128,25 +127,12 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
127 """Abstract class for attention metadata builders."""
128
129 @abstractmethod
131 - def __init__(self, input_builder) -> None:
130 + def __init__(self, input_builder: "ModelRunnerInputBuilderBase") -> None:
131 raise NotImplementedError
132
133 @abstractmethod
135 - def add_seq_group(self, seq_group_metadata: "SequenceGroupMetadata",
136 - token_lens: List[int], seq_lens: List[int],
137 - curr_seq_lens: List[int], query_lens: List[int],
138 - context_lens: List[int],
139 - curr_sliding_window_blocks: List[int],
140 - prefix_cache_hit: bool, chunked_prefill_enabled: bool):
141 - """Add a sequence group to the metadata and update
142 - corresponding fields (in Python objects).
143 - """
144 - raise NotImplementedError
145 -
146 - @abstractmethod
147 - def build(self, runner: "ModelRunnerInputBuilderBase", seq_lens: List[int],
148 - query_lens: List[int], cuda_graph_pad_size: int,
149 - batch_size: int) -> T:
134 + def build(self, seq_lens: List[int], query_lens: List[int],
135 + cuda_graph_pad_size: int, batch_size: int) -> T:
136 """Build attention metadata with on-device tensors."""
137 raise NotImplementedError
138
vllm/attention/backends/flash_attn.py
+22 -21
@@ -13,12 +13,10 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
14 compute_slot_mapping_start_idx,
15 is_block_tables_empty)
16 -from vllm.sequence import SequenceGroupMetadata
16 from vllm.utils import make_tensor_with_pad
17
18 if TYPE_CHECKING:
20 - from vllm.worker.model_runner import (GPUModelRunnerBase,
21 - ModelInputForGPUBuilder)
19 + from vllm.worker.model_runner import ModelInputForGPUBuilder
20
21
22 class FlashAttentionBackend(AttentionBackend):
@@ -212,30 +210,30 @@ class FlashAttentionMetadataBuilder(
210 self.num_prefill_tokens = 0
211 self.num_decode_tokens = 0
212
213 + self.input_builder = input_builder
214 + self.runner = input_builder.runner
215 self.sliding_window = input_builder.sliding_window
216 self.block_size = input_builder.block_size
217 self.use_v2_block_manager = (
218 input_builder.scheduler_config.use_v2_block_manager)
219
220 - def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
221 - token_lens: List[int], seq_lens: List[int],
222 - curr_seq_lens: List[int], query_lens: List[int],
223 - context_lens: List[int],
224 - curr_sliding_window_blocks: List[int],
225 - prefix_cache_hit: bool, chunked_prefill_enabled: bool):
220 + def _add_seq_group(
221 + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
222 + chunked_prefill_enabled: bool):
223 """Add a sequence group to the metadata. Specifically update/append
224 1. context length.
225 2. block table.
226 3. slot mapping.
227 """
231 - is_prompt = seq_group_metadata.is_prompt
232 - block_tables = seq_group_metadata.block_tables
228 + is_prompt = inter_data.is_prompt
229 + block_tables = inter_data.block_tables
230
231 for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
232 curr_sliding_window_block) in zip(
236 - seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
237 - curr_seq_lens, query_lens, context_lens,
238 - curr_sliding_window_blocks):
233 + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
234 + inter_data.orig_seq_lens, inter_data.seq_lens,
235 + inter_data.query_lens, inter_data.context_lens,
236 + inter_data.curr_sliding_window_blocks):
237 self.context_lens.append(context_len)
238
239 if is_prompt:
@@ -254,7 +252,7 @@ class FlashAttentionMetadataBuilder(
252 # only allowing multiple of block_size chunk size.
253 # NOTE: This only works for oooooooxxx style attention.
254 block_table = []
257 - if prefix_cache_hit:
255 + if inter_data.prefix_cache_hit:
256 # NOTE(woosuk): For flash-attn, the block table should
257 # include the entries for the incoming prefill tokens.
258 block_table = block_tables[seq_id]
@@ -270,16 +268,19 @@ class FlashAttentionMetadataBuilder(
268 self.use_v2_block_manager)
269 compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
270 seq_len, context_len, start_idx,
273 - self.block_size,
274 - seq_group_metadata.block_tables)
271 + self.block_size, inter_data.block_tables)
272
276 - def build(self, runner: "GPUModelRunnerBase", seq_lens, query_lens,
273 + def build(self, seq_lens: List[int], query_lens: List[int],
274 cuda_graph_pad_size: int, batch_size: int):
275 """Build attention metadata with on-device tensors."""
279 - device = runner.device
276 + for inter_data in self.input_builder.inter_data_list:
277 + self._add_seq_group(inter_data,
278 + self.input_builder.chunked_prefill_enabled)
279 +
280 + device = self.runner.device
281 use_captured_graph = cuda_graph_pad_size != -1
282
282 - logits_soft_cap = getattr(runner.model_config.hf_config,
283 + logits_soft_cap = getattr(self.runner.model_config.hf_config,
284 "attn_logit_softcapping", None)
285 if logits_soft_cap is not None:
286 raise ValueError(
@@ -300,7 +301,7 @@ class FlashAttentionMetadataBuilder(
301
302 # The shape of graph_block_tables is
303 # [max batch size, max context len // block size].
303 - input_block_tables = runner.graph_block_tables[:batch_size]
304 + input_block_tables = self.runner.graph_block_tables[:batch_size]
305 for i, block_table in enumerate(self.block_tables):
306 if block_table:
307 input_block_tables[i, :len(block_table)] = block_table
vllm/attention/backends/flashinfer.py
+31 -29
@@ -21,12 +21,10 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21 compute_slot_mapping_start_idx,
22 is_block_tables_empty)
23 from vllm.attention.ops.paged_attn import PagedAttention
24 -from vllm.sequence import SequenceGroupMetadata
24 from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
25
26 if TYPE_CHECKING:
28 - from vllm.worker.model_runner import (GPUModelRunnerBase,
29 - ModelInputForGPUBuilder)
27 + from vllm.worker.model_runner import ModelInputForGPUBuilder
28
29
30 class FlashInferBackend(AttentionBackend):
@@ -216,6 +214,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
214 self.num_prefill_tokens = 0
215 self.num_decode_tokens = 0
216
217 + self.input_builder = input_builder
218 + self.runner = input_builder.runner
219 +
220 self.sliding_window = input_builder.sliding_window
221 self.block_size = input_builder.block_size
222 self.use_v2_block_manager = (
@@ -238,26 +239,24 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
239 # paged_kv_last_page_len is the length of the last page of each request
240 self.paged_kv_last_page_len: List[int] = []
241
241 - def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
242 - token_lens: List[int], seq_lens: List[int],
243 - curr_seq_lens: List[int], query_lens: List[int],
244 - context_lens: List[int],
245 - curr_sliding_window_blocks: List[int],
246 - prefix_cache_hit: bool, chunked_prefill_enabled: bool):
242 + def _add_seq_group(
243 + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
244 + chunked_prefill_enabled: bool):
245 """Add a sequence group to the metadata. Specifically update/append
246 1. context length.
247 2. block table.
248 3. slot mapping.
249 """
252 - is_prompt = seq_group_metadata.is_prompt
253 - block_tables = seq_group_metadata.block_tables
254 - computed_block_nums = seq_group_metadata.computed_block_nums
250 + is_prompt = inter_data.is_prompt
251 + block_tables = inter_data.block_tables
252 + computed_block_nums = inter_data.computed_block_nums
253
254 for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
255 curr_sliding_window_block) in zip(
258 - seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
259 - curr_seq_lens, query_lens, context_lens,
260 - curr_sliding_window_blocks):
256 + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
257 + inter_data.orig_seq_lens, inter_data.seq_lens,
258 + inter_data.query_lens, inter_data.context_lens,
259 + inter_data.curr_sliding_window_blocks):
260 self.context_lens.append(context_len)
261 if is_prompt:
262 self.num_prefills += 1
@@ -275,7 +274,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
274 # only allowing multiple of block_size chunk size.
275 # NOTE: This only works for oooooooxxx style attention.
276 block_table = []
278 - if prefix_cache_hit:
277 + if inter_data.prefix_cache_hit:
278 block_table = computed_block_nums
279 elif ((chunked_prefill_enabled or not is_prompt)
280 and block_tables is not None):
@@ -290,8 +289,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
289 self.use_v2_block_manager)
290 compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
291 seq_len, context_len, start_idx,
293 - self.block_size,
294 - seq_group_metadata.block_tables)
292 + self.block_size, inter_data.block_tables)
293
294 # It is not necessary to add paged_kv_indices, paged_kv_indptr,
295 # and paged_kv_last_page_len for profile run because we will
@@ -317,9 +315,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
315 last_page_len = self.block_size
316 self.paged_kv_last_page_len.append(last_page_len)
317
320 - def build(self, runner: "GPUModelRunnerBase", seq_lens, query_lens,
318 + def build(self, seq_lens: List[int], query_lens: List[int],
319 cuda_graph_pad_size: int, batch_size: int):
322 - device = runner.device
320 + for inter_data in self.input_builder.inter_data_list:
321 + self._add_seq_group(inter_data,
322 + self.input_builder.chunked_prefill_enabled)
323 +
324 + device = self.runner.device
325 use_captured_graph = cuda_graph_pad_size != -1
326
327 max_query_len = max(query_lens)
@@ -333,7 +335,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
335
336 # The shape of graph_block_tables is
337 # [max batch size, max context len // block size].
336 - input_block_tables = runner.graph_block_tables[:batch_size]
338 + input_block_tables = self.runner.graph_block_tables[:batch_size]
339 for i, block_table in enumerate(self.block_tables):
340 if block_table:
341 input_block_tables[i, :len(block_table)] = block_table
@@ -377,7 +379,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
379 dtype=torch.long,
380 device=device)
381
380 - logits_soft_cap = getattr(runner.model_config.hf_config,
382 + logits_soft_cap = getattr(self.runner.model_config.hf_config,
383 "attn_logit_softcapping", None)
384
385 if len(self.paged_kv_indptr) > 0:
@@ -394,8 +396,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
396 paged_kv_indptr_tensor = None
397 paged_kv_last_page_len_tensor = None
398
397 - kv_cache_dtype = get_kv_cache_torch_dtype(runner.kv_cache_dtype,
398 - runner.model_config.dtype)
399 + kv_cache_dtype = get_kv_cache_torch_dtype(
400 + self.runner.kv_cache_dtype, self.runner.model_config.dtype)
401 return FlashInferMetadata(
402 num_prefills=self.num_prefills,
403 slot_mapping=slot_mapping_tensor,
@@ -406,11 +408,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
408 paged_kv_indptr=paged_kv_indptr_tensor,
409 paged_kv_indices=paged_kv_indices_tensor,
410 paged_kv_last_page_len=paged_kv_last_page_len_tensor,
409 - num_qo_heads=runner.model_config.get_num_attention_heads(
410 - runner.parallel_config),
411 - num_kv_heads=runner.model_config.get_num_kv_heads(
412 - runner.parallel_config),
413 - head_dim=runner.model_config.get_head_size(),
411 + num_qo_heads=self.runner.model_config.get_num_attention_heads(
412 + self.runner.parallel_config),
413 + num_kv_heads=self.runner.model_config.get_num_kv_heads(
414 + self.runner.parallel_config),
415 + head_dim=self.runner.model_config.get_head_size(),
416 page_size=self.block_size,
417 seq_start_loc=seq_start_loc,
418 query_start_loc=query_start_loc,
vllm/attention/backends/utils.py
+25 -24
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
4 import torch
5
6 from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7 -from vllm.sequence import SequenceGroupMetadata
7 from vllm.utils import make_tensor_with_pad
8
9 # Error string(s) for encoder/decoder
@@ -15,8 +14,7 @@ STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
14 PAD_SLOT_ID = -1
15
16 if TYPE_CHECKING:
18 - from vllm.worker.model_runner import (GPUModelRunnerBase,
19 - ModelInputForGPUBuilder)
17 + from vllm.worker.model_runner import ModelInputForGPUBuilder
18
19
20 def is_block_tables_empty(block_tables: Union[None, Dict]):
@@ -95,26 +93,27 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
93 self.num_prefill_tokens = 0
94 self.num_decode_tokens = 0
95
96 + self.input_builder = input_builder
97 + self.runner = input_builder.runner
98 +
99 self.sliding_window = input_builder.sliding_window
100 self.block_size = input_builder.block_size
101 self.use_v2_block_manager = (
102 input_builder.scheduler_config.use_v2_block_manager)
103
103 - def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata,
104 - token_lens: List[int], seq_lens: List[int],
105 - curr_seq_lens: List[int], query_lens: List[int],
106 - context_lens: List[int],
107 - curr_sliding_window_blocks: List[int], prefix_cache_hit,
108 - chunked_prefill_enabled):
109 - is_prompt = seq_group_metadata.is_prompt
110 - block_tables = seq_group_metadata.block_tables
111 - computed_block_nums = seq_group_metadata.computed_block_nums
104 + def _add_seq_group(
105 + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
106 + chunked_prefill_enabled: bool):
107 + is_prompt = inter_data.is_prompt
108 + block_tables = inter_data.block_tables
109 + computed_block_nums = inter_data.computed_block_nums
110
111 for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
112 curr_sliding_window_block) in zip(
115 - seq_group_metadata.seq_data.keys(), token_lens, seq_lens,
116 - curr_seq_lens, query_lens, context_lens,
117 - curr_sliding_window_blocks):
113 + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
114 + inter_data.orig_seq_lens, inter_data.seq_lens,
115 + inter_data.query_lens, inter_data.context_lens,
116 + inter_data.curr_sliding_window_blocks):
117 self.context_lens.append(context_len)
118 if is_prompt:
119 self.num_prefills += 1
@@ -132,7 +131,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
131 # only allowing multiple of block_size chunk size.
132 # NOTE: This only works for oooooooxxx style attention.
133 block_table = []
135 - if prefix_cache_hit:
134 + if inter_data.prefix_cache_hit:
135 block_table = computed_block_nums
136 elif ((chunked_prefill_enabled or not is_prompt)
137 and block_tables is not None):
@@ -146,16 +145,18 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
145 self.use_v2_block_manager)
146 compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id,
147 seq_len, context_len, start_idx,
149 - self.block_size,
150 - seq_group_metadata.block_tables)
148 + self.block_size, inter_data.block_tables)
149 +
150 + def build(self, seq_lens: List[int], query_lens: List[int],
151 + cuda_graph_pad_size: int, batch_size: int):
152 + for inter_data in self.input_builder.inter_data_list:
153 + self._add_seq_group(inter_data,
154 + self.input_builder.chunked_prefill_enabled)
155
152 - def build(self, runner: "GPUModelRunnerBase", seq_lens: List[int],
153 - query_lens: List[int], cuda_graph_pad_size: int,
154 - batch_size: int):
155 - device = runner.device
156 + device = self.runner.device
157 use_captured_graph = cuda_graph_pad_size != -1
158
158 - logits_soft_cap = getattr(runner.model_config.hf_config,
159 + logits_soft_cap = getattr(self.runner.model_config.hf_config,
160 "attn_logit_softcapping", None)
161 if logits_soft_cap is not None:
162 raise ValueError(
@@ -176,7 +177,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
177
178 # The shape of graph_block_tables is
179 # [max batch size, max context len // block size].
179 - input_block_tables = runner.graph_block_tables[:batch_size]
180 + input_block_tables = self.runner.graph_block_tables[:batch_size]
181 for i, block_table in enumerate(self.block_tables):
182 if block_table:
183 input_block_tables[i, :len(block_table)] = block_table
vllm/utils.py
+5
@@ -719,6 +719,11 @@ def merge_dicts(dict1: Dict[K, List[T]],
719 return dict(merged_dict)
720
721
722 +def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
723 + """Flatten a list of lists to a single list."""
724 + return [item for sublist in lists for item in sublist]
725 +
726 +
727 def init_cached_hf_modules() -> None:
728 """
729 Lazy initialization of the Hugging Face modules.
vllm/worker/model_runner.py
+323 -207
@@ -3,7 +3,7 @@ import gc
3 import time
4 import warnings
5 import weakref
6 -from collections import defaultdict
6 +from dataclasses import dataclass, field
7 from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8 Tuple, Type, TypeVar, Union)
9
@@ -49,7 +49,8 @@ from vllm.prompt_adapter.worker_manager import (
49 from vllm.sampling_params import SamplingParams
50 from vllm.sequence import (IntermediateTensors, SamplerOutput,
51 SequenceGroupMetadata)
52 -from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
52 +from vllm.utils import (CudaMemoryProfiler, flatten_2d_lists,
53 + get_kv_cache_torch_dtype, is_hip,
54 is_pin_memory_available)
55 from vllm.worker.model_runner_base import (
56 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -76,7 +77,7 @@ _NUM_WARMUP_ITERS = 2
77 TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
78
79
79 -@dataclasses.dataclass(frozen=True)
80 +@dataclass(frozen=True)
81 class ModelInputForGPU(ModelRunnerInputBase):
82 """
83 This base class contains metadata needed for the base model forward pass
@@ -126,7 +127,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
127 return cls(**tensor_dict)
128
129
129 -@dataclasses.dataclass(frozen=True)
130 +@dataclass(frozen=True)
131 class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
132 """
133 Used by the ModelRunner.
@@ -168,12 +169,84 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
169
170
171 class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
171 - """TBA"""
172 + """Build ModelInputForGPU from SequenceGroupMetadata."""
173 +
174 + @dataclass
175 + class InterDataForSeqGroup:
176 + """Intermediate data for the current sequence group."""
177 + # From sequence group metadata.
178 + request_id: str
179 + seq_ids: List[int]
180 + is_prompt: bool
181 + block_tables: Optional[Dict[int, List[int]]]
182 + computed_block_nums: List[int]
183 + n_seqs: int = 0
184 +
185 + # Input tokens and positions.
186 + input_tokens: List[List[int]] = field(default_factory=list)
187 + input_positions: List[List[int]] = field(default_factory=list)
188 +
189 + # The sequence length (may be capped to the sliding window).
190 + seq_lens: List[int] = field(default_factory=list)
191 + # The original sequence length (before applying sliding window).
192 + # This is used to compute slot mapping.
193 + orig_seq_lens: List[int] = field(default_factory=list)
194 + # The query length.
195 + query_lens: List[int] = field(default_factory=list)
196 + # The number of tokens that are already computed.
197 + context_lens: List[int] = field(default_factory=list)
198 + # The current sliding window block.
199 + curr_sliding_window_blocks: List[int] = field(default_factory=list)
200 +
201 + # LoRA inputs.
202 + lora_index_mapping: List[List[int]] = field(default_factory=list)
203 + lora_prompt_mapping: List[List[int]] = field(default_factory=list)
204 + lora_requests: Set[LoRARequest] = field(default_factory=set)
205 +
206 + # Prompt adapter inputs.
207 + prompt_adapter_index_mapping: List[int] = field(default_factory=list)
208 + prompt_adapter_prompt_mapping: List[int] = field(default_factory=list)
209 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
210 +
211 + # Multi-modal inputs.
212 + multi_modal_inputs: Optional[MultiModalInputs] = None
213 +
214 + # Whether the prefix cache is hit (prefill only).
215 + prefix_cache_hit: bool = False
216 +
217 + def __post_init__(self):
218 + self.n_seqs = len(self.seq_ids)
219 +
220 + self.input_tokens = [[] for _ in range(self.n_seqs)]
221 + self.input_positions = [[] for _ in range(self.n_seqs)]
222 + self.seq_lens = [0] * self.n_seqs
223 + self.orig_seq_lens = [0] * self.n_seqs
224 + self.query_lens = [0] * self.n_seqs
225 + self.context_lens = [0] * self.n_seqs
226 + self.curr_sliding_window_blocks = [0] * self.n_seqs
227 +
228 + self.lora_index_mapping = [[] for _ in range(self.n_seqs)]
229 + self.lora_prompt_mapping = [[] for _ in range(self.n_seqs)]
230
231 def __init__(self,
232 runner: "GPUModelRunnerBase",
233 finished_requests_ids: Optional[List[str]] = None):
234 super().__init__()
235 + # Compute functions for each sequence in a sequence group.
236 + # WARNING: The order of the functions matters!
237 + self.per_seq_compute_fns = [
238 + self._compute_lens,
239 + self._compute_for_prefix_cache_hit,
240 + self._compute_for_sliding_window,
241 + self._compute_lora_input,
242 + ]
243 + # Compute functions for each sequence group.
244 + # WARNING: The order of the functions matters!
245 + self.per_seq_group_compute_fns = [
246 + self._compute_prompt_adapter_input,
247 + self._compute_multi_modal_input,
248 + ]
249 +
250 self.runner = runner
251 self.model_input_cls = self.runner._model_input_cls
252 self.attn_backend = self.runner.attn_backend
@@ -187,30 +260,14 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
260 self.finished_requests_ids = finished_requests_ids
261 self.decode_only = True
262
190 - # Common inputs.
191 - self.input_tokens: List[int] = []
192 - self.input_positions: List[int] = []
193 - self.seq_lens: List[int] = []
194 - self.query_lens: List[int] = []
195 - self.max_decode_seq_len: int = 0
196 - self.request_ids_to_seq_ids: Dict[str, List[int]] = defaultdict(list)
197 -
198 - # LoRA inputs.
199 - self.lora_index_mapping: List[int] = []
200 - self.lora_prompt_mapping: List[int] = []
201 - self.lora_requests: Set[LoRARequest] = set()
202 -
203 - # Prompt adapter inputs.
204 - self.prompt_adapter_index_mapping: List[int] = []
205 - self.prompt_adapter_prompt_mapping: List[int] = []
206 - self.prompt_adapter_requests: Set[PromptAdapterRequest] = set()
207 -
208 - # Multi-modal inputs.
209 - self.multi_modal_inputs_list: List[MultiModalInputs] = []
263 + # Intermediate data (data in CPU before going to GPU) for
264 + # the current sequence group.
265 + self.inter_data_list: List[
266 + ModelInputForGPUBuilder.InterDataForSeqGroup] = []
267
268 # Attention metadata inputs.
269 self.attn_metadata_builder = self.attn_backend.make_metadata_builder(
213 - self)
270 + weakref.proxy(self))
271
272 # Engine/Model configurations.
273 self.chunked_prefill_enabled = (
@@ -222,175 +279,222 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
279 self.block_aligned_sliding_window = \
280 self.sliding_window_blocks * self.block_size
281
225 - def _compute_len_for_sliding_window(self, seq_len: int):
226 - curr_sliding_window_blocks = 0
227 - sliding_seq_len = seq_len
282 + def _compute_lens(self, inter_data: InterDataForSeqGroup, seq_idx: int,
283 + seq_group_metadata: SequenceGroupMetadata):
284 + """Compute context length, sequence length and tokens
285 + for the given sequence data.
286 + """
287 + seq_data = seq_group_metadata.seq_data[inter_data.seq_ids[seq_idx]]
288 + token_chunk_size = seq_group_metadata.token_chunk_size
289
229 - # TODO(sang): This is a hack to make sliding window work with
230 - # paged attn. We can remove it if we make paged attn kernel
231 - # to properly handle slinding window attn.
232 - if self.sliding_window is not None:
233 - curr_sliding_window_blocks = self.sliding_window_blocks
290 + # Compute context length (the number of tokens that are
291 + # already computed) and sequence length (total number of tokens).
292 + seq_len = seq_data.get_len()
293 + if inter_data.is_prompt:
294 + context_len = seq_data.get_num_computed_tokens()
295 + else:
296 + # get_num_computed_tokens is incorrect for spec decoding.
297 + # So, we should have a special logic here.
298 + # TODO(sang): Fix it.
299 + context_len = seq_len - 1
300 + seq_len = min(seq_len, context_len + token_chunk_size)
301 +
302 + # Compute tokens.
303 + if inter_data.is_prompt:
304 + tokens = seq_data.get_token_ids()[context_len:seq_len]
305 + else:
306 + # Optimization. get_token_ids requires the entire copy of
307 + # tokens.
308 + tokens = [seq_data.get_last_token_id()]
309 +
310 + inter_data.seq_lens[seq_idx] = seq_len
311 + inter_data.orig_seq_lens[seq_idx] = seq_len
312 + inter_data.context_lens[seq_idx] = context_len
313 + inter_data.input_tokens[seq_idx] = tokens
314 + inter_data.input_positions[seq_idx] = list(range(context_len, seq_len))
315 + inter_data.query_lens[
316 + seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
317 +
318 + def _compute_for_prefix_cache_hit(
319 + self, inter_data: InterDataForSeqGroup, seq_idx: int,
320 + seq_group_metadata: SequenceGroupMetadata):
321 + """Check if hit prefix cache (i.e., some blocks are already computed).
322 + If hit, update input tokens and positions to only compute the
323 + remaining blocks.
324 + """
325 + computed_block_nums = inter_data.computed_block_nums
326 +
327 + # Note that prefix caching does not support sliding window.
328 + prefix_cache_hit = (computed_block_nums is not None
329 + and len(computed_block_nums) > 0
330 + and self.sliding_window is None
331 + and inter_data.is_prompt)
332 + inter_data.prefix_cache_hit = prefix_cache_hit
333 + if self.chunked_prefill_enabled and prefix_cache_hit:
334 + raise RuntimeError(
335 + "chunked prefill cannot be used with prefix caching now.")
336 +
337 + # If prefix cache is hit, advance context length to bypass
338 + # hit blocks. Accordingly, input tokens, position and query length
339 + # have to be updated.
340 + if prefix_cache_hit:
341 + assert computed_block_nums is not None
342 + context_len = len(computed_block_nums) * self.block_size
343 + inter_data.input_tokens[seq_idx] = inter_data.input_tokens[
344 + seq_idx][context_len:]
345 + inter_data.input_positions[seq_idx] = inter_data.input_positions[
346 + seq_idx][context_len:]
347 + inter_data.context_lens[seq_idx] = context_len
348 + inter_data.query_lens[
349 + seq_idx] = inter_data.seq_lens[seq_idx] - context_len
350 +
351 + def _compute_for_sliding_window(self, inter_data: InterDataForSeqGroup,
352 + seq_idx: int,
353 + seq_group_metadata: SequenceGroupMetadata):
354 + """Update seq_len and curr_sliding_window_block for the given
355 + sequence data (only required by decoding) if sliding window is enabled.
356 + """
357 + curr_sliding_window_block = 0
358 + sliding_seq_len = inter_data.seq_lens[seq_idx]
359 + if not inter_data.is_prompt and self.sliding_window is not None:
360 + # TODO(sang): This is a hack to make sliding window work with
361 + # paged attn. We can remove it if we make paged attn kernel
362 + # to properly handle slinding window attn.
363 + curr_sliding_window_block = self.sliding_window_blocks
364 if self.scheduler_config.use_v2_block_manager:
365 # number of elements in last block
236 - suff_len = seq_len % self.block_size
366 + suff_len = inter_data.seq_lens[seq_idx] % self.block_size
367 sliding_seq_len = min(
238 - seq_len, self.block_aligned_sliding_window + suff_len)
368 + inter_data.seq_lens[seq_idx],
369 + self.block_aligned_sliding_window + suff_len)
370 if suff_len > 0:
240 - curr_sliding_window_blocks += 1
371 + curr_sliding_window_block += 1
372 else:
242 - sliding_seq_len = min(seq_len, self.sliding_window)
243 - return curr_sliding_window_blocks, sliding_seq_len
373 + sliding_seq_len = min(inter_data.seq_lens[seq_idx],
374 + self.sliding_window)
375 +
376 + inter_data.curr_sliding_window_blocks[
377 + seq_idx] = curr_sliding_window_block
378 + inter_data.seq_lens[seq_idx] = sliding_seq_len
379 +
380 + def _compute_lora_input(self, inter_data: InterDataForSeqGroup,
381 + seq_idx: int,
382 + seq_group_metadata: SequenceGroupMetadata):
383 + """If LoRA is enabled, compute LoRA index and prompt mapping."""
384 + if not self.enable_lora:
385 + return
386 +
387 + lora_id = seq_group_metadata.lora_int_id
388 + if lora_id > 0:
389 + inter_data.lora_requests.add(seq_group_metadata.lora_request)
390 + query_len = inter_data.query_lens[seq_idx]
391 + inter_data.lora_index_mapping.append([lora_id] * query_len)
392 + inter_data.lora_prompt_mapping.append(
393 + [lora_id] *
394 + (query_len if seq_group_metadata.sampling_params
395 + and seq_group_metadata.sampling_params.prompt_logprobs is not None
396 + else 1))
397 +
398 + def _compute_prompt_adapter_input(
399 + self, inter_data: InterDataForSeqGroup,
400 + seq_group_metadata: SequenceGroupMetadata):
401 + """If prompt adapter is enabled, compute index and prompt mapping.
402 + """
403 + # Note that when is_prompt=True, we expect only one sequence
404 + # in the group.
405 + if not self.enable_prompt_adapter:
406 + return
407 +
408 + prompt_adapter_id = seq_group_metadata.prompt_adapter_id
409 + if prompt_adapter_id <= 0 or not inter_data.is_prompt:
410 + return
411 +
412 + # We expect only one sequence in the group when is_prompt=True.
413 + assert inter_data.n_seqs == 1
414 + query_len = inter_data.query_lens[0]
415 + inter_data.prompt_adapter_request = (
416 + seq_group_metadata.prompt_adapter_request)
417 +
418 + num_tokens = seq_group_metadata.prompt_adapter_num_virtual_tokens
419 + inter_data.prompt_adapter_index_mapping = [
420 + prompt_adapter_id
421 + ] * num_tokens + [0] * (query_len - num_tokens)
422 + inter_data.prompt_adapter_prompt_mapping = [prompt_adapter_id] * (
423 + query_len if seq_group_metadata.sampling_params
424 + and seq_group_metadata.sampling_params.prompt_logprobs else 1)
425 +
426 + def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
427 + seq_group_metadata: SequenceGroupMetadata):
428 + """If multi-modal data is given, add it to the input."""
429 + mm_data = seq_group_metadata.multi_modal_data
430 + if not mm_data:
431 + return
432 +
433 + mm_kwargs = self.multi_modal_input_mapper(mm_data)
434 + inter_data.multi_modal_inputs = mm_kwargs
435
436 def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
437 + """Add a sequence group to the builder."""
438 seq_ids = list(seq_group_metadata.seq_data.keys())
439 n_seqs = len(seq_ids)
440 is_prompt = seq_group_metadata.is_prompt
249 - token_chunk_size = seq_group_metadata.token_chunk_size
441
442 if is_prompt:
443 assert n_seqs == 1
444 self.decode_only = False
445
255 - # Mapping from request IDs to sequence IDs. Used for Jamba models
256 - # that manages the cache by itself.
257 - self.request_ids_to_seq_ids[seq_group_metadata.request_id] = []
258 - # The number of input tokens in each sequence.
259 - token_lens: List[int] = []
260 - # The number of tokens that are already computed.
261 - context_lens: List[int] = []
262 - # The current sliding window block for each sequence.
263 - curr_sliding_window_blocks: List[int] = []
264 - # The original sequence length (before applying sliding window)
265 - # for each sequence.
266 - orig_seq_lens: List[int] = []
267 - # The sequence length (may be capped to the sliding window).
268 - curr_seq_lens: List[int] = []
269 - for seq_id in seq_ids:
270 - seq_data = seq_group_metadata.seq_data[seq_id]
271 - self.request_ids_to_seq_ids[seq_group_metadata.request_id].append(
272 - seq_id)
273 - computed_block_nums = seq_group_metadata.computed_block_nums
274 -
275 - # Check if hit prefix cache (i.e., some blocks are already computed)
276 - # Note that prefix caching does not support sliding window.
277 - prefix_cache_hit = (computed_block_nums is not None
278 - and len(computed_block_nums) > 0
279 - and self.sliding_window is None and is_prompt)
280 - if self.chunked_prefill_enabled and prefix_cache_hit:
281 - raise RuntimeError(
282 - "chunked prefill cannot be used with prefix caching now.")
283 -
284 - # Compute context length (the number of tokens that are
285 - # already computed) and sequence length (total number of tokens).
286 - seq_len = seq_data.get_len()
287 - if is_prompt:
288 - context_len = seq_data.get_num_computed_tokens()
289 - else:
290 - # get_num_computed_tokens is incorrect for spec decoding.
291 - # So, we should have a special logic here.
292 - # TODO(sang): Fix it.
293 - context_len = seq_len - 1
294 - seq_len = min(seq_len, context_len + token_chunk_size)
295 -
296 - # Compute tokens.
297 - if is_prompt:
298 - tokens = seq_data.get_token_ids()[context_len:seq_len]
299 - else:
300 - # Optimization. get_token_ids requires the entire copy of
301 - # tokens.
302 - tokens = [seq_data.get_last_token_id()]
303 - if prefix_cache_hit:
304 - assert computed_block_nums is not None
305 - context_len = len(computed_block_nums) * self.block_size
306 - tokens = tokens[context_len:]
307 -
308 - # These are seq_len/context_len capped to the sliding window.
309 - # They are passed to decode kernel.
310 - # We still need original seq_len/context_len to compute slot
311 - # mapping (and input position) below.
312 - if is_prompt:
313 - curr_sliding_window_block = 0
314 - sliding_seq_len = seq_len
315 - query_len = seq_len - context_len
316 - else:
317 - curr_sliding_window_block, sliding_seq_len = (
318 - self._compute_len_for_sliding_window(seq_len))
319 - query_len = 1
320 -
321 - self.seq_lens.append(sliding_seq_len)
322 - if not is_prompt:
323 - self.max_decode_seq_len = max(self.max_decode_seq_len,
324 - sliding_seq_len)
325 - self.query_lens.append(query_len)
326 - self.input_tokens.extend(tokens)
327 - self.input_positions.extend(list(range(context_len, seq_len)))
328 -
329 - # Intermediate data of the current sequence group for
330 - # the attention metadata.
331 - token_lens.append(len(tokens))
332 - context_lens.append(context_len)
333 - curr_seq_lens.append(sliding_seq_len)
334 - curr_sliding_window_blocks.append(curr_sliding_window_block)
335 - orig_seq_lens.append(seq_len)
336 -
337 - # Update attention metadata. Note that input builder attributes
338 - # (self.xxx) include all added sequences, so we need to slice
339 - # the last n_seqs sequences.
340 - self.attn_metadata_builder.add_seq_group(
341 - seq_group_metadata, token_lens, orig_seq_lens, curr_seq_lens,
342 - self.query_lens[-n_seqs:], context_lens,
343 - curr_sliding_window_blocks, prefix_cache_hit,
344 - self.chunked_prefill_enabled)
345 -
346 - # LoRA data.
347 - if self.enable_lora:
348 - lora_id = seq_group_metadata.lora_int_id
349 - for query_len in self.query_lens[-n_seqs:]:
350 - if lora_id > 0:
351 - self.lora_requests.add(seq_group_metadata.lora_request)
352 - self.lora_index_mapping += [lora_id] * query_len
353 - self.lora_prompt_mapping.extend(
354 - [lora_id] *
355 - (query_len if seq_group_metadata.sampling_params
356 - and seq_group_metadata.sampling_params.prompt_logprobs
357 - is not None else 1))
358 -
359 - # Prompt adapter data. Note that when is_prompt=True,
360 - # we expect only one sequence in the group.
361 - if self.enable_prompt_adapter:
362 - prompt_adapter_id = seq_group_metadata.prompt_adapter_id
363 - if prompt_adapter_id > 0 and is_prompt:
364 - query_len = self.query_lens[-1]
365 - self.prompt_adapter_requests.add(
366 - seq_group_metadata.prompt_adapter_request)
367 -
368 - num_tokens = seq_group_metadata.\
369 - prompt_adapter_num_virtual_tokens
370 - pm = [prompt_adapter_id
371 - ] * num_tokens + [0] * (query_len - num_tokens)
372 - self.prompt_adapter_index_mapping += pm
373 - self.prompt_adapter_prompt_mapping.extend(
374 - [prompt_adapter_id] *
375 - (query_len if seq_group_metadata.sampling_params
376 - and seq_group_metadata.sampling_params.prompt_logprobs
377 - else 1))
446 + inter_data = self.InterDataForSeqGroup(
447 + request_id=seq_group_metadata.request_id,
448 + seq_ids=seq_ids,
449 + is_prompt=is_prompt,
450 + block_tables=seq_group_metadata.block_tables,
451 + computed_block_nums=seq_group_metadata.computed_block_nums)
452 + self.inter_data_list.append(inter_data)
453
379 - # Multi-modal data.
380 - mm_data = seq_group_metadata.multi_modal_data
381 - if mm_data:
382 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
383 - self.multi_modal_inputs_list.append(mm_kwargs)
454 + for seq_idx in range(n_seqs):
455 + for per_seq_fn in self.per_seq_compute_fns:
456 + per_seq_fn(inter_data, seq_idx, seq_group_metadata)
457 + for per_seq_group_fn in self.per_seq_group_compute_fns:
458 + per_seq_group_fn(inter_data, seq_group_metadata)
459
460 def build(self) -> ModelInputForGPU:
386 - if not self.input_tokens:
461 + """Finalize the builder intermediate data and
462 + create on-device tensors.
463 + """
464 + # Combine and flatten intermediate data.
465 + input_tokens = flatten_2d_lists([
466 + flatten_2d_lists(inter_data.input_tokens)
467 + for inter_data in self.inter_data_list
468 + ])
469 + if not input_tokens:
470 + # This may happen when all prefill requests hit
471 + # prefix caching and there is no decode request.
472 return self.model_input_cls()
473 + input_positions = flatten_2d_lists([
474 + flatten_2d_lists(inter_data.input_positions)
475 + for inter_data in self.inter_data_list
476 + ])
477 + seq_lens = []
478 + max_decode_seq_len = 0
479 + for inter_data in self.inter_data_list:
480 + seq_lens.extend(inter_data.seq_lens)
481 + if not inter_data.is_prompt:
482 + max_decode_seq_len = max(max_decode_seq_len,
483 + max(inter_data.seq_lens))
484 + query_lens = flatten_2d_lists(
485 + [inter_data.query_lens for inter_data in self.inter_data_list])
486 + # Mapping from request IDs to sequence IDs. Used for Jamba models
487 + # that manages the cache by itself.
488 + request_ids_to_seq_ids = {
489 + data.request_id: data.seq_ids
490 + for data in self.inter_data_list
491 + }
492
389 - batch_size = len(self.input_tokens)
493 + batch_size = len(input_tokens)
494 use_captured_graph = (
495 self.decode_only and not self.runner.model_config.enforce_eager
496 and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
393 - and self.max_decode_seq_len <= self.runner.max_seq_len_to_capture)
497 + and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
498
499 # If cuda graph can be used, pad tensors accordingly.
500 # See `capture_model` API for more details.
@@ -403,60 +507,84 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
507 batch_size = graph_batch_size
508
509 # Tokens and positions.
406 - self.input_tokens.extend([0] * cuda_graph_pad_size)
407 - self.input_positions.extend([0] * cuda_graph_pad_size)
408 - input_tokens_tensor = torch.tensor(self.input_tokens,
510 + input_tokens.extend([0] * cuda_graph_pad_size)
511 + input_positions.extend([0] * cuda_graph_pad_size)
512 + input_tokens_tensor = torch.tensor(input_tokens,
513 dtype=torch.long,
514 device=self.runner.device)
411 - input_positions_tensor = torch.tensor(self.input_positions,
515 + input_positions_tensor = torch.tensor(input_positions,
516 dtype=torch.long,
517 device=self.runner.device)
518
519 # Sequence and query lengths.
416 - self.seq_lens.extend([1] * cuda_graph_pad_size)
520 + seq_lens.extend([1] * cuda_graph_pad_size)
521
522 # Attention metadata.
523 attn_metadata = self.attn_metadata_builder.build(
420 - self.runner, self.seq_lens, self.query_lens, cuda_graph_pad_size,
421 - batch_size)
524 + seq_lens, query_lens, cuda_graph_pad_size, batch_size)
525
526 # LoRA data.
527 + lora_requests = set()
528 + lora_mapping = None
529 if self.enable_lora:
425 - self.lora_index_mapping.extend([0] * cuda_graph_pad_size)
530 + lora_requests = set(r for data in self.inter_data_list
531 + for r in data.lora_requests)
532 + lora_index_mapping = flatten_2d_lists([
533 + flatten_2d_lists(inter_data.lora_index_mapping)
534 + for inter_data in self.inter_data_list
535 + ])
536 + lora_index_mapping.extend([0] * cuda_graph_pad_size)
537 + lora_prompt_mapping = flatten_2d_lists([
538 + flatten_2d_lists(inter_data.lora_prompt_mapping)
539 + for inter_data in self.inter_data_list
540 + ])
541 lora_mapping = LoRAMapping(
427 - self.lora_index_mapping,
428 - self.lora_prompt_mapping,
542 + lora_index_mapping,
543 + lora_prompt_mapping,
544 )
430 - else:
431 - lora_mapping = None
545
546 # Prompt adapter data.
547 + prompt_adapter_requests: Set[PromptAdapterRequest] = set()
548 + prompt_adapter_mapping = None
549 if self.enable_prompt_adapter:
435 - self.prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
550 + prompt_adapter_requests = set(
551 + data.prompt_adapter_request for data in self.inter_data_list
552 + if data.prompt_adapter_request is not None)
553 + prompt_adapter_index_mapping = flatten_2d_lists([
554 + inter_data.prompt_adapter_index_mapping
555 + for inter_data in self.inter_data_list
556 + ])
557 + prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
558 + prompt_adapter_prompt_mapping = flatten_2d_lists([
559 + inter_data.prompt_adapter_prompt_mapping
560 + for inter_data in self.inter_data_list
561 + ])
562 prompt_adapter_mapping = PromptAdapterMapping(
437 - self.prompt_adapter_index_mapping,
438 - self.prompt_adapter_prompt_mapping,
563 + prompt_adapter_index_mapping,
564 + prompt_adapter_prompt_mapping,
565 )
440 - else:
441 - prompt_adapter_mapping = None
566
567 # Multi-modal data.
444 - multi_modal_kwargs = MultiModalInputs.batch(
445 - self.multi_modal_inputs_list, device=self.runner.device)
568 + multi_modal_inputs_list = [
569 + data.multi_modal_inputs for data in self.inter_data_list
570 + if data.multi_modal_inputs is not None
571 + ]
572 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
573 + device=self.runner.device)
574
575 return self.model_input_cls(
576 input_tokens=input_tokens_tensor,
577 input_positions=input_positions_tensor,
578 attn_metadata=attn_metadata,
451 - seq_lens=self.seq_lens,
452 - query_lens=self.query_lens,
579 + seq_lens=seq_lens,
580 + query_lens=query_lens,
581 lora_mapping=lora_mapping,
454 - lora_requests=self.lora_requests,
582 + lora_requests=lora_requests,
583 multi_modal_kwargs=multi_modal_kwargs,
456 - request_ids_to_seq_ids=self.request_ids_to_seq_ids,
584 + request_ids_to_seq_ids=request_ids_to_seq_ids,
585 finished_requests_ids=self.finished_requests_ids,
586 prompt_adapter_mapping=prompt_adapter_mapping,
459 - prompt_adapter_requests=self.prompt_adapter_requests)
587 + prompt_adapter_requests=prompt_adapter_requests)
588
589
590 class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
@@ -1393,15 +1521,3 @@ def _get_graph_batch_size(batch_size: int) -> int:
1521 else:
1522 return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
1523 _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
1396 -
1397 -
1398 -def _is_block_tables_empty(block_tables: Union[None, Dict]):
1399 - """
1400 - Check if block_tables is None or a dictionary with all None values.
1401 - """
1402 - if block_tables is None:
1403 - return True
1404 - if isinstance(block_tables, dict) and all(
1405 - value is None for value in block_tables.values()):
1406 - return True
1407 - return False