A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Tweaks to model runner/input builder developer APIs (#6712)
Antoni Baum committed
Jul 24, 2024 at 12:17 UTC
5448f67635570cee6fc23c7cd166e9d8f7595009
3 files changed
+109
-64
vllm/attention/backends/flashinfer.py
+19
-16
@@ -297,23 +297,26 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
297
if is_profile_run:
298
return
299
300
- # Get the number of valid blocks based on sequence length.
301
- # If seq_len = 16, block_size = 16,
302
- # block_table_bound is 1 with 1 valid block.
303
- # If seq_len = 15, block_size = 16,
304
- # block_table_bound is 0 + 1 with 1 valid block.
305
- block_table_bound = seq_len // self.block_size + 1 \
306
- if seq_len % self.block_size != 0 \
307
- else seq_len // self.block_size
300
block_table = block_tables[seq_id]
309
- self.paged_kv_indices.extend(block_table[:block_table_bound])
310
- self.paged_kv_indptr.append(self.paged_kv_indptr[-1] +
311
- block_table_bound)
312
-
313
- last_page_len = seq_len % self.block_size
314
- if last_page_len == 0:
315
- last_page_len = self.block_size
316
- self.paged_kv_last_page_len.append(last_page_len)
301
+ self._update_paged_kv_tensors(block_table, seq_len)
302
+
303
+ def _update_paged_kv_tensors(self, block_table: List[int], seq_len: int):
304
+ # Get the number of valid blocks based on sequence length.
305
+ # If seq_len = 16, block_size = 16,
306
+ # block_table_bound is 1 with 1 valid block.
307
+ # If seq_len = 15, block_size = 16,
308
+ # block_table_bound is 0 + 1 with 1 valid block.
309
+ block_table_bound = seq_len // self.block_size + 1 \
310
+ if seq_len % self.block_size != 0 \
311
+ else seq_len // self.block_size
312
+ self.paged_kv_indices.extend(block_table[:block_table_bound])
313
+ self.paged_kv_indptr.append(self.paged_kv_indptr[-1] +
314
+ block_table_bound)
315
+
316
+ last_page_len = seq_len % self.block_size
317
+ if last_page_len == 0:
318
+ last_page_len = self.block_size
319
+ self.paged_kv_last_page_len.append(last_page_len)
320
321
def build(self, seq_lens: List[int], query_lens: List[int],
322
cuda_graph_pad_size: int, batch_size: int):
vllm/worker/embedding_model_runner.py
+3
-1
@@ -11,7 +11,8 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.pooling_params import PoolingParams
12
from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
13
SequenceGroupMetadata)
14
-from vllm.worker.model_runner import GPUModelRunnerBase, ModelInputForGPU
14
+from vllm.worker.model_runner import (GPUModelRunnerBase, ModelInputForGPU,
15
+ ModelInputForGPUBuilder)
16
17
logger = init_logger(__name__)
18
@@ -28,6 +29,7 @@ class EmbeddingModelRunner(
29
GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]):
30
_model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = (
31
ModelInputForGPUWithPoolingMetadata)
32
+ _builder_cls: Type[ModelInputForGPUBuilder] = ModelInputForGPUBuilder
33
34
def __init__(
35
self,
vllm/worker/model_runner.py
+87
-47
@@ -3,7 +3,7 @@ import gc
3
import time
4
import warnings
5
import weakref
6
-from dataclasses import dataclass, field
6
+from dataclasses import dataclass
7
from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8
Tuple, Type, TypeVar, Union)
9
@@ -171,48 +171,83 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
171
class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
172
"""Build ModelInputForGPU from SequenceGroupMetadata."""
173
174
- @dataclass
174
+ # Note: ideally we would be using a dataclass(kw_only=True)
175
+ # here, so that this can be subclassed easily,
176
+ # but kw_only is not supported in python<3.10.
177
class InterDataForSeqGroup:
178
"""Intermediate data for the current sequence group."""
177
- # From sequence group metadata.
178
- request_id: str
179
- seq_ids: List[int]
180
- is_prompt: bool
181
- block_tables: Optional[Dict[int, List[int]]]
182
- computed_block_nums: List[int]
183
- n_seqs: int = 0
184
-
185
- # Input tokens and positions.
186
- input_tokens: List[List[int]] = field(default_factory=list)
187
- input_positions: List[List[int]] = field(default_factory=list)
188
-
189
- # The sequence length (may be capped to the sliding window).
190
- seq_lens: List[int] = field(default_factory=list)
191
- # The original sequence length (before applying sliding window).
192
- # This is used to compute slot mapping.
193
- orig_seq_lens: List[int] = field(default_factory=list)
194
- # The query length.
195
- query_lens: List[int] = field(default_factory=list)
196
- # The number of tokens that are already computed.
197
- context_lens: List[int] = field(default_factory=list)
198
- # The current sliding window block.
199
- curr_sliding_window_blocks: List[int] = field(default_factory=list)
200
-
201
- # LoRA inputs.
202
- lora_index_mapping: List[List[int]] = field(default_factory=list)
203
- lora_prompt_mapping: List[List[int]] = field(default_factory=list)
204
- lora_requests: Set[LoRARequest] = field(default_factory=set)
205
-
206
- # Prompt adapter inputs.
207
- prompt_adapter_index_mapping: List[int] = field(default_factory=list)
208
- prompt_adapter_prompt_mapping: List[int] = field(default_factory=list)
209
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
210
-
211
- # Multi-modal inputs.
212
- multi_modal_inputs: Optional[MultiModalInputs] = None
213
-
214
- # Whether the prefix cache is hit (prefill only).
215
- prefix_cache_hit: bool = False
179
+
180
+ def __init__(
181
+ self,
182
+ *,
183
+ # From sequence group metadata.
184
+ request_id: str,
185
+ seq_ids: List[int],
186
+ is_prompt: bool,
187
+ block_tables: Optional[Dict[int, List[int]]],
188
+ computed_block_nums: List[int],
189
+ n_seqs: int = 0,
190
+
191
+ # Input tokens and positions.
192
+ input_tokens: Optional[List[List[int]]] = None,
193
+ input_positions: Optional[List[List[int]]] = None,
194
+
195
+ # The sequence length (may be capped to the sliding window).
196
+ seq_lens: Optional[List[int]] = None,
197
+ # The original sequence length (before applying sliding window).
198
+ # This is used to compute slot mapping.
199
+ orig_seq_lens: Optional[List[int]] = None,
200
+ # The query length.
201
+ query_lens: Optional[List[int]] = None,
202
+ # The number of tokens that are already computed.
203
+ context_lens: Optional[List[int]] = None,
204
+ # The current sliding window block.
205
+ curr_sliding_window_blocks: Optional[List[int]] = None,
206
+
207
+ # LoRA inputs.
208
+ lora_index_mapping: Optional[List[List[int]]] = None,
209
+ lora_prompt_mapping: Optional[List[List[int]]] = None,
210
+ lora_requests: Optional[Set[LoRARequest]] = None,
211
+
212
+ # Prompt adapter inputs.
213
+ prompt_adapter_index_mapping: Optional[List[int]] = None,
214
+ prompt_adapter_prompt_mapping: Optional[List[int]] = None,
215
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
216
+
217
+ # Multi-modal inputs.
218
+ multi_modal_inputs: Optional[MultiModalInputs] = None,
219
+
220
+ # Whether the prefix cache is hit (prefill only).
221
+ prefix_cache_hit: bool = False,
222
+ ):
223
+ self.request_id = request_id
224
+ self.seq_ids = seq_ids
225
+ self.is_prompt = is_prompt
226
+ self.block_tables = block_tables
227
+ self.computed_block_nums = computed_block_nums
228
+ self.n_seqs = n_seqs
229
+ self.input_tokens = input_tokens or []
230
+ self.input_positions = input_positions or []
231
+ self.seq_lens = seq_lens or []
232
+ self.orig_seq_lens = orig_seq_lens or []
233
+ self.query_lens = query_lens or []
234
+ self.context_lens = context_lens or []
235
+ self.curr_sliding_window_blocks = curr_sliding_window_blocks or []
236
+
237
+ self.lora_index_mapping = lora_index_mapping or []
238
+ self.lora_prompt_mapping = lora_prompt_mapping or []
239
+ self.lora_requests = lora_requests or set()
240
+
241
+ self.prompt_adapter_index_mapping = (prompt_adapter_index_mapping
242
+ or [])
243
+ self.prompt_adapter_prompt_mapping = (prompt_adapter_prompt_mapping
244
+ or [])
245
+ self.prompt_adapter_request = prompt_adapter_request
246
+
247
+ self.multi_modal_inputs = multi_modal_inputs
248
+ self.prefix_cache_hit = prefix_cache_hit
249
+
250
+ self.__post_init__()
251
252
def __post_init__(self):
253
self.n_seqs = len(self.seq_ids)
@@ -457,6 +492,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
492
for per_seq_group_fn in self.per_seq_group_compute_fns:
493
per_seq_group_fn(inter_data, seq_group_metadata)
494
495
+ def _use_captured_graph(self, batch_size: int,
496
+ max_decode_seq_len: int) -> bool:
497
+ return (self.decode_only and not self.runner.model_config.enforce_eager
498
+ and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
499
+ and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
500
+
501
def build(self) -> ModelInputForGPU:
502
"""Finalize the builder intermediate data and
503
create on-device tensors.
@@ -491,10 +532,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
532
}
533
534
batch_size = len(input_tokens)
494
- use_captured_graph = (
495
- self.decode_only and not self.runner.model_config.enforce_eager
496
- and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
497
- and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
535
+ use_captured_graph = self._use_captured_graph(batch_size,
536
+ max_decode_seq_len)
537
538
# If cuda graph can be used, pad tensors accordingly.
539
# See `capture_model` API for more details.
@@ -592,6 +631,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
631
Helper class for shared methods between GPU model runners.
632
"""
633
_model_input_cls: Type[TModelInputForGPU]
634
+ _builder_cls: Type[ModelInputForGPUBuilder]
635
636
def __init__(
637
self,
@@ -794,8 +834,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
834
835
If cuda graph is required, this API automatically pads inputs.
836
"""
797
- builder = ModelInputForGPUBuilder(weakref.proxy(self),
798
- finished_requests_ids)
837
+ builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
838
for seq_group_metadata in seq_group_metadata_list:
839
builder.add_seq_group(seq_group_metadata)
840
return builder.build() # type: ignore
@@ -1191,6 +1230,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1230
"""
1231
_model_input_cls: Type[ModelInputForGPUWithSamplingMetadata] = (
1232
ModelInputForGPUWithSamplingMetadata)
1233
+ _builder_cls: Type[ModelInputForGPUBuilder] = ModelInputForGPUBuilder
1234
1235
def make_model_input_from_broadcasted_tensor_dict(
1236
self,