@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5448f67635
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Tweaks to model runner/input builder developer APIs (#6712)

Antoni Baum committed Jul 24, 2024 at 12:17 UTC 5448f67635570cee6fc23c7cd166e9d8f7595009
3 files changed +109 -64
vllm/attention/backends/flashinfer.py
+19 -16
@@ -297,23 +297,26 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
297 if is_profile_run:
298 return
299
300 - # Get the number of valid blocks based on sequence length.
301 - # If seq_len = 16, block_size = 16,
302 - # block_table_bound is 1 with 1 valid block.
303 - # If seq_len = 15, block_size = 16,
304 - # block_table_bound is 0 + 1 with 1 valid block.
305 - block_table_bound = seq_len // self.block_size + 1 \
306 - if seq_len % self.block_size != 0 \
307 - else seq_len // self.block_size
300 block_table = block_tables[seq_id]
309 - self.paged_kv_indices.extend(block_table[:block_table_bound])
310 - self.paged_kv_indptr.append(self.paged_kv_indptr[-1] +
311 - block_table_bound)
312 -
313 - last_page_len = seq_len % self.block_size
314 - if last_page_len == 0:
315 - last_page_len = self.block_size
316 - self.paged_kv_last_page_len.append(last_page_len)
301 + self._update_paged_kv_tensors(block_table, seq_len)
302 +
303 + def _update_paged_kv_tensors(self, block_table: List[int], seq_len: int):
304 + # Get the number of valid blocks based on sequence length.
305 + # If seq_len = 16, block_size = 16,
306 + # block_table_bound is 1 with 1 valid block.
307 + # If seq_len = 15, block_size = 16,
308 + # block_table_bound is 0 + 1 with 1 valid block.
309 + block_table_bound = seq_len // self.block_size + 1 \
310 + if seq_len % self.block_size != 0 \
311 + else seq_len // self.block_size
312 + self.paged_kv_indices.extend(block_table[:block_table_bound])
313 + self.paged_kv_indptr.append(self.paged_kv_indptr[-1] +
314 + block_table_bound)
315 +
316 + last_page_len = seq_len % self.block_size
317 + if last_page_len == 0:
318 + last_page_len = self.block_size
319 + self.paged_kv_last_page_len.append(last_page_len)
320
321 def build(self, seq_lens: List[int], query_lens: List[int],
322 cuda_graph_pad_size: int, batch_size: int):
vllm/worker/embedding_model_runner.py
+3 -1
@@ -11,7 +11,8 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.pooling_params import PoolingParams
12 from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
13 SequenceGroupMetadata)
14 -from vllm.worker.model_runner import GPUModelRunnerBase, ModelInputForGPU
14 +from vllm.worker.model_runner import (GPUModelRunnerBase, ModelInputForGPU,
15 + ModelInputForGPUBuilder)
16
17 logger = init_logger(__name__)
18
@@ -28,6 +29,7 @@ class EmbeddingModelRunner(
29 GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]):
30 _model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = (
31 ModelInputForGPUWithPoolingMetadata)
32 + _builder_cls: Type[ModelInputForGPUBuilder] = ModelInputForGPUBuilder
33
34 def __init__(
35 self,
vllm/worker/model_runner.py
+87 -47
@@ -3,7 +3,7 @@ import gc
3 import time
4 import warnings
5 import weakref
6 -from dataclasses import dataclass, field
6 +from dataclasses import dataclass
7 from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
8 Tuple, Type, TypeVar, Union)
9
@@ -171,48 +171,83 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
171 class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
172 """Build ModelInputForGPU from SequenceGroupMetadata."""
173
174 - @dataclass
174 + # Note: ideally we would be using a dataclass(kw_only=True)
175 + # here, so that this can be subclassed easily,
176 + # but kw_only is not supported in python<3.10.
177 class InterDataForSeqGroup:
178 """Intermediate data for the current sequence group."""
177 - # From sequence group metadata.
178 - request_id: str
179 - seq_ids: List[int]
180 - is_prompt: bool
181 - block_tables: Optional[Dict[int, List[int]]]
182 - computed_block_nums: List[int]
183 - n_seqs: int = 0
184 -
185 - # Input tokens and positions.
186 - input_tokens: List[List[int]] = field(default_factory=list)
187 - input_positions: List[List[int]] = field(default_factory=list)
188 -
189 - # The sequence length (may be capped to the sliding window).
190 - seq_lens: List[int] = field(default_factory=list)
191 - # The original sequence length (before applying sliding window).
192 - # This is used to compute slot mapping.
193 - orig_seq_lens: List[int] = field(default_factory=list)
194 - # The query length.
195 - query_lens: List[int] = field(default_factory=list)
196 - # The number of tokens that are already computed.
197 - context_lens: List[int] = field(default_factory=list)
198 - # The current sliding window block.
199 - curr_sliding_window_blocks: List[int] = field(default_factory=list)
200 -
201 - # LoRA inputs.
202 - lora_index_mapping: List[List[int]] = field(default_factory=list)
203 - lora_prompt_mapping: List[List[int]] = field(default_factory=list)
204 - lora_requests: Set[LoRARequest] = field(default_factory=set)
205 -
206 - # Prompt adapter inputs.
207 - prompt_adapter_index_mapping: List[int] = field(default_factory=list)
208 - prompt_adapter_prompt_mapping: List[int] = field(default_factory=list)
209 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
210 -
211 - # Multi-modal inputs.
212 - multi_modal_inputs: Optional[MultiModalInputs] = None
213 -
214 - # Whether the prefix cache is hit (prefill only).
215 - prefix_cache_hit: bool = False
179 +
180 + def __init__(
181 + self,
182 + *,
183 + # From sequence group metadata.
184 + request_id: str,
185 + seq_ids: List[int],
186 + is_prompt: bool,
187 + block_tables: Optional[Dict[int, List[int]]],
188 + computed_block_nums: List[int],
189 + n_seqs: int = 0,
190 +
191 + # Input tokens and positions.
192 + input_tokens: Optional[List[List[int]]] = None,
193 + input_positions: Optional[List[List[int]]] = None,
194 +
195 + # The sequence length (may be capped to the sliding window).
196 + seq_lens: Optional[List[int]] = None,
197 + # The original sequence length (before applying sliding window).
198 + # This is used to compute slot mapping.
199 + orig_seq_lens: Optional[List[int]] = None,
200 + # The query length.
201 + query_lens: Optional[List[int]] = None,
202 + # The number of tokens that are already computed.
203 + context_lens: Optional[List[int]] = None,
204 + # The current sliding window block.
205 + curr_sliding_window_blocks: Optional[List[int]] = None,
206 +
207 + # LoRA inputs.
208 + lora_index_mapping: Optional[List[List[int]]] = None,
209 + lora_prompt_mapping: Optional[List[List[int]]] = None,
210 + lora_requests: Optional[Set[LoRARequest]] = None,
211 +
212 + # Prompt adapter inputs.
213 + prompt_adapter_index_mapping: Optional[List[int]] = None,
214 + prompt_adapter_prompt_mapping: Optional[List[int]] = None,
215 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
216 +
217 + # Multi-modal inputs.
218 + multi_modal_inputs: Optional[MultiModalInputs] = None,
219 +
220 + # Whether the prefix cache is hit (prefill only).
221 + prefix_cache_hit: bool = False,
222 + ):
223 + self.request_id = request_id
224 + self.seq_ids = seq_ids
225 + self.is_prompt = is_prompt
226 + self.block_tables = block_tables
227 + self.computed_block_nums = computed_block_nums
228 + self.n_seqs = n_seqs
229 + self.input_tokens = input_tokens or []
230 + self.input_positions = input_positions or []
231 + self.seq_lens = seq_lens or []
232 + self.orig_seq_lens = orig_seq_lens or []
233 + self.query_lens = query_lens or []
234 + self.context_lens = context_lens or []
235 + self.curr_sliding_window_blocks = curr_sliding_window_blocks or []
236 +
237 + self.lora_index_mapping = lora_index_mapping or []
238 + self.lora_prompt_mapping = lora_prompt_mapping or []
239 + self.lora_requests = lora_requests or set()
240 +
241 + self.prompt_adapter_index_mapping = (prompt_adapter_index_mapping
242 + or [])
243 + self.prompt_adapter_prompt_mapping = (prompt_adapter_prompt_mapping
244 + or [])
245 + self.prompt_adapter_request = prompt_adapter_request
246 +
247 + self.multi_modal_inputs = multi_modal_inputs
248 + self.prefix_cache_hit = prefix_cache_hit
249 +
250 + self.__post_init__()
251
252 def __post_init__(self):
253 self.n_seqs = len(self.seq_ids)
@@ -457,6 +492,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
492 for per_seq_group_fn in self.per_seq_group_compute_fns:
493 per_seq_group_fn(inter_data, seq_group_metadata)
494
495 + def _use_captured_graph(self, batch_size: int,
496 + max_decode_seq_len: int) -> bool:
497 + return (self.decode_only and not self.runner.model_config.enforce_eager
498 + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
499 + and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
500 +
501 def build(self) -> ModelInputForGPU:
502 """Finalize the builder intermediate data and
503 create on-device tensors.
@@ -491,10 +532,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
532 }
533
534 batch_size = len(input_tokens)
494 - use_captured_graph = (
495 - self.decode_only and not self.runner.model_config.enforce_eager
496 - and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
497 - and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
535 + use_captured_graph = self._use_captured_graph(batch_size,
536 + max_decode_seq_len)
537
538 # If cuda graph can be used, pad tensors accordingly.
539 # See `capture_model` API for more details.
@@ -592,6 +631,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
631 Helper class for shared methods between GPU model runners.
632 """
633 _model_input_cls: Type[TModelInputForGPU]
634 + _builder_cls: Type[ModelInputForGPUBuilder]
635
636 def __init__(
637 self,
@@ -794,8 +834,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
834
835 If cuda graph is required, this API automatically pads inputs.
836 """
797 - builder = ModelInputForGPUBuilder(weakref.proxy(self),
798 - finished_requests_ids)
837 + builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
838 for seq_group_metadata in seq_group_metadata_list:
839 builder.add_seq_group(seq_group_metadata)
840 return builder.build() # type: ignore
@@ -1191,6 +1230,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1230 """
1231 _model_input_cls: Type[ModelInputForGPUWithSamplingMetadata] = (
1232 ModelInputForGPUWithSamplingMetadata)
1233 + _builder_cls: Type[ModelInputForGPUBuilder] = ModelInputForGPUBuilder
1234
1235 def make_model_input_from_broadcasted_tensor_dict(
1236 self,