@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66818e5b63
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] separate builder init and builder prepare for each batch (#12253)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 22, 2025 at 14:13 UTC 66818e5b63818a286756653816772faa8622ad89
10 files changed +90 -47
vllm/attention/backends/abstract.py
+6 -5
@@ -65,11 +65,6 @@ class AttentionBackend(ABC):
65 def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
66 raise NotImplementedError
67
68 - @classmethod
69 - def make_metadata_builder(cls, *args,
70 - **kwargs) -> "AttentionMetadataBuilder":
71 - return cls.get_builder_cls()(*args, **kwargs)
72 -
68 @staticmethod
69 @abstractmethod
70 def get_kv_cache_shape(
@@ -214,6 +209,12 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
209
210 @abstractmethod
211 def __init__(self, input_builder: "ModelRunnerInputBuilderBase") -> None:
212 + """Create the builder, remember some configuration and parameters."""
213 + raise NotImplementedError
214 +
215 + @abstractmethod
216 + def prepare(self) -> None:
217 + """Prepare for one batch."""
218 raise NotImplementedError
219
220 @abstractmethod
vllm/attention/backends/flash_attn.py
+6 -5
@@ -375,6 +375,12 @@ class FlashAttentionMetadataBuilder(
375 AttentionMetadataBuilder[FlashAttentionMetadata]):
376
377 def __init__(self, input_builder: "ModelInputForGPUBuilder"):
378 + self.input_builder = input_builder
379 + self.runner = input_builder.runner
380 + self.sliding_window = input_builder.sliding_window
381 + self.block_size = input_builder.block_size
382 +
383 + def prepare(self):
384 self.slot_mapping: List[int] = []
385 self.prefill_seq_lens: List[int] = []
386 self.context_lens: List[int] = []
@@ -388,11 +394,6 @@ class FlashAttentionMetadataBuilder(
394 self.num_decode_tokens = 0
395 self.has_prefix_cache_hit = False
396
391 - self.input_builder = input_builder
392 - self.runner = input_builder.runner
393 - self.sliding_window = input_builder.sliding_window
394 - self.block_size = input_builder.block_size
395 -
397 def _add_seq_group(
398 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
399 chunked_prefill_enabled: bool, prefix_cache_hit: bool):
vllm/attention/backends/flashinfer.py
+8 -6
@@ -488,6 +488,14 @@ class FlashInferMetadata(AttentionMetadata):
488 class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
489
490 def __init__(self, input_builder: "ModelInputForGPUBuilder"):
491 +
492 + self.input_builder = input_builder
493 + self.runner = input_builder.runner
494 +
495 + self.sliding_window = input_builder.sliding_window
496 + self.block_size = input_builder.block_size
497 +
498 + def prepare(self):
499 self.slot_mapping: List[int] = []
500 self.prefill_seq_lens: List[int] = []
501 self.context_lens: List[int] = []
@@ -500,12 +508,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
508 self.num_prefill_tokens = 0
509 self.num_decode_tokens = 0
510
503 - self.input_builder = input_builder
504 - self.runner = input_builder.runner
505 -
506 - self.sliding_window = input_builder.sliding_window
507 - self.block_size = input_builder.block_size
508 -
511 # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
512 # for the precise definition of the following fields.
513 # An example:
vllm/attention/backends/placeholder_attn.py
+5 -3
@@ -253,6 +253,11 @@ class PlaceholderAttentionMetadataBuilder(
253 AttentionMetadataBuilder[PlaceholderAttentionMetadata]):
254
255 def __init__(self, input_builder: "ModelInputForGPUBuilder"):
256 +
257 + self.input_builder = input_builder
258 + self.runner = input_builder.runner
259 +
260 + def prepare(self):
261 self.prefill_seq_lens: List[int] = []
262 self.context_lens: List[int] = []
263 self.curr_seq_lens: List[int] = []
@@ -263,9 +268,6 @@ class PlaceholderAttentionMetadataBuilder(
268 self.num_prefill_tokens = 0
269 self.num_decode_tokens = 0
270
266 - self.input_builder = input_builder
267 - self.runner = input_builder.runner
268 -
271 def _add_seq_group(
272 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
273 chunked_prefill_enabled: bool):
vllm/attention/backends/torch_sdpa.py
+4 -1
@@ -282,7 +282,10 @@ class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
282
283 def __init__(self, input_builder: ModelInputForCPUBuilder) -> None:
284 self.chunked_prefill = input_builder.chunked_prefill
285 - self.input_data = input_builder.input_data
285 + self.input_builder = input_builder
286 +
287 + def prepare(self):
288 + self.input_data = self.input_builder.input_data
289
290 def build(self, seq_lens: List[int], query_lens: List[int],
291 cuda_graph_pad_size: int, batch_size: int) -> TorchSDPAMetadata:
vllm/attention/backends/utils.py
+7 -6
@@ -122,6 +122,13 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
122 _metadata_cls: Type[TAttentionMetadata]
123
124 def __init__(self, input_builder: "ModelInputForGPUBuilder"):
125 + self.input_builder = input_builder
126 + self.runner = input_builder.runner
127 +
128 + self.sliding_window = input_builder.sliding_window
129 + self.block_size = input_builder.block_size
130 +
131 + def prepare(self):
132 self.slot_mapping: List[int] = []
133 self.prefill_seq_lens: List[int] = []
134 self.context_lens: List[int] = []
@@ -134,12 +141,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
141 self.num_prefill_tokens = 0
142 self.num_decode_tokens = 0
143
137 - self.input_builder = input_builder
138 - self.runner = input_builder.runner
139 -
140 - self.sliding_window = input_builder.sliding_window
141 - self.block_size = input_builder.block_size
142 -
144 def _add_seq_group(
145 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
146 chunked_prefill_enabled: bool):
vllm/worker/cpu_model_runner.py
+17 -7
@@ -144,9 +144,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
144 runner: "CPUModelRunner",
145 finished_requests_ids: Optional[List[str]] = None) -> None:
146 super().__init__()
147 - self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
147 self.runner = runner
149 -
148 self.chunked_prefill = (runner.scheduler_config.chunked_prefill_enabled
149 or runner.cache_config.enable_prefix_caching)
150 self.model_input_cls = self.runner._model_input_cls
@@ -156,10 +154,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
154 self.device = self.runner.device
155 self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
156 self.enable_lora = self.runner.lora_config is not None
157 + if self.runner.attn_backend is not None:
158 + # spec decode (e.g. Medusa) does not have atten backend
159 + attn_backend = self.runner.attn_backend
160 + self.att_metadata_builder = attn_backend.get_builder_cls()(self)
161 +
162 + def prepare(self,
163 + finished_requests_ids: Optional[List[str]] = None) -> None:
164 + self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
165 self.input_data = ModelInputForCPUBuilder.ModelInputData(
166 self.runner.model_config.uses_mrope)
161 - self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
162 - self)
167 + self.att_metadata_builder.prepare()
168
169 def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
170 self.seq_group_metadata_list.append(seq_group_metadata)
@@ -431,6 +436,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
436 """
437 _model_input_cls: Type[TModelInputForCPU]
438 _builder_cls: Type[ModelInputForCPUBuilder]
439 + builder: ModelInputForCPUBuilder
440
441 def __init__(
442 self,
@@ -477,6 +483,10 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
483 # Set after load_model.
484 self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
485
486 + if hasattr(self, "_builder_cls"):
487 + # multi-step model runner does not have `_builder_cls`
488 + self.builder = self._builder_cls(weakref.proxy(self))
489 +
490 def load_model(self) -> None:
491 self.model = get_model(vllm_config=self.vllm_config)
492
@@ -522,10 +532,10 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
532 metadata for possible additional steps, e.g., sampling.
533
534 """
525 - builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
526 - builder.set_seq_group_list(seq_group_metadata_list)
535 + self.builder.prepare(finished_requests_ids)
536 + self.builder.set_seq_group_list(seq_group_metadata_list)
537
528 - return builder.build() # type: ignore
538 + return self.builder.build() # type: ignore
539
540 # sampler property will be used by spec_decode_worker
541 @property
vllm/worker/model_runner.py
+24 -12
@@ -457,17 +457,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
457 self.enable_prompt_adapter = (self.runner.prompt_adapter_config
458 is not None)
459 self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
460 - self.finished_requests_ids = finished_requests_ids
460 self.decode_only = True
461
463 - # Intermediate data (data in CPU before going to GPU) for
464 - # the current sequence group.
465 - self.inter_data_list: List[
466 - ModelInputForGPUBuilder.InterDataForSeqGroup] = []
467 -
462 # Attention metadata inputs.
469 - self.attn_metadata_builder = self.attn_backend.make_metadata_builder(
470 - weakref.proxy(self))
463 + if self.attn_backend is not None:
464 + # spec decode (e.g. Medusa) does not have atten backend
465 + self.attn_metadata_builder = self.attn_backend.get_builder_cls()(
466 + weakref.proxy(self))
467
468 # Engine/Model configurations.
469 self.chunked_prefill_enabled = (
@@ -479,6 +475,17 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
475 self.block_aligned_sliding_window = \
476 self.sliding_window_blocks * self.block_size
477
478 + def prepare(self,
479 + finished_requests_ids: Optional[List[str]] = None) -> None:
480 + self.finished_requests_ids = finished_requests_ids
481 +
482 + # Intermediate data (data in CPU before going to GPU) for
483 + # the current sequence group.
484 + self.inter_data_list: List[
485 + ModelInputForGPUBuilder.InterDataForSeqGroup] = []
486 +
487 + self.attn_metadata_builder.prepare()
488 +
489 def _compute_lens(self, inter_data: InterDataForSeqGroup, seq_idx: int,
490 seq_group_metadata: SequenceGroupMetadata):
491 """Compute context length, sequence length and tokens
@@ -993,6 +1000,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1000 """
1001 _model_input_cls: Type[TModelInputForGPU]
1002 _builder_cls: Type[ModelInputForGPUBuilder]
1003 + builder: ModelInputForGPUBuilder
1004
1005 def __init__(
1006 self,
@@ -1093,6 +1101,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1101 SamplingMetadataCache() \
1102 if self.parallel_config.pipeline_parallel_size == 1 else None
1103
1104 + if hasattr(self, "_builder_cls"):
1105 + # multi-step model runner does not have `_builder_cls`
1106 + self.builder = self._builder_cls(weakref.proxy(self))
1107 +
1108 def load_model(self) -> None:
1109 logger.info("Starting to load model %s...", self.model_config.model)
1110 with DeviceMemoryProfiler() as m:
@@ -1226,13 +1238,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1238
1239 If cuda graph is required, this API automatically pads inputs.
1240 """
1229 - builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
1241 + self.builder.prepare(finished_requests_ids)
1242 for seq_group_metadata in seq_group_metadata_list:
1231 - builder.add_seq_group(seq_group_metadata)
1243 + self.builder.add_seq_group(seq_group_metadata)
1244
1233 - builder.reset_cached_inter_data()
1245 + self.builder.reset_cached_inter_data()
1246
1235 - return builder.build() # type: ignore
1247 + return self.builder.build() # type: ignore
1248
1249 @contextmanager
1250 def set_in_profile_run(self):
vllm/worker/model_runner_base.py
+5
@@ -200,6 +200,11 @@ class ModelRunnerInputBuilderBase(ABC, Generic[T]):
200 """A builder to create ModelRunnerInputBase objects.
201 """
202
203 + @abstractmethod
204 + def prepare(self,
205 + finished_requests_ids: Optional[List[str]] = None) -> None:
206 + raise NotImplementedError
207 +
208 @abstractmethod
209 def add_seq_group(self, seq_group_metadata):
210 """TBA"""
vllm/worker/xpu_model_runner.py
+8 -2
@@ -113,7 +113,6 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
113 runner: "XPUModelRunner",
114 finished_requests_ids: Optional[List[str]] = None) -> None:
115 super().__init__()
116 - self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
116 self.runner = runner
117 self.model_input_cls = self.runner._model_input_cls
118 self.attn_backend = self.runner.attn_backend
@@ -121,6 +120,10 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
120 self.block_size = self.runner.block_size
121 self.device = self.runner.device
122
123 + def prepare(self,
124 + finished_requests_ids: Optional[List[str]] = None) -> None:
125 + self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
126 +
127 def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
128 self.seq_group_metadata_list.append(seq_group_metadata)
129
@@ -408,6 +411,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
411 SamplingMetadataCache() \
412 if self.parallel_config.pipeline_parallel_size == 1 else None
413
414 + self.builder = self._builder_cls(weakref.proxy(self))
415 +
416 def load_model(self) -> None:
417 with DeviceMemoryProfiler() as m:
418 self.model = get_model(vllm_config=self.vllm_config)
@@ -517,7 +522,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
522 metadata for possible additional steps, e.g., sampling.
523
524 """
520 - builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
525 + builder = self.builder
526 + builder.prepare(finished_requests_ids)
527 for seq_group_metadata in seq_group_metadata_list:
528 builder.add_seq_group(seq_group_metadata)
529