A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core] separate builder init and builder prepare for each batch (#12253)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 22, 2025 at 14:13 UTC
66818e5b63818a286756653816772faa8622ad89
10 files changed
+90
-47
vllm/attention/backends/abstract.py
+6
-5
@@ -65,11 +65,6 @@ class AttentionBackend(ABC):
65
def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
66
raise NotImplementedError
67
68
- @classmethod
69
- def make_metadata_builder(cls, *args,
70
- **kwargs) -> "AttentionMetadataBuilder":
71
- return cls.get_builder_cls()(*args, **kwargs)
72
-
68
@staticmethod
69
@abstractmethod
70
def get_kv_cache_shape(
@@ -214,6 +209,12 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
209
210
@abstractmethod
211
def __init__(self, input_builder: "ModelRunnerInputBuilderBase") -> None:
212
+ """Create the builder, remember some configuration and parameters."""
213
+ raise NotImplementedError
214
+
215
+ @abstractmethod
216
+ def prepare(self) -> None:
217
+ """Prepare for one batch."""
218
raise NotImplementedError
219
220
@abstractmethod
vllm/attention/backends/flash_attn.py
+6
-5
@@ -375,6 +375,12 @@ class FlashAttentionMetadataBuilder(
375
AttentionMetadataBuilder[FlashAttentionMetadata]):
376
377
def __init__(self, input_builder: "ModelInputForGPUBuilder"):
378
+ self.input_builder = input_builder
379
+ self.runner = input_builder.runner
380
+ self.sliding_window = input_builder.sliding_window
381
+ self.block_size = input_builder.block_size
382
+
383
+ def prepare(self):
384
self.slot_mapping: List[int] = []
385
self.prefill_seq_lens: List[int] = []
386
self.context_lens: List[int] = []
@@ -388,11 +394,6 @@ class FlashAttentionMetadataBuilder(
394
self.num_decode_tokens = 0
395
self.has_prefix_cache_hit = False
396
391
- self.input_builder = input_builder
392
- self.runner = input_builder.runner
393
- self.sliding_window = input_builder.sliding_window
394
- self.block_size = input_builder.block_size
395
-
397
def _add_seq_group(
398
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
399
chunked_prefill_enabled: bool, prefix_cache_hit: bool):
vllm/attention/backends/flashinfer.py
+8
-6
@@ -488,6 +488,14 @@ class FlashInferMetadata(AttentionMetadata):
488
class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
489
490
def __init__(self, input_builder: "ModelInputForGPUBuilder"):
491
+
492
+ self.input_builder = input_builder
493
+ self.runner = input_builder.runner
494
+
495
+ self.sliding_window = input_builder.sliding_window
496
+ self.block_size = input_builder.block_size
497
+
498
+ def prepare(self):
499
self.slot_mapping: List[int] = []
500
self.prefill_seq_lens: List[int] = []
501
self.context_lens: List[int] = []
@@ -500,12 +508,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
508
self.num_prefill_tokens = 0
509
self.num_decode_tokens = 0
510
503
- self.input_builder = input_builder
504
- self.runner = input_builder.runner
505
-
506
- self.sliding_window = input_builder.sliding_window
507
- self.block_size = input_builder.block_size
508
-
511
# Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
512
# for the precise definition of the following fields.
513
# An example:
vllm/attention/backends/placeholder_attn.py
+5
-3
@@ -253,6 +253,11 @@ class PlaceholderAttentionMetadataBuilder(
253
AttentionMetadataBuilder[PlaceholderAttentionMetadata]):
254
255
def __init__(self, input_builder: "ModelInputForGPUBuilder"):
256
+
257
+ self.input_builder = input_builder
258
+ self.runner = input_builder.runner
259
+
260
+ def prepare(self):
261
self.prefill_seq_lens: List[int] = []
262
self.context_lens: List[int] = []
263
self.curr_seq_lens: List[int] = []
@@ -263,9 +268,6 @@ class PlaceholderAttentionMetadataBuilder(
268
self.num_prefill_tokens = 0
269
self.num_decode_tokens = 0
270
266
- self.input_builder = input_builder
267
- self.runner = input_builder.runner
268
-
271
def _add_seq_group(
272
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
273
chunked_prefill_enabled: bool):
vllm/attention/backends/torch_sdpa.py
+4
-1
@@ -282,7 +282,10 @@ class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
282
283
def __init__(self, input_builder: ModelInputForCPUBuilder) -> None:
284
self.chunked_prefill = input_builder.chunked_prefill
285
- self.input_data = input_builder.input_data
285
+ self.input_builder = input_builder
286
+
287
+ def prepare(self):
288
+ self.input_data = self.input_builder.input_data
289
290
def build(self, seq_lens: List[int], query_lens: List[int],
291
cuda_graph_pad_size: int, batch_size: int) -> TorchSDPAMetadata:
vllm/attention/backends/utils.py
+7
-6
@@ -122,6 +122,13 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
122
_metadata_cls: Type[TAttentionMetadata]
123
124
def __init__(self, input_builder: "ModelInputForGPUBuilder"):
125
+ self.input_builder = input_builder
126
+ self.runner = input_builder.runner
127
+
128
+ self.sliding_window = input_builder.sliding_window
129
+ self.block_size = input_builder.block_size
130
+
131
+ def prepare(self):
132
self.slot_mapping: List[int] = []
133
self.prefill_seq_lens: List[int] = []
134
self.context_lens: List[int] = []
@@ -134,12 +141,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
141
self.num_prefill_tokens = 0
142
self.num_decode_tokens = 0
143
137
- self.input_builder = input_builder
138
- self.runner = input_builder.runner
139
-
140
- self.sliding_window = input_builder.sliding_window
141
- self.block_size = input_builder.block_size
142
-
144
def _add_seq_group(
145
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
146
chunked_prefill_enabled: bool):
vllm/worker/cpu_model_runner.py
+17
-7
@@ -144,9 +144,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
144
runner: "CPUModelRunner",
145
finished_requests_ids: Optional[List[str]] = None) -> None:
146
super().__init__()
147
- self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
147
self.runner = runner
149
-
148
self.chunked_prefill = (runner.scheduler_config.chunked_prefill_enabled
149
or runner.cache_config.enable_prefix_caching)
150
self.model_input_cls = self.runner._model_input_cls
@@ -156,10 +154,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
154
self.device = self.runner.device
155
self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
156
self.enable_lora = self.runner.lora_config is not None
157
+ if self.runner.attn_backend is not None:
158
+ # spec decode (e.g. Medusa) does not have atten backend
159
+ attn_backend = self.runner.attn_backend
160
+ self.att_metadata_builder = attn_backend.get_builder_cls()(self)
161
+
162
+ def prepare(self,
163
+ finished_requests_ids: Optional[List[str]] = None) -> None:
164
+ self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
165
self.input_data = ModelInputForCPUBuilder.ModelInputData(
166
self.runner.model_config.uses_mrope)
161
- self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
162
- self)
167
+ self.att_metadata_builder.prepare()
168
169
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
170
self.seq_group_metadata_list.append(seq_group_metadata)
@@ -431,6 +436,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
436
"""
437
_model_input_cls: Type[TModelInputForCPU]
438
_builder_cls: Type[ModelInputForCPUBuilder]
439
+ builder: ModelInputForCPUBuilder
440
441
def __init__(
442
self,
@@ -477,6 +483,10 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
483
# Set after load_model.
484
self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
485
486
+ if hasattr(self, "_builder_cls"):
487
+ # multi-step model runner does not have `_builder_cls`
488
+ self.builder = self._builder_cls(weakref.proxy(self))
489
+
490
def load_model(self) -> None:
491
self.model = get_model(vllm_config=self.vllm_config)
492
@@ -522,10 +532,10 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
532
metadata for possible additional steps, e.g., sampling.
533
534
"""
525
- builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
526
- builder.set_seq_group_list(seq_group_metadata_list)
535
+ self.builder.prepare(finished_requests_ids)
536
+ self.builder.set_seq_group_list(seq_group_metadata_list)
537
528
- return builder.build() # type: ignore
538
+ return self.builder.build() # type: ignore
539
540
# sampler property will be used by spec_decode_worker
541
@property
vllm/worker/model_runner.py
+24
-12
@@ -457,17 +457,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
457
self.enable_prompt_adapter = (self.runner.prompt_adapter_config
458
is not None)
459
self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
460
- self.finished_requests_ids = finished_requests_ids
460
self.decode_only = True
461
463
- # Intermediate data (data in CPU before going to GPU) for
464
- # the current sequence group.
465
- self.inter_data_list: List[
466
- ModelInputForGPUBuilder.InterDataForSeqGroup] = []
467
-
462
# Attention metadata inputs.
469
- self.attn_metadata_builder = self.attn_backend.make_metadata_builder(
470
- weakref.proxy(self))
463
+ if self.attn_backend is not None:
464
+ # spec decode (e.g. Medusa) does not have atten backend
465
+ self.attn_metadata_builder = self.attn_backend.get_builder_cls()(
466
+ weakref.proxy(self))
467
468
# Engine/Model configurations.
469
self.chunked_prefill_enabled = (
@@ -479,6 +475,17 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
475
self.block_aligned_sliding_window = \
476
self.sliding_window_blocks * self.block_size
477
478
+ def prepare(self,
479
+ finished_requests_ids: Optional[List[str]] = None) -> None:
480
+ self.finished_requests_ids = finished_requests_ids
481
+
482
+ # Intermediate data (data in CPU before going to GPU) for
483
+ # the current sequence group.
484
+ self.inter_data_list: List[
485
+ ModelInputForGPUBuilder.InterDataForSeqGroup] = []
486
+
487
+ self.attn_metadata_builder.prepare()
488
+
489
def _compute_lens(self, inter_data: InterDataForSeqGroup, seq_idx: int,
490
seq_group_metadata: SequenceGroupMetadata):
491
"""Compute context length, sequence length and tokens
@@ -993,6 +1000,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1000
"""
1001
_model_input_cls: Type[TModelInputForGPU]
1002
_builder_cls: Type[ModelInputForGPUBuilder]
1003
+ builder: ModelInputForGPUBuilder
1004
1005
def __init__(
1006
self,
@@ -1093,6 +1101,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1101
SamplingMetadataCache() \
1102
if self.parallel_config.pipeline_parallel_size == 1 else None
1103
1104
+ if hasattr(self, "_builder_cls"):
1105
+ # multi-step model runner does not have `_builder_cls`
1106
+ self.builder = self._builder_cls(weakref.proxy(self))
1107
+
1108
def load_model(self) -> None:
1109
logger.info("Starting to load model %s...", self.model_config.model)
1110
with DeviceMemoryProfiler() as m:
@@ -1226,13 +1238,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1238
1239
If cuda graph is required, this API automatically pads inputs.
1240
"""
1229
- builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
1241
+ self.builder.prepare(finished_requests_ids)
1242
for seq_group_metadata in seq_group_metadata_list:
1231
- builder.add_seq_group(seq_group_metadata)
1243
+ self.builder.add_seq_group(seq_group_metadata)
1244
1233
- builder.reset_cached_inter_data()
1245
+ self.builder.reset_cached_inter_data()
1246
1235
- return builder.build() # type: ignore
1247
+ return self.builder.build() # type: ignore
1248
1249
@contextmanager
1250
def set_in_profile_run(self):
vllm/worker/model_runner_base.py
+5
@@ -200,6 +200,11 @@ class ModelRunnerInputBuilderBase(ABC, Generic[T]):
200
"""A builder to create ModelRunnerInputBase objects.
201
"""
202
203
+ @abstractmethod
204
+ def prepare(self,
205
+ finished_requests_ids: Optional[List[str]] = None) -> None:
206
+ raise NotImplementedError
207
+
208
@abstractmethod
209
def add_seq_group(self, seq_group_metadata):
210
"""TBA"""
vllm/worker/xpu_model_runner.py
+8
-2
@@ -113,7 +113,6 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
113
runner: "XPUModelRunner",
114
finished_requests_ids: Optional[List[str]] = None) -> None:
115
super().__init__()
116
- self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
116
self.runner = runner
117
self.model_input_cls = self.runner._model_input_cls
118
self.attn_backend = self.runner.attn_backend
@@ -121,6 +120,10 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
120
self.block_size = self.runner.block_size
121
self.device = self.runner.device
122
123
+ def prepare(self,
124
+ finished_requests_ids: Optional[List[str]] = None) -> None:
125
+ self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
126
+
127
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
128
self.seq_group_metadata_list.append(seq_group_metadata)
129
@@ -408,6 +411,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
411
SamplingMetadataCache() \
412
if self.parallel_config.pipeline_parallel_size == 1 else None
413
414
+ self.builder = self._builder_cls(weakref.proxy(self))
415
+
416
def load_model(self) -> None:
417
with DeviceMemoryProfiler() as m:
418
self.model = get_model(vllm_config=self.vllm_config)
@@ -517,7 +522,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
522
metadata for possible additional steps, e.g., sampling.
523
524
"""
520
- builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
525
+ builder = self.builder
526
+ builder.prepare(finished_requests_ids)
527
for seq_group_metadata in seq_group_metadata_list:
528
builder.add_seq_group(seq_group_metadata)
529