A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Hardware][CPU] Refactor CPU model runner (#8729)
Isotr0py committed
Sep 23, 2024 at 20:12 UTC
e551ca1555b64ba1ecb2310ea658f3e25c62571d
1 file changed
+193
-109
vllm/worker/cpu_model_runner.py
+193
-109
@@ -1,3 +1,5 @@
1
+import dataclasses
2
+import weakref
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
5
@@ -17,7 +19,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
20
from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
21
from vllm.worker.model_runner_base import (
20
- ModelRunnerBase, ModelRunnerInputBase,
22
+ ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
23
_add_attn_metadata_broadcastable_dict,
24
_add_sampling_metadata_broadcastable_dict,
25
_init_attn_metadata_from_tensor_dict,
@@ -32,16 +34,17 @@ _PAD_SLOT_ID = -1
34
35
36
@dataclass(frozen=True)
35
-class CPUModelInput(ModelRunnerInputBase):
37
+class ModelInputForCPU(ModelRunnerInputBase):
38
"""
37
- Used by the CPUModelRunner.
39
+ Base class contains metadata needed for the base model forward pass on CPU
40
"""
41
input_tokens: Optional[torch.Tensor] = None
42
input_positions: Optional[torch.Tensor] = None
43
attn_metadata: Optional["AttentionMetadata"] = None
42
- sampling_metadata: Optional["SamplingMetadata"] = None
44
multi_modal_kwargs: Optional[BatchedTensorInputs] = None
45
virtual_engine: Optional[int] = None
46
+ seq_lens: Optional[List[int]] = None
47
+ query_lens: Optional[List[int]] = None
48
49
def as_broadcastable_tensor_dict(
50
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -51,88 +54,96 @@ class CPUModelInput(ModelRunnerInputBase):
54
"multi_modal_kwargs": self.multi_modal_kwargs,
55
}
56
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
54
- _add_sampling_metadata_broadcastable_dict(tensor_dict,
55
- self.sampling_metadata)
57
+
58
return tensor_dict
59
60
@classmethod
61
def from_broadcasted_tensor_dict(
60
- cls: Type["CPUModelInput"],
61
- tensor_dict: Dict[str, Any],
62
- attn_backend: Optional["AttentionBackend"] = None
63
- ) -> "CPUModelInput":
64
- tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
62
+ cls: Type["ModelInputForCPU"],
63
+ tensor_dict: Dict[str, Any],
64
+ attn_backend: Optional["AttentionBackend"] = None
65
+ ) -> "ModelInputForCPU":
66
if attn_backend is not None:
67
tensor_dict = _init_attn_metadata_from_tensor_dict(
68
attn_backend, tensor_dict)
69
return cls(**tensor_dict)
70
71
71
-class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
72
+@dataclass(frozen=True)
73
+class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
74
+ """
75
+ Used by the ModelRunner.
76
+ """
77
+ sampling_metadata: Optional["SamplingMetadata"] = None
78
73
- def __init__(
74
- self,
75
- model_config: ModelConfig,
76
- parallel_config: ParallelConfig,
77
- scheduler_config: SchedulerConfig,
78
- device_config: DeviceConfig,
79
- cache_config: CacheConfig,
80
- load_config: LoadConfig,
81
- lora_config: Optional[LoRAConfig],
82
- kv_cache_dtype: Optional[str] = "auto",
83
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
84
- is_driver_worker: bool = False,
85
- *args,
86
- **kwargs,
87
- ):
88
- self.model_config = model_config
89
- self.parallel_config = parallel_config
90
- self.scheduler_config = scheduler_config
91
- # Currently, CPU worker doesn't support chunked prefill.
92
- assert self.scheduler_config.chunked_prefill_enabled is False
93
- self.device_config = device_config
94
- self.cache_config = cache_config
95
- self.lora_config = lora_config
96
- self.prompt_adapter_config = prompt_adapter_config
97
- self.load_config = load_config
98
- self.is_driver_worker = is_driver_worker
79
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
80
+ tensor_dict = {
81
+ "input_tokens": self.input_tokens,
82
+ "input_positions": self.input_positions,
83
+ }
84
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
85
+ _add_sampling_metadata_broadcastable_dict(tensor_dict,
86
+ self.sampling_metadata)
87
+ return tensor_dict
88
100
- self.device = self.device_config.device
89
+ @classmethod
90
+ def from_broadcasted_tensor_dict(
91
+ cls,
92
+ tensor_dict: Dict[str, Any],
93
+ attn_backend: Optional["AttentionBackend"] = None,
94
+ ) -> "ModelInputForCPUWithSamplingMetadata":
95
+ tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
96
+ if attn_backend is not None:
97
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
98
+ attn_backend, tensor_dict)
99
+ return cls(**tensor_dict)
100
102
- self.kv_cache_dtype = kv_cache_dtype
103
- self.sliding_window = model_config.get_sliding_window()
104
- self.block_size = cache_config.block_size
105
- self.attn_backend = get_attn_backend(
106
- self.model_config.get_num_attention_heads(self.parallel_config),
107
- self.model_config.get_head_size(),
108
- self.model_config.get_num_kv_heads(self.parallel_config),
109
- self.model_config.get_sliding_window(),
110
- self.model_config.dtype,
111
- self.kv_cache_dtype,
112
- self.block_size,
113
- )
101
115
- # Multi-modal data support
116
- self.mm_registry = MULTIMODAL_REGISTRY
117
- self.multi_modal_input_mapper = self.mm_registry \
118
- .create_input_mapper(self.model_config)
119
- self.mm_registry.init_mm_limits_per_prompt(self.model_config)
102
+class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
103
121
- # Lazy initialization.
122
- self.model: nn.Module # Set after init_Model
104
+ def __init__(self,
105
+ runner: "CPUModelRunner",
106
+ finished_requests_ids: Optional[List[str]] = None) -> None:
107
+ super().__init__()
108
+ self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
109
+ self.runner = runner
110
+ self.model_input_cls = self.runner._model_input_cls
111
+ self.attn_backend = self.runner.attn_backend
112
+ self.sliding_window = self.runner.sliding_window
113
+ self.block_size = self.runner.block_size
114
+ self.device = self.runner.device
115
+ self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
116
124
- if self.model_config.is_encoder_decoder_model:
125
- raise NotImplementedError(
126
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
117
+ def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
118
+ self.seq_group_metadata_list.append(seq_group_metadata)
119
128
- def load_model(self) -> None:
129
- self.model = get_model(model_config=self.model_config,
130
- load_config=self.load_config,
131
- device_config=self.device_config,
132
- lora_config=self.lora_config,
133
- parallel_config=self.parallel_config,
134
- scheduler_config=self.scheduler_config,
135
- cache_config=self.cache_config)
120
+ def build(self) -> ModelInputForCPU:
121
+ multi_modal_kwargs = None
122
+ # NOTE: We assume that all sequences in the group are all prompts or
123
+ # all decodes.
124
+ is_prompt = self.seq_group_metadata_list[0].is_prompt
125
+ # Prepare input tensors.
126
+ if is_prompt:
127
+ (input_tokens, input_positions, attn_metadata, seq_lens,
128
+ multi_modal_kwargs) = self._prepare_prompt(
129
+ self.seq_group_metadata_list)
130
+ else:
131
+ (input_tokens, input_positions,
132
+ attn_metadata) = self._prepare_decode(
133
+ self.seq_group_metadata_list)
134
+ seq_lens = []
135
+
136
+ return self.model_input_cls(
137
+ input_tokens=input_tokens,
138
+ input_positions=input_positions,
139
+ attn_metadata=attn_metadata,
140
+ multi_modal_kwargs=multi_modal_kwargs,
141
+ # query_lens is not needed if chunked prefill is not
142
+ # supported. Since CPU worker doesn't support chunked prefill
143
+ # just use seq_lens instead.
144
+ seq_lens=seq_lens,
145
+ query_lens=seq_lens,
146
+ )
147
148
def _prepare_prompt(
149
self,
@@ -165,8 +176,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
176
# is always the first token in the sequence.
177
input_positions.extend(list(range(computed_len, seq_len)))
178
168
- mm_data = seq_group_metadata.multi_modal_data
169
- if mm_data:
179
+ if (mm_data := seq_group_metadata.multi_modal_data):
180
mm_kwargs = self.multi_modal_input_mapper(mm_data)
181
multi_modal_inputs_list.append(mm_kwargs)
182
@@ -302,56 +312,130 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
312
attn_metadata,
313
)
314
315
+
316
+class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
317
+ _model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
318
+ ModelInputForCPUWithSamplingMetadata)
319
+ _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
320
+
321
+ def __init__(
322
+ self,
323
+ model_config: ModelConfig,
324
+ parallel_config: ParallelConfig,
325
+ scheduler_config: SchedulerConfig,
326
+ device_config: DeviceConfig,
327
+ cache_config: CacheConfig,
328
+ load_config: LoadConfig,
329
+ lora_config: Optional[LoRAConfig],
330
+ kv_cache_dtype: Optional[str] = "auto",
331
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
332
+ is_driver_worker: bool = False,
333
+ *args,
334
+ **kwargs,
335
+ ):
336
+ self.model_config = model_config
337
+ self.parallel_config = parallel_config
338
+ self.scheduler_config = scheduler_config
339
+ # Currently, CPU worker doesn't support chunked prefill.
340
+ assert self.scheduler_config.chunked_prefill_enabled is False
341
+ self.device_config = device_config
342
+ self.cache_config = cache_config
343
+ self.lora_config = lora_config
344
+ self.prompt_adapter_config = prompt_adapter_config
345
+ self.load_config = load_config
346
+ self.is_driver_worker = is_driver_worker
347
+
348
+ self.device = self.device_config.device
349
+
350
+ self.kv_cache_dtype = kv_cache_dtype
351
+ self.sliding_window = model_config.get_sliding_window()
352
+ self.block_size = cache_config.block_size
353
+ self.attn_backend = get_attn_backend(
354
+ self.model_config.get_num_attention_heads(self.parallel_config),
355
+ self.model_config.get_head_size(),
356
+ self.model_config.get_num_kv_heads(self.parallel_config),
357
+ self.model_config.get_sliding_window(),
358
+ self.model_config.dtype,
359
+ self.kv_cache_dtype,
360
+ self.block_size,
361
+ )
362
+
363
+ # Multi-modal data support
364
+ self.mm_registry = MULTIMODAL_REGISTRY
365
+ self.multi_modal_input_mapper = self.mm_registry \
366
+ .create_input_mapper(self.model_config)
367
+ self.mm_registry.init_mm_limits_per_prompt(self.model_config)
368
+
369
+ # Lazy initialization.
370
+ self.model: nn.Module # Set after init_Model
371
+
372
+ if self.model_config.is_encoder_decoder_model:
373
+ raise NotImplementedError(
374
+ STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
375
+
376
+ def load_model(self) -> None:
377
+ self.model = get_model(model_config=self.model_config,
378
+ load_config=self.load_config,
379
+ device_config=self.device_config,
380
+ lora_config=self.lora_config,
381
+ parallel_config=self.parallel_config,
382
+ scheduler_config=self.scheduler_config,
383
+ cache_config=self.cache_config)
384
+
385
def make_model_input_from_broadcasted_tensor_dict(
386
self,
387
tensor_dict: Dict[str, Any],
308
- ) -> CPUModelInput:
309
- return CPUModelInput.from_broadcasted_tensor_dict(
388
+ ) -> ModelInputForCPU:
389
+ return ModelInputForCPU.from_broadcasted_tensor_dict(
390
tensor_dict,
391
attn_backend=self.attn_backend,
392
)
393
394
+ def _prepare_model_input_tensors(
395
+ self,
396
+ seq_group_metadata_list: List[SequenceGroupMetadata],
397
+ finished_requests_ids: Optional[List[str]] = None
398
+ ) -> ModelInputForCPUWithSamplingMetadata:
399
+ """Helper method to prepare the model input based on a given sequence
400
+ group. Prepares metadata needed for the base model forward pass but not
401
+ metadata for possible additional steps, e.g., sampling.
402
+
403
+ """
404
+ builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
405
+ for seq_group_metadata in seq_group_metadata_list:
406
+ builder.add_seq_group(seq_group_metadata)
407
+
408
+ return builder.build() # type: ignore
409
+
410
def prepare_model_input(
315
- self,
316
- seq_group_metadata_list: List[SequenceGroupMetadata],
317
- virtual_engine: int = 0,
318
- finished_requests_ids: Optional[List[str]] = None
319
- ) -> CPUModelInput:
320
- multi_modal_kwargs = None
321
- # NOTE: We assume that all sequences in the group are all prompts or
322
- # all decodes.
323
- is_prompt = seq_group_metadata_list[0].is_prompt
324
- # Prepare input tensors.
325
- if is_prompt:
326
- (input_tokens, input_positions, attn_metadata, seq_lens,
327
- multi_modal_kwargs
328
- ) = self._prepare_prompt(seq_group_metadata_list)
329
- else:
330
- (input_tokens, input_positions,
331
- attn_metadata) = self._prepare_decode(seq_group_metadata_list)
332
- seq_lens = []
333
- sampling_metadata = SamplingMetadata.prepare(
334
- seq_group_metadata_list,
335
- seq_lens,
336
- # query_lens is not needed if chunked prefill is not
337
- # supported. Since CPU worker doesn't support chunked prefill
338
- # just use seq_lens instead.
339
- seq_lens,
340
- self.device,
341
- pin_memory=False,
342
- generators=self.get_generators(finished_requests_ids))
343
- return CPUModelInput(
344
- input_tokens=input_tokens,
345
- input_positions=input_positions,
346
- attn_metadata=attn_metadata,
347
- sampling_metadata=sampling_metadata,
348
- multi_modal_kwargs=multi_modal_kwargs,
349
- )
411
+ self,
412
+ seq_group_metadata_list: List[SequenceGroupMetadata],
413
+ virtual_engine: int = 0,
414
+ finished_requests_ids: Optional[List[str]] = None
415
+ ) -> ModelInputForCPUWithSamplingMetadata:
416
+ """Prepare the model input based on a given sequence group, including
417
+ metadata for the sampling step.
418
+
419
+ """
420
+ model_input = self._prepare_model_input_tensors(
421
+ seq_group_metadata_list, finished_requests_ids)
422
+ # Sampling metadata is only required for the final pp group
423
+ generators = self.get_generators(finished_requests_ids)
424
+ sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
425
+ model_input.seq_lens,
426
+ model_input.query_lens,
427
+ self.device,
428
+ pin_memory=False,
429
+ generators=generators)
430
+
431
+ return dataclasses.replace(model_input,
432
+ sampling_metadata=sampling_metadata,
433
+ virtual_engine=virtual_engine)
434
435
@torch.no_grad()
436
def execute_model(
437
self,
354
- model_input: CPUModelInput,
438
+ model_input: ModelInputForCPUWithSamplingMetadata,
439
kv_caches: List[torch.Tensor],
440
intermediate_tensors: Optional[IntermediateTensors] = None,
441
num_steps: int = 1,