A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core] Refactor Worker and ModelRunner to consolidate control plane communication (#5408)
Signed-off-by: Stephanie Wang <swang@cs.berkeley.edu> Signed-off-by: Stephanie <swang@anyscale.com> Co-authored-by: Stephanie <swang@anyscale.com>
Stephanie Wang committed
Jun 25, 2024 at 20:30 UTC
dda4811591fdb90d263bc9b8ac522436369aef13
29 files changed
+1108
-575
tests/worker/test_model_input.py
new
+152
@@ -0,0 +1,152 @@
1
+import dataclasses
2
+from typing import List, Tuple, Type
3
+
4
+import torch
5
+
6
+from vllm.attention import AttentionMetadata
7
+from vllm.attention.backends.abstract import AttentionBackend
8
+from vllm.model_executor import SamplingMetadata
9
+from vllm.model_executor.pooling_metadata import PoolingMetadata
10
+from vllm.worker.embedding_model_runner import (
11
+ ModelInputForGPUWithPoolingMetadata)
12
+from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
13
+
14
+
15
+class MockAttentionBackend(AttentionBackend):
16
+
17
+ @staticmethod
18
+ def get_name() -> str:
19
+ raise NotImplementedError
20
+
21
+ @staticmethod
22
+ def get_impl_cls():
23
+ raise NotImplementedError
24
+
25
+ @staticmethod
26
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
27
+ return AttentionMetadata
28
+
29
+ @staticmethod
30
+ def get_kv_cache_shape(
31
+ num_blocks: int,
32
+ block_size: int,
33
+ num_kv_heads: int,
34
+ head_size: int,
35
+ ) -> Tuple[int, ...]:
36
+ raise NotImplementedError
37
+
38
+ @staticmethod
39
+ def swap_blocks(
40
+ src_kv_cache: torch.Tensor,
41
+ dst_kv_cache: torch.Tensor,
42
+ src_to_dst: torch.Tensor,
43
+ ) -> None:
44
+ pass
45
+
46
+ @staticmethod
47
+ def copy_blocks(
48
+ kv_caches: List[torch.Tensor],
49
+ src_to_dists: torch.Tensor,
50
+ ) -> None:
51
+ pass
52
+
53
+
54
+def test_model_runner_input():
55
+ sampling_metadata = SamplingMetadata(
56
+ ["seq_group"],
57
+ "selected_token_indices",
58
+ "categorized_sample_indices",
59
+ "num_prompts",
60
+ )
61
+ attn_metadata = AttentionMetadata(
62
+ num_prefills=1,
63
+ num_prefill_tokens=2,
64
+ num_decode_tokens=3,
65
+ slot_mapping=torch.zeros(1),
66
+ )
67
+ model_input = ModelInputForGPUWithSamplingMetadata(
68
+ input_tokens=torch.ones(10),
69
+ input_positions=torch.ones(10),
70
+ sampling_metadata=sampling_metadata,
71
+ attn_metadata=attn_metadata)
72
+
73
+ assert isinstance(model_input, ModelInputForGPUWithSamplingMetadata)
74
+
75
+ # Test round trip serialization.
76
+ tensor_dict = model_input.as_broadcastable_tensor_dict()
77
+ attn_backend = MockAttentionBackend()
78
+ received_model_input = (
79
+ ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
80
+ tensor_dict, attn_backend=attn_backend))
81
+ # Check that received copy has correct values.
82
+ assert isinstance(received_model_input,
83
+ ModelInputForGPUWithSamplingMetadata)
84
+ assert received_model_input.input_tokens is not None
85
+ assert (
86
+ received_model_input.input_tokens == model_input.input_tokens).all()
87
+ assert received_model_input.input_positions is not None
88
+ assert (received_model_input.input_positions == model_input.input_positions
89
+ ).all()
90
+ assert received_model_input.multi_modal_kwargs is None
91
+ assert (received_model_input.multi_modal_kwargs ==
92
+ model_input.multi_modal_kwargs)
93
+ assert received_model_input.lora_requests is None
94
+ assert received_model_input.lora_requests == model_input.lora_requests
95
+ assert received_model_input.lora_mapping is None
96
+ assert received_model_input.lora_mapping == model_input.lora_mapping
97
+ for field in dataclasses.fields(AttentionMetadata):
98
+ assert getattr(received_model_input.attn_metadata, field.name,
99
+ None) == getattr(attn_metadata, field.name, None)
100
+ # For sampling metadata, only selected_token_indices is copied.
101
+ assert (received_model_input.sampling_metadata.selected_token_indices ==
102
+ sampling_metadata.selected_token_indices)
103
+ assert received_model_input.sampling_metadata.seq_groups is None
104
+
105
+
106
+def test_embedding_model_runner_input():
107
+ pooling_metadata = PoolingMetadata(
108
+ seq_groups=[[0]],
109
+ seq_data={},
110
+ prompt_lens=[1],
111
+ )
112
+ attn_metadata = AttentionMetadata(
113
+ num_prefills=1,
114
+ num_prefill_tokens=2,
115
+ num_decode_tokens=3,
116
+ slot_mapping=torch.zeros(1),
117
+ )
118
+ model_input = ModelInputForGPUWithPoolingMetadata(
119
+ input_tokens=torch.ones(10),
120
+ input_positions=torch.ones(10),
121
+ pooling_metadata=pooling_metadata,
122
+ attn_metadata=attn_metadata)
123
+
124
+ assert isinstance(model_input, ModelInputForGPUWithPoolingMetadata)
125
+
126
+ # Test round trip serialization.
127
+ tensor_dict = model_input.as_broadcastable_tensor_dict()
128
+ attn_backend = MockAttentionBackend()
129
+ received_model_input = (
130
+ ModelInputForGPUWithPoolingMetadata.from_broadcasted_tensor_dict(
131
+ tensor_dict, attn_backend=attn_backend))
132
+ # Check that received copy has correct values.
133
+ assert isinstance(received_model_input,
134
+ ModelInputForGPUWithPoolingMetadata)
135
+ assert received_model_input.input_tokens is not None
136
+ assert (
137
+ received_model_input.input_tokens == model_input.input_tokens).all()
138
+ assert received_model_input.input_positions is not None
139
+ assert (received_model_input.input_positions == model_input.input_positions
140
+ ).all()
141
+ assert received_model_input.multi_modal_kwargs is None
142
+ assert (received_model_input.multi_modal_kwargs ==
143
+ model_input.multi_modal_kwargs)
144
+ assert received_model_input.lora_requests is None
145
+ assert received_model_input.lora_requests == model_input.lora_requests
146
+ assert received_model_input.lora_mapping is None
147
+ assert received_model_input.lora_mapping == model_input.lora_mapping
148
+ for field in dataclasses.fields(AttentionMetadata):
149
+ assert getattr(received_model_input.attn_metadata, field.name,
150
+ None) == getattr(attn_metadata, field.name, None)
151
+ # Pooling metadata is not broadcast.
152
+ assert received_model_input.pooling_metadata is None
tests/worker/test_model_runner.py
+30
-27
@@ -61,12 +61,13 @@ def test_prepare_prompt(batch_size):
61
expected_selected_token_indices.append(selected_token_start_idx +
62
seq_len - 1)
63
selected_token_start_idx += seq_len
64
- model_input = model_runner._prepare_model_input(seq_group_metadata_list)
64
+ model_input = model_runner._prepare_model_input_tensors(
65
+ seq_group_metadata_list)
66
input_tokens = model_input.input_tokens
67
input_positions = model_input.input_positions
68
attn_metadata = model_input.attn_metadata
69
return_seq_lens = model_input.seq_lens
69
- slot_mapping = model_input.slot_mapping
70
+ slot_mapping = attn_metadata.slot_mapping
71
assert return_seq_lens == seq_lens
72
assert len(slot_mapping) == len(input_tokens)
73
@@ -174,10 +175,11 @@ def test_prepare_decode_cuda_graph(batch_size):
175
assert seq_group_metadata.token_chunk_size == 1
176
seq_group_metadata_list.append(seq_group_metadata)
177
177
- model_input = model_runner._prepare_model_input(seq_group_metadata_list)
178
+ model_input = model_runner._prepare_model_input_tensors(
179
+ seq_group_metadata_list)
180
input_tokens, input_positions, attn_metadata, slot_mapping = (
181
model_input.input_tokens, model_input.input_positions,
180
- model_input.attn_metadata, model_input.slot_mapping)
182
+ model_input.attn_metadata, model_input.attn_metadata.slot_mapping)
183
assert len(slot_mapping) == len(input_tokens)
184
185
expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
@@ -259,32 +261,29 @@ def test_empty_seq_group():
261
enforce_eager=False,
262
)
263
seq_group_metadata_list: List[SequenceGroupMetadata] = []
262
- model_input = model_runner._prepare_model_input(seq_group_metadata_list)
263
- input_tokens, input_positions, attn_metadata, slot_mapping = (
264
+ model_input = model_runner._prepare_model_input_tensors(
265
+ seq_group_metadata_list)
266
+ input_tokens, input_positions, attn_metadata = (
267
model_input.input_tokens,
268
model_input.input_positions,
269
model_input.attn_metadata,
267
- model_input.slot_mapping,
270
)
269
- assert len(input_tokens) == 0
270
- assert len(input_positions) == 0
271
+ assert input_tokens is None
272
+ assert input_positions is None
273
assert attn_metadata is None
272
- assert len(slot_mapping) == 0
273
-
274
- model_input = model_runner._prepare_model_input(seq_group_metadata_list)
275
- (input_tokens, input_positions, attn_metadata, slot_mapping,
276
- return_seq_lens) = (
277
- model_input.input_tokens,
278
- model_input.input_positions,
279
- model_input.attn_metadata,
280
- model_input.slot_mapping,
281
- model_input.seq_lens,
282
- )
283
- assert len(input_tokens) == 0
284
- assert len(input_positions) == 0
274
+
275
+ model_input = model_runner._prepare_model_input_tensors(
276
+ seq_group_metadata_list)
277
+ (input_tokens, input_positions, attn_metadata, return_seq_lens) = (
278
+ model_input.input_tokens,
279
+ model_input.input_positions,
280
+ model_input.attn_metadata,
281
+ model_input.seq_lens,
282
+ )
283
+ assert input_tokens is None
284
+ assert input_positions is None
285
assert attn_metadata is None
286
- assert len(slot_mapping) == 0
287
- assert len(return_seq_lens) == 0
286
+ assert return_seq_lens is None
287
288
289
@pytest.fixture
@@ -353,8 +352,12 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
352
seq_group_metadata_list.append(seq_group_metadata)
353
decode_metadata_list.append(seq_group_metadata)
354
356
- (input_tokens, input_positions, attn_metadata, _, _, _,
357
- _) = model_runner.prepare_input_tensors(seq_group_metadata_list)
355
+ model_input = model_runner.prepare_model_input(seq_group_metadata_list)
356
+ (input_tokens, input_positions, attn_metadata) = (
357
+ model_input.input_tokens,
358
+ model_input.input_positions,
359
+ model_input.attn_metadata,
360
+ )
361
362
prefill_meta_actual = attn_metadata.prefill_metadata
363
decode_meta_actual = attn_metadata.decode_metadata
@@ -367,7 +370,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
370
371
# Verify attn metadata is consistent. We don't need to test individual
372
# values here because they are tested above.
370
- attn_metadata = model_runner._prepare_model_input(
373
+ attn_metadata = model_runner._prepare_model_input_tensors(
374
seq_group_metadata_list).attn_metadata
375
376
for attr_expected, attr_actual in zip(vars(attn_metadata.prefill_metadata),
vllm/attention/backends/abstract.py
+5
-1
@@ -21,9 +21,13 @@ class AttentionBackend(ABC):
21
22
@staticmethod
23
@abstractmethod
24
- def make_metadata(*args, **kwargs) -> "AttentionMetadata":
24
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
25
raise NotImplementedError
26
27
+ @classmethod
28
+ def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":
29
+ return cls.get_metadata_cls()(*args, **kwargs)
30
+
31
@staticmethod
32
@abstractmethod
33
def get_kv_cache_shape(
vllm/attention/backends/blocksparse_attn.py
+2
-2
@@ -90,8 +90,8 @@ class BlocksparseFlashAttentionBackend(AttentionBackend):
90
return BlocksparseFlashAttentionImpl
91
92
@staticmethod
93
- def make_metadata(*args, **kwargs) -> "BlocksparseFlashAttentionMetadata":
94
- return BlocksparseFlashAttentionMetadata(*args, **kwargs)
93
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
94
+ return BlocksparseFlashAttentionMetadata
95
96
@staticmethod
97
def get_kv_cache_shape(
vllm/attention/backends/flash_attn.py
+2
-2
@@ -25,8 +25,8 @@ class FlashAttentionBackend(AttentionBackend):
25
return FlashAttentionImpl
26
27
@staticmethod
28
- def make_metadata(*args, **kwargs) -> "FlashAttentionMetadata":
29
- return FlashAttentionMetadata(*args, **kwargs)
28
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
29
+ return FlashAttentionMetadata
30
31
@staticmethod
32
def get_kv_cache_shape(
vllm/attention/backends/flashinfer.py
+2
-2
@@ -22,8 +22,8 @@ class FlashInferBackend(AttentionBackend):
22
return FlashInferImpl
23
24
@staticmethod
25
- def make_metadata(*args, **kwargs) -> "FlashInferMetadata":
26
- return FlashInferMetadata(*args, **kwargs)
25
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
26
+ return FlashInferMetadata
27
28
@staticmethod
29
def get_kv_cache_shape(
vllm/attention/backends/ipex_attn.py
+2
-2
@@ -25,8 +25,8 @@ class IpexAttnBackend(AttentionBackend):
25
return IpexAttnBackendImpl
26
27
@staticmethod
28
- def make_metadata(*args, **kwargs) -> "IpexAttnMetadata":
29
- return IpexAttnMetadata(*args, **kwargs)
28
+ def get_metadata_cls() -> Type["IpexAttnMetadata"]:
29
+ return IpexAttnMetadata
30
31
@staticmethod
32
def get_kv_cache_shape(
vllm/attention/backends/pallas.py
+2
-2
@@ -16,8 +16,8 @@ class PallasAttentionBackend(AttentionBackend):
16
return PallasAttentionBackendImpl
17
18
@staticmethod
19
- def make_metadata(*args, **kwargs) -> "PallasMetadata":
20
- return PallasMetadata(*args, **kwargs)
19
+ def get_metadata_cls() -> Type["PallasMetadata"]:
20
+ return PallasMetadata
21
22
@staticmethod
23
def get_kv_cache_shape(
vllm/attention/backends/rocm_flash_attn.py
+2
-2
@@ -25,8 +25,8 @@ class ROCmFlashAttentionBackend(AttentionBackend):
25
return ROCmFlashAttentionImpl
26
27
@staticmethod
28
- def make_metadata(*args, **kwargs) -> "ROCmFlashAttentionMetadata":
29
- return ROCmFlashAttentionMetadata(*args, **kwargs)
28
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
29
+ return ROCmFlashAttentionMetadata
30
31
@staticmethod
32
def get_kv_cache_shape(
vllm/attention/backends/torch_sdpa.py
+2
-2
@@ -31,8 +31,8 @@ class TorchSDPABackend(AttentionBackend):
31
return TorchSDPABackendImpl
32
33
@staticmethod
34
- def make_metadata(*args, **kwargs) -> "TorchSDPAMetadata":
35
- return TorchSDPAMetadata(*args, **kwargs)
34
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
35
+ return TorchSDPAMetadata
36
37
@staticmethod
38
def get_kv_cache_shape(
vllm/attention/backends/xformers.py
+2
-2
@@ -28,8 +28,8 @@ class XFormersBackend(AttentionBackend):
28
return XFormersImpl
29
30
@staticmethod
31
- def make_metadata(*args, **kwargs) -> "XFormersMetadata":
32
- return XFormersMetadata(*args, **kwargs)
31
+ def get_metadata_cls() -> Type["AttentionMetadata"]:
32
+ return XFormersMetadata
33
34
@staticmethod
35
def get_kv_cache_shape(
vllm/executor/distributed_gpu_executor.py
+8
-8
@@ -64,8 +64,8 @@ class DistributedGPUExecutor(GPUExecutor):
64
num_cpu_blocks=num_cpu_blocks)
65
66
def execute_model(
67
- self,
68
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
67
+ self, execute_model_req: ExecuteModelRequest
68
+ ) -> Optional[List[SamplerOutput]]:
69
if self.parallel_worker_tasks is None:
70
self.parallel_worker_tasks = self._run_workers(
71
"start_worker_execution_loop",
@@ -79,7 +79,7 @@ class DistributedGPUExecutor(GPUExecutor):
79
if self.parallel_worker_tasks is None:
80
return
81
82
- self._driver_execute_model()
82
+ self._driver_execute_model(execute_model_req=None)
83
parallel_worker_tasks = self.parallel_worker_tasks
84
self.parallel_worker_tasks = None
85
# Ensure that workers exit model loop cleanly
@@ -123,13 +123,13 @@ class DistributedGPUExecutor(GPUExecutor):
123
124
@abstractmethod
125
def _driver_execute_model(
126
- self,
127
- execute_model_req: Optional[ExecuteModelRequest] = None
128
- ) -> List[SamplerOutput]:
126
+ self, execute_model_req: Optional[ExecuteModelRequest]
127
+ ) -> Optional[List[SamplerOutput]]:
128
"""Run execute_model in the driver worker.
129
131
- Passing None will cause the driver to stop the model execution
132
- loop running in each of the remote workers.
130
+ Passing None will cause the driver to stop the model execution loop
131
+ running in each of the remote workers. In this case, this method
132
+ returns None. Otherwise, this method returns the model output.
133
"""
134
raise NotImplementedError
135
vllm/executor/executor_base.py
+2
-2
@@ -69,8 +69,8 @@ class ExecutorBase(ABC):
69
70
@abstractmethod
71
def execute_model(
72
- self,
73
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
72
+ self, execute_model_req: ExecuteModelRequest
73
+ ) -> Optional[List[SamplerOutput]]:
74
"""Executes at least one model step on the given sequences."""
75
raise NotImplementedError
76
vllm/executor/gpu_executor.py
+1
-1
@@ -87,7 +87,7 @@ class GPUExecutor(ExecutorBase):
87
88
def execute_model(
89
self, execute_model_req: ExecuteModelRequest
90
- ) -> List[Union[SamplerOutput, PoolerOutput]]:
90
+ ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
91
output = self.driver_worker.execute_model(execute_model_req)
92
return output
93
vllm/executor/multiproc_gpu_executor.py
+3
-5
@@ -78,16 +78,14 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
78
worker_monitor.close()
79
80
def _driver_execute_model(
81
- self,
82
- execute_model_req: Optional[ExecuteModelRequest] = None
83
- ) -> List[SamplerOutput]:
81
+ self, execute_model_req: Optional[ExecuteModelRequest]
82
+ ) -> Optional[List[SamplerOutput]]:
83
"""Run execute_model in the driver worker.
84
85
Passing None will cause the driver to stop the model execution
86
loop running in each of the remote workers.
87
"""
89
- return self.driver_worker.execute_model(
90
- execute_model_req=execute_model_req)
88
+ return self.driver_worker.execute_model(execute_model_req)
89
90
def _run_workers(
91
self,
vllm/executor/neuron_executor.py
+1
-2
@@ -55,8 +55,7 @@ class NeuronExecutor(ExecutorBase):
55
assert execute_model_req.num_lookahead_slots == 0, (
56
"lookahead not supported for Neuron backend.")
57
58
- output = self.driver_worker.execute_model(
59
- execute_model_req.seq_group_metadata_list)
58
+ output = self.driver_worker.execute_model(execute_model_req)
59
return output
60
61
def add_lora(self, lora_request: LoRARequest) -> bool:
vllm/executor/ray_gpu_executor.py
+2
-3
@@ -190,9 +190,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
190
max_parallel_loading_workers)
191
192
def _driver_execute_model(
193
- self,
194
- execute_model_req: Optional[ExecuteModelRequest] = None
195
- ) -> List[SamplerOutput]:
193
+ self, execute_model_req: Optional[ExecuteModelRequest]
194
+ ) -> Optional[List[SamplerOutput]]:
195
"""Run execute_model in the driver worker.
196
197
Passing None will cause the driver to stop the model execution
vllm/sequence.py
+2
-1
@@ -887,7 +887,8 @@ class HiddenStates:
887
888
@dataclass
889
class ExecuteModelRequest:
890
- """The model execution request."""
890
+ """The model execution request, containing CPU metadata only. The LLM
891
+ engine should create an instance of this class for each request batch."""
892
# The sequence group metadata list.
893
seq_group_metadata_list: List[SequenceGroupMetadata]
894
# Blocks to swap in. List of CPU -> GPU block number.
vllm/spec_decode/mlp_speculator_worker.py
+1
-2
@@ -7,7 +7,6 @@ from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7
SequenceGroupMetadata)
8
from vllm.spec_decode.multi_step_worker import MultiStepWorker
9
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10
-from vllm.worker.model_runner import ModelInput
10
11
12
class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
@@ -56,7 +55,7 @@ class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
55
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
56
) -> Tuple[torch.Tensor, List[int], List[int]]:
57
if not seq_group_metadata_list:
59
- return ModelInput.empty(self.device)
58
+ return torch.empty(0, device=self.device), [], []
59
60
input_tokens: List[int] = []
61
seq_lens: List[int] = []
vllm/worker/cpu_model_runner.py
+95
-66
@@ -1,5 +1,6 @@
1
from collections import defaultdict
2
-from typing import Dict, List, Optional, Tuple
2
+from dataclasses import dataclass
3
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
4
5
import torch
6
from torch import nn
@@ -8,20 +9,64 @@ from vllm.attention import AttentionMetadata, get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, ParallelConfig, SchedulerConfig,
11
VisionLanguageConfig)
11
-from vllm.distributed import broadcast_tensor_dict
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader import get_model
15
from vllm.multimodal import MULTIMODAL_REGISTRY
16
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17
from vllm.utils import make_tensor_with_pad
18
+from vllm.worker.model_runner_base import (
19
+ ModelRunnerBase, ModelRunnerInputBase,
20
+ _add_attn_metadata_broadcastable_dict,
21
+ _add_sampling_metadata_broadcastable_dict,
22
+ _init_attn_metadata_from_tensor_dict,
23
+ _init_sampling_metadata_from_tensor_dict)
24
+
25
+if TYPE_CHECKING:
26
+ from vllm.attention.backends.abstract import AttentionBackend
27
28
logger = init_logger(__name__)
29
30
_PAD_SLOT_ID = -1
31
32
24
-class CPUModelRunner:
33
+@dataclass(frozen=True)
34
+class CPUModelInput(ModelRunnerInputBase):
35
+ """
36
+ Used by the CPUModelRunner.
37
+ """
38
+ input_tokens: Optional[torch.Tensor] = None
39
+ input_positions: Optional[torch.Tensor] = None
40
+ attn_metadata: Optional["AttentionMetadata"] = None
41
+ sampling_metadata: Optional["SamplingMetadata"] = None
42
+ multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
43
+
44
+ def as_broadcastable_tensor_dict(
45
+ self) -> Dict[str, Union[int, torch.Tensor]]:
46
+ tensor_dict = {
47
+ "input_tokens": self.input_tokens,
48
+ "input_positions": self.input_positions,
49
+ "multi_modal_kwargs": self.multi_modal_kwargs,
50
+ }
51
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
52
+ _add_sampling_metadata_broadcastable_dict(tensor_dict,
53
+ self.sampling_metadata)
54
+ return tensor_dict
55
+
56
+ @classmethod
57
+ def from_broadcasted_tensor_dict(
58
+ cls: Type["CPUModelInput"],
59
+ tensor_dict: Dict[str, Any],
60
+ attn_backend: Optional["AttentionBackend"] = None
61
+ ) -> "CPUModelInput":
62
+ tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
63
+ if attn_backend is not None:
64
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
65
+ attn_backend, tensor_dict)
66
+ return cls(**tensor_dict)
67
+
68
+
69
+class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
70
71
def __init__(
72
self,
@@ -270,86 +315,70 @@ class CPUModelRunner:
315
attn_metadata,
316
)
317
273
- def prepare_input_tensors(
318
+ def make_model_input_from_broadcasted_tensor_dict(
319
+ self,
320
+ tensor_dict: Dict[str, Any],
321
+ ) -> CPUModelInput:
322
+ return CPUModelInput.from_broadcasted_tensor_dict(
323
+ tensor_dict,
324
+ attn_backend=self.attn_backend,
325
+ )
326
+
327
+ def prepare_model_input(
328
self,
329
seq_group_metadata_list: List[SequenceGroupMetadata],
276
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
277
- Optional[Dict[str, torch.Tensor]]]:
330
+ ) -> CPUModelInput:
331
multi_modal_kwargs = None
279
- if self.is_driver_worker:
280
- # NOTE: We assume that all sequences in the group are all prompts or
281
- # all decodes.
282
- is_prompt = seq_group_metadata_list[0].is_prompt
283
- # Prepare input tensors.
284
- if is_prompt:
285
- (input_tokens, input_positions, attn_metadata, seq_lens,
286
- multi_modal_kwargs
287
- ) = self._prepare_prompt(seq_group_metadata_list)
288
- else:
289
- (input_tokens, input_positions,
290
- attn_metadata) = self._prepare_decode(seq_group_metadata_list)
291
- seq_lens = []
292
- sampling_metadata = SamplingMetadata.prepare(
293
- seq_group_metadata_list,
294
- seq_lens,
295
- # query_lens is not needed if chunked prefill is not
296
- # supported. Since CPU worker doesn't support chunked prefill
297
- # just use seq_lens instead.
298
- seq_lens,
299
- self.device,
300
- pin_memory=False)
301
- # Broadcast the metadata.
302
- metadata_dict = {
303
- "input_tokens": input_tokens,
304
- "input_positions": input_positions,
305
- "selected_token_indices":
306
- sampling_metadata.selected_token_indices,
307
- }
308
- metadata_dict.update(attn_metadata.asdict_zerocopy())
309
- broadcast_tensor_dict(metadata_dict, src=0)
332
+ # NOTE: We assume that all sequences in the group are all prompts or
333
+ # all decodes.
334
+ is_prompt = seq_group_metadata_list[0].is_prompt
335
+ # Prepare input tensors.
336
+ if is_prompt:
337
+ (input_tokens, input_positions, attn_metadata, seq_lens,
338
+ multi_modal_kwargs
339
+ ) = self._prepare_prompt(seq_group_metadata_list)
340
else:
311
- metadata_dict = broadcast_tensor_dict(src=0)
312
- input_tokens = metadata_dict.pop("input_tokens")
313
- input_positions = metadata_dict.pop("input_positions")
314
- selected_token_indices = metadata_dict.pop(
315
- "selected_token_indices")
316
- attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
317
- sampling_metadata = SamplingMetadata(
318
- seq_groups=None,
319
- seq_data=None,
320
- seq_lens=None,
321
- selected_token_indices=selected_token_indices,
322
- categorized_sample_indices=None,
323
- generators=None,
324
- )
325
-
326
- return (input_tokens, input_positions, attn_metadata,
327
- sampling_metadata, multi_modal_kwargs)
341
+ (input_tokens, input_positions,
342
+ attn_metadata) = self._prepare_decode(seq_group_metadata_list)
343
+ seq_lens = []
344
+ sampling_metadata = SamplingMetadata.prepare(
345
+ seq_group_metadata_list,
346
+ seq_lens,
347
+ # query_lens is not needed if chunked prefill is not
348
+ # supported. Since CPU worker doesn't support chunked prefill
349
+ # just use seq_lens instead.
350
+ seq_lens,
351
+ self.device,
352
+ pin_memory=False)
353
+ return CPUModelInput(
354
+ input_tokens=input_tokens,
355
+ input_positions=input_positions,
356
+ attn_metadata=attn_metadata,
357
+ sampling_metadata=sampling_metadata,
358
+ )
359
360
@torch.inference_mode()
361
def execute_model(
362
self,
332
- seq_group_metadata_list: List[SequenceGroupMetadata],
363
+ model_input: CPUModelInput,
364
kv_caches: List[torch.Tensor],
365
) -> Optional[SamplerOutput]:
335
- (input_tokens, input_positions, attn_metadata, sampling_metadata,
336
- multi_modal_input
337
- ) = self.prepare_input_tensors(seq_group_metadata_list)
338
-
366
model_executable = self.model
367
execute_model_kwargs = {
341
- "input_ids": input_tokens,
342
- "positions": input_positions,
368
+ "input_ids": model_input.input_tokens,
369
+ "positions": model_input.input_positions,
370
"kv_caches": kv_caches,
344
- "attn_metadata": attn_metadata,
371
+ "attn_metadata": model_input.attn_metadata,
372
}
346
- if self.vision_language_config and multi_modal_input is not None:
347
- execute_model_kwargs.update(multi_modal_input)
373
+ if (self.vision_language_config
374
+ and model_input.multi_modal_kwargs is not None):
375
+ execute_model_kwargs.update(model_input.multi_modal_kwargs)
376
377
hidden_states = model_executable(**execute_model_kwargs)
378
379
# Compute the logits.
352
- logits = self.model.compute_logits(hidden_states, sampling_metadata)
380
+ logits = self.model.compute_logits(hidden_states,
381
+ model_input.sampling_metadata)
382
383
# Only perform sampling in the driver worker.
384
if not self.is_driver_worker:
@@ -358,6 +387,6 @@ class CPUModelRunner:
387
# Sample the next token.
388
output = self.model.sample(
389
logits=logits,
361
- sampling_metadata=sampling_metadata,
390
+ sampling_metadata=model_input.sampling_metadata,
391
)
392
return output
vllm/worker/cpu_worker.py
+34
-51
@@ -1,5 +1,5 @@
1
"""A CPU worker class."""
2
-from typing import Any, Dict, List, Optional, Tuple
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
import torch.distributed
@@ -8,15 +8,15 @@ from vllm.attention import get_attn_backend
8
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
ModelConfig, ParallelConfig, SchedulerConfig,
10
VisionLanguageConfig)
11
-from vllm.distributed import (broadcast_tensor_dict,
12
- ensure_model_parallel_initialized,
11
+from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment)
13
from vllm.logger import init_logger
14
from vllm.model_executor import set_random_seed
16
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
15
+from vllm.sequence import ExecuteModelRequest
16
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17
from vllm.worker.cpu_model_runner import CPUModelRunner
19
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
18
+from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
19
+ LoraNotSupportedWorkerBase, WorkerInput)
20
21
logger = init_logger(__name__)
22
@@ -110,7 +110,7 @@ class CPUCacheEngine:
110
return dtype_size * total
111
112
113
-class CPUWorker(LoraNotSupportedWorkerBase):
113
+class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
114
"""A worker class that executes (a partition of) the model on a CPU socket.
115
116
Each worker is associated with a single CPU socket. The worker is
@@ -154,7 +154,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
154
# note: lazy import to avoid importing torch before initializing
155
from vllm.utils import init_cached_hf_modules
156
init_cached_hf_modules()
157
- self.model_runner = CPUModelRunner(
157
+ self.model_runner: CPUModelRunner = CPUModelRunner(
158
model_config,
159
parallel_config,
160
scheduler_config,
@@ -255,54 +255,37 @@ class CPUWorker(LoraNotSupportedWorkerBase):
255
for layer_cache in self.cpu_cache:
256
layer_cache.fill_(0)
257
258
- def cache_copy(
258
+ @property
259
+ def do_metadata_broadcast(self) -> bool:
260
+ return self.parallel_config.tensor_parallel_size > 1
261
+
262
+ @property
263
+ def kv_cache(self) -> Optional[List[torch.Tensor]]:
264
+ return self.cpu_cache
265
+
266
+ def execute_worker(
267
self,
260
- blocks_to_copy: torch.Tensor,
268
+ worker_input: WorkerInput,
269
) -> None:
262
- if blocks_to_copy.numel() > 0:
263
- self.cache_engine.copy(blocks_to_copy)
270
+ if (worker_input.blocks_to_copy is not None
271
+ and worker_input.blocks_to_copy.numel() > 0):
272
+ self.cache_engine.copy(worker_input.blocks_to_copy)
273
274
@torch.inference_mode()
266
- def execute_model(
267
- self,
268
- execute_model_req: Optional[ExecuteModelRequest] = None,
269
- ) -> List[SamplerOutput]:
270
-
271
- if execute_model_req is None:
272
- seq_group_metadata_list = None
273
- else:
274
- seq_group_metadata_list = execute_model_req.seq_group_metadata_list
275
-
276
- if self.is_driver_worker:
277
- assert seq_group_metadata_list is not None
278
- num_seq_groups: int = len(seq_group_metadata_list)
279
- assert execute_model_req is not None
280
- blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
281
- device="cpu",
282
- dtype=torch.int64).view(-1, 2)
283
- assert len(execute_model_req.blocks_to_swap_in) == 0
284
- assert len(execute_model_req.blocks_to_swap_out) == 0
285
- data: Dict[str, Any] = {
286
- "num_seq_groups": num_seq_groups,
287
- "blocks_to_copy": execute_model_req.blocks_to_copy,
288
- }
289
- broadcast_tensor_dict(data, src=0)
290
- else:
291
- data = broadcast_tensor_dict(src=0)
292
- num_seq_groups = data["num_seq_groups"]
293
- blocks_to_copy = data["blocks_to_copy"]
294
-
295
- self.cache_copy(blocks_to_copy)
296
-
297
- # If there is no input, we don't need to execute the model.
298
- if num_seq_groups == 0:
299
- return []
300
-
301
- output = self.model_runner.execute_model(seq_group_metadata_list,
302
- self.cpu_cache)
303
-
304
- # CPU worker only supports single-step execution.
305
- return [output]
275
+ def prepare_worker_input(
276
+ self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
277
+ assert execute_model_req is not None
278
+ num_seq_groups: int = len(execute_model_req.seq_group_metadata_list)
279
+ blocks_to_copy = execute_model_req.blocks_to_copy
280
+ blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
281
+ device="cpu",
282
+ dtype=torch.int64).view(-1, 2)
283
+ assert len(execute_model_req.blocks_to_swap_in) == 0
284
+ assert len(execute_model_req.blocks_to_swap_out) == 0
285
+ return WorkerInput(
286
+ num_seq_groups=num_seq_groups,
287
+ blocks_to_copy=blocks_to_copy,
288
+ )
289
290
def init_distributed_environment(self) -> None:
291
"""Initialize the distributed environment."""
vllm/worker/embedding_model_runner.py
+52
-77
@@ -1,24 +1,32 @@
1
-from typing import Dict, List, Optional, Set, Tuple
1
+import dataclasses
2
+from typing import Any, Dict, List, Optional, Tuple, Type
3
4
import torch
5
5
-from vllm.attention import AttentionMetadata
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
ModelConfig, ParallelConfig, SchedulerConfig,
8
VisionLanguageConfig)
9
-from vllm.distributed import broadcast_tensor_dict
9
from vllm.logger import init_logger
11
-from vllm.lora.layers import LoRAMapping
12
-from vllm.lora.request import LoRARequest
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.pooling_params import PoolingParams
12
from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
16
-from vllm.worker.model_runner import ModelRunner
13
+from vllm.worker.model_runner import GPUModelRunnerBase, ModelInputForGPU
14
15
logger = init_logger(__name__)
16
17
21
-class EmbeddingModelRunner(ModelRunner):
18
+@dataclasses.dataclass(frozen=True)
19
+class ModelInputForGPUWithPoolingMetadata(ModelInputForGPU):
20
+ """
21
+ Used by the EmbeddingModelRunner.
22
+ """
23
+ pooling_metadata: Optional["PoolingMetadata"] = None
24
+
25
+
26
+class EmbeddingModelRunner(
27
+ GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]):
28
+ _model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = (
29
+ ModelInputForGPUWithPoolingMetadata)
30
31
def __init__(
32
self,
@@ -47,21 +55,22 @@ class EmbeddingModelRunner(ModelRunner):
55
@torch.inference_mode()
56
def execute_model(
57
self,
50
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
58
+ model_input: ModelInputForGPUWithPoolingMetadata,
59
kv_caches: List[torch.Tensor],
60
) -> Optional[PoolerOutput]:
53
- (input_tokens, input_positions, attn_metadata, pooling_metadata,
54
- lora_requests, lora_mapping, multi_modal_input
55
- ) = self.prepare_input_tensors(seq_group_metadata_list)
56
-
61
if self.lora_config:
58
- self.set_active_loras(lora_requests, lora_mapping)
62
+ assert model_input.lora_requests is not None
63
+ assert model_input.lora_mapping is not None
64
+ self.set_active_loras(model_input.lora_requests,
65
+ model_input.lora_mapping)
66
67
# Currently cuda graph is only supported by the decode phase.
61
- prefill_meta = attn_metadata.prefill_metadata
62
- decode_meta = attn_metadata.decode_metadata
68
+ assert model_input.attn_metadata is not None
69
+ prefill_meta = model_input.attn_metadata.prefill_metadata
70
+ decode_meta = model_input.attn_metadata.decode_metadata
71
if prefill_meta is None and decode_meta.use_cuda_graph:
64
- graph_batch_size = input_tokens.shape[0]
72
+ assert model_input.input_tokens is not None
73
+ graph_batch_size = model_input.input_tokens.shape[0]
74
model_executable = self.graph_runners[graph_batch_size]
75
else:
76
model_executable = self.model
@@ -70,13 +79,14 @@ class EmbeddingModelRunner(ModelRunner):
79
kv_caches = [None] * num_layers
80
81
execute_model_kwargs = {
73
- "input_ids": input_tokens,
74
- "positions": input_positions,
82
+ "input_ids": model_input.input_tokens,
83
+ "positions": model_input.input_positions,
84
"kv_caches": kv_caches,
76
- "attn_metadata": attn_metadata,
85
+ "attn_metadata": model_input.attn_metadata,
86
}
87
if self.vision_language_config:
79
- execute_model_kwargs.update({"image_input": multi_modal_input})
88
+ multi_modal_kwargs = model_input.multi_modal_kwargs or {}
89
+ execute_model_kwargs.update({"image_input": multi_modal_kwargs})
90
hidden_states = model_executable(**execute_model_kwargs)
91
92
# Only perform pooling in the driver worker.
@@ -84,66 +94,31 @@ class EmbeddingModelRunner(ModelRunner):
94
return None
95
96
return self.model.pooler(hidden_states=hidden_states,
87
- pooling_metadata=pooling_metadata)
97
+ pooling_metadata=model_input.pooling_metadata)
98
+
99
+ def make_model_input_from_broadcasted_tensor_dict(
100
+ self,
101
+ tensor_dict: Dict[str,
102
+ Any]) -> ModelInputForGPUWithPoolingMetadata:
103
+ return ModelInputForGPUWithPoolingMetadata.from_broadcasted_tensor_dict(
104
+ tensor_dict,
105
+ attn_backend=self.attn_backend,
106
+ )
107
89
- def prepare_input_tensors(
108
+ def prepare_model_input(
109
self,
110
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
92
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
93
- Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
94
- if self.is_driver_worker:
95
- assert seq_group_metadata_list is not None
96
- # Prepare input tensors.
97
- (
98
- input_tokens,
99
- input_positions,
100
- attn_metadata,
101
- seq_lens,
102
- _,
103
- lora_mapping,
104
- lora_requests,
105
- multi_modal_kwargs,
106
- slot_mapping,
107
- num_prefill_tokens,
108
- num_decode_tokens,
109
- num_prefills,
110
- ) = self._prepare_model_input(seq_group_metadata_list)
111
- # Prepare PoolingMetadata
112
- pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
113
- seq_lens)
114
-
115
- metadata_dict = {
116
- "input_tokens": input_tokens,
117
- "input_positions": input_positions,
118
- "lora_requests": lora_requests,
119
- "lora_mapping": lora_mapping,
120
- "multi_modal_kwargs": multi_modal_kwargs,
121
- "num_prefill_tokens": num_prefill_tokens,
122
- "num_decode_tokens": num_decode_tokens,
123
- "slot_mapping": slot_mapping,
124
- "num_prefills": num_prefills,
125
- }
126
- if attn_metadata:
127
- metadata_dict.update(attn_metadata.asdict_zerocopy())
128
- broadcast_tensor_dict(metadata_dict, src=0)
129
- else:
130
- metadata_dict = broadcast_tensor_dict(src=0)
131
- input_tokens = metadata_dict.pop("input_tokens")
132
- input_positions = metadata_dict.pop("input_positions")
133
- lora_mapping = metadata_dict.pop("lora_mapping")
134
- lora_requests = metadata_dict.pop("lora_requests")
135
- multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
136
- if metadata_dict:
137
- attn_metadata = self.attn_backend.make_metadata(
138
- **metadata_dict)
139
- else:
140
- attn_metadata = None
141
- pooling_metadata = PoolingMetadata(seq_groups=None,
142
- seq_data=None,
143
- prompt_lens=None)
144
-
145
- return (input_tokens, input_positions, attn_metadata, pooling_metadata,
146
- lora_requests, lora_mapping, multi_modal_kwargs)
111
+ ) -> ModelInputForGPUWithPoolingMetadata:
112
+ assert seq_group_metadata_list is not None
113
+ model_input = self._prepare_model_input_tensors(
114
+ seq_group_metadata_list)
115
+ # Prepare PoolingMetadata.
116
+ assert model_input.seq_lens is not None
117
+ pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
118
+ model_input.seq_lens)
119
+
120
+ return dataclasses.replace(model_input,
121
+ pooling_metadata=pooling_metadata)
122
123
def _prepare_pooling(
124
self,
vllm/worker/model_runner.py
+205
-162
@@ -1,8 +1,10 @@
1
+import dataclasses
2
import gc
3
import time
4
import warnings
5
from collections import defaultdict
5
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
6
+from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
7
+ TypeVar, Union)
8
9
import numpy as np
10
import torch
@@ -12,7 +14,6 @@ from vllm.attention import AttentionMetadata, get_attn_backend
14
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
ModelConfig, ParallelConfig, SchedulerConfig,
16
VisionLanguageConfig)
15
-from vllm.distributed import broadcast_tensor_dict
17
from vllm.distributed.parallel_state import graph_capture
18
from vllm.logger import init_logger
19
from vllm.lora.layers import LoRAMapping
@@ -26,6 +27,15 @@ from vllm.sampling_params import SamplingParams
27
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
28
from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
29
is_pin_memory_available, make_tensor_with_pad)
30
+from vllm.worker.model_runner_base import (
31
+ ModelRunnerBase, ModelRunnerInputBase,
32
+ _add_attn_metadata_broadcastable_dict,
33
+ _add_sampling_metadata_broadcastable_dict,
34
+ _init_attn_metadata_from_tensor_dict,
35
+ _init_sampling_metadata_from_tensor_dict)
36
+
37
+if TYPE_CHECKING:
38
+ from vllm.attention.backends.abstract import AttentionBackend
39
40
logger = init_logger(__name__)
41
@@ -39,40 +49,90 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
49
]
50
_NUM_WARMUP_ITERS = 2
51
52
+TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
53
43
-class ModelInput(NamedTuple):
44
- input_tokens: torch.Tensor
45
- input_positions: torch.Tensor
46
- attn_metadata: Optional[AttentionMetadata]
47
- seq_lens: List[int]
48
- query_lens: List[int]
49
- lora_mapping: Optional[LoRAMapping]
50
- lora_requests: Set[LoRARequest]
51
- multi_modal_kwargs: Dict[str, torch.Tensor]
52
- slot_mapping: torch.Tensor
53
- num_prefill_tokens: int
54
- num_decode_tokens: int
55
- num_prefills: int
54
57
- @classmethod
58
- def empty(cls, device):
59
- return ModelInput(
60
- input_tokens=torch.empty(0, device=device),
61
- input_positions=torch.empty(0, device=device),
62
- attn_metadata=None,
63
- seq_lens=[],
64
- query_lens=[],
65
- lora_mapping=None,
66
- lora_requests=set(),
67
- multi_modal_kwargs={},
68
- slot_mapping=torch.empty(0, device=device),
69
- num_prefill_tokens=0,
70
- num_decode_tokens=0,
71
- num_prefills=0,
72
- )
55
+@dataclasses.dataclass(frozen=True)
56
+class ModelInputForGPU(ModelRunnerInputBase):
57
+ """
58
+ This base class contains metadata needed for the base model forward pass
59
+ but not metadata for possible additional steps, e.g., sampling. Model
60
+ runners that run additional steps should subclass this method to add
61
+ additional fields.
62
+ """
63
+ input_tokens: Optional[torch.Tensor] = None
64
+ input_positions: Optional[torch.Tensor] = None
65
+ seq_lens: Optional[List[int]] = None
66
+ query_lens: Optional[List[int]] = None
67
+ lora_mapping: Optional["LoRAMapping"] = None
68
+ lora_requests: Optional[Set[LoRARequest]] = None
69
+ attn_metadata: Optional["AttentionMetadata"] = None
70
+ multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
71
+
72
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
73
+ tensor_dict = {
74
+ "input_tokens": self.input_tokens,
75
+ "input_positions": self.input_positions,
76
+ "lora_requests": self.lora_requests,
77
+ "lora_mapping": self.lora_mapping,
78
+ "multi_modal_kwargs": self.multi_modal_kwargs,
79
+ }
80
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
81
+ return tensor_dict
82
83
+ @classmethod
84
+ def from_broadcasted_tensor_dict(
85
+ cls: Type[TModelInputForGPU],
86
+ tensor_dict: Dict[str, Any],
87
+ attn_backend: Optional["AttentionBackend"] = None,
88
+ ) -> TModelInputForGPU:
89
+ if attn_backend is not None:
90
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
91
+ attn_backend, tensor_dict)
92
+ return cls(**tensor_dict)
93
+
94
+
95
+@dataclasses.dataclass(frozen=True)
96
+class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
97
+ """
98
+ Used by the ModelRunner.
99
+ """
100
+ sampling_metadata: Optional["SamplingMetadata"] = None
101
+ # Used for speculative decoding. We do not broadcast it because it is only
102
+ # used by the driver worker.
103
+ is_prompt: Optional[bool] = None
104
+
105
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
106
+ tensor_dict = {
107
+ "input_tokens": self.input_tokens,
108
+ "input_positions": self.input_positions,
109
+ "lora_requests": self.lora_requests,
110
+ "lora_mapping": self.lora_mapping,
111
+ "multi_modal_kwargs": self.multi_modal_kwargs,
112
+ }
113
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
114
+ _add_sampling_metadata_broadcastable_dict(tensor_dict,
115
+ self.sampling_metadata)
116
+ return tensor_dict
117
75
-class ModelRunner:
118
+ @classmethod
119
+ def from_broadcasted_tensor_dict(
120
+ cls,
121
+ tensor_dict: Dict[str, Any],
122
+ attn_backend: Optional["AttentionBackend"] = None,
123
+ ) -> "ModelInputForGPUWithSamplingMetadata":
124
+ tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
125
+ if attn_backend is not None:
126
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
127
+ attn_backend, tensor_dict)
128
+ return cls(**tensor_dict)
129
+
130
+
131
+class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
132
+ """
133
+ Helper class for shared methods between GPU model runners.
134
+ """
135
+ _model_input_cls: Type[TModelInputForGPU]
136
137
def __init__(
138
self,
@@ -241,11 +301,13 @@ class ModelRunner:
301
block_size = self.block_size
302
return (self.max_seq_len_to_capture + block_size - 1) // block_size
303
244
- def _prepare_model_input(
304
+ def _prepare_model_input_tensors(
305
self,
306
seq_group_metadata_list: List[SequenceGroupMetadata],
247
- ) -> ModelInput:
248
- """Prepare the model input based on a given sequence group.
307
+ ) -> TModelInputForGPU:
308
+ """Helper method to prepare the model input based on a given sequence
309
+ group. Prepares metadata needed for the base model forward pass but not
310
+ metadata for possible additional steps, e.g., sampling.
311
312
The API assumes seq_group_metadata_list is sorted by prefill -> decode.
313
@@ -296,7 +358,7 @@ class ModelRunner:
358
paged_kv_last_page_len: List[int] = []
359
360
if len(seq_group_metadata_list) == 0:
299
- return ModelInput.empty(self.device)
361
+ return self._model_input_cls()
362
363
if self.sliding_window is not None:
364
sliding_window_blocks = (self.sliding_window + self.block_size -
@@ -646,7 +708,7 @@ class ModelRunner:
708
for k, v in multi_modal_kwargs_list.items()
709
}
710
649
- return ModelInput(
711
+ return self._model_input_cls(
712
input_tokens=input_tokens_tensor,
713
input_positions=input_positions_tensor,
714
attn_metadata=attn_metadata,
@@ -655,132 +717,8 @@ class ModelRunner:
717
lora_mapping=lora_mapping,
718
lora_requests=lora_requests,
719
multi_modal_kwargs=multi_modal_kwargs,
658
- slot_mapping=slot_mapping_tensor,
659
- num_prefill_tokens=num_prefill_tokens,
660
- num_decode_tokens=num_decode_tokens,
661
- num_prefills=num_prefills,
662
- )
663
-
664
- def prepare_input_tensors(
665
- self,
666
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
667
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
668
- Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
669
- if self.is_driver_worker:
670
- assert seq_group_metadata_list is not None
671
- # Prepare input tensors.
672
- (
673
- input_tokens,
674
- input_positions,
675
- attn_metadata,
676
- seq_lens,
677
- query_lens,
678
- lora_mapping,
679
- lora_requests,
680
- multi_modal_kwargs,
681
- slot_mapping,
682
- num_prefill_tokens,
683
- num_decode_tokens,
684
- num_prefills,
685
- ) = self._prepare_model_input(seq_group_metadata_list)
686
- sampling_metadata = SamplingMetadata.prepare(
687
- seq_group_metadata_list, seq_lens, query_lens, self.device,
688
- self.pin_memory)
689
-
690
- metadata_dict = {
691
- "input_tokens": input_tokens,
692
- "input_positions": input_positions,
693
- "selected_token_indices":
694
- sampling_metadata.selected_token_indices,
695
- "lora_requests": lora_requests,
696
- "lora_mapping": lora_mapping,
697
- "multi_modal_kwargs": multi_modal_kwargs,
698
- "num_prefill_tokens": num_prefill_tokens,
699
- "num_decode_tokens": num_decode_tokens,
700
- "slot_mapping": slot_mapping,
701
- "num_prefills": num_prefills,
702
- }
703
- if attn_metadata:
704
- metadata_dict.update(attn_metadata.asdict_zerocopy())
705
- broadcast_tensor_dict(metadata_dict, src=0)
706
- else:
707
- metadata_dict = broadcast_tensor_dict(src=0)
708
- input_tokens = metadata_dict.pop("input_tokens")
709
- input_positions = metadata_dict.pop("input_positions")
710
- selected_token_indices = metadata_dict.pop(
711
- "selected_token_indices")
712
- lora_mapping = metadata_dict.pop("lora_mapping")
713
- lora_requests = metadata_dict.pop("lora_requests")
714
- multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
715
- if metadata_dict:
716
- attn_metadata = self.attn_backend.make_metadata(
717
- **metadata_dict)
718
- else:
719
- attn_metadata = None
720
- sampling_metadata = SamplingMetadata(
721
- seq_groups=None,
722
- selected_token_indices=selected_token_indices,
723
- categorized_sample_indices=None,
724
- num_prompts=0,
725
- )
726
-
727
- return (input_tokens, input_positions, attn_metadata,
728
- sampling_metadata, lora_requests, lora_mapping,
729
- multi_modal_kwargs)
730
-
731
- @torch.inference_mode()
732
- def execute_model(
733
- self,
734
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
735
- kv_caches: List[torch.Tensor],
736
- ) -> Optional[SamplerOutput]:
737
- (input_tokens, input_positions, attn_metadata, sampling_metadata,
738
- lora_requests, lora_mapping, multi_modal_kwargs
739
- ) = self.prepare_input_tensors(seq_group_metadata_list)
740
-
741
- if self.lora_config:
742
- self.set_active_loras(lora_requests, lora_mapping)
743
-
744
- # Currently cuda graph is only supported by the decode phase.
745
- prefill_meta = attn_metadata.prefill_metadata
746
- decode_meta = attn_metadata.decode_metadata
747
- if prefill_meta is None and decode_meta.use_cuda_graph:
748
- graph_batch_size = input_tokens.shape[0]
749
- model_executable = self.graph_runners[graph_batch_size]
750
- else:
751
- model_executable = self.model
752
-
753
- hidden_states = model_executable(
754
- input_ids=input_tokens,
755
- positions=input_positions,
756
- kv_caches=kv_caches,
757
- attn_metadata=attn_metadata,
758
- **multi_modal_kwargs,
759
- )
760
-
761
- # Compute the logits.
762
- logits = self.model.compute_logits(hidden_states, sampling_metadata)
763
-
764
- # Only perform sampling in the driver worker.
765
- if not self.is_driver_worker:
766
- return None
767
-
768
- # Sample the next token.
769
- output: SamplerOutput = self.model.sample(
770
- logits=logits,
771
- sampling_metadata=sampling_metadata,
720
)
721
774
- if self.return_hidden_states:
775
- # we only need to pass hidden states of most recent token
776
- assert seq_group_metadata_list is not None
777
- if seq_group_metadata_list[0].is_prompt:
778
- hidden_states = hidden_states.index_select(
779
- 0, sampling_metadata.selected_token_indices)
780
- output.hidden_states = hidden_states
781
-
782
- return output
783
-
722
@torch.inference_mode()
723
def profile_run(self) -> None:
724
# Enable top-k sampling to reflect the accurate memory usage.
@@ -853,7 +791,8 @@ class ModelRunner:
791
# Run the model with the dummy inputs.
792
num_layers = self.model_config.get_num_layers(self.parallel_config)
793
kv_caches = [None] * num_layers
856
- self.execute_model(seqs, kv_caches)
794
+ model_input = self.prepare_model_input(seqs)
795
+ self.execute_model(model_input, kv_caches)
796
torch.cuda.synchronize()
797
return
798
@@ -986,6 +925,110 @@ class ModelRunner:
925
return self.model_config.get_vocab_size()
926
927
928
+class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
929
+ """
930
+ GPU model runner with sampling step.
931
+ """
932
+ _model_input_cls: Type[ModelInputForGPUWithSamplingMetadata] = (
933
+ ModelInputForGPUWithSamplingMetadata)
934
+
935
+ def make_model_input_from_broadcasted_tensor_dict(
936
+ self,
937
+ tensor_dict: Dict[str, Any],
938
+ ) -> ModelInputForGPUWithSamplingMetadata:
939
+ return (
940
+ ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
941
+ tensor_dict,
942
+ attn_backend=self.attn_backend,
943
+ ))
944
+
945
+ def prepare_model_input(
946
+ self,
947
+ seq_group_metadata_list: List[SequenceGroupMetadata],
948
+ ) -> ModelInputForGPUWithSamplingMetadata:
949
+ """Prepare the model input based on a given sequence group, including
950
+ metadata for the sampling step.
951
+
952
+ The API assumes seq_group_metadata_list is sorted by prefill -> decode.
953
+
954
+ The result tensors and data structure also batches input in prefill
955
+ -> decode order. For example,
956
+
957
+ - input_tokens[:num_prefill_tokens] contains prefill tokens.
958
+ - input_tokens[num_prefill_tokens:] contains decode tokens.
959
+
960
+ If cuda graph is required, this API automatically pads inputs.
961
+ """
962
+ model_input = self._prepare_model_input_tensors(
963
+ seq_group_metadata_list)
964
+ sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
965
+ model_input.seq_lens,
966
+ model_input.query_lens,
967
+ self.device,
968
+ self.pin_memory)
969
+ is_prompt = (seq_group_metadata_list[0].is_prompt
970
+ if seq_group_metadata_list else None)
971
+ return dataclasses.replace(model_input,
972
+ sampling_metadata=sampling_metadata,
973
+ is_prompt=is_prompt)
974
+
975
+ @torch.inference_mode()
976
+ def execute_model(
977
+ self,
978
+ model_input: ModelInputForGPUWithSamplingMetadata,
979
+ kv_caches: List[torch.Tensor],
980
+ ) -> SamplerOutput:
981
+ if self.lora_config:
982
+ assert model_input.lora_requests is not None
983
+ assert model_input.lora_mapping is not None
984
+ self.set_active_loras(model_input.lora_requests,
985
+ model_input.lora_mapping)
986
+
987
+ # Currently cuda graph is only supported by the decode phase.
988
+ assert model_input.attn_metadata is not None
989
+ prefill_meta = model_input.attn_metadata.prefill_metadata
990
+ decode_meta = model_input.attn_metadata.decode_metadata
991
+ if prefill_meta is None and decode_meta.use_cuda_graph:
992
+ assert model_input.input_tokens is not None
993
+ graph_batch_size = model_input.input_tokens.shape[0]
994
+ model_executable = self.graph_runners[graph_batch_size]
995
+ else:
996
+ model_executable = self.model
997
+
998
+ multi_modal_kwargs = model_input.multi_modal_kwargs or {}
999
+ hidden_states = model_executable(
1000
+ input_ids=model_input.input_tokens,
1001
+ positions=model_input.input_positions,
1002
+ kv_caches=kv_caches,
1003
+ attn_metadata=model_input.attn_metadata,
1004
+ **multi_modal_kwargs,
1005
+ )
1006
+
1007
+ # Compute the logits.
1008
+ logits = self.model.compute_logits(hidden_states,
1009
+ model_input.sampling_metadata)
1010
+
1011
+ # Only perform sampling in the driver worker.
1012
+ if not self.is_driver_worker:
1013
+ return None
1014
+
1015
+ # Sample the next token.
1016
+ output: SamplerOutput = self.model.sample(
1017
+ logits=logits,
1018
+ sampling_metadata=model_input.sampling_metadata,
1019
+ )
1020
+
1021
+ if self.return_hidden_states:
1022
+ # we only need to pass hidden states of most recent token
1023
+ if model_input.is_prompt:
1024
+ assert model_input.sampling_metadata is not None
1025
+ hidden_states = hidden_states.index_select(
1026
+ 0, model_input.sampling_metadata.selected_token_indices)
1027
+ output.hidden_states = hidden_states
1028
+
1029
+ return output
1030
+
1031
+
1032
class CUDAGraphRunner:
1033
1034
def __init__(self, model: nn.Module):
vllm/worker/model_runner_base.py
new
+157
@@ -0,0 +1,157 @@
1
+import dataclasses
2
+from abc import ABC, abstractmethod
3
+from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
4
+ TypeVar)
5
+
6
+import torch
7
+
8
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
+
10
+if TYPE_CHECKING:
11
+ from vllm.attention import AttentionMetadata
12
+ from vllm.attention.backends.abstract import AttentionBackend
13
+ from vllm.model_executor import SamplingMetadata
14
+
15
+T = TypeVar('T', bound="ModelRunnerInputBase")
16
+
17
+
18
+def _add_attn_metadata_broadcastable_dict(
19
+ tensor_dict: Dict[str, Any],
20
+ attn_metadata: Optional["AttentionMetadata"]) -> None:
21
+ """
22
+ Helper method to update tensor_dict with broadcastable
23
+ AttentionMetadata fields.
24
+ """
25
+ if attn_metadata is not None:
26
+ tensor_dict.update(attn_metadata.asdict_zerocopy())
27
+
28
+
29
+def _init_attn_metadata_from_tensor_dict(
30
+ attn_backend: "AttentionBackend",
31
+ tensor_dict: Dict[str, Any],
32
+) -> Dict[str, Any]:
33
+ """
34
+ Helper method to initialize AttentionMetadata based on an
35
+ AttentionBackend and broadcastable AttentionMetadata fields.
36
+ """
37
+ # Extract the fields used to create AttentionMetadata.
38
+ valid_attn_kwargs = {}
39
+ for field in dataclasses.fields(attn_backend.get_metadata_cls()):
40
+ val = tensor_dict.pop(field.name, None)
41
+ if val is not None:
42
+ valid_attn_kwargs[field.name] = val
43
+
44
+ attn_metadata = attn_backend.make_metadata(**valid_attn_kwargs)
45
+ tensor_dict["attn_metadata"] = attn_metadata
46
+ return tensor_dict
47
+
48
+
49
+def _init_sampling_metadata_from_tensor_dict( # type: ignore
50
+ tensor_dict: Dict[str, Any]) -> Dict[str, Any]:
51
+ """
52
+ Helper method to initialize SamplingMetadata based on broadcastable
53
+ SamplingMetadata fields.
54
+ """
55
+ from vllm.model_executor import SamplingMetadata
56
+
57
+ selected_token_indices = tensor_dict.pop("selected_token_indices", None)
58
+ # An empty SamplingMetadata to signal that the worker should skip
59
+ # sampling.
60
+ if selected_token_indices is not None:
61
+ tensor_dict["sampling_metadata"] = SamplingMetadata(
62
+ seq_groups=None,
63
+ selected_token_indices=selected_token_indices,
64
+ categorized_sample_indices=None,
65
+ num_prompts=0,
66
+ )
67
+ return tensor_dict
68
+
69
+
70
+def _add_sampling_metadata_broadcastable_dict(
71
+ tensor_dict: Dict[str, Any],
72
+ sampling_metadata: Optional["SamplingMetadata"]) -> None:
73
+ """
74
+ Helper method to update tensor_dict with broadcastable
75
+ SamplingMetadata fields.
76
+ """
77
+ if sampling_metadata is not None:
78
+ tensor_dict["selected_token_indices"] = (
79
+ sampling_metadata.selected_token_indices)
80
+
81
+
82
+@dataclasses.dataclass(frozen=True)
83
+class ModelRunnerInputBase(ABC):
84
+ """Local inputs to each worker's model runner. May contain
85
+ device-specific data. Different worker backends may have different methods
86
+ of converting from the global ExecuteModelRequest produced by the LLM
87
+ engine to the worker-local ModelRunnerInputBase objects.
88
+
89
+ Model runners that support multi-GPU execution should define a
90
+ ModelRunnerInputBase subclass, add their required fields, and specify how to
91
+ serialize/deserialize a ModelInput for broadcast between workers.
92
+ """
93
+
94
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
95
+ """
96
+ Extract broadcastable fields. Override for fields that require some
97
+ custom deserialization.
98
+ """
99
+ raise NotImplementedError
100
+
101
+ @classmethod
102
+ @abstractmethod
103
+ def from_broadcasted_tensor_dict(
104
+ cls: Type[T],
105
+ tensor_dict: Dict[str, Any],
106
+ attn_backend: Optional["AttentionBackend"] = None,
107
+ ) -> T:
108
+ """
109
+ Pop fields from the given tensor_dict and populate a new instance of
110
+ ModelRunnerInputBase.
111
+ """
112
+ raise NotImplementedError
113
+
114
+
115
+class ModelRunnerBase(ABC, Generic[T]):
116
+ """
117
+ Model runner interface that abstracts a particular hardware and/or type of
118
+ model. Model execution may communicate data with model runners in other
119
+ processes, but it should not include control plane metadata communication.
120
+
121
+ Each ModelRunnerBase subclass should define a corresponding
122
+ ModelRunnerInputBase subclass.
123
+ """
124
+
125
+ @abstractmethod
126
+ def make_model_input_from_broadcasted_tensor_dict(
127
+ self,
128
+ tensor_dict: Dict[str, Any],
129
+ ) -> T:
130
+ """
131
+ Make an instance of a ModelRunnerInputBase from the broadcasted tensor
132
+ dict.
133
+ """
134
+ raise NotImplementedError
135
+
136
+ @abstractmethod
137
+ def prepare_model_input(
138
+ self,
139
+ seq_group_metadata_list: List[SequenceGroupMetadata],
140
+ ) -> T:
141
+ """
142
+ Prepare the inputs to ModelRunnerBase.execute_model from an execution
143
+ request. This method may move data to the worker's local device. It is
144
+ not allowed to communicate with other workers or devices.
145
+ """
146
+ raise NotImplementedError
147
+
148
+ @torch.inference_mode()
149
+ def execute_model(
150
+ self,
151
+ model_input: T,
152
+ kv_caches: Optional[List[torch.Tensor]],
153
+ ) -> Optional[SamplerOutput]:
154
+ """
155
+ Execute the model on the given input.
156
+ """
157
+ raise NotImplementedError
vllm/worker/neuron_model_runner.py
+49
-15
@@ -1,4 +1,5 @@
1
-from typing import List, Optional, Tuple
1
+from dataclasses import dataclass
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
3
4
import torch
5
from torch import nn
@@ -10,11 +11,39 @@ from vllm.model_executor import SamplingMetadata
11
from vllm.model_executor.model_loader.neuron import get_neuron_model
12
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
13
from vllm.utils import is_pin_memory_available, make_tensor_with_pad
14
+from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
15
+
16
+if TYPE_CHECKING:
17
+ from vllm.attention.backends.abstract import AttentionBackend
18
19
logger = init_logger(__name__)
20
21
17
-class NeuronModelRunner:
22
+@dataclass(frozen=True)
23
+class ModelInputForNeuron(ModelRunnerInputBase):
24
+ """
25
+ Used by the NeuronModelRunner.
26
+ """
27
+ input_tokens: Optional[torch.Tensor] = None
28
+ input_positions: Optional[torch.Tensor] = None
29
+ input_block_ids: Optional[torch.Tensor] = None
30
+ sampling_metadata: Optional["SamplingMetadata"] = None
31
+
32
+ def as_broadcastable_tensor_dict(
33
+ self) -> Dict[str, Union[int, torch.Tensor]]:
34
+ raise NotImplementedError("ModelInputForNeuron cannot be broadcast.")
35
+
36
+ @classmethod
37
+ def from_broadcasted_tensor_dict(
38
+ cls,
39
+ tensor_dict: Dict[str, Any],
40
+ attn_backend: Optional["AttentionBackend"] = None,
41
+ ) -> "ModelInputForNeuron":
42
+ assert attn_backend is None
43
+ return cls.from_broadcasted_tensor_dict(tensor_dict)
44
+
45
+
46
+class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
47
48
def __init__(
49
self,
@@ -139,10 +168,14 @@ class NeuronModelRunner:
168
169
return input_tokens, input_positions, input_block_ids
170
142
- def prepare_input_tensors(
171
+ def make_model_input_from_broadcasted_tensor_dict(
172
+ self, tensor_dict: Dict[str, Any]) -> ModelInputForNeuron:
173
+ return ModelInputForNeuron.from_broadcasted_tensor_dict(tensor_dict)
174
+
175
+ def prepare_model_input(
176
self,
177
seq_group_metadata_list: List[SequenceGroupMetadata],
145
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, SamplingMetadata]:
178
+ ) -> ModelInputForNeuron:
179
# NOTE: We assume that all sequences in the group are all prompts or
180
# all decodes.
181
is_prompt = seq_group_metadata_list[0].is_prompt
@@ -164,30 +197,31 @@ class NeuronModelRunner:
197
self.device,
198
self.pin_memory)
199
167
- return (input_tokens, input_positions, input_block_ids,
168
- sampling_metadata)
200
+ return ModelInputForNeuron(input_tokens=input_tokens,
201
+ input_positions=input_positions,
202
+ input_block_ids=input_block_ids,
203
+ sampling_metadata=sampling_metadata)
204
205
@torch.inference_mode()
206
def execute_model(
207
self,
173
- seq_group_metadata_list: List[SequenceGroupMetadata],
208
+ model_input: ModelInputForNeuron,
209
+ kv_caches: Optional[List[torch.Tensor]] = None,
210
) -> Optional[SamplerOutput]:
175
- (input_tokens, input_positions, input_block_ids, sampling_metadata
176
- ) = self.prepare_input_tensors(seq_group_metadata_list)
177
-
211
hidden_states = self.model(
179
- input_ids=input_tokens,
180
- positions=input_positions,
181
- input_block_ids=input_block_ids,
212
+ input_ids=model_input.input_tokens,
213
+ positions=model_input.input_positions,
214
+ input_block_ids=model_input.input_block_ids,
215
)
216
217
# Compute the logits.
185
- logits = self.model.compute_logits(hidden_states, sampling_metadata)
218
+ logits = self.model.compute_logits(hidden_states,
219
+ model_input.sampling_metadata)
220
221
# Sample the next token.
222
output = self.model.sample(
223
logits=logits,
190
- sampling_metadata=sampling_metadata,
224
+ sampling_metadata=model_input.sampling_metadata,
225
)
226
return output
227
vllm/worker/neuron_worker.py
+19
-20
@@ -1,5 +1,5 @@
1
"""A Neuron worker class."""
2
-from typing import List, Tuple
2
+from typing import List, Optional, Tuple
3
4
import torch
5
import torch.distributed
@@ -7,12 +7,13 @@ import torch.distributed
7
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8
ParallelConfig, SchedulerConfig)
9
from vllm.model_executor import set_random_seed
10
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10
+from vllm.sequence import ExecuteModelRequest
11
from vllm.worker.neuron_model_runner import NeuronModelRunner
12
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
12
+from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
13
+ LoraNotSupportedWorkerBase, WorkerInput)
14
15
15
-class NeuronWorker(LoraNotSupportedWorkerBase):
16
+class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
17
"""A worker class that executes the model on a group of neuron cores.
18
"""
19
@@ -34,8 +35,9 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
35
from vllm.utils import init_cached_hf_modules
36
init_cached_hf_modules()
37
37
- self.model_runner = NeuronModelRunner(model_config, parallel_config,
38
- scheduler_config, device_config)
38
+ self.model_runner: NeuronModelRunner = NeuronModelRunner(
39
+ model_config, parallel_config, scheduler_config, device_config)
40
+ self.is_driver_worker = True
41
42
def init_device(self) -> None:
43
# Set random seed.
@@ -73,22 +75,19 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
75
self.cache_config.num_gpu_blocks = num_gpu_blocks
76
self.cache_config.num_cpu_blocks = num_cpu_blocks
77
76
- @torch.inference_mode()
77
- def execute_model(
78
- self,
79
- seq_group_metadata_list: List[SequenceGroupMetadata],
80
- ) -> List[SamplerOutput]:
81
- num_seq_groups = len(seq_group_metadata_list)
78
+ @property
79
+ def do_metadata_broadcast(self) -> bool:
80
+ return False
81
83
- # If there is no input, we don't need to execute the model.
84
- if num_seq_groups == 0:
85
- return []
82
+ @property
83
+ def kv_cache(self) -> Optional[List[torch.Tensor]]:
84
+ return None
85
87
- output = self.model_runner.execute_model(seq_group_metadata_list)
88
-
89
- # Neuron worker only supports single-step output. Wrap the output in a
90
- # list to conform to interface.
91
- return [output]
86
+ @torch.inference_mode()
87
+ def prepare_worker_input(
88
+ self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
89
+ return WorkerInput(num_seq_groups=len(
90
+ execute_model_req.seq_group_metadata_list), )
91
92
def get_cache_block_size_bytes(self) -> int:
93
"""Determine the size in bytes of a cache block.
vllm/worker/worker.py
+37
-92
@@ -1,7 +1,7 @@
1
"""A GPU worker class."""
2
import gc
3
import os
4
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
4
+from typing import List, Optional, Set, Tuple, Type
5
6
import torch
7
import torch.distributed
@@ -9,21 +9,20 @@ import torch.distributed
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, ParallelConfig, SchedulerConfig,
11
SpeculativeConfig, VisionLanguageConfig)
12
-from vllm.distributed import (broadcast_tensor_dict,
13
- ensure_model_parallel_initialized,
12
+from vllm.distributed import (ensure_model_parallel_initialized,
13
init_distributed_environment,
14
set_custom_all_reduce)
15
from vllm.lora.request import LoRARequest
16
from vllm.model_executor import set_random_seed
17
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19
-from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
18
+from vllm.sequence import ExecuteModelRequest
19
from vllm.worker.cache_engine import CacheEngine
20
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22
-from vllm.worker.model_runner import ModelRunner
23
-from vllm.worker.worker_base import WorkerBase
21
+from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
22
+from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
23
24
26
-class Worker(WorkerBase):
25
+class Worker(LocalOrDistributedWorkerBase):
26
"""A worker class that executes (a partition of) the model on a GPU.
27
28
Each worker is associated with a single GPU. The worker is responsible for
@@ -78,9 +77,10 @@ class Worker(WorkerBase):
77
or (speculative_config.draft_model_config.hf_config.model_type !=
78
"mlp_speculator") else {"return_hidden_states": True}
79
81
- ModelRunnerClass = (EmbeddingModelRunner if
82
- self.model_config.embedding_mode else ModelRunner)
83
- self.model_runner = ModelRunnerClass(
80
+ ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
81
+ if self.model_config.embedding_mode:
82
+ ModelRunnerClass = EmbeddingModelRunner
83
+ self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
84
model_config,
85
parallel_config,
86
scheduler_config,
@@ -225,40 +225,18 @@ class Worker(WorkerBase):
225
# the model initialization and profiling.
226
set_random_seed(self.model_config.seed)
227
228
- def cache_swap(
229
- self,
230
- blocks_to_swap_in: torch.Tensor,
231
- blocks_to_swap_out: torch.Tensor,
232
- blocks_to_copy: torch.Tensor,
233
- ) -> None:
234
- # Issue cache operations.
235
- if blocks_to_swap_in.numel() > 0:
236
- self.cache_engine.swap_in(blocks_to_swap_in)
237
- if blocks_to_swap_out.numel() > 0:
238
- self.cache_engine.swap_out(blocks_to_swap_out)
239
- if blocks_to_copy.numel() > 0:
240
- self.cache_engine.copy(blocks_to_copy)
228
+ @property
229
+ def do_metadata_broadcast(self) -> bool:
230
+ return self.parallel_config.tensor_parallel_size > 1
231
+
232
+ @property
233
+ def kv_cache(self) -> Optional[List[torch.Tensor]]:
234
+ return self.gpu_cache
235
236
@torch.inference_mode()
243
- def execute_model(
244
- self,
245
- execute_model_req: Optional[ExecuteModelRequest] = None
246
- ) -> List[Union[SamplerOutput, PoolerOutput]]:
247
- if not self.is_driver_worker:
248
- self._execute_model_non_driver()
249
- return []
250
-
251
- if execute_model_req is None:
252
- # This signals that there's no more requests to process for now.
253
- # All workers are running infinite loop with broadcast_tensor_dict,
254
- # and it stops the loop when the driver broadcasts an empty input.
255
- # Send an empty input to notify all other workers to stop their
256
- # execution loop.
257
- broadcast_tensor_dict({}, src=0)
258
- return []
259
-
260
- seq_group_metadata_list = execute_model_req.seq_group_metadata_list
261
- num_seq_groups = len(seq_group_metadata_list)
237
+ def prepare_worker_input(
238
+ self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
239
+ num_seq_groups = len(execute_model_req.seq_group_metadata_list)
240
# `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
241
# they contain parameters to launch cudamemcpyasync.
242
blocks_to_swap_in = torch.tensor(execute_model_req.blocks_to_swap_in,
@@ -273,59 +251,26 @@ class Worker(WorkerBase):
251
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
252
device=self.device,
253
dtype=torch.int64).view(-1, 2)
276
- data: Dict[str, Any] = {
277
- "num_seq_groups": num_seq_groups,
278
- "blocks_to_swap_in": blocks_to_swap_in,
279
- "blocks_to_swap_out": blocks_to_swap_out,
280
- "blocks_to_copy": blocks_to_copy,
281
- }
282
- broadcast_tensor_dict(data, src=0)
283
-
284
- self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
285
-
286
- # If there is no input, we don't need to execute the model.
287
- if num_seq_groups == 0:
288
- return []
254
290
- output = self.model_runner.execute_model(seq_group_metadata_list,
291
- self.gpu_cache)
292
-
293
- # Worker only supports single-step execution. Wrap the output in a list
294
- # to conform to interface.
295
- return [output]
255
+ return WorkerInput(
256
+ num_seq_groups=num_seq_groups,
257
+ blocks_to_swap_in=blocks_to_swap_in,
258
+ blocks_to_swap_out=blocks_to_swap_out,
259
+ blocks_to_copy=blocks_to_copy,
260
+ )
261
262
@torch.inference_mode()
298
- def start_worker_execution_loop(self) -> None:
299
- """Execute model loop in parallel worker.
300
-
301
- You can stop the loop by executing a driver worker with an empty output.
302
- See `stop_remote_worker_execution_loop` for more details.
303
- """
304
- while self._execute_model_non_driver():
305
- pass
306
-
307
- def _execute_model_non_driver(self) -> bool:
308
- """Execute model in parallel worker.
309
-
310
- Returns True iff there are remaining sequences to process.
311
- """
312
- assert not self.is_driver_worker
313
- data = broadcast_tensor_dict(src=0)
314
- if not data:
315
- return False
316
-
317
- num_seq_groups = data.get("num_seq_groups", 0)
318
- blocks_to_swap_in = data.get("blocks_to_swap_in")
319
- blocks_to_swap_out = data.get("blocks_to_swap_out")
320
- blocks_to_copy = data.get("blocks_to_copy")
321
- self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
322
-
323
- # If there is no input, we don't need to execute the model.
324
- if num_seq_groups == 0:
325
- return False
326
-
327
- self.model_runner.execute_model(None, self.gpu_cache)
328
- return True
263
+ def execute_worker(self, worker_input: WorkerInput) -> None:
264
+ # Issue cache operations.
265
+ if (worker_input.blocks_to_swap_in is not None
266
+ and worker_input.blocks_to_swap_in.numel() > 0):
267
+ self.cache_engine.swap_in(worker_input.blocks_to_swap_in)
268
+ if (worker_input.blocks_to_swap_out is not None
269
+ and worker_input.blocks_to_swap_out.numel() > 0):
270
+ self.cache_engine.swap_out(worker_input.blocks_to_swap_out)
271
+ if (worker_input.blocks_to_copy is not None
272
+ and worker_input.blocks_to_copy.numel() > 0):
273
+ self.cache_engine.copy(worker_input.blocks_to_copy)
274
275
def add_lora(self, lora_request: LoRARequest) -> bool:
276
return self.model_runner.add_lora(lora_request)
vllm/worker/worker_base.py
+165
-5
@@ -1,20 +1,26 @@
1
+import dataclasses
2
import importlib
3
import os
4
from abc import ABC, abstractmethod
4
-from typing import Dict, List, Optional, Set, Tuple
5
+from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7
+import torch
8
+
9
+from vllm.distributed import broadcast_tensor_dict
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.sequence import ExecuteModelRequest, SamplerOutput
13
from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
14
update_environment_variables)
15
+from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
16
17
logger = init_logger(__name__)
18
19
20
class WorkerBase(ABC):
21
"""Worker interface that allows vLLM to cleanly separate implementations for
17
- different hardware.
22
+ different hardware. Also abstracts control plane communication, e.g., to
23
+ communicate request metadata to other workers.
24
"""
25
26
@abstractmethod
@@ -46,13 +52,23 @@ class WorkerBase(ABC):
52
"""
53
raise NotImplementedError
54
55
+ @torch.inference_mode()
56
+ def start_worker_execution_loop(self) -> None:
57
+ """Execute model loop in parallel worker.
58
+
59
+ You can stop the loop by executing a driver worker with an empty output.
60
+ See `stop_remote_worker_execution_loop` for more details.
61
+ """
62
+ while True:
63
+ output = self.execute_model(execute_model_req=None)
64
+ if output is None:
65
+ return None
66
+
67
@abstractmethod
68
def execute_model(
69
self,
70
execute_model_req: Optional[ExecuteModelRequest] = None
53
- ) -> List[SamplerOutput]:
54
- """Executes at least one model step on the given sequences, unless no
55
- sequences are provided."""
71
+ ) -> Optional[List[SamplerOutput]]:
72
raise NotImplementedError
73
74
@abstractmethod
@@ -98,6 +114,150 @@ class LoraNotSupportedWorkerBase(WorkerBase):
114
raise ValueError(f"{type(self)} does not support LoRA")
115
116
117
+@dataclasses.dataclass(frozen=True)
118
+class WorkerInput:
119
+ """Local inputs to each worker. May contain device-specific data. These
120
+ fields should be broadcastable to other workers.
121
+ """
122
+
123
+ num_seq_groups: Optional[int] = None
124
+ blocks_to_swap_in: Optional[torch.Tensor] = None
125
+ blocks_to_swap_out: Optional[torch.Tensor] = None
126
+ blocks_to_copy: Optional[torch.Tensor] = None
127
+
128
+ @classmethod
129
+ def from_broadcasted_tensor_dict(
130
+ cls: Type["WorkerInput"],
131
+ tensor_dict: Dict[str, Any],
132
+ ) -> "WorkerInput":
133
+ """
134
+ Pop fields from the given tensor_dict and populate a new instance of
135
+ WorkerInput.
136
+ """
137
+ return cls(
138
+ num_seq_groups=tensor_dict.pop("num_seq_groups"),
139
+ blocks_to_swap_in=tensor_dict.pop("blocks_to_swap_in"),
140
+ blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"),
141
+ blocks_to_copy=tensor_dict.pop("blocks_to_copy"),
142
+ )
143
+
144
+ def as_broadcastable_tensor_dict(
145
+ self) -> Dict[str, Union[int, torch.Tensor]]:
146
+ """
147
+ Extract broadcastable fields.
148
+ """
149
+ tensor_dict = {
150
+ "num_seq_groups": self.num_seq_groups,
151
+ "blocks_to_swap_in": self.blocks_to_swap_in,
152
+ "blocks_to_swap_out": self.blocks_to_swap_out,
153
+ "blocks_to_copy": self.blocks_to_copy,
154
+ }
155
+
156
+ return tensor_dict
157
+
158
+
159
+class LocalOrDistributedWorkerBase(WorkerBase):
160
+ """
161
+ Partial implementation of WorkerBase that has a default `execute_model`
162
+ definition to perform metadata transfer between workers when in distributed
163
+ mode. Subclasses of this interface should use model runners that inherit
164
+ from ModelRunnerBase, and should only need to implement worker-local logic.
165
+ If custom control plane logic is needed to transfer metadata, or if the
166
+ model runner cannot inherit from ModelRunnerBase, use WorkerBase instead.
167
+ """
168
+ is_driver_worker: bool
169
+ model_runner: ModelRunnerBase
170
+
171
+ @property
172
+ @abstractmethod
173
+ def do_metadata_broadcast(self) -> bool:
174
+ """
175
+ Used by the default `execute_model` to check whether broadcast is
176
+ needed to transfer request inputs from the driver worker to other
177
+ workers in the TP group. If WorkerBase subclass only supports
178
+ single-worker execution, then this method should return False.
179
+ """
180
+ raise NotImplementedError
181
+
182
+ @property
183
+ @abstractmethod
184
+ def kv_cache(self) -> Optional[List[torch.Tensor]]:
185
+ """
186
+ Get the kv cache to pass to the worker's model runner. Used by the
187
+ default `execute_model`. If the worker's model runner does not follow
188
+ the ModelRunnerBase interface, then inherit from WorkerBase instead.
189
+ """
190
+ raise NotImplementedError
191
+
192
+ @abstractmethod
193
+ def prepare_worker_input(
194
+ self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
195
+ """
196
+ Prepare the inputs to WorkerBase.execute_worker from an execution
197
+ request. This method may move data to the worker's local device. It is
198
+ not allowed to communicate with other workers or devices.
199
+ """
200
+ raise NotImplementedError
201
+
202
+ @abstractmethod
203
+ def execute_worker(self, worker_input: WorkerInput) -> None:
204
+ """
205
+ Process an execution request.
206
+ """
207
+ raise NotImplementedError
208
+
209
+ def execute_model(
210
+ self,
211
+ execute_model_req: Optional[ExecuteModelRequest] = None
212
+ ) -> Optional[List[SamplerOutput]]:
213
+ """Executes at least one model step on the given sequences, unless no
214
+ sequences are provided."""
215
+ if self.is_driver_worker:
216
+ if execute_model_req is None:
217
+ if self.do_metadata_broadcast:
218
+ # This signals that there's no more requests to process for
219
+ # now. All workers are running infinite loop with
220
+ # broadcast_tensor_dict, and it stops the loop when the
221
+ # driver broadcasts an empty input. Send an empty input to
222
+ # notify all other workers to stop their execution loop.
223
+ broadcast_tensor_dict({}, src=0)
224
+ return None
225
+
226
+ worker_input: WorkerInput = self.prepare_worker_input(
227
+ execute_model_req=execute_model_req)
228
+ model_input: ModelRunnerInputBase = (
229
+ self.model_runner.prepare_model_input(
230
+ execute_model_req.seq_group_metadata_list))
231
+
232
+ if self.do_metadata_broadcast:
233
+ broadcast_data = worker_input.as_broadcastable_tensor_dict()
234
+ broadcast_data.update(
235
+ model_input.as_broadcastable_tensor_dict())
236
+ broadcast_tensor_dict(broadcast_data, src=0)
237
+ else:
238
+ assert self.do_metadata_broadcast
239
+ broadcast_data = broadcast_tensor_dict(src=0)
240
+ if not broadcast_data:
241
+ return None
242
+
243
+ worker_input = WorkerInput.from_broadcasted_tensor_dict(
244
+ broadcast_data)
245
+ model_input = (
246
+ self.model_runner.
247
+ make_model_input_from_broadcasted_tensor_dict(broadcast_data))
248
+
249
+ self.execute_worker(worker_input)
250
+
251
+ # If there is no input, we don't need to execute the model.
252
+ if worker_input.num_seq_groups == 0:
253
+ return []
254
+
255
+ output = self.model_runner.execute_model(model_input, self.kv_cache)
256
+ # Worker only supports single-step execution. Wrap the output in a
257
+ # list to conform to interface.
258
+ return [output]
259
+
260
+
261
class WorkerWrapperBase:
262
"""
263
The whole point of this class is to lazily initialize the worker.
vllm/worker/xpu_model_runner.py
+72
-19
@@ -1,4 +1,5 @@
1
-from typing import List, Optional, Tuple
1
+from dataclasses import dataclass
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
import torch.nn as nn
@@ -14,6 +15,15 @@ from vllm.sampling_params import SamplingParams
15
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
16
from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
17
from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
18
+from vllm.worker.model_runner_base import (
19
+ ModelRunnerBase, ModelRunnerInputBase,
20
+ _add_attn_metadata_broadcastable_dict,
21
+ _add_sampling_metadata_broadcastable_dict,
22
+ _init_attn_metadata_from_tensor_dict,
23
+ _init_sampling_metadata_from_tensor_dict)
24
+
25
+if TYPE_CHECKING:
26
+ from vllm.attention.backends.abstract import AttentionBackend
27
28
logger = init_logger(__name__)
29
@@ -24,7 +34,42 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
34
]
35
36
27
-class XPUModelRunner:
37
+@dataclass(frozen=True)
38
+class ModelInputForXPU(ModelRunnerInputBase):
39
+ """
40
+ Used by the NeuronModelRunner.
41
+ """
42
+ input_tokens: Optional[torch.Tensor] = None
43
+ input_positions: Optional[torch.Tensor] = None
44
+ attn_metadata: Optional["AttentionMetadata"] = None
45
+ sampling_metadata: Optional["SamplingMetadata"] = None
46
+ multi_modal_input: Optional[Dict[str, torch.Tensor]] = None
47
+
48
+ def as_broadcastable_tensor_dict(
49
+ self) -> Dict[str, Union[int, torch.Tensor]]:
50
+ tensor_dict = {
51
+ "input_tokens": self.input_tokens,
52
+ "input_positions": self.input_positions,
53
+ }
54
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
55
+ _add_sampling_metadata_broadcastable_dict(tensor_dict,
56
+ self.sampling_metadata)
57
+ return tensor_dict
58
+
59
+ @classmethod
60
+ def from_broadcasted_tensor_dict(
61
+ cls: Type["ModelInputForXPU"],
62
+ tensor_dict: Dict[str, Any],
63
+ attn_backend: Optional["AttentionBackend"] = None,
64
+ ) -> "ModelInputForXPU":
65
+ tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
66
+ if attn_backend is not None:
67
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
68
+ attn_backend, tensor_dict)
69
+ return cls(**tensor_dict)
70
+
71
+
72
+class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
73
74
def __init__(
75
self,
@@ -130,15 +175,22 @@ class XPUModelRunner:
175
# Run the model with the dummy inputs.
176
num_layers = self.model_config.get_num_layers(self.parallel_config)
177
kv_caches = [None] * num_layers
133
- self.execute_model(seqs, kv_caches)
178
+ model_input = self.prepare_model_input(seqs)
179
+ self.execute_model(model_input, kv_caches)
180
torch.xpu.synchronize()
181
return
182
137
- def prepare_input_tensors(
183
+ def make_model_input_from_broadcasted_tensor_dict(
184
+ self, tensor_dict: Dict[str, Any]) -> ModelInputForXPU:
185
+ return (ModelInputForXPU.from_broadcasted_tensor_dict(
186
+ tensor_dict,
187
+ attn_backend=self.attn_backend,
188
+ ))
189
+
190
+ def prepare_model_input(
191
self,
192
seq_group_metadata_list: List[SequenceGroupMetadata],
140
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
141
- Optional[torch.Tensor]]:
193
+ ) -> ModelInputForXPU:
194
multi_modal_input = None
195
if self.is_driver_worker:
196
# NOTE: We assume that all sequences in the group are all prompts or
@@ -185,8 +237,11 @@ class XPUModelRunner:
237
num_prompts=0,
238
)
239
188
- return (input_tokens, input_positions, attn_metadata,
189
- sampling_metadata, multi_modal_input)
240
+ return ModelInputForXPU(input_tokens=input_tokens,
241
+ input_positions=input_positions,
242
+ attn_metadata=attn_metadata,
243
+ sampling_metadata=sampling_metadata,
244
+ multi_modal_input=multi_modal_input)
245
246
def _prepare_decode(
247
self,
@@ -277,27 +332,25 @@ class XPUModelRunner:
332
@torch.inference_mode()
333
def execute_model(
334
self,
280
- seq_group_metadata_list: List[SequenceGroupMetadata],
335
+ model_input: ModelInputForXPU,
336
kv_caches: List[torch.Tensor],
337
) -> Optional[SamplerOutput]:
283
- (input_tokens, input_positions, attn_metadata, sampling_metadata,
284
- multi_modal_input
285
- ) = self.prepare_input_tensors(seq_group_metadata_list)
286
-
338
model_executable = self.model
339
execute_model_kwargs = {
289
- "input_ids": input_tokens,
290
- "positions": input_positions,
340
+ "input_ids": model_input.input_tokens,
341
+ "positions": model_input.input_positions,
342
"kv_caches": kv_caches,
292
- "attn_metadata": attn_metadata,
343
+ "attn_metadata": model_input.attn_metadata,
344
}
345
if self.vision_language_config:
295
- execute_model_kwargs.update({"image_input": multi_modal_input})
346
+ execute_model_kwargs.update(
347
+ {"image_input": model_input.multi_modal_input})
348
349
hidden_states = model_executable(**execute_model_kwargs)
350
351
# Compute the logits.
300
- logits = self.model.compute_logits(hidden_states, sampling_metadata)
352
+ logits = self.model.compute_logits(hidden_states,
353
+ model_input.sampling_metadata)
354
355
# Only perform sampling in the driver worker.
356
if not self.is_driver_worker:
@@ -306,7 +359,7 @@ class XPUModelRunner:
359
# Sample the next token.
360
output = self.model.sample(
361
logits=logits,
309
- sampling_metadata=sampling_metadata,
362
+ sampling_metadata=model_input.sampling_metadata,
363
)
364
return output
365