@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dda4811591
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Core] Refactor Worker and ModelRunner to consolidate control plane communication (#5408)

Signed-off-by: Stephanie Wang <swang@cs.berkeley.edu> Signed-off-by: Stephanie <swang@anyscale.com> Co-authored-by: Stephanie <swang@anyscale.com>

Stephanie Wang committed Jun 25, 2024 at 20:30 UTC dda4811591fdb90d263bc9b8ac522436369aef13
29 files changed +1108 -575
tests/worker/test_model_input.py new
+152
@@ -0,0 +1,152 @@
1 +import dataclasses
2 +from typing import List, Tuple, Type
3 +
4 +import torch
5 +
6 +from vllm.attention import AttentionMetadata
7 +from vllm.attention.backends.abstract import AttentionBackend
8 +from vllm.model_executor import SamplingMetadata
9 +from vllm.model_executor.pooling_metadata import PoolingMetadata
10 +from vllm.worker.embedding_model_runner import (
11 + ModelInputForGPUWithPoolingMetadata)
12 +from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
13 +
14 +
15 +class MockAttentionBackend(AttentionBackend):
16 +
17 + @staticmethod
18 + def get_name() -> str:
19 + raise NotImplementedError
20 +
21 + @staticmethod
22 + def get_impl_cls():
23 + raise NotImplementedError
24 +
25 + @staticmethod
26 + def get_metadata_cls() -> Type["AttentionMetadata"]:
27 + return AttentionMetadata
28 +
29 + @staticmethod
30 + def get_kv_cache_shape(
31 + num_blocks: int,
32 + block_size: int,
33 + num_kv_heads: int,
34 + head_size: int,
35 + ) -> Tuple[int, ...]:
36 + raise NotImplementedError
37 +
38 + @staticmethod
39 + def swap_blocks(
40 + src_kv_cache: torch.Tensor,
41 + dst_kv_cache: torch.Tensor,
42 + src_to_dst: torch.Tensor,
43 + ) -> None:
44 + pass
45 +
46 + @staticmethod
47 + def copy_blocks(
48 + kv_caches: List[torch.Tensor],
49 + src_to_dists: torch.Tensor,
50 + ) -> None:
51 + pass
52 +
53 +
54 +def test_model_runner_input():
55 + sampling_metadata = SamplingMetadata(
56 + ["seq_group"],
57 + "selected_token_indices",
58 + "categorized_sample_indices",
59 + "num_prompts",
60 + )
61 + attn_metadata = AttentionMetadata(
62 + num_prefills=1,
63 + num_prefill_tokens=2,
64 + num_decode_tokens=3,
65 + slot_mapping=torch.zeros(1),
66 + )
67 + model_input = ModelInputForGPUWithSamplingMetadata(
68 + input_tokens=torch.ones(10),
69 + input_positions=torch.ones(10),
70 + sampling_metadata=sampling_metadata,
71 + attn_metadata=attn_metadata)
72 +
73 + assert isinstance(model_input, ModelInputForGPUWithSamplingMetadata)
74 +
75 + # Test round trip serialization.
76 + tensor_dict = model_input.as_broadcastable_tensor_dict()
77 + attn_backend = MockAttentionBackend()
78 + received_model_input = (
79 + ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
80 + tensor_dict, attn_backend=attn_backend))
81 + # Check that received copy has correct values.
82 + assert isinstance(received_model_input,
83 + ModelInputForGPUWithSamplingMetadata)
84 + assert received_model_input.input_tokens is not None
85 + assert (
86 + received_model_input.input_tokens == model_input.input_tokens).all()
87 + assert received_model_input.input_positions is not None
88 + assert (received_model_input.input_positions == model_input.input_positions
89 + ).all()
90 + assert received_model_input.multi_modal_kwargs is None
91 + assert (received_model_input.multi_modal_kwargs ==
92 + model_input.multi_modal_kwargs)
93 + assert received_model_input.lora_requests is None
94 + assert received_model_input.lora_requests == model_input.lora_requests
95 + assert received_model_input.lora_mapping is None
96 + assert received_model_input.lora_mapping == model_input.lora_mapping
97 + for field in dataclasses.fields(AttentionMetadata):
98 + assert getattr(received_model_input.attn_metadata, field.name,
99 + None) == getattr(attn_metadata, field.name, None)
100 + # For sampling metadata, only selected_token_indices is copied.
101 + assert (received_model_input.sampling_metadata.selected_token_indices ==
102 + sampling_metadata.selected_token_indices)
103 + assert received_model_input.sampling_metadata.seq_groups is None
104 +
105 +
106 +def test_embedding_model_runner_input():
107 + pooling_metadata = PoolingMetadata(
108 + seq_groups=[[0]],
109 + seq_data={},
110 + prompt_lens=[1],
111 + )
112 + attn_metadata = AttentionMetadata(
113 + num_prefills=1,
114 + num_prefill_tokens=2,
115 + num_decode_tokens=3,
116 + slot_mapping=torch.zeros(1),
117 + )
118 + model_input = ModelInputForGPUWithPoolingMetadata(
119 + input_tokens=torch.ones(10),
120 + input_positions=torch.ones(10),
121 + pooling_metadata=pooling_metadata,
122 + attn_metadata=attn_metadata)
123 +
124 + assert isinstance(model_input, ModelInputForGPUWithPoolingMetadata)
125 +
126 + # Test round trip serialization.
127 + tensor_dict = model_input.as_broadcastable_tensor_dict()
128 + attn_backend = MockAttentionBackend()
129 + received_model_input = (
130 + ModelInputForGPUWithPoolingMetadata.from_broadcasted_tensor_dict(
131 + tensor_dict, attn_backend=attn_backend))
132 + # Check that received copy has correct values.
133 + assert isinstance(received_model_input,
134 + ModelInputForGPUWithPoolingMetadata)
135 + assert received_model_input.input_tokens is not None
136 + assert (
137 + received_model_input.input_tokens == model_input.input_tokens).all()
138 + assert received_model_input.input_positions is not None
139 + assert (received_model_input.input_positions == model_input.input_positions
140 + ).all()
141 + assert received_model_input.multi_modal_kwargs is None
142 + assert (received_model_input.multi_modal_kwargs ==
143 + model_input.multi_modal_kwargs)
144 + assert received_model_input.lora_requests is None
145 + assert received_model_input.lora_requests == model_input.lora_requests
146 + assert received_model_input.lora_mapping is None
147 + assert received_model_input.lora_mapping == model_input.lora_mapping
148 + for field in dataclasses.fields(AttentionMetadata):
149 + assert getattr(received_model_input.attn_metadata, field.name,
150 + None) == getattr(attn_metadata, field.name, None)
151 + # Pooling metadata is not broadcast.
152 + assert received_model_input.pooling_metadata is None
tests/worker/test_model_runner.py
+30 -27
@@ -61,12 +61,13 @@ def test_prepare_prompt(batch_size):
61 expected_selected_token_indices.append(selected_token_start_idx +
62 seq_len - 1)
63 selected_token_start_idx += seq_len
64 - model_input = model_runner._prepare_model_input(seq_group_metadata_list)
64 + model_input = model_runner._prepare_model_input_tensors(
65 + seq_group_metadata_list)
66 input_tokens = model_input.input_tokens
67 input_positions = model_input.input_positions
68 attn_metadata = model_input.attn_metadata
69 return_seq_lens = model_input.seq_lens
69 - slot_mapping = model_input.slot_mapping
70 + slot_mapping = attn_metadata.slot_mapping
71 assert return_seq_lens == seq_lens
72 assert len(slot_mapping) == len(input_tokens)
73
@@ -174,10 +175,11 @@ def test_prepare_decode_cuda_graph(batch_size):
175 assert seq_group_metadata.token_chunk_size == 1
176 seq_group_metadata_list.append(seq_group_metadata)
177
177 - model_input = model_runner._prepare_model_input(seq_group_metadata_list)
178 + model_input = model_runner._prepare_model_input_tensors(
179 + seq_group_metadata_list)
180 input_tokens, input_positions, attn_metadata, slot_mapping = (
181 model_input.input_tokens, model_input.input_positions,
180 - model_input.attn_metadata, model_input.slot_mapping)
182 + model_input.attn_metadata, model_input.attn_metadata.slot_mapping)
183 assert len(slot_mapping) == len(input_tokens)
184
185 expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
@@ -259,32 +261,29 @@ def test_empty_seq_group():
261 enforce_eager=False,
262 )
263 seq_group_metadata_list: List[SequenceGroupMetadata] = []
262 - model_input = model_runner._prepare_model_input(seq_group_metadata_list)
263 - input_tokens, input_positions, attn_metadata, slot_mapping = (
264 + model_input = model_runner._prepare_model_input_tensors(
265 + seq_group_metadata_list)
266 + input_tokens, input_positions, attn_metadata = (
267 model_input.input_tokens,
268 model_input.input_positions,
269 model_input.attn_metadata,
267 - model_input.slot_mapping,
270 )
269 - assert len(input_tokens) == 0
270 - assert len(input_positions) == 0
271 + assert input_tokens is None
272 + assert input_positions is None
273 assert attn_metadata is None
272 - assert len(slot_mapping) == 0
273 -
274 - model_input = model_runner._prepare_model_input(seq_group_metadata_list)
275 - (input_tokens, input_positions, attn_metadata, slot_mapping,
276 - return_seq_lens) = (
277 - model_input.input_tokens,
278 - model_input.input_positions,
279 - model_input.attn_metadata,
280 - model_input.slot_mapping,
281 - model_input.seq_lens,
282 - )
283 - assert len(input_tokens) == 0
284 - assert len(input_positions) == 0
274 +
275 + model_input = model_runner._prepare_model_input_tensors(
276 + seq_group_metadata_list)
277 + (input_tokens, input_positions, attn_metadata, return_seq_lens) = (
278 + model_input.input_tokens,
279 + model_input.input_positions,
280 + model_input.attn_metadata,
281 + model_input.seq_lens,
282 + )
283 + assert input_tokens is None
284 + assert input_positions is None
285 assert attn_metadata is None
286 - assert len(slot_mapping) == 0
287 - assert len(return_seq_lens) == 0
286 + assert return_seq_lens is None
287
288
289 @pytest.fixture
@@ -353,8 +352,12 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
352 seq_group_metadata_list.append(seq_group_metadata)
353 decode_metadata_list.append(seq_group_metadata)
354
356 - (input_tokens, input_positions, attn_metadata, _, _, _,
357 - _) = model_runner.prepare_input_tensors(seq_group_metadata_list)
355 + model_input = model_runner.prepare_model_input(seq_group_metadata_list)
356 + (input_tokens, input_positions, attn_metadata) = (
357 + model_input.input_tokens,
358 + model_input.input_positions,
359 + model_input.attn_metadata,
360 + )
361
362 prefill_meta_actual = attn_metadata.prefill_metadata
363 decode_meta_actual = attn_metadata.decode_metadata
@@ -367,7 +370,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
370
371 # Verify attn metadata is consistent. We don't need to test individual
372 # values here because they are tested above.
370 - attn_metadata = model_runner._prepare_model_input(
373 + attn_metadata = model_runner._prepare_model_input_tensors(
374 seq_group_metadata_list).attn_metadata
375
376 for attr_expected, attr_actual in zip(vars(attn_metadata.prefill_metadata),
vllm/attention/backends/abstract.py
+5 -1
@@ -21,9 +21,13 @@ class AttentionBackend(ABC):
21
22 @staticmethod
23 @abstractmethod
24 - def make_metadata(*args, **kwargs) -> "AttentionMetadata":
24 + def get_metadata_cls() -> Type["AttentionMetadata"]:
25 raise NotImplementedError
26
27 + @classmethod
28 + def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":
29 + return cls.get_metadata_cls()(*args, **kwargs)
30 +
31 @staticmethod
32 @abstractmethod
33 def get_kv_cache_shape(
vllm/attention/backends/blocksparse_attn.py
+2 -2
@@ -90,8 +90,8 @@ class BlocksparseFlashAttentionBackend(AttentionBackend):
90 return BlocksparseFlashAttentionImpl
91
92 @staticmethod
93 - def make_metadata(*args, **kwargs) -> "BlocksparseFlashAttentionMetadata":
94 - return BlocksparseFlashAttentionMetadata(*args, **kwargs)
93 + def get_metadata_cls() -> Type["AttentionMetadata"]:
94 + return BlocksparseFlashAttentionMetadata
95
96 @staticmethod
97 def get_kv_cache_shape(
vllm/attention/backends/flash_attn.py
+2 -2
@@ -25,8 +25,8 @@ class FlashAttentionBackend(AttentionBackend):
25 return FlashAttentionImpl
26
27 @staticmethod
28 - def make_metadata(*args, **kwargs) -> "FlashAttentionMetadata":
29 - return FlashAttentionMetadata(*args, **kwargs)
28 + def get_metadata_cls() -> Type["AttentionMetadata"]:
29 + return FlashAttentionMetadata
30
31 @staticmethod
32 def get_kv_cache_shape(
vllm/attention/backends/flashinfer.py
+2 -2
@@ -22,8 +22,8 @@ class FlashInferBackend(AttentionBackend):
22 return FlashInferImpl
23
24 @staticmethod
25 - def make_metadata(*args, **kwargs) -> "FlashInferMetadata":
26 - return FlashInferMetadata(*args, **kwargs)
25 + def get_metadata_cls() -> Type["AttentionMetadata"]:
26 + return FlashInferMetadata
27
28 @staticmethod
29 def get_kv_cache_shape(
vllm/attention/backends/ipex_attn.py
+2 -2
@@ -25,8 +25,8 @@ class IpexAttnBackend(AttentionBackend):
25 return IpexAttnBackendImpl
26
27 @staticmethod
28 - def make_metadata(*args, **kwargs) -> "IpexAttnMetadata":
29 - return IpexAttnMetadata(*args, **kwargs)
28 + def get_metadata_cls() -> Type["IpexAttnMetadata"]:
29 + return IpexAttnMetadata
30
31 @staticmethod
32 def get_kv_cache_shape(
vllm/attention/backends/pallas.py
+2 -2
@@ -16,8 +16,8 @@ class PallasAttentionBackend(AttentionBackend):
16 return PallasAttentionBackendImpl
17
18 @staticmethod
19 - def make_metadata(*args, **kwargs) -> "PallasMetadata":
20 - return PallasMetadata(*args, **kwargs)
19 + def get_metadata_cls() -> Type["PallasMetadata"]:
20 + return PallasMetadata
21
22 @staticmethod
23 def get_kv_cache_shape(
vllm/attention/backends/rocm_flash_attn.py
+2 -2
@@ -25,8 +25,8 @@ class ROCmFlashAttentionBackend(AttentionBackend):
25 return ROCmFlashAttentionImpl
26
27 @staticmethod
28 - def make_metadata(*args, **kwargs) -> "ROCmFlashAttentionMetadata":
29 - return ROCmFlashAttentionMetadata(*args, **kwargs)
28 + def get_metadata_cls() -> Type["AttentionMetadata"]:
29 + return ROCmFlashAttentionMetadata
30
31 @staticmethod
32 def get_kv_cache_shape(
vllm/attention/backends/torch_sdpa.py
+2 -2
@@ -31,8 +31,8 @@ class TorchSDPABackend(AttentionBackend):
31 return TorchSDPABackendImpl
32
33 @staticmethod
34 - def make_metadata(*args, **kwargs) -> "TorchSDPAMetadata":
35 - return TorchSDPAMetadata(*args, **kwargs)
34 + def get_metadata_cls() -> Type["AttentionMetadata"]:
35 + return TorchSDPAMetadata
36
37 @staticmethod
38 def get_kv_cache_shape(
vllm/attention/backends/xformers.py
+2 -2
@@ -28,8 +28,8 @@ class XFormersBackend(AttentionBackend):
28 return XFormersImpl
29
30 @staticmethod
31 - def make_metadata(*args, **kwargs) -> "XFormersMetadata":
32 - return XFormersMetadata(*args, **kwargs)
31 + def get_metadata_cls() -> Type["AttentionMetadata"]:
32 + return XFormersMetadata
33
34 @staticmethod
35 def get_kv_cache_shape(
vllm/executor/distributed_gpu_executor.py
+8 -8
@@ -64,8 +64,8 @@ class DistributedGPUExecutor(GPUExecutor):
64 num_cpu_blocks=num_cpu_blocks)
65
66 def execute_model(
67 - self,
68 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
67 + self, execute_model_req: ExecuteModelRequest
68 + ) -> Optional[List[SamplerOutput]]:
69 if self.parallel_worker_tasks is None:
70 self.parallel_worker_tasks = self._run_workers(
71 "start_worker_execution_loop",
@@ -79,7 +79,7 @@ class DistributedGPUExecutor(GPUExecutor):
79 if self.parallel_worker_tasks is None:
80 return
81
82 - self._driver_execute_model()
82 + self._driver_execute_model(execute_model_req=None)
83 parallel_worker_tasks = self.parallel_worker_tasks
84 self.parallel_worker_tasks = None
85 # Ensure that workers exit model loop cleanly
@@ -123,13 +123,13 @@ class DistributedGPUExecutor(GPUExecutor):
123
124 @abstractmethod
125 def _driver_execute_model(
126 - self,
127 - execute_model_req: Optional[ExecuteModelRequest] = None
128 - ) -> List[SamplerOutput]:
126 + self, execute_model_req: Optional[ExecuteModelRequest]
127 + ) -> Optional[List[SamplerOutput]]:
128 """Run execute_model in the driver worker.
129
131 - Passing None will cause the driver to stop the model execution
132 - loop running in each of the remote workers.
130 + Passing None will cause the driver to stop the model execution loop
131 + running in each of the remote workers. In this case, this method
132 + returns None. Otherwise, this method returns the model output.
133 """
134 raise NotImplementedError
135
vllm/executor/executor_base.py
+2 -2
@@ -69,8 +69,8 @@ class ExecutorBase(ABC):
69
70 @abstractmethod
71 def execute_model(
72 - self,
73 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
72 + self, execute_model_req: ExecuteModelRequest
73 + ) -> Optional[List[SamplerOutput]]:
74 """Executes at least one model step on the given sequences."""
75 raise NotImplementedError
76
vllm/executor/gpu_executor.py
+1 -1
@@ -87,7 +87,7 @@ class GPUExecutor(ExecutorBase):
87
88 def execute_model(
89 self, execute_model_req: ExecuteModelRequest
90 - ) -> List[Union[SamplerOutput, PoolerOutput]]:
90 + ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
91 output = self.driver_worker.execute_model(execute_model_req)
92 return output
93
vllm/executor/multiproc_gpu_executor.py
+3 -5
@@ -78,16 +78,14 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
78 worker_monitor.close()
79
80 def _driver_execute_model(
81 - self,
82 - execute_model_req: Optional[ExecuteModelRequest] = None
83 - ) -> List[SamplerOutput]:
81 + self, execute_model_req: Optional[ExecuteModelRequest]
82 + ) -> Optional[List[SamplerOutput]]:
83 """Run execute_model in the driver worker.
84
85 Passing None will cause the driver to stop the model execution
86 loop running in each of the remote workers.
87 """
89 - return self.driver_worker.execute_model(
90 - execute_model_req=execute_model_req)
88 + return self.driver_worker.execute_model(execute_model_req)
89
90 def _run_workers(
91 self,
vllm/executor/neuron_executor.py
+1 -2
@@ -55,8 +55,7 @@ class NeuronExecutor(ExecutorBase):
55 assert execute_model_req.num_lookahead_slots == 0, (
56 "lookahead not supported for Neuron backend.")
57
58 - output = self.driver_worker.execute_model(
59 - execute_model_req.seq_group_metadata_list)
58 + output = self.driver_worker.execute_model(execute_model_req)
59 return output
60
61 def add_lora(self, lora_request: LoRARequest) -> bool:
vllm/executor/ray_gpu_executor.py
+2 -3
@@ -190,9 +190,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
190 max_parallel_loading_workers)
191
192 def _driver_execute_model(
193 - self,
194 - execute_model_req: Optional[ExecuteModelRequest] = None
195 - ) -> List[SamplerOutput]:
193 + self, execute_model_req: Optional[ExecuteModelRequest]
194 + ) -> Optional[List[SamplerOutput]]:
195 """Run execute_model in the driver worker.
196
197 Passing None will cause the driver to stop the model execution
vllm/sequence.py
+2 -1
@@ -887,7 +887,8 @@ class HiddenStates:
887
888 @dataclass
889 class ExecuteModelRequest:
890 - """The model execution request."""
890 + """The model execution request, containing CPU metadata only. The LLM
891 + engine should create an instance of this class for each request batch."""
892 # The sequence group metadata list.
893 seq_group_metadata_list: List[SequenceGroupMetadata]
894 # Blocks to swap in. List of CPU -> GPU block number.
vllm/spec_decode/mlp_speculator_worker.py
+1 -2
@@ -7,7 +7,6 @@ from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7 SequenceGroupMetadata)
8 from vllm.spec_decode.multi_step_worker import MultiStepWorker
9 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10 -from vllm.worker.model_runner import ModelInput
10
11
12 class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
@@ -56,7 +55,7 @@ class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
55 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
56 ) -> Tuple[torch.Tensor, List[int], List[int]]:
57 if not seq_group_metadata_list:
59 - return ModelInput.empty(self.device)
58 + return torch.empty(0, device=self.device), [], []
59
60 input_tokens: List[int] = []
61 seq_lens: List[int] = []
vllm/worker/cpu_model_runner.py
+95 -66
@@ -1,5 +1,6 @@
1 from collections import defaultdict
2 -from typing import Dict, List, Optional, Tuple
2 +from dataclasses import dataclass
3 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
4
5 import torch
6 from torch import nn
@@ -8,20 +9,64 @@ from vllm.attention import AttentionMetadata, get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, ParallelConfig, SchedulerConfig,
11 VisionLanguageConfig)
11 -from vllm.distributed import broadcast_tensor_dict
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader import get_model
15 from vllm.multimodal import MULTIMODAL_REGISTRY
16 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17 from vllm.utils import make_tensor_with_pad
18 +from vllm.worker.model_runner_base import (
19 + ModelRunnerBase, ModelRunnerInputBase,
20 + _add_attn_metadata_broadcastable_dict,
21 + _add_sampling_metadata_broadcastable_dict,
22 + _init_attn_metadata_from_tensor_dict,
23 + _init_sampling_metadata_from_tensor_dict)
24 +
25 +if TYPE_CHECKING:
26 + from vllm.attention.backends.abstract import AttentionBackend
27
28 logger = init_logger(__name__)
29
30 _PAD_SLOT_ID = -1
31
32
24 -class CPUModelRunner:
33 +@dataclass(frozen=True)
34 +class CPUModelInput(ModelRunnerInputBase):
35 + """
36 + Used by the CPUModelRunner.
37 + """
38 + input_tokens: Optional[torch.Tensor] = None
39 + input_positions: Optional[torch.Tensor] = None
40 + attn_metadata: Optional["AttentionMetadata"] = None
41 + sampling_metadata: Optional["SamplingMetadata"] = None
42 + multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
43 +
44 + def as_broadcastable_tensor_dict(
45 + self) -> Dict[str, Union[int, torch.Tensor]]:
46 + tensor_dict = {
47 + "input_tokens": self.input_tokens,
48 + "input_positions": self.input_positions,
49 + "multi_modal_kwargs": self.multi_modal_kwargs,
50 + }
51 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
52 + _add_sampling_metadata_broadcastable_dict(tensor_dict,
53 + self.sampling_metadata)
54 + return tensor_dict
55 +
56 + @classmethod
57 + def from_broadcasted_tensor_dict(
58 + cls: Type["CPUModelInput"],
59 + tensor_dict: Dict[str, Any],
60 + attn_backend: Optional["AttentionBackend"] = None
61 + ) -> "CPUModelInput":
62 + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
63 + if attn_backend is not None:
64 + tensor_dict = _init_attn_metadata_from_tensor_dict(
65 + attn_backend, tensor_dict)
66 + return cls(**tensor_dict)
67 +
68 +
69 +class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
70
71 def __init__(
72 self,
@@ -270,86 +315,70 @@ class CPUModelRunner:
315 attn_metadata,
316 )
317
273 - def prepare_input_tensors(
318 + def make_model_input_from_broadcasted_tensor_dict(
319 + self,
320 + tensor_dict: Dict[str, Any],
321 + ) -> CPUModelInput:
322 + return CPUModelInput.from_broadcasted_tensor_dict(
323 + tensor_dict,
324 + attn_backend=self.attn_backend,
325 + )
326 +
327 + def prepare_model_input(
328 self,
329 seq_group_metadata_list: List[SequenceGroupMetadata],
276 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
277 - Optional[Dict[str, torch.Tensor]]]:
330 + ) -> CPUModelInput:
331 multi_modal_kwargs = None
279 - if self.is_driver_worker:
280 - # NOTE: We assume that all sequences in the group are all prompts or
281 - # all decodes.
282 - is_prompt = seq_group_metadata_list[0].is_prompt
283 - # Prepare input tensors.
284 - if is_prompt:
285 - (input_tokens, input_positions, attn_metadata, seq_lens,
286 - multi_modal_kwargs
287 - ) = self._prepare_prompt(seq_group_metadata_list)
288 - else:
289 - (input_tokens, input_positions,
290 - attn_metadata) = self._prepare_decode(seq_group_metadata_list)
291 - seq_lens = []
292 - sampling_metadata = SamplingMetadata.prepare(
293 - seq_group_metadata_list,
294 - seq_lens,
295 - # query_lens is not needed if chunked prefill is not
296 - # supported. Since CPU worker doesn't support chunked prefill
297 - # just use seq_lens instead.
298 - seq_lens,
299 - self.device,
300 - pin_memory=False)
301 - # Broadcast the metadata.
302 - metadata_dict = {
303 - "input_tokens": input_tokens,
304 - "input_positions": input_positions,
305 - "selected_token_indices":
306 - sampling_metadata.selected_token_indices,
307 - }
308 - metadata_dict.update(attn_metadata.asdict_zerocopy())
309 - broadcast_tensor_dict(metadata_dict, src=0)
332 + # NOTE: We assume that all sequences in the group are all prompts or
333 + # all decodes.
334 + is_prompt = seq_group_metadata_list[0].is_prompt
335 + # Prepare input tensors.
336 + if is_prompt:
337 + (input_tokens, input_positions, attn_metadata, seq_lens,
338 + multi_modal_kwargs
339 + ) = self._prepare_prompt(seq_group_metadata_list)
340 else:
311 - metadata_dict = broadcast_tensor_dict(src=0)
312 - input_tokens = metadata_dict.pop("input_tokens")
313 - input_positions = metadata_dict.pop("input_positions")
314 - selected_token_indices = metadata_dict.pop(
315 - "selected_token_indices")
316 - attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
317 - sampling_metadata = SamplingMetadata(
318 - seq_groups=None,
319 - seq_data=None,
320 - seq_lens=None,
321 - selected_token_indices=selected_token_indices,
322 - categorized_sample_indices=None,
323 - generators=None,
324 - )
325 -
326 - return (input_tokens, input_positions, attn_metadata,
327 - sampling_metadata, multi_modal_kwargs)
341 + (input_tokens, input_positions,
342 + attn_metadata) = self._prepare_decode(seq_group_metadata_list)
343 + seq_lens = []
344 + sampling_metadata = SamplingMetadata.prepare(
345 + seq_group_metadata_list,
346 + seq_lens,
347 + # query_lens is not needed if chunked prefill is not
348 + # supported. Since CPU worker doesn't support chunked prefill
349 + # just use seq_lens instead.
350 + seq_lens,
351 + self.device,
352 + pin_memory=False)
353 + return CPUModelInput(
354 + input_tokens=input_tokens,
355 + input_positions=input_positions,
356 + attn_metadata=attn_metadata,
357 + sampling_metadata=sampling_metadata,
358 + )
359
360 @torch.inference_mode()
361 def execute_model(
362 self,
332 - seq_group_metadata_list: List[SequenceGroupMetadata],
363 + model_input: CPUModelInput,
364 kv_caches: List[torch.Tensor],
365 ) -> Optional[SamplerOutput]:
335 - (input_tokens, input_positions, attn_metadata, sampling_metadata,
336 - multi_modal_input
337 - ) = self.prepare_input_tensors(seq_group_metadata_list)
338 -
366 model_executable = self.model
367 execute_model_kwargs = {
341 - "input_ids": input_tokens,
342 - "positions": input_positions,
368 + "input_ids": model_input.input_tokens,
369 + "positions": model_input.input_positions,
370 "kv_caches": kv_caches,
344 - "attn_metadata": attn_metadata,
371 + "attn_metadata": model_input.attn_metadata,
372 }
346 - if self.vision_language_config and multi_modal_input is not None:
347 - execute_model_kwargs.update(multi_modal_input)
373 + if (self.vision_language_config
374 + and model_input.multi_modal_kwargs is not None):
375 + execute_model_kwargs.update(model_input.multi_modal_kwargs)
376
377 hidden_states = model_executable(**execute_model_kwargs)
378
379 # Compute the logits.
352 - logits = self.model.compute_logits(hidden_states, sampling_metadata)
380 + logits = self.model.compute_logits(hidden_states,
381 + model_input.sampling_metadata)
382
383 # Only perform sampling in the driver worker.
384 if not self.is_driver_worker:
@@ -358,6 +387,6 @@ class CPUModelRunner:
387 # Sample the next token.
388 output = self.model.sample(
389 logits=logits,
361 - sampling_metadata=sampling_metadata,
390 + sampling_metadata=model_input.sampling_metadata,
391 )
392 return output
vllm/worker/cpu_worker.py
+34 -51
@@ -1,5 +1,5 @@
1 """A CPU worker class."""
2 -from typing import Any, Dict, List, Optional, Tuple
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5 import torch.distributed
@@ -8,15 +8,15 @@ from vllm.attention import get_attn_backend
8 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 ModelConfig, ParallelConfig, SchedulerConfig,
10 VisionLanguageConfig)
11 -from vllm.distributed import (broadcast_tensor_dict,
12 - ensure_model_parallel_initialized,
11 +from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment)
13 from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
16 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
15 +from vllm.sequence import ExecuteModelRequest
16 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17 from vllm.worker.cpu_model_runner import CPUModelRunner
19 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
18 +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
19 + LoraNotSupportedWorkerBase, WorkerInput)
20
21 logger = init_logger(__name__)
22
@@ -110,7 +110,7 @@ class CPUCacheEngine:
110 return dtype_size * total
111
112
113 -class CPUWorker(LoraNotSupportedWorkerBase):
113 +class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
114 """A worker class that executes (a partition of) the model on a CPU socket.
115
116 Each worker is associated with a single CPU socket. The worker is
@@ -154,7 +154,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
154 # note: lazy import to avoid importing torch before initializing
155 from vllm.utils import init_cached_hf_modules
156 init_cached_hf_modules()
157 - self.model_runner = CPUModelRunner(
157 + self.model_runner: CPUModelRunner = CPUModelRunner(
158 model_config,
159 parallel_config,
160 scheduler_config,
@@ -255,54 +255,37 @@ class CPUWorker(LoraNotSupportedWorkerBase):
255 for layer_cache in self.cpu_cache:
256 layer_cache.fill_(0)
257
258 - def cache_copy(
258 + @property
259 + def do_metadata_broadcast(self) -> bool:
260 + return self.parallel_config.tensor_parallel_size > 1
261 +
262 + @property
263 + def kv_cache(self) -> Optional[List[torch.Tensor]]:
264 + return self.cpu_cache
265 +
266 + def execute_worker(
267 self,
260 - blocks_to_copy: torch.Tensor,
268 + worker_input: WorkerInput,
269 ) -> None:
262 - if blocks_to_copy.numel() > 0:
263 - self.cache_engine.copy(blocks_to_copy)
270 + if (worker_input.blocks_to_copy is not None
271 + and worker_input.blocks_to_copy.numel() > 0):
272 + self.cache_engine.copy(worker_input.blocks_to_copy)
273
274 @torch.inference_mode()
266 - def execute_model(
267 - self,
268 - execute_model_req: Optional[ExecuteModelRequest] = None,
269 - ) -> List[SamplerOutput]:
270 -
271 - if execute_model_req is None:
272 - seq_group_metadata_list = None
273 - else:
274 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list
275 -
276 - if self.is_driver_worker:
277 - assert seq_group_metadata_list is not None
278 - num_seq_groups: int = len(seq_group_metadata_list)
279 - assert execute_model_req is not None
280 - blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
281 - device="cpu",
282 - dtype=torch.int64).view(-1, 2)
283 - assert len(execute_model_req.blocks_to_swap_in) == 0
284 - assert len(execute_model_req.blocks_to_swap_out) == 0
285 - data: Dict[str, Any] = {
286 - "num_seq_groups": num_seq_groups,
287 - "blocks_to_copy": execute_model_req.blocks_to_copy,
288 - }
289 - broadcast_tensor_dict(data, src=0)
290 - else:
291 - data = broadcast_tensor_dict(src=0)
292 - num_seq_groups = data["num_seq_groups"]
293 - blocks_to_copy = data["blocks_to_copy"]
294 -
295 - self.cache_copy(blocks_to_copy)
296 -
297 - # If there is no input, we don't need to execute the model.
298 - if num_seq_groups == 0:
299 - return []
300 -
301 - output = self.model_runner.execute_model(seq_group_metadata_list,
302 - self.cpu_cache)
303 -
304 - # CPU worker only supports single-step execution.
305 - return [output]
275 + def prepare_worker_input(
276 + self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
277 + assert execute_model_req is not None
278 + num_seq_groups: int = len(execute_model_req.seq_group_metadata_list)
279 + blocks_to_copy = execute_model_req.blocks_to_copy
280 + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
281 + device="cpu",
282 + dtype=torch.int64).view(-1, 2)
283 + assert len(execute_model_req.blocks_to_swap_in) == 0
284 + assert len(execute_model_req.blocks_to_swap_out) == 0
285 + return WorkerInput(
286 + num_seq_groups=num_seq_groups,
287 + blocks_to_copy=blocks_to_copy,
288 + )
289
290 def init_distributed_environment(self) -> None:
291 """Initialize the distributed environment."""
vllm/worker/embedding_model_runner.py
+52 -77
@@ -1,24 +1,32 @@
1 -from typing import Dict, List, Optional, Set, Tuple
1 +import dataclasses
2 +from typing import Any, Dict, List, Optional, Tuple, Type
3
4 import torch
5
5 -from vllm.attention import AttentionMetadata
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 ModelConfig, ParallelConfig, SchedulerConfig,
8 VisionLanguageConfig)
9 -from vllm.distributed import broadcast_tensor_dict
9 from vllm.logger import init_logger
11 -from vllm.lora.layers import LoRAMapping
12 -from vllm.lora.request import LoRARequest
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.pooling_params import PoolingParams
12 from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
16 -from vllm.worker.model_runner import ModelRunner
13 +from vllm.worker.model_runner import GPUModelRunnerBase, ModelInputForGPU
14
15 logger = init_logger(__name__)
16
17
21 -class EmbeddingModelRunner(ModelRunner):
18 +@dataclasses.dataclass(frozen=True)
19 +class ModelInputForGPUWithPoolingMetadata(ModelInputForGPU):
20 + """
21 + Used by the EmbeddingModelRunner.
22 + """
23 + pooling_metadata: Optional["PoolingMetadata"] = None
24 +
25 +
26 +class EmbeddingModelRunner(
27 + GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]):
28 + _model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = (
29 + ModelInputForGPUWithPoolingMetadata)
30
31 def __init__(
32 self,
@@ -47,21 +55,22 @@ class EmbeddingModelRunner(ModelRunner):
55 @torch.inference_mode()
56 def execute_model(
57 self,
50 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
58 + model_input: ModelInputForGPUWithPoolingMetadata,
59 kv_caches: List[torch.Tensor],
60 ) -> Optional[PoolerOutput]:
53 - (input_tokens, input_positions, attn_metadata, pooling_metadata,
54 - lora_requests, lora_mapping, multi_modal_input
55 - ) = self.prepare_input_tensors(seq_group_metadata_list)
56 -
61 if self.lora_config:
58 - self.set_active_loras(lora_requests, lora_mapping)
62 + assert model_input.lora_requests is not None
63 + assert model_input.lora_mapping is not None
64 + self.set_active_loras(model_input.lora_requests,
65 + model_input.lora_mapping)
66
67 # Currently cuda graph is only supported by the decode phase.
61 - prefill_meta = attn_metadata.prefill_metadata
62 - decode_meta = attn_metadata.decode_metadata
68 + assert model_input.attn_metadata is not None
69 + prefill_meta = model_input.attn_metadata.prefill_metadata
70 + decode_meta = model_input.attn_metadata.decode_metadata
71 if prefill_meta is None and decode_meta.use_cuda_graph:
64 - graph_batch_size = input_tokens.shape[0]
72 + assert model_input.input_tokens is not None
73 + graph_batch_size = model_input.input_tokens.shape[0]
74 model_executable = self.graph_runners[graph_batch_size]
75 else:
76 model_executable = self.model
@@ -70,13 +79,14 @@ class EmbeddingModelRunner(ModelRunner):
79 kv_caches = [None] * num_layers
80
81 execute_model_kwargs = {
73 - "input_ids": input_tokens,
74 - "positions": input_positions,
82 + "input_ids": model_input.input_tokens,
83 + "positions": model_input.input_positions,
84 "kv_caches": kv_caches,
76 - "attn_metadata": attn_metadata,
85 + "attn_metadata": model_input.attn_metadata,
86 }
87 if self.vision_language_config:
79 - execute_model_kwargs.update({"image_input": multi_modal_input})
88 + multi_modal_kwargs = model_input.multi_modal_kwargs or {}
89 + execute_model_kwargs.update({"image_input": multi_modal_kwargs})
90 hidden_states = model_executable(**execute_model_kwargs)
91
92 # Only perform pooling in the driver worker.
@@ -84,66 +94,31 @@ class EmbeddingModelRunner(ModelRunner):
94 return None
95
96 return self.model.pooler(hidden_states=hidden_states,
87 - pooling_metadata=pooling_metadata)
97 + pooling_metadata=model_input.pooling_metadata)
98 +
99 + def make_model_input_from_broadcasted_tensor_dict(
100 + self,
101 + tensor_dict: Dict[str,
102 + Any]) -> ModelInputForGPUWithPoolingMetadata:
103 + return ModelInputForGPUWithPoolingMetadata.from_broadcasted_tensor_dict(
104 + tensor_dict,
105 + attn_backend=self.attn_backend,
106 + )
107
89 - def prepare_input_tensors(
108 + def prepare_model_input(
109 self,
110 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
92 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
93 - Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
94 - if self.is_driver_worker:
95 - assert seq_group_metadata_list is not None
96 - # Prepare input tensors.
97 - (
98 - input_tokens,
99 - input_positions,
100 - attn_metadata,
101 - seq_lens,
102 - _,
103 - lora_mapping,
104 - lora_requests,
105 - multi_modal_kwargs,
106 - slot_mapping,
107 - num_prefill_tokens,
108 - num_decode_tokens,
109 - num_prefills,
110 - ) = self._prepare_model_input(seq_group_metadata_list)
111 - # Prepare PoolingMetadata
112 - pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
113 - seq_lens)
114 -
115 - metadata_dict = {
116 - "input_tokens": input_tokens,
117 - "input_positions": input_positions,
118 - "lora_requests": lora_requests,
119 - "lora_mapping": lora_mapping,
120 - "multi_modal_kwargs": multi_modal_kwargs,
121 - "num_prefill_tokens": num_prefill_tokens,
122 - "num_decode_tokens": num_decode_tokens,
123 - "slot_mapping": slot_mapping,
124 - "num_prefills": num_prefills,
125 - }
126 - if attn_metadata:
127 - metadata_dict.update(attn_metadata.asdict_zerocopy())
128 - broadcast_tensor_dict(metadata_dict, src=0)
129 - else:
130 - metadata_dict = broadcast_tensor_dict(src=0)
131 - input_tokens = metadata_dict.pop("input_tokens")
132 - input_positions = metadata_dict.pop("input_positions")
133 - lora_mapping = metadata_dict.pop("lora_mapping")
134 - lora_requests = metadata_dict.pop("lora_requests")
135 - multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
136 - if metadata_dict:
137 - attn_metadata = self.attn_backend.make_metadata(
138 - **metadata_dict)
139 - else:
140 - attn_metadata = None
141 - pooling_metadata = PoolingMetadata(seq_groups=None,
142 - seq_data=None,
143 - prompt_lens=None)
144 -
145 - return (input_tokens, input_positions, attn_metadata, pooling_metadata,
146 - lora_requests, lora_mapping, multi_modal_kwargs)
111 + ) -> ModelInputForGPUWithPoolingMetadata:
112 + assert seq_group_metadata_list is not None
113 + model_input = self._prepare_model_input_tensors(
114 + seq_group_metadata_list)
115 + # Prepare PoolingMetadata.
116 + assert model_input.seq_lens is not None
117 + pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
118 + model_input.seq_lens)
119 +
120 + return dataclasses.replace(model_input,
121 + pooling_metadata=pooling_metadata)
122
123 def _prepare_pooling(
124 self,
vllm/worker/model_runner.py
+205 -162
@@ -1,8 +1,10 @@
1 +import dataclasses
2 import gc
3 import time
4 import warnings
5 from collections import defaultdict
5 -from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
6 +from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
7 + TypeVar, Union)
8
9 import numpy as np
10 import torch
@@ -12,7 +14,6 @@ from vllm.attention import AttentionMetadata, get_attn_backend
14 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 ModelConfig, ParallelConfig, SchedulerConfig,
16 VisionLanguageConfig)
15 -from vllm.distributed import broadcast_tensor_dict
17 from vllm.distributed.parallel_state import graph_capture
18 from vllm.logger import init_logger
19 from vllm.lora.layers import LoRAMapping
@@ -26,6 +27,15 @@ from vllm.sampling_params import SamplingParams
27 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
28 from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
29 is_pin_memory_available, make_tensor_with_pad)
30 +from vllm.worker.model_runner_base import (
31 + ModelRunnerBase, ModelRunnerInputBase,
32 + _add_attn_metadata_broadcastable_dict,
33 + _add_sampling_metadata_broadcastable_dict,
34 + _init_attn_metadata_from_tensor_dict,
35 + _init_sampling_metadata_from_tensor_dict)
36 +
37 +if TYPE_CHECKING:
38 + from vllm.attention.backends.abstract import AttentionBackend
39
40 logger = init_logger(__name__)
41
@@ -39,40 +49,90 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
49 ]
50 _NUM_WARMUP_ITERS = 2
51
52 +TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
53
43 -class ModelInput(NamedTuple):
44 - input_tokens: torch.Tensor
45 - input_positions: torch.Tensor
46 - attn_metadata: Optional[AttentionMetadata]
47 - seq_lens: List[int]
48 - query_lens: List[int]
49 - lora_mapping: Optional[LoRAMapping]
50 - lora_requests: Set[LoRARequest]
51 - multi_modal_kwargs: Dict[str, torch.Tensor]
52 - slot_mapping: torch.Tensor
53 - num_prefill_tokens: int
54 - num_decode_tokens: int
55 - num_prefills: int
54
57 - @classmethod
58 - def empty(cls, device):
59 - return ModelInput(
60 - input_tokens=torch.empty(0, device=device),
61 - input_positions=torch.empty(0, device=device),
62 - attn_metadata=None,
63 - seq_lens=[],
64 - query_lens=[],
65 - lora_mapping=None,
66 - lora_requests=set(),
67 - multi_modal_kwargs={},
68 - slot_mapping=torch.empty(0, device=device),
69 - num_prefill_tokens=0,
70 - num_decode_tokens=0,
71 - num_prefills=0,
72 - )
55 +@dataclasses.dataclass(frozen=True)
56 +class ModelInputForGPU(ModelRunnerInputBase):
57 + """
58 + This base class contains metadata needed for the base model forward pass
59 + but not metadata for possible additional steps, e.g., sampling. Model
60 + runners that run additional steps should subclass this method to add
61 + additional fields.
62 + """
63 + input_tokens: Optional[torch.Tensor] = None
64 + input_positions: Optional[torch.Tensor] = None
65 + seq_lens: Optional[List[int]] = None
66 + query_lens: Optional[List[int]] = None
67 + lora_mapping: Optional["LoRAMapping"] = None
68 + lora_requests: Optional[Set[LoRARequest]] = None
69 + attn_metadata: Optional["AttentionMetadata"] = None
70 + multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
71 +
72 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
73 + tensor_dict = {
74 + "input_tokens": self.input_tokens,
75 + "input_positions": self.input_positions,
76 + "lora_requests": self.lora_requests,
77 + "lora_mapping": self.lora_mapping,
78 + "multi_modal_kwargs": self.multi_modal_kwargs,
79 + }
80 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
81 + return tensor_dict
82
83 + @classmethod
84 + def from_broadcasted_tensor_dict(
85 + cls: Type[TModelInputForGPU],
86 + tensor_dict: Dict[str, Any],
87 + attn_backend: Optional["AttentionBackend"] = None,
88 + ) -> TModelInputForGPU:
89 + if attn_backend is not None:
90 + tensor_dict = _init_attn_metadata_from_tensor_dict(
91 + attn_backend, tensor_dict)
92 + return cls(**tensor_dict)
93 +
94 +
95 +@dataclasses.dataclass(frozen=True)
96 +class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
97 + """
98 + Used by the ModelRunner.
99 + """
100 + sampling_metadata: Optional["SamplingMetadata"] = None
101 + # Used for speculative decoding. We do not broadcast it because it is only
102 + # used by the driver worker.
103 + is_prompt: Optional[bool] = None
104 +
105 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
106 + tensor_dict = {
107 + "input_tokens": self.input_tokens,
108 + "input_positions": self.input_positions,
109 + "lora_requests": self.lora_requests,
110 + "lora_mapping": self.lora_mapping,
111 + "multi_modal_kwargs": self.multi_modal_kwargs,
112 + }
113 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
114 + _add_sampling_metadata_broadcastable_dict(tensor_dict,
115 + self.sampling_metadata)
116 + return tensor_dict
117
75 -class ModelRunner:
118 + @classmethod
119 + def from_broadcasted_tensor_dict(
120 + cls,
121 + tensor_dict: Dict[str, Any],
122 + attn_backend: Optional["AttentionBackend"] = None,
123 + ) -> "ModelInputForGPUWithSamplingMetadata":
124 + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
125 + if attn_backend is not None:
126 + tensor_dict = _init_attn_metadata_from_tensor_dict(
127 + attn_backend, tensor_dict)
128 + return cls(**tensor_dict)
129 +
130 +
131 +class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
132 + """
133 + Helper class for shared methods between GPU model runners.
134 + """
135 + _model_input_cls: Type[TModelInputForGPU]
136
137 def __init__(
138 self,
@@ -241,11 +301,13 @@ class ModelRunner:
301 block_size = self.block_size
302 return (self.max_seq_len_to_capture + block_size - 1) // block_size
303
244 - def _prepare_model_input(
304 + def _prepare_model_input_tensors(
305 self,
306 seq_group_metadata_list: List[SequenceGroupMetadata],
247 - ) -> ModelInput:
248 - """Prepare the model input based on a given sequence group.
307 + ) -> TModelInputForGPU:
308 + """Helper method to prepare the model input based on a given sequence
309 + group. Prepares metadata needed for the base model forward pass but not
310 + metadata for possible additional steps, e.g., sampling.
311
312 The API assumes seq_group_metadata_list is sorted by prefill -> decode.
313
@@ -296,7 +358,7 @@ class ModelRunner:
358 paged_kv_last_page_len: List[int] = []
359
360 if len(seq_group_metadata_list) == 0:
299 - return ModelInput.empty(self.device)
361 + return self._model_input_cls()
362
363 if self.sliding_window is not None:
364 sliding_window_blocks = (self.sliding_window + self.block_size -
@@ -646,7 +708,7 @@ class ModelRunner:
708 for k, v in multi_modal_kwargs_list.items()
709 }
710
649 - return ModelInput(
711 + return self._model_input_cls(
712 input_tokens=input_tokens_tensor,
713 input_positions=input_positions_tensor,
714 attn_metadata=attn_metadata,
@@ -655,132 +717,8 @@ class ModelRunner:
717 lora_mapping=lora_mapping,
718 lora_requests=lora_requests,
719 multi_modal_kwargs=multi_modal_kwargs,
658 - slot_mapping=slot_mapping_tensor,
659 - num_prefill_tokens=num_prefill_tokens,
660 - num_decode_tokens=num_decode_tokens,
661 - num_prefills=num_prefills,
662 - )
663 -
664 - def prepare_input_tensors(
665 - self,
666 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
667 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
668 - Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
669 - if self.is_driver_worker:
670 - assert seq_group_metadata_list is not None
671 - # Prepare input tensors.
672 - (
673 - input_tokens,
674 - input_positions,
675 - attn_metadata,
676 - seq_lens,
677 - query_lens,
678 - lora_mapping,
679 - lora_requests,
680 - multi_modal_kwargs,
681 - slot_mapping,
682 - num_prefill_tokens,
683 - num_decode_tokens,
684 - num_prefills,
685 - ) = self._prepare_model_input(seq_group_metadata_list)
686 - sampling_metadata = SamplingMetadata.prepare(
687 - seq_group_metadata_list, seq_lens, query_lens, self.device,
688 - self.pin_memory)
689 -
690 - metadata_dict = {
691 - "input_tokens": input_tokens,
692 - "input_positions": input_positions,
693 - "selected_token_indices":
694 - sampling_metadata.selected_token_indices,
695 - "lora_requests": lora_requests,
696 - "lora_mapping": lora_mapping,
697 - "multi_modal_kwargs": multi_modal_kwargs,
698 - "num_prefill_tokens": num_prefill_tokens,
699 - "num_decode_tokens": num_decode_tokens,
700 - "slot_mapping": slot_mapping,
701 - "num_prefills": num_prefills,
702 - }
703 - if attn_metadata:
704 - metadata_dict.update(attn_metadata.asdict_zerocopy())
705 - broadcast_tensor_dict(metadata_dict, src=0)
706 - else:
707 - metadata_dict = broadcast_tensor_dict(src=0)
708 - input_tokens = metadata_dict.pop("input_tokens")
709 - input_positions = metadata_dict.pop("input_positions")
710 - selected_token_indices = metadata_dict.pop(
711 - "selected_token_indices")
712 - lora_mapping = metadata_dict.pop("lora_mapping")
713 - lora_requests = metadata_dict.pop("lora_requests")
714 - multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
715 - if metadata_dict:
716 - attn_metadata = self.attn_backend.make_metadata(
717 - **metadata_dict)
718 - else:
719 - attn_metadata = None
720 - sampling_metadata = SamplingMetadata(
721 - seq_groups=None,
722 - selected_token_indices=selected_token_indices,
723 - categorized_sample_indices=None,
724 - num_prompts=0,
725 - )
726 -
727 - return (input_tokens, input_positions, attn_metadata,
728 - sampling_metadata, lora_requests, lora_mapping,
729 - multi_modal_kwargs)
730 -
731 - @torch.inference_mode()
732 - def execute_model(
733 - self,
734 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
735 - kv_caches: List[torch.Tensor],
736 - ) -> Optional[SamplerOutput]:
737 - (input_tokens, input_positions, attn_metadata, sampling_metadata,
738 - lora_requests, lora_mapping, multi_modal_kwargs
739 - ) = self.prepare_input_tensors(seq_group_metadata_list)
740 -
741 - if self.lora_config:
742 - self.set_active_loras(lora_requests, lora_mapping)
743 -
744 - # Currently cuda graph is only supported by the decode phase.
745 - prefill_meta = attn_metadata.prefill_metadata
746 - decode_meta = attn_metadata.decode_metadata
747 - if prefill_meta is None and decode_meta.use_cuda_graph:
748 - graph_batch_size = input_tokens.shape[0]
749 - model_executable = self.graph_runners[graph_batch_size]
750 - else:
751 - model_executable = self.model
752 -
753 - hidden_states = model_executable(
754 - input_ids=input_tokens,
755 - positions=input_positions,
756 - kv_caches=kv_caches,
757 - attn_metadata=attn_metadata,
758 - **multi_modal_kwargs,
759 - )
760 -
761 - # Compute the logits.
762 - logits = self.model.compute_logits(hidden_states, sampling_metadata)
763 -
764 - # Only perform sampling in the driver worker.
765 - if not self.is_driver_worker:
766 - return None
767 -
768 - # Sample the next token.
769 - output: SamplerOutput = self.model.sample(
770 - logits=logits,
771 - sampling_metadata=sampling_metadata,
720 )
721
774 - if self.return_hidden_states:
775 - # we only need to pass hidden states of most recent token
776 - assert seq_group_metadata_list is not None
777 - if seq_group_metadata_list[0].is_prompt:
778 - hidden_states = hidden_states.index_select(
779 - 0, sampling_metadata.selected_token_indices)
780 - output.hidden_states = hidden_states
781 -
782 - return output
783 -
722 @torch.inference_mode()
723 def profile_run(self) -> None:
724 # Enable top-k sampling to reflect the accurate memory usage.
@@ -853,7 +791,8 @@ class ModelRunner:
791 # Run the model with the dummy inputs.
792 num_layers = self.model_config.get_num_layers(self.parallel_config)
793 kv_caches = [None] * num_layers
856 - self.execute_model(seqs, kv_caches)
794 + model_input = self.prepare_model_input(seqs)
795 + self.execute_model(model_input, kv_caches)
796 torch.cuda.synchronize()
797 return
798
@@ -986,6 +925,110 @@ class ModelRunner:
925 return self.model_config.get_vocab_size()
926
927
928 +class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
929 + """
930 + GPU model runner with sampling step.
931 + """
932 + _model_input_cls: Type[ModelInputForGPUWithSamplingMetadata] = (
933 + ModelInputForGPUWithSamplingMetadata)
934 +
935 + def make_model_input_from_broadcasted_tensor_dict(
936 + self,
937 + tensor_dict: Dict[str, Any],
938 + ) -> ModelInputForGPUWithSamplingMetadata:
939 + return (
940 + ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
941 + tensor_dict,
942 + attn_backend=self.attn_backend,
943 + ))
944 +
945 + def prepare_model_input(
946 + self,
947 + seq_group_metadata_list: List[SequenceGroupMetadata],
948 + ) -> ModelInputForGPUWithSamplingMetadata:
949 + """Prepare the model input based on a given sequence group, including
950 + metadata for the sampling step.
951 +
952 + The API assumes seq_group_metadata_list is sorted by prefill -> decode.
953 +
954 + The result tensors and data structure also batches input in prefill
955 + -> decode order. For example,
956 +
957 + - input_tokens[:num_prefill_tokens] contains prefill tokens.
958 + - input_tokens[num_prefill_tokens:] contains decode tokens.
959 +
960 + If cuda graph is required, this API automatically pads inputs.
961 + """
962 + model_input = self._prepare_model_input_tensors(
963 + seq_group_metadata_list)
964 + sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
965 + model_input.seq_lens,
966 + model_input.query_lens,
967 + self.device,
968 + self.pin_memory)
969 + is_prompt = (seq_group_metadata_list[0].is_prompt
970 + if seq_group_metadata_list else None)
971 + return dataclasses.replace(model_input,
972 + sampling_metadata=sampling_metadata,
973 + is_prompt=is_prompt)
974 +
975 + @torch.inference_mode()
976 + def execute_model(
977 + self,
978 + model_input: ModelInputForGPUWithSamplingMetadata,
979 + kv_caches: List[torch.Tensor],
980 + ) -> SamplerOutput:
981 + if self.lora_config:
982 + assert model_input.lora_requests is not None
983 + assert model_input.lora_mapping is not None
984 + self.set_active_loras(model_input.lora_requests,
985 + model_input.lora_mapping)
986 +
987 + # Currently cuda graph is only supported by the decode phase.
988 + assert model_input.attn_metadata is not None
989 + prefill_meta = model_input.attn_metadata.prefill_metadata
990 + decode_meta = model_input.attn_metadata.decode_metadata
991 + if prefill_meta is None and decode_meta.use_cuda_graph:
992 + assert model_input.input_tokens is not None
993 + graph_batch_size = model_input.input_tokens.shape[0]
994 + model_executable = self.graph_runners[graph_batch_size]
995 + else:
996 + model_executable = self.model
997 +
998 + multi_modal_kwargs = model_input.multi_modal_kwargs or {}
999 + hidden_states = model_executable(
1000 + input_ids=model_input.input_tokens,
1001 + positions=model_input.input_positions,
1002 + kv_caches=kv_caches,
1003 + attn_metadata=model_input.attn_metadata,
1004 + **multi_modal_kwargs,
1005 + )
1006 +
1007 + # Compute the logits.
1008 + logits = self.model.compute_logits(hidden_states,
1009 + model_input.sampling_metadata)
1010 +
1011 + # Only perform sampling in the driver worker.
1012 + if not self.is_driver_worker:
1013 + return None
1014 +
1015 + # Sample the next token.
1016 + output: SamplerOutput = self.model.sample(
1017 + logits=logits,
1018 + sampling_metadata=model_input.sampling_metadata,
1019 + )
1020 +
1021 + if self.return_hidden_states:
1022 + # we only need to pass hidden states of most recent token
1023 + if model_input.is_prompt:
1024 + assert model_input.sampling_metadata is not None
1025 + hidden_states = hidden_states.index_select(
1026 + 0, model_input.sampling_metadata.selected_token_indices)
1027 + output.hidden_states = hidden_states
1028 +
1029 + return output
1030 +
1031 +
1032 class CUDAGraphRunner:
1033
1034 def __init__(self, model: nn.Module):
vllm/worker/model_runner_base.py new
+157
@@ -0,0 +1,157 @@
1 +import dataclasses
2 +from abc import ABC, abstractmethod
3 +from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
4 + TypeVar)
5 +
6 +import torch
7 +
8 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9 +
10 +if TYPE_CHECKING:
11 + from vllm.attention import AttentionMetadata
12 + from vllm.attention.backends.abstract import AttentionBackend
13 + from vllm.model_executor import SamplingMetadata
14 +
15 +T = TypeVar('T', bound="ModelRunnerInputBase")
16 +
17 +
18 +def _add_attn_metadata_broadcastable_dict(
19 + tensor_dict: Dict[str, Any],
20 + attn_metadata: Optional["AttentionMetadata"]) -> None:
21 + """
22 + Helper method to update tensor_dict with broadcastable
23 + AttentionMetadata fields.
24 + """
25 + if attn_metadata is not None:
26 + tensor_dict.update(attn_metadata.asdict_zerocopy())
27 +
28 +
29 +def _init_attn_metadata_from_tensor_dict(
30 + attn_backend: "AttentionBackend",
31 + tensor_dict: Dict[str, Any],
32 +) -> Dict[str, Any]:
33 + """
34 + Helper method to initialize AttentionMetadata based on an
35 + AttentionBackend and broadcastable AttentionMetadata fields.
36 + """
37 + # Extract the fields used to create AttentionMetadata.
38 + valid_attn_kwargs = {}
39 + for field in dataclasses.fields(attn_backend.get_metadata_cls()):
40 + val = tensor_dict.pop(field.name, None)
41 + if val is not None:
42 + valid_attn_kwargs[field.name] = val
43 +
44 + attn_metadata = attn_backend.make_metadata(**valid_attn_kwargs)
45 + tensor_dict["attn_metadata"] = attn_metadata
46 + return tensor_dict
47 +
48 +
49 +def _init_sampling_metadata_from_tensor_dict( # type: ignore
50 + tensor_dict: Dict[str, Any]) -> Dict[str, Any]:
51 + """
52 + Helper method to initialize SamplingMetadata based on broadcastable
53 + SamplingMetadata fields.
54 + """
55 + from vllm.model_executor import SamplingMetadata
56 +
57 + selected_token_indices = tensor_dict.pop("selected_token_indices", None)
58 + # An empty SamplingMetadata to signal that the worker should skip
59 + # sampling.
60 + if selected_token_indices is not None:
61 + tensor_dict["sampling_metadata"] = SamplingMetadata(
62 + seq_groups=None,
63 + selected_token_indices=selected_token_indices,
64 + categorized_sample_indices=None,
65 + num_prompts=0,
66 + )
67 + return tensor_dict
68 +
69 +
70 +def _add_sampling_metadata_broadcastable_dict(
71 + tensor_dict: Dict[str, Any],
72 + sampling_metadata: Optional["SamplingMetadata"]) -> None:
73 + """
74 + Helper method to update tensor_dict with broadcastable
75 + SamplingMetadata fields.
76 + """
77 + if sampling_metadata is not None:
78 + tensor_dict["selected_token_indices"] = (
79 + sampling_metadata.selected_token_indices)
80 +
81 +
82 +@dataclasses.dataclass(frozen=True)
83 +class ModelRunnerInputBase(ABC):
84 + """Local inputs to each worker's model runner. May contain
85 + device-specific data. Different worker backends may have different methods
86 + of converting from the global ExecuteModelRequest produced by the LLM
87 + engine to the worker-local ModelRunnerInputBase objects.
88 +
89 + Model runners that support multi-GPU execution should define a
90 + ModelRunnerInputBase subclass, add their required fields, and specify how to
91 + serialize/deserialize a ModelInput for broadcast between workers.
92 + """
93 +
94 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
95 + """
96 + Extract broadcastable fields. Override for fields that require some
97 + custom deserialization.
98 + """
99 + raise NotImplementedError
100 +
101 + @classmethod
102 + @abstractmethod
103 + def from_broadcasted_tensor_dict(
104 + cls: Type[T],
105 + tensor_dict: Dict[str, Any],
106 + attn_backend: Optional["AttentionBackend"] = None,
107 + ) -> T:
108 + """
109 + Pop fields from the given tensor_dict and populate a new instance of
110 + ModelRunnerInputBase.
111 + """
112 + raise NotImplementedError
113 +
114 +
115 +class ModelRunnerBase(ABC, Generic[T]):
116 + """
117 + Model runner interface that abstracts a particular hardware and/or type of
118 + model. Model execution may communicate data with model runners in other
119 + processes, but it should not include control plane metadata communication.
120 +
121 + Each ModelRunnerBase subclass should define a corresponding
122 + ModelRunnerInputBase subclass.
123 + """
124 +
125 + @abstractmethod
126 + def make_model_input_from_broadcasted_tensor_dict(
127 + self,
128 + tensor_dict: Dict[str, Any],
129 + ) -> T:
130 + """
131 + Make an instance of a ModelRunnerInputBase from the broadcasted tensor
132 + dict.
133 + """
134 + raise NotImplementedError
135 +
136 + @abstractmethod
137 + def prepare_model_input(
138 + self,
139 + seq_group_metadata_list: List[SequenceGroupMetadata],
140 + ) -> T:
141 + """
142 + Prepare the inputs to ModelRunnerBase.execute_model from an execution
143 + request. This method may move data to the worker's local device. It is
144 + not allowed to communicate with other workers or devices.
145 + """
146 + raise NotImplementedError
147 +
148 + @torch.inference_mode()
149 + def execute_model(
150 + self,
151 + model_input: T,
152 + kv_caches: Optional[List[torch.Tensor]],
153 + ) -> Optional[SamplerOutput]:
154 + """
155 + Execute the model on the given input.
156 + """
157 + raise NotImplementedError
vllm/worker/neuron_model_runner.py
+49 -15
@@ -1,4 +1,5 @@
1 -from typing import List, Optional, Tuple
1 +from dataclasses import dataclass
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
3
4 import torch
5 from torch import nn
@@ -10,11 +11,39 @@ from vllm.model_executor import SamplingMetadata
11 from vllm.model_executor.model_loader.neuron import get_neuron_model
12 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
13 from vllm.utils import is_pin_memory_available, make_tensor_with_pad
14 +from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
15 +
16 +if TYPE_CHECKING:
17 + from vllm.attention.backends.abstract import AttentionBackend
18
19 logger = init_logger(__name__)
20
21
17 -class NeuronModelRunner:
22 +@dataclass(frozen=True)
23 +class ModelInputForNeuron(ModelRunnerInputBase):
24 + """
25 + Used by the NeuronModelRunner.
26 + """
27 + input_tokens: Optional[torch.Tensor] = None
28 + input_positions: Optional[torch.Tensor] = None
29 + input_block_ids: Optional[torch.Tensor] = None
30 + sampling_metadata: Optional["SamplingMetadata"] = None
31 +
32 + def as_broadcastable_tensor_dict(
33 + self) -> Dict[str, Union[int, torch.Tensor]]:
34 + raise NotImplementedError("ModelInputForNeuron cannot be broadcast.")
35 +
36 + @classmethod
37 + def from_broadcasted_tensor_dict(
38 + cls,
39 + tensor_dict: Dict[str, Any],
40 + attn_backend: Optional["AttentionBackend"] = None,
41 + ) -> "ModelInputForNeuron":
42 + assert attn_backend is None
43 + return cls.from_broadcasted_tensor_dict(tensor_dict)
44 +
45 +
46 +class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
47
48 def __init__(
49 self,
@@ -139,10 +168,14 @@ class NeuronModelRunner:
168
169 return input_tokens, input_positions, input_block_ids
170
142 - def prepare_input_tensors(
171 + def make_model_input_from_broadcasted_tensor_dict(
172 + self, tensor_dict: Dict[str, Any]) -> ModelInputForNeuron:
173 + return ModelInputForNeuron.from_broadcasted_tensor_dict(tensor_dict)
174 +
175 + def prepare_model_input(
176 self,
177 seq_group_metadata_list: List[SequenceGroupMetadata],
145 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, SamplingMetadata]:
178 + ) -> ModelInputForNeuron:
179 # NOTE: We assume that all sequences in the group are all prompts or
180 # all decodes.
181 is_prompt = seq_group_metadata_list[0].is_prompt
@@ -164,30 +197,31 @@ class NeuronModelRunner:
197 self.device,
198 self.pin_memory)
199
167 - return (input_tokens, input_positions, input_block_ids,
168 - sampling_metadata)
200 + return ModelInputForNeuron(input_tokens=input_tokens,
201 + input_positions=input_positions,
202 + input_block_ids=input_block_ids,
203 + sampling_metadata=sampling_metadata)
204
205 @torch.inference_mode()
206 def execute_model(
207 self,
173 - seq_group_metadata_list: List[SequenceGroupMetadata],
208 + model_input: ModelInputForNeuron,
209 + kv_caches: Optional[List[torch.Tensor]] = None,
210 ) -> Optional[SamplerOutput]:
175 - (input_tokens, input_positions, input_block_ids, sampling_metadata
176 - ) = self.prepare_input_tensors(seq_group_metadata_list)
177 -
211 hidden_states = self.model(
179 - input_ids=input_tokens,
180 - positions=input_positions,
181 - input_block_ids=input_block_ids,
212 + input_ids=model_input.input_tokens,
213 + positions=model_input.input_positions,
214 + input_block_ids=model_input.input_block_ids,
215 )
216
217 # Compute the logits.
185 - logits = self.model.compute_logits(hidden_states, sampling_metadata)
218 + logits = self.model.compute_logits(hidden_states,
219 + model_input.sampling_metadata)
220
221 # Sample the next token.
222 output = self.model.sample(
223 logits=logits,
190 - sampling_metadata=sampling_metadata,
224 + sampling_metadata=model_input.sampling_metadata,
225 )
226 return output
227
vllm/worker/neuron_worker.py
+19 -20
@@ -1,5 +1,5 @@
1 """A Neuron worker class."""
2 -from typing import List, Tuple
2 +from typing import List, Optional, Tuple
3
4 import torch
5 import torch.distributed
@@ -7,12 +7,13 @@ import torch.distributed
7 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8 ParallelConfig, SchedulerConfig)
9 from vllm.model_executor import set_random_seed
10 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10 +from vllm.sequence import ExecuteModelRequest
11 from vllm.worker.neuron_model_runner import NeuronModelRunner
12 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
12 +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
13 + LoraNotSupportedWorkerBase, WorkerInput)
14
15
15 -class NeuronWorker(LoraNotSupportedWorkerBase):
16 +class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
17 """A worker class that executes the model on a group of neuron cores.
18 """
19
@@ -34,8 +35,9 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
35 from vllm.utils import init_cached_hf_modules
36 init_cached_hf_modules()
37
37 - self.model_runner = NeuronModelRunner(model_config, parallel_config,
38 - scheduler_config, device_config)
38 + self.model_runner: NeuronModelRunner = NeuronModelRunner(
39 + model_config, parallel_config, scheduler_config, device_config)
40 + self.is_driver_worker = True
41
42 def init_device(self) -> None:
43 # Set random seed.
@@ -73,22 +75,19 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
75 self.cache_config.num_gpu_blocks = num_gpu_blocks
76 self.cache_config.num_cpu_blocks = num_cpu_blocks
77
76 - @torch.inference_mode()
77 - def execute_model(
78 - self,
79 - seq_group_metadata_list: List[SequenceGroupMetadata],
80 - ) -> List[SamplerOutput]:
81 - num_seq_groups = len(seq_group_metadata_list)
78 + @property
79 + def do_metadata_broadcast(self) -> bool:
80 + return False
81
83 - # If there is no input, we don't need to execute the model.
84 - if num_seq_groups == 0:
85 - return []
82 + @property
83 + def kv_cache(self) -> Optional[List[torch.Tensor]]:
84 + return None
85
87 - output = self.model_runner.execute_model(seq_group_metadata_list)
88 -
89 - # Neuron worker only supports single-step output. Wrap the output in a
90 - # list to conform to interface.
91 - return [output]
86 + @torch.inference_mode()
87 + def prepare_worker_input(
88 + self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
89 + return WorkerInput(num_seq_groups=len(
90 + execute_model_req.seq_group_metadata_list), )
91
92 def get_cache_block_size_bytes(self) -> int:
93 """Determine the size in bytes of a cache block.
vllm/worker/worker.py
+37 -92
@@ -1,7 +1,7 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -from typing import Any, Dict, List, Optional, Set, Tuple, Union
4 +from typing import List, Optional, Set, Tuple, Type
5
6 import torch
7 import torch.distributed
@@ -9,21 +9,20 @@ import torch.distributed
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, ParallelConfig, SchedulerConfig,
11 SpeculativeConfig, VisionLanguageConfig)
12 -from vllm.distributed import (broadcast_tensor_dict,
13 - ensure_model_parallel_initialized,
12 +from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment,
14 set_custom_all_reduce)
15 from vllm.lora.request import LoRARequest
16 from vllm.model_executor import set_random_seed
17 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19 -from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
18 +from vllm.sequence import ExecuteModelRequest
19 from vllm.worker.cache_engine import CacheEngine
20 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22 -from vllm.worker.model_runner import ModelRunner
23 -from vllm.worker.worker_base import WorkerBase
21 +from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
22 +from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
23
24
26 -class Worker(WorkerBase):
25 +class Worker(LocalOrDistributedWorkerBase):
26 """A worker class that executes (a partition of) the model on a GPU.
27
28 Each worker is associated with a single GPU. The worker is responsible for
@@ -78,9 +77,10 @@ class Worker(WorkerBase):
77 or (speculative_config.draft_model_config.hf_config.model_type !=
78 "mlp_speculator") else {"return_hidden_states": True}
79
81 - ModelRunnerClass = (EmbeddingModelRunner if
82 - self.model_config.embedding_mode else ModelRunner)
83 - self.model_runner = ModelRunnerClass(
80 + ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
81 + if self.model_config.embedding_mode:
82 + ModelRunnerClass = EmbeddingModelRunner
83 + self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
84 model_config,
85 parallel_config,
86 scheduler_config,
@@ -225,40 +225,18 @@ class Worker(WorkerBase):
225 # the model initialization and profiling.
226 set_random_seed(self.model_config.seed)
227
228 - def cache_swap(
229 - self,
230 - blocks_to_swap_in: torch.Tensor,
231 - blocks_to_swap_out: torch.Tensor,
232 - blocks_to_copy: torch.Tensor,
233 - ) -> None:
234 - # Issue cache operations.
235 - if blocks_to_swap_in.numel() > 0:
236 - self.cache_engine.swap_in(blocks_to_swap_in)
237 - if blocks_to_swap_out.numel() > 0:
238 - self.cache_engine.swap_out(blocks_to_swap_out)
239 - if blocks_to_copy.numel() > 0:
240 - self.cache_engine.copy(blocks_to_copy)
228 + @property
229 + def do_metadata_broadcast(self) -> bool:
230 + return self.parallel_config.tensor_parallel_size > 1
231 +
232 + @property
233 + def kv_cache(self) -> Optional[List[torch.Tensor]]:
234 + return self.gpu_cache
235
236 @torch.inference_mode()
243 - def execute_model(
244 - self,
245 - execute_model_req: Optional[ExecuteModelRequest] = None
246 - ) -> List[Union[SamplerOutput, PoolerOutput]]:
247 - if not self.is_driver_worker:
248 - self._execute_model_non_driver()
249 - return []
250 -
251 - if execute_model_req is None:
252 - # This signals that there's no more requests to process for now.
253 - # All workers are running infinite loop with broadcast_tensor_dict,
254 - # and it stops the loop when the driver broadcasts an empty input.
255 - # Send an empty input to notify all other workers to stop their
256 - # execution loop.
257 - broadcast_tensor_dict({}, src=0)
258 - return []
259 -
260 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list
261 - num_seq_groups = len(seq_group_metadata_list)
237 + def prepare_worker_input(
238 + self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
239 + num_seq_groups = len(execute_model_req.seq_group_metadata_list)
240 # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
241 # they contain parameters to launch cudamemcpyasync.
242 blocks_to_swap_in = torch.tensor(execute_model_req.blocks_to_swap_in,
@@ -273,59 +251,26 @@ class Worker(WorkerBase):
251 blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
252 device=self.device,
253 dtype=torch.int64).view(-1, 2)
276 - data: Dict[str, Any] = {
277 - "num_seq_groups": num_seq_groups,
278 - "blocks_to_swap_in": blocks_to_swap_in,
279 - "blocks_to_swap_out": blocks_to_swap_out,
280 - "blocks_to_copy": blocks_to_copy,
281 - }
282 - broadcast_tensor_dict(data, src=0)
283 -
284 - self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
285 -
286 - # If there is no input, we don't need to execute the model.
287 - if num_seq_groups == 0:
288 - return []
254
290 - output = self.model_runner.execute_model(seq_group_metadata_list,
291 - self.gpu_cache)
292 -
293 - # Worker only supports single-step execution. Wrap the output in a list
294 - # to conform to interface.
295 - return [output]
255 + return WorkerInput(
256 + num_seq_groups=num_seq_groups,
257 + blocks_to_swap_in=blocks_to_swap_in,
258 + blocks_to_swap_out=blocks_to_swap_out,
259 + blocks_to_copy=blocks_to_copy,
260 + )
261
262 @torch.inference_mode()
298 - def start_worker_execution_loop(self) -> None:
299 - """Execute model loop in parallel worker.
300 -
301 - You can stop the loop by executing a driver worker with an empty output.
302 - See `stop_remote_worker_execution_loop` for more details.
303 - """
304 - while self._execute_model_non_driver():
305 - pass
306 -
307 - def _execute_model_non_driver(self) -> bool:
308 - """Execute model in parallel worker.
309 -
310 - Returns True iff there are remaining sequences to process.
311 - """
312 - assert not self.is_driver_worker
313 - data = broadcast_tensor_dict(src=0)
314 - if not data:
315 - return False
316 -
317 - num_seq_groups = data.get("num_seq_groups", 0)
318 - blocks_to_swap_in = data.get("blocks_to_swap_in")
319 - blocks_to_swap_out = data.get("blocks_to_swap_out")
320 - blocks_to_copy = data.get("blocks_to_copy")
321 - self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
322 -
323 - # If there is no input, we don't need to execute the model.
324 - if num_seq_groups == 0:
325 - return False
326 -
327 - self.model_runner.execute_model(None, self.gpu_cache)
328 - return True
263 + def execute_worker(self, worker_input: WorkerInput) -> None:
264 + # Issue cache operations.
265 + if (worker_input.blocks_to_swap_in is not None
266 + and worker_input.blocks_to_swap_in.numel() > 0):
267 + self.cache_engine.swap_in(worker_input.blocks_to_swap_in)
268 + if (worker_input.blocks_to_swap_out is not None
269 + and worker_input.blocks_to_swap_out.numel() > 0):
270 + self.cache_engine.swap_out(worker_input.blocks_to_swap_out)
271 + if (worker_input.blocks_to_copy is not None
272 + and worker_input.blocks_to_copy.numel() > 0):
273 + self.cache_engine.copy(worker_input.blocks_to_copy)
274
275 def add_lora(self, lora_request: LoRARequest) -> bool:
276 return self.model_runner.add_lora(lora_request)
vllm/worker/worker_base.py
+165 -5
@@ -1,20 +1,26 @@
1 +import dataclasses
2 import importlib
3 import os
4 from abc import ABC, abstractmethod
4 -from typing import Dict, List, Optional, Set, Tuple
5 +from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7 +import torch
8 +
9 +from vllm.distributed import broadcast_tensor_dict
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.sequence import ExecuteModelRequest, SamplerOutput
13 from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
14 update_environment_variables)
15 +from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
16
17 logger = init_logger(__name__)
18
19
20 class WorkerBase(ABC):
21 """Worker interface that allows vLLM to cleanly separate implementations for
17 - different hardware.
22 + different hardware. Also abstracts control plane communication, e.g., to
23 + communicate request metadata to other workers.
24 """
25
26 @abstractmethod
@@ -46,13 +52,23 @@ class WorkerBase(ABC):
52 """
53 raise NotImplementedError
54
55 + @torch.inference_mode()
56 + def start_worker_execution_loop(self) -> None:
57 + """Execute model loop in parallel worker.
58 +
59 + You can stop the loop by executing a driver worker with an empty output.
60 + See `stop_remote_worker_execution_loop` for more details.
61 + """
62 + while True:
63 + output = self.execute_model(execute_model_req=None)
64 + if output is None:
65 + return None
66 +
67 @abstractmethod
68 def execute_model(
69 self,
70 execute_model_req: Optional[ExecuteModelRequest] = None
53 - ) -> List[SamplerOutput]:
54 - """Executes at least one model step on the given sequences, unless no
55 - sequences are provided."""
71 + ) -> Optional[List[SamplerOutput]]:
72 raise NotImplementedError
73
74 @abstractmethod
@@ -98,6 +114,150 @@ class LoraNotSupportedWorkerBase(WorkerBase):
114 raise ValueError(f"{type(self)} does not support LoRA")
115
116
117 +@dataclasses.dataclass(frozen=True)
118 +class WorkerInput:
119 + """Local inputs to each worker. May contain device-specific data. These
120 + fields should be broadcastable to other workers.
121 + """
122 +
123 + num_seq_groups: Optional[int] = None
124 + blocks_to_swap_in: Optional[torch.Tensor] = None
125 + blocks_to_swap_out: Optional[torch.Tensor] = None
126 + blocks_to_copy: Optional[torch.Tensor] = None
127 +
128 + @classmethod
129 + def from_broadcasted_tensor_dict(
130 + cls: Type["WorkerInput"],
131 + tensor_dict: Dict[str, Any],
132 + ) -> "WorkerInput":
133 + """
134 + Pop fields from the given tensor_dict and populate a new instance of
135 + WorkerInput.
136 + """
137 + return cls(
138 + num_seq_groups=tensor_dict.pop("num_seq_groups"),
139 + blocks_to_swap_in=tensor_dict.pop("blocks_to_swap_in"),
140 + blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"),
141 + blocks_to_copy=tensor_dict.pop("blocks_to_copy"),
142 + )
143 +
144 + def as_broadcastable_tensor_dict(
145 + self) -> Dict[str, Union[int, torch.Tensor]]:
146 + """
147 + Extract broadcastable fields.
148 + """
149 + tensor_dict = {
150 + "num_seq_groups": self.num_seq_groups,
151 + "blocks_to_swap_in": self.blocks_to_swap_in,
152 + "blocks_to_swap_out": self.blocks_to_swap_out,
153 + "blocks_to_copy": self.blocks_to_copy,
154 + }
155 +
156 + return tensor_dict
157 +
158 +
159 +class LocalOrDistributedWorkerBase(WorkerBase):
160 + """
161 + Partial implementation of WorkerBase that has a default `execute_model`
162 + definition to perform metadata transfer between workers when in distributed
163 + mode. Subclasses of this interface should use model runners that inherit
164 + from ModelRunnerBase, and should only need to implement worker-local logic.
165 + If custom control plane logic is needed to transfer metadata, or if the
166 + model runner cannot inherit from ModelRunnerBase, use WorkerBase instead.
167 + """
168 + is_driver_worker: bool
169 + model_runner: ModelRunnerBase
170 +
171 + @property
172 + @abstractmethod
173 + def do_metadata_broadcast(self) -> bool:
174 + """
175 + Used by the default `execute_model` to check whether broadcast is
176 + needed to transfer request inputs from the driver worker to other
177 + workers in the TP group. If WorkerBase subclass only supports
178 + single-worker execution, then this method should return False.
179 + """
180 + raise NotImplementedError
181 +
182 + @property
183 + @abstractmethod
184 + def kv_cache(self) -> Optional[List[torch.Tensor]]:
185 + """
186 + Get the kv cache to pass to the worker's model runner. Used by the
187 + default `execute_model`. If the worker's model runner does not follow
188 + the ModelRunnerBase interface, then inherit from WorkerBase instead.
189 + """
190 + raise NotImplementedError
191 +
192 + @abstractmethod
193 + def prepare_worker_input(
194 + self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
195 + """
196 + Prepare the inputs to WorkerBase.execute_worker from an execution
197 + request. This method may move data to the worker's local device. It is
198 + not allowed to communicate with other workers or devices.
199 + """
200 + raise NotImplementedError
201 +
202 + @abstractmethod
203 + def execute_worker(self, worker_input: WorkerInput) -> None:
204 + """
205 + Process an execution request.
206 + """
207 + raise NotImplementedError
208 +
209 + def execute_model(
210 + self,
211 + execute_model_req: Optional[ExecuteModelRequest] = None
212 + ) -> Optional[List[SamplerOutput]]:
213 + """Executes at least one model step on the given sequences, unless no
214 + sequences are provided."""
215 + if self.is_driver_worker:
216 + if execute_model_req is None:
217 + if self.do_metadata_broadcast:
218 + # This signals that there's no more requests to process for
219 + # now. All workers are running infinite loop with
220 + # broadcast_tensor_dict, and it stops the loop when the
221 + # driver broadcasts an empty input. Send an empty input to
222 + # notify all other workers to stop their execution loop.
223 + broadcast_tensor_dict({}, src=0)
224 + return None
225 +
226 + worker_input: WorkerInput = self.prepare_worker_input(
227 + execute_model_req=execute_model_req)
228 + model_input: ModelRunnerInputBase = (
229 + self.model_runner.prepare_model_input(
230 + execute_model_req.seq_group_metadata_list))
231 +
232 + if self.do_metadata_broadcast:
233 + broadcast_data = worker_input.as_broadcastable_tensor_dict()
234 + broadcast_data.update(
235 + model_input.as_broadcastable_tensor_dict())
236 + broadcast_tensor_dict(broadcast_data, src=0)
237 + else:
238 + assert self.do_metadata_broadcast
239 + broadcast_data = broadcast_tensor_dict(src=0)
240 + if not broadcast_data:
241 + return None
242 +
243 + worker_input = WorkerInput.from_broadcasted_tensor_dict(
244 + broadcast_data)
245 + model_input = (
246 + self.model_runner.
247 + make_model_input_from_broadcasted_tensor_dict(broadcast_data))
248 +
249 + self.execute_worker(worker_input)
250 +
251 + # If there is no input, we don't need to execute the model.
252 + if worker_input.num_seq_groups == 0:
253 + return []
254 +
255 + output = self.model_runner.execute_model(model_input, self.kv_cache)
256 + # Worker only supports single-step execution. Wrap the output in a
257 + # list to conform to interface.
258 + return [output]
259 +
260 +
261 class WorkerWrapperBase:
262 """
263 The whole point of this class is to lazily initialize the worker.
vllm/worker/xpu_model_runner.py
+72 -19
@@ -1,4 +1,5 @@
1 -from typing import List, Optional, Tuple
1 +from dataclasses import dataclass
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
4 import torch
5 import torch.nn as nn
@@ -14,6 +15,15 @@ from vllm.sampling_params import SamplingParams
15 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
16 from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
17 from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
18 +from vllm.worker.model_runner_base import (
19 + ModelRunnerBase, ModelRunnerInputBase,
20 + _add_attn_metadata_broadcastable_dict,
21 + _add_sampling_metadata_broadcastable_dict,
22 + _init_attn_metadata_from_tensor_dict,
23 + _init_sampling_metadata_from_tensor_dict)
24 +
25 +if TYPE_CHECKING:
26 + from vllm.attention.backends.abstract import AttentionBackend
27
28 logger = init_logger(__name__)
29
@@ -24,7 +34,42 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
34 ]
35
36
27 -class XPUModelRunner:
37 +@dataclass(frozen=True)
38 +class ModelInputForXPU(ModelRunnerInputBase):
39 + """
40 + Used by the NeuronModelRunner.
41 + """
42 + input_tokens: Optional[torch.Tensor] = None
43 + input_positions: Optional[torch.Tensor] = None
44 + attn_metadata: Optional["AttentionMetadata"] = None
45 + sampling_metadata: Optional["SamplingMetadata"] = None
46 + multi_modal_input: Optional[Dict[str, torch.Tensor]] = None
47 +
48 + def as_broadcastable_tensor_dict(
49 + self) -> Dict[str, Union[int, torch.Tensor]]:
50 + tensor_dict = {
51 + "input_tokens": self.input_tokens,
52 + "input_positions": self.input_positions,
53 + }
54 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
55 + _add_sampling_metadata_broadcastable_dict(tensor_dict,
56 + self.sampling_metadata)
57 + return tensor_dict
58 +
59 + @classmethod
60 + def from_broadcasted_tensor_dict(
61 + cls: Type["ModelInputForXPU"],
62 + tensor_dict: Dict[str, Any],
63 + attn_backend: Optional["AttentionBackend"] = None,
64 + ) -> "ModelInputForXPU":
65 + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
66 + if attn_backend is not None:
67 + tensor_dict = _init_attn_metadata_from_tensor_dict(
68 + attn_backend, tensor_dict)
69 + return cls(**tensor_dict)
70 +
71 +
72 +class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
73
74 def __init__(
75 self,
@@ -130,15 +175,22 @@ class XPUModelRunner:
175 # Run the model with the dummy inputs.
176 num_layers = self.model_config.get_num_layers(self.parallel_config)
177 kv_caches = [None] * num_layers
133 - self.execute_model(seqs, kv_caches)
178 + model_input = self.prepare_model_input(seqs)
179 + self.execute_model(model_input, kv_caches)
180 torch.xpu.synchronize()
181 return
182
137 - def prepare_input_tensors(
183 + def make_model_input_from_broadcasted_tensor_dict(
184 + self, tensor_dict: Dict[str, Any]) -> ModelInputForXPU:
185 + return (ModelInputForXPU.from_broadcasted_tensor_dict(
186 + tensor_dict,
187 + attn_backend=self.attn_backend,
188 + ))
189 +
190 + def prepare_model_input(
191 self,
192 seq_group_metadata_list: List[SequenceGroupMetadata],
140 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
141 - Optional[torch.Tensor]]:
193 + ) -> ModelInputForXPU:
194 multi_modal_input = None
195 if self.is_driver_worker:
196 # NOTE: We assume that all sequences in the group are all prompts or
@@ -185,8 +237,11 @@ class XPUModelRunner:
237 num_prompts=0,
238 )
239
188 - return (input_tokens, input_positions, attn_metadata,
189 - sampling_metadata, multi_modal_input)
240 + return ModelInputForXPU(input_tokens=input_tokens,
241 + input_positions=input_positions,
242 + attn_metadata=attn_metadata,
243 + sampling_metadata=sampling_metadata,
244 + multi_modal_input=multi_modal_input)
245
246 def _prepare_decode(
247 self,
@@ -277,27 +332,25 @@ class XPUModelRunner:
332 @torch.inference_mode()
333 def execute_model(
334 self,
280 - seq_group_metadata_list: List[SequenceGroupMetadata],
335 + model_input: ModelInputForXPU,
336 kv_caches: List[torch.Tensor],
337 ) -> Optional[SamplerOutput]:
283 - (input_tokens, input_positions, attn_metadata, sampling_metadata,
284 - multi_modal_input
285 - ) = self.prepare_input_tensors(seq_group_metadata_list)
286 -
338 model_executable = self.model
339 execute_model_kwargs = {
289 - "input_ids": input_tokens,
290 - "positions": input_positions,
340 + "input_ids": model_input.input_tokens,
341 + "positions": model_input.input_positions,
342 "kv_caches": kv_caches,
292 - "attn_metadata": attn_metadata,
343 + "attn_metadata": model_input.attn_metadata,
344 }
345 if self.vision_language_config:
295 - execute_model_kwargs.update({"image_input": multi_modal_input})
346 + execute_model_kwargs.update(
347 + {"image_input": model_input.multi_modal_input})
348
349 hidden_states = model_executable(**execute_model_kwargs)
350
351 # Compute the logits.
300 - logits = self.model.compute_logits(hidden_states, sampling_metadata)
352 + logits = self.model.compute_logits(hidden_states,
353 + model_input.sampling_metadata)
354
355 # Only perform sampling in the driver worker.
356 if not self.is_driver_worker:
@@ -306,7 +359,7 @@ class XPUModelRunner:
359 # Sample the next token.
360 output = self.model.sample(
361 logits=logits,
309 - sampling_metadata=sampling_metadata,
362 + sampling_metadata=model_input.sampling_metadata,
363 )
364 return output
365