@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc5ebbd1d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Hardware][Intel GPU] refactor xpu_model_runner for tp (#7712)

Kunshang Ji committed Aug 23, 2024 at 11:06 UTC fc5ebbd1d3453461ea6e00a78faf87c41d1aa625
3 files changed +370 -650
vllm/executor/ray_xpu_executor.py
+17 -366
@@ -1,386 +1,37 @@
1 import asyncio
2 -import os
3 -from collections import defaultdict
4 -from itertools import islice, repeat
5 -from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
6 - Tuple, Union)
2 +from typing import List, Optional
3
4 import vllm.envs as envs
9 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, PromptAdapterConfig,
11 - SchedulerConfig, SpeculativeConfig)
12 -from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13 - DistributedGPUExecutor, DistributedGPUExecutorAsync)
14 -from vllm.executor.ray_utils import RayWorkerWrapper, ray
5 +from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
6 +from vllm.executor.xpu_executor import XPUExecutor
7 from vllm.logger import init_logger
16 -from vllm.lora.request import LoRARequest
17 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
18 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
19 - make_async)
20 -
21 -if ray is not None:
22 - from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
23 -
24 -if TYPE_CHECKING:
25 - from ray.util.placement_group import PlacementGroup
8 +from vllm.utils import get_vllm_instance_id, make_async
9
10 logger = init_logger(__name__)
11
29 -# If the env var is set, it uses the Ray's compiled DAG API
30 -# which optimizes the control plane overhead.
31 -# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
32 -USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
33 -
34 -
35 -class RayXPUExecutor(DistributedGPUExecutor):
36 -
37 - uses_ray: bool = True
38 -
39 - def __init__(
40 - self,
41 - model_config: ModelConfig,
42 - cache_config: CacheConfig,
43 - parallel_config: ParallelConfig,
44 - scheduler_config: SchedulerConfig,
45 - device_config: DeviceConfig,
46 - load_config: LoadConfig,
47 - lora_config: Optional[LoRAConfig],
48 - prompt_adapter_config: Optional[PromptAdapterConfig],
49 - speculative_config: Optional[SpeculativeConfig],
50 - ) -> None:
51 - assert device_config.device_type == "xpu"
52 - assert (not speculative_config
53 - ), "Speculative decoding not yet supported for XPU backend"
54 -
55 - self.model_config = model_config
56 - self.cache_config = cache_config
57 - self.load_config = load_config
58 - self.lora_config = lora_config
59 - self.parallel_config = parallel_config
60 - self.scheduler_config = scheduler_config
61 - self.device_config = device_config
62 - self.prompt_adapter_config = prompt_adapter_config
63 -
64 - placement_group = self.parallel_config.placement_group
65 -
66 - # Disable Ray usage stats collection.
67 - ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
68 - if ray_usage != "1":
69 - os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
70 -
71 - # Create the parallel GPU workers.
72 - self._init_workers_ray(placement_group)
73 -
74 - self.forward_dag = None
75 - if USE_RAY_COMPILED_DAG:
76 - self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
77 -
78 - # This is non-None when the execute model loop is running
79 - # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
80 - self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
81 - # Updated by implementations that require additional args to be passed
82 - # to the _run_workers execute_model call
83 - self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
84 -
85 - def _init_executor(self) -> None:
86 - pass
87 -
88 - def determine_num_available_blocks(self) -> Tuple[int, int]:
89 - """Determine the number of available KV blocks.
12
91 - This invokes `determine_num_available_blocks` on each worker and takes
92 - the min of the results, guaranteeing that the selected cache sizes are
93 - compatible with all workers.
94 -
95 - Returns:
96 - - Tuple[num_gpu_blocks, num_cpu_blocks]
97 - """
98 - # Get the maximum number of blocks that can be allocated on GPU and CPU.
99 - num_blocks = self._run_workers("determine_num_available_blocks", )
100 -
101 - # Since we use a shared centralized controller, we take the minimum
102 - # number of blocks across all workers to make sure all the memory
103 - # operators can be applied to all workers.
104 - num_gpu_blocks = min(b[0] for b in num_blocks)
105 - num_cpu_blocks = min(b[1] for b in num_blocks)
106 -
107 - return num_gpu_blocks, num_cpu_blocks
108 -
109 - def _get_worker_wrapper_args(self) -> Dict[str, Any]:
110 - return dict(
111 - worker_module_name="vllm.worker.xpu_worker",
112 - worker_class_name="XPUWorker",
113 - trust_remote_code=self.model_config.trust_remote_code,
114 - )
115 -
116 - def _init_workers_ray(self, placement_group: "PlacementGroup",
117 - **ray_remote_kwargs):
118 - if self.parallel_config.tensor_parallel_size == 1:
119 - # For single GPU case, we use a ray worker with constrained memory.
120 - num_gpus = self.cache_config.gpu_memory_utilization
121 - else:
122 - # Otherwise, the ray workers are allocated with a full GPU.
123 - num_gpus = 1
124 -
125 - # The driver dummy worker does not actually use any resources.
126 - # It holds the resource for the driver worker.
127 - self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
128 - # The remaining workers are the actual ray actors.
129 - self.workers: List[RayWorkerWrapper] = []
130 -
131 - # Create the workers.
132 - driver_ip = get_ip()
133 - worker_wrapper_kwargs = self._get_worker_wrapper_args()
134 - for bundle_id, bundle in enumerate(placement_group.bundle_specs):
135 - if not bundle.get("GPU", 0):
136 - continue
137 - scheduling_strategy = PlacementGroupSchedulingStrategy(
138 - placement_group=placement_group,
139 - placement_group_capture_child_tasks=True,
140 - placement_group_bundle_index=bundle_id,
141 - )
142 - worker = ray.remote(
143 - num_cpus=0,
144 - num_gpus=num_gpus,
145 - scheduling_strategy=scheduling_strategy,
146 - **ray_remote_kwargs,
147 - )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
148 -
149 - worker_ip = ray.get(worker.get_node_ip.remote())
150 - if worker_ip == driver_ip and self.driver_dummy_worker is None:
151 - # If the worker is on the same node as the driver, we use it
152 - # as the resource holder for the driver process.
153 - self.driver_dummy_worker = worker
154 - self.driver_worker = RayWorkerWrapper(**worker_wrapper_kwargs)
155 - else:
156 - # Else, added to the list of workers.
157 - self.workers.append(worker)
158 - if self.driver_dummy_worker is None:
159 - raise ValueError(
160 - "Ray does not allocate any GPUs on the driver node. Consider "
161 - "adjusting the Ray placement group or running the driver on a "
162 - "GPU node.")
13 +class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
14
15 + def _get_env_vars_to_be_updated(self):
16 # Get the set of GPU IDs used on each node.
17 worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
18 use_dummy_driver=True)
19
168 - node_workers = defaultdict(list)
169 - node_gpus = defaultdict(list)
170 -
171 - for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
172 - node_workers[node_id].append(i)
173 - node_gpus[node_id].extend(gpu_ids)
174 - for node_id, gpu_ids in node_gpus.items():
175 - node_gpus[node_id] = sorted(gpu_ids)
176 -
177 - # TODO: add env var for xpu
178 -
179 - distributed_init_method = get_distributed_init_method(
180 - driver_ip, get_open_port())
181 -
182 - def collect_arg_helper_func(**kwargs):
183 - # avoid writing `{"name": value}` manually
184 - return kwargs
185 -
186 - init_worker_all_kwargs = []
187 -
188 - # Initialize the actual workers inside worker wrapper.
189 - for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids, ):
190 - local_rank = node_workers[node_id].index(rank)
191 - init_worker_all_kwargs.append(
192 - collect_arg_helper_func(
193 - model_config=self.model_config,
194 - parallel_config=self.parallel_config,
195 - scheduler_config=self.scheduler_config,
196 - device_config=self.device_config,
197 - cache_config=self.cache_config,
198 - load_config=self.load_config,
199 - local_rank=local_rank,
200 - rank=rank,
201 - distributed_init_method=distributed_init_method,
202 - lora_config=self.lora_config,
203 - is_driver_worker=rank == 0,
204 - ))
205 - self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
20 + VLLM_INSTANCE_ID = get_vllm_instance_id()
21
207 - self._run_workers("init_device")
208 - self._run_workers(
209 - "load_model",
210 - max_concurrent_workers=self.parallel_config.
211 - max_parallel_loading_workers,
212 - )
22 + # Set environment variables for the driver and workers.
23 + all_args_to_update_environment_variables = [({
24 + "VLLM_INSTANCE_ID":
25 + VLLM_INSTANCE_ID,
26 + "VLLM_TRACE_FUNCTION":
27 + str(envs.VLLM_TRACE_FUNCTION),
28 + }, ) for (_, _) in worker_node_and_gpu_ids]
29 + return all_args_to_update_environment_variables
30
214 - def initialize_cache(self, num_gpu_blocks: int,
215 - num_cpu_blocks: int) -> None:
216 - """Initialize the KV cache in all workers.
217 - """
31
219 - # NOTE: We log here to avoid multiple logs when number of workers is
220 - # greater than one. We could log in the engine, but not all executors
221 - # have GPUs.
222 - logger.info("# GPU blocks: %d, "
223 - "# CPU blocks: %d", num_gpu_blocks, num_cpu_blocks)
224 -
225 - self.cache_config.num_gpu_blocks = num_gpu_blocks
226 - self.cache_config.num_cpu_blocks = num_cpu_blocks
227 -
228 - self._run_workers("initialize_cache",
229 - num_gpu_blocks=num_gpu_blocks,
230 - num_cpu_blocks=num_cpu_blocks)
231 -
232 - def _driver_execute_model(
233 - self,
234 - execute_model_req: Optional[ExecuteModelRequest] = None
235 - ) -> List[SamplerOutput]:
236 - """Run execute_model in the driver worker.
237 -
238 - Passing None will cause the driver to stop the model execution
239 - loop running in each of the remote workers.
240 - """
241 - return self.driver_worker.execute_method("execute_model",
242 - execute_model_req)
243 -
244 - def add_lora(self, lora_request: LoRARequest) -> bool:
245 - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
246 - return self._run_workers(
247 - "add_lora",
248 - lora_request=lora_request,
249 - )
250 -
251 - def remove_lora(self, lora_id: int) -> bool:
252 - assert lora_id > 0, "lora_id must be greater than 0."
253 - return self._run_workers(
254 - "remove_lora",
255 - lora_id=lora_id,
256 - )
257 -
258 - def list_loras(self) -> Set[int]:
259 - return self._run_workers("list_loras")
260 -
261 - def _run_workers(
262 - self,
263 - method: str,
264 - *args,
265 - async_run_remote_workers_only: bool = False,
266 - all_args: Optional[List[Tuple[Any, ...]]] = None,
267 - all_kwargs: Optional[List[Dict[str, Any]]] = None,
268 - use_dummy_driver: bool = False,
269 - max_concurrent_workers: Optional[int] = None,
270 - **kwargs,
271 - ) -> Any:
272 - """Runs the given method on all workers. Can be used in the following
273 - ways:
274 -
275 - - args/kwargs: All workers share the same args/kwargs
276 - - args/kwargs and driver_args/driver_kwargs: Driver worker has
277 - different args
278 - - all_args/all_kwargs: args/kwargs for each worker are specified
279 - individually
280 - """
281 -
282 - if max_concurrent_workers:
283 - raise NotImplementedError(
284 - "max_concurrent_workers is not supported yet.")
285 -
286 - count = len(self.workers)
287 - all_worker_args = repeat(args, count) if all_args is None \
288 - else islice(all_args, 1, None)
289 - all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
290 - else islice(all_kwargs, 1, None)
291 -
292 - # Start the ray workers first.
293 - ray_worker_outputs = [
294 - worker.execute_method.remote(method, *worker_args, **worker_kwargs)
295 - for (worker, worker_args, worker_kwargs
296 - ) in zip(self.workers, all_worker_args, all_worker_kwargs)
297 - ]
298 -
299 - if async_run_remote_workers_only:
300 - # Just return futures
301 - return ray_worker_outputs
302 -
303 - driver_worker_output = []
304 - driver_args = args if all_args is None else all_args[0]
305 - driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
306 - # Start the driver worker after all the ray workers.
307 - if not use_dummy_driver:
308 - driver_worker_output = self.driver_worker.execute_method(
309 - method, *driver_args, **driver_kwargs)
310 - else:
311 - assert self.driver_dummy_worker is not None
312 - driver_worker_output = ray.get(
313 - self.driver_dummy_worker.execute_method.remote(
314 - method, *driver_args, **driver_kwargs))
315 - # Get the results of the ray workers.
316 - if self.workers:
317 - ray_worker_outputs = ray.get(ray_worker_outputs)
318 -
319 - return driver_worker_output + ray_worker_outputs
320 -
321 - def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
322 - """Wait for futures returned from _run_workers() with
323 - async_run_remote_workers_only to complete."""
324 - ray.get(parallel_worker_tasks)
325 -
326 - def _compiled_ray_dag(self, enable_asyncio: bool):
327 - import pkg_resources
328 - from packaging import version
329 -
330 - required_version = version.parse("2.32")
331 - current_version = version.parse(
332 - pkg_resources.get_distribution("ray").version)
333 - if current_version < required_version:
334 - raise ValueError(f"Ray version {required_version} or greater is "
335 - f"required, but found {current_version}")
336 -
337 - from ray.dag import InputNode, MultiOutputNode
338 - assert self.parallel_config.use_ray
339 -
340 - # Right now, compiled DAG requires at least 1 arg. We send
341 - # a dummy value for now. It will be fixed soon.
342 - with InputNode() as input_data:
343 - forward_dag = MultiOutputNode([
344 - worker.execute_model_compiled_dag_remote.
345 - bind( # type: ignore[attr-defined]
346 - input_data) for worker in self.workers
347 - ])
348 - return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
349 -
350 - def check_health(self) -> None:
351 - """Raises an error if engine is unhealthy."""
352 - self._check_if_any_actor_is_dead()
353 -
354 - def _check_if_any_actor_is_dead(self):
355 - if not self.workers:
356 - return
357 -
358 - dead_actors = []
359 - for actor in self.workers:
360 - actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
361 - if actor_state["State"] == "DEAD":
362 - dead_actors.append(actor)
363 - if dead_actors:
364 - raise RuntimeError("At least one Worker is dead. "
365 - f"Dead Workers: {dead_actors}. ")
366 -
367 -
368 -class RayXPUExecutorAsync(RayXPUExecutor, DistributedGPUExecutorAsync):
32 +class RayXPUExecutorAsync(RayXPUExecutor, RayGPUExecutorAsync):
33
34 def __init__(self, *args, **kwargs):
35 super().__init__(*args, **kwargs)
36 self.driver_exec_method = make_async(self.driver_worker.execute_method)
373 -
374 - async def _driver_execute_model_async(
375 - self,
376 - execute_model_req: Optional[ExecuteModelRequest] = None
377 - ) -> List[SamplerOutput]:
378 - return await self.driver_exec_method("execute_model",
379 - execute_model_req)
380 -
381 - async def _start_worker_execution_loop(self):
382 - coros = [
383 - worker.execute_method.remote("start_worker_execution_loop")
384 - for worker in self.workers
385 - ]
386 - return await asyncio.gather(*coros)
37 + self.pp_locks: Optional[List[asyncio.Lock]] = None
vllm/worker/xpu_model_runner.py
+350 -278
@@ -1,14 +1,17 @@
1 +import dataclasses
2 +import time
3 +import weakref
4 from dataclasses import dataclass
2 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
5 +from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6 + TypeVar)
7
8 import torch
9 import torch.nn as nn
10
11 from vllm.attention import get_attn_backend
12 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 - ModelConfig, MultiModalConfig, ParallelConfig,
13 + ModelConfig, ObservabilityConfig, ParallelConfig,
14 PromptAdapterConfig, SchedulerConfig)
11 -from vllm.distributed import broadcast_tensor_dict
15 from vllm.inputs import INPUT_REGISTRY, InputRegistry
16 from vllm.logger import init_logger
17 from vllm.model_executor.model_loader import get_model
@@ -20,7 +23,7 @@ from vllm.sequence import (IntermediateTensors, SamplerOutput,
23 from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
24 from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
25 from vllm.worker.model_runner_base import (
23 - ModelRunnerBase, ModelRunnerInputBase,
26 + ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
27 _add_attn_metadata_broadcastable_dict,
28 _add_sampling_metadata_broadcastable_dict,
29 _init_attn_metadata_from_tensor_dict,
@@ -37,6 +40,8 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
40 _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
41 ]
42
43 +TModelInputForXPU = TypeVar('TModelInputForXPU', bound="ModelInputForXPU")
44 +
45
46 @dataclass(frozen=True)
47 class ModelInputForXPU(ModelRunnerInputBase):
@@ -46,11 +51,40 @@ class ModelInputForXPU(ModelRunnerInputBase):
51 input_tokens: Optional[torch.Tensor] = None
52 input_positions: Optional[torch.Tensor] = None
53 attn_metadata: Optional["AttentionMetadata"] = None
49 - sampling_metadata: Optional["SamplingMetadata"] = None
54 multi_modal_kwargs: Optional[BatchedTensorInputs] = None
55 + virtual_engine: Optional[int] = None
56 + seq_lens: Optional[List[int]] = None
57 + query_lens: Optional[List[int]] = None
58 +
59 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
60 + tensor_dict = {
61 + "input_tokens": self.input_tokens,
62 + "input_positions": self.input_positions,
63 + }
64 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
65 +
66 + return tensor_dict
67 +
68 + @classmethod
69 + def from_broadcasted_tensor_dict(
70 + cls: Type[TModelInputForXPU],
71 + tensor_dict: Dict[str, Any],
72 + attn_backend: Optional["AttentionBackend"] = None,
73 + ) -> TModelInputForXPU:
74 + if attn_backend is not None:
75 + tensor_dict = _init_attn_metadata_from_tensor_dict(
76 + attn_backend, tensor_dict)
77 + return cls(**tensor_dict)
78 +
79 +
80 +@dataclass(frozen=True)
81 +class ModelInputForXPUWithSamplingMetadata(ModelInputForXPU):
82 + """
83 + Used by the ModelRunner.
84 + """
85 + sampling_metadata: Optional["SamplingMetadata"] = None
86
52 - def as_broadcastable_tensor_dict(
53 - self) -> Dict[str, Union[int, torch.Tensor]]:
87 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
88 tensor_dict = {
89 "input_tokens": self.input_tokens,
90 "input_positions": self.input_positions,
@@ -62,10 +96,10 @@ class ModelInputForXPU(ModelRunnerInputBase):
96
97 @classmethod
98 def from_broadcasted_tensor_dict(
65 - cls: Type["ModelInputForXPU"],
99 + cls,
100 tensor_dict: Dict[str, Any],
101 attn_backend: Optional["AttentionBackend"] = None,
68 - ) -> "ModelInputForXPU":
102 + ) -> "ModelInputForXPUWithSamplingMetadata":
103 tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
104 if attn_backend is not None:
105 tensor_dict = _init_attn_metadata_from_tensor_dict(
@@ -73,7 +107,230 @@ class ModelInputForXPU(ModelRunnerInputBase):
107 return cls(**tensor_dict)
108
109
76 -class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
110 +class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
111 +
112 + def __init__(self,
113 + runner: "XPUModelRunner",
114 + finished_requests_ids: Optional[List[str]] = None) -> None:
115 + super().__init__()
116 + self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
117 + self.runner = runner
118 + self.model_input_cls = self.runner._model_input_cls
119 + self.attn_backend = self.runner.attn_backend
120 + self.sliding_window = self.runner.sliding_window
121 + self.block_size = self.runner.block_size
122 + self.device = self.runner.device
123 +
124 + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
125 + self.seq_group_metadata_list.append(seq_group_metadata)
126 +
127 + def build(self) -> ModelInputForXPU:
128 + is_prompt = self.seq_group_metadata_list[0].is_prompt
129 + # Prepare input tensors.
130 + if is_prompt:
131 + (input_tokens, input_positions, attn_metadata, seq_lens,
132 + multi_modal_kwargs) = self._prepare_prompt(
133 + self.seq_group_metadata_list)
134 + else:
135 + (input_tokens, input_positions,
136 + attn_metadata) = self._prepare_decode(
137 + self.seq_group_metadata_list)
138 + seq_lens = []
139 + multi_modal_kwargs = None
140 +
141 + return self.model_input_cls(
142 + input_tokens=input_tokens,
143 + input_positions=input_positions,
144 + attn_metadata=attn_metadata,
145 + multi_modal_kwargs=multi_modal_kwargs,
146 + seq_lens=seq_lens,
147 + query_lens=seq_lens,
148 + )
149 +
150 + def _prepare_prompt(
151 + self,
152 + seq_group_metadata_list: List[SequenceGroupMetadata],
153 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
154 + BatchedTensorInputs]:
155 + assert len(seq_group_metadata_list) > 0
156 + input_tokens: List[int] = []
157 + input_positions: List[int] = []
158 + slot_mapping: List[int] = []
159 + seq_lens: List[int] = []
160 + multi_modal_inputs_list: List[MultiModalInputs] = []
161 +
162 + for seq_group_metadata in seq_group_metadata_list:
163 + assert seq_group_metadata.is_prompt
164 + seq_ids = list(seq_group_metadata.seq_data.keys())
165 + assert len(seq_ids) == 1
166 + seq_id = seq_ids[0]
167 +
168 + seq_data = seq_group_metadata.seq_data[seq_id]
169 + prompt_tokens = seq_data.get_token_ids()
170 + computed_len = seq_data.get_num_computed_tokens()
171 + seq_len = len(prompt_tokens)
172 +
173 + seq_lens.append(seq_len) # Prompt token num
174 + input_tokens.extend(prompt_tokens) # Token ids
175 +
176 + # Token position ids
177 + # NOTE(woosuk): Here we assume that the first token in the prompt
178 + # is always the first token in the sequence.
179 + input_positions.extend(list(range(computed_len, seq_len)))
180 +
181 + if seq_group_metadata.block_tables is None:
182 + # During memory profiling, the block tables are not initialized
183 + # yet. In this case, we just use a dummy slot mapping.
184 + slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
185 + continue
186 +
187 + # Compute the slot mapping.
188 + block_table = seq_group_metadata.block_tables[seq_id]
189 + # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
190 + # where start_idx is max(0, seq_len - sliding_window).
191 + # For example, if the prompt len is 10, sliding window is 8, and
192 + # block size is 4, the first two tokens are masked and the slot
193 + # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
194 + start_idx = 0
195 + if self.sliding_window is not None:
196 + start_idx = max(0, seq_len - self.sliding_window)
197 +
198 + for i in range(computed_len, seq_len):
199 + if i < start_idx:
200 + slot_mapping.append(_PAD_SLOT_ID)
201 + continue
202 +
203 + block_number = block_table[i //
204 + self.block_size] # type: ignore
205 + block_offset = i % self.block_size # type: ignore
206 + slot = block_number * self.block_size + block_offset
207 + slot_mapping.append(slot)
208 +
209 + num_prompt_tokens = len(input_tokens)
210 +
211 + input_tokens = torch.tensor(input_tokens,
212 + dtype=torch.long,
213 + device=self.device) # type: ignore
214 + input_positions = torch.tensor(input_positions,
215 + dtype=torch.long,
216 + device=self.device) # type: ignore
217 + slot_mapping = torch.tensor(slot_mapping,
218 + dtype=torch.long,
219 + device=self.device) # type: ignore
220 +
221 + max_seqlen = max(seq_lens)
222 + tmp = [0]
223 + tmp.extend(seq_lens)
224 + seqlen = torch.tensor(tmp)
225 + seqlen_q = torch.cumsum(seqlen, dim=0).to(device=self.device)
226 +
227 + attn_metadata = self.attn_backend.make_metadata(
228 + is_prompt=True,
229 + slot_mapping=slot_mapping,
230 + seq_lens=seq_lens,
231 + seqlen_q=seqlen_q,
232 + max_seqlen=max_seqlen,
233 + seq_lens_tensor=torch.tensor([]),
234 + max_decode_seq_len=0,
235 + num_prefills=len(seq_lens),
236 + num_prefill_tokens=num_prompt_tokens,
237 + num_decode_tokens=0,
238 + block_tables=torch.tensor([], device=self.device, dtype=torch.int),
239 + )
240 +
241 + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
242 +
243 + return (input_tokens, input_positions, attn_metadata, seq_lens,
244 + multi_modal_kwargs)
245 +
246 + def _prepare_decode(
247 + self,
248 + seq_group_metadata_list: List[SequenceGroupMetadata],
249 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
250 + assert len(seq_group_metadata_list) > 0
251 + input_tokens: List[int] = []
252 + input_positions: List[int] = []
253 + slot_mapping: List[int] = []
254 + seq_lens: List[int] = []
255 + block_tables: List[List[int]] = []
256 +
257 + for seq_group_metadata in seq_group_metadata_list:
258 + assert not seq_group_metadata.is_prompt
259 + assert seq_group_metadata.token_chunk_size == 1
260 +
261 + seq_ids = list(seq_group_metadata.seq_data.keys())
262 +
263 + for seq_id in seq_ids:
264 + seq_data = seq_group_metadata.seq_data[seq_id]
265 + generation_token = seq_data.get_last_token_id()
266 + input_tokens.append(generation_token)
267 +
268 + seq_len = seq_data.get_len()
269 + position = seq_len - 1
270 + input_positions.append(position)
271 +
272 + seq_len = seq_len if self.sliding_window is None else min(
273 + seq_len, self.sliding_window)
274 + seq_lens.append(seq_len)
275 +
276 + block_table = seq_group_metadata.block_tables[seq_id]
277 + block_number = block_table[position // self.block_size]
278 + block_offset = position % self.block_size
279 + slot = block_number * self.block_size + block_offset
280 + slot_mapping.append(slot)
281 +
282 + if self.sliding_window is not None:
283 + sliding_window_blocks = (self.sliding_window //
284 + self.block_size)
285 + block_table = block_table[-sliding_window_blocks:]
286 + block_tables.append(block_table)
287 +
288 + max_decode_seq_len = max(seq_lens)
289 +
290 + input_tokens = torch.tensor(input_tokens,
291 + dtype=torch.long,
292 + device=self.device)
293 + input_positions = torch.tensor(input_positions,
294 + dtype=torch.long,
295 + device=self.device)
296 + slot_mapping = torch.tensor(slot_mapping,
297 + dtype=torch.long,
298 + device=self.device)
299 + seq_lens_tensor = torch.tensor(seq_lens,
300 + dtype=torch.int,
301 + device=self.device)
302 +
303 + block_tables = make_tensor_with_pad(
304 + block_tables,
305 + pad=0,
306 + dtype=torch.int,
307 + device=self.device,
308 + )
309 +
310 + attn_metadata = self.attn_backend.make_metadata(
311 + is_prompt=False,
312 + slot_mapping=slot_mapping,
313 + seq_lens=seq_lens,
314 + seqlen_q=torch.tensor([]),
315 + max_seqlen=0,
316 + seq_lens_tensor=seq_lens_tensor,
317 + max_decode_seq_len=max_decode_seq_len,
318 + num_prefill_tokens=0,
319 + num_decode_tokens=len(input_tokens),
320 + num_prefills=0,
321 + block_tables=block_tables,
322 + )
323 + return (
324 + input_tokens,
325 + input_positions,
326 + attn_metadata,
327 + )
328 +
329 +
330 +class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
331 + _model_input_cls: Type[ModelInputForXPUWithSamplingMetadata] = (
332 + ModelInputForXPUWithSamplingMetadata)
333 + _builder_cls: Type[ModelInputForXPUBuilder] = ModelInputForXPUBuilder
334
335 def __init__(
336 self,
@@ -84,30 +341,32 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
341 cache_config: CacheConfig,
342 load_config: LoadConfig,
343 lora_config: Optional[LoRAConfig],
87 - multimodal_config: Optional[MultiModalConfig],
344 kv_cache_dtype: Optional[str] = "auto",
89 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
345 is_driver_worker: bool = False,
346 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
347 + return_hidden_states: bool = False,
348 + observability_config: Optional[ObservabilityConfig] = None,
349 input_registry: InputRegistry = INPUT_REGISTRY,
350 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
93 - *args,
94 - **kwargs,
351 ):
352 self.model_config = model_config
353 self.parallel_config = parallel_config
354 self.scheduler_config = scheduler_config
355 + self.device_config = device_config
356 + self.cache_config = cache_config
357 self.lora_config = lora_config
358 self.load_config = load_config
101 - self.cache_config = cache_config
102 - self.prompt_adapter_config = prompt_adapter_config
103 - self.multimodal_config = multimodal_config
359 self.is_driver_worker = is_driver_worker
360 + self.prompt_adapter_config = prompt_adapter_config
361 + self.observability_config = observability_config
362 + if self.observability_config is not None:
363 + print(f"observability_config is {self.observability_config}")
364 + self.return_hidden_states = return_hidden_states
365
106 - self.sliding_window = model_config.get_sliding_window()
107 - self.device_config = device_config
366 self.device = self.device_config.device
367
368 self.kv_cache_dtype = kv_cache_dtype
369 + self.sliding_window = model_config.get_sliding_window()
370 self.block_size = cache_config.block_size
371
372 self.attn_backend = get_attn_backend(
@@ -203,166 +462,68 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
462 # Run the model with the dummy inputs.
463 num_layers = self.model_config.get_num_layers(self.parallel_config)
464 kv_caches = [None] * num_layers
206 - model_input = self.prepare_model_input(seqs)
465 + finished_requests_ids = [seq.request_id for seq in seqs]
466 + model_input = self.prepare_model_input(
467 + seqs, finished_requests_ids=finished_requests_ids)
468 self.execute_model(model_input, kv_caches)
469 torch.xpu.synchronize()
470 return
471
472 def make_model_input_from_broadcasted_tensor_dict(
212 - self, tensor_dict: Dict[str, Any]) -> ModelInputForXPU:
213 - return (ModelInputForXPU.from_broadcasted_tensor_dict(
214 - tensor_dict,
215 - attn_backend=self.attn_backend,
216 - ))
217 -
218 - def prepare_model_input(
473 self,
220 - seq_group_metadata_list: List[SequenceGroupMetadata],
221 - virtual_engine: int = 0,
222 - finished_requests_ids: Optional[List[str]] = None
223 - ) -> ModelInputForXPU:
224 - multi_modal_kwargs = None
225 - if self.is_driver_worker:
226 - # NOTE: We assume that all sequences in the group are all prompts or
227 - # all decodes.
228 - is_prompt = seq_group_metadata_list[0].is_prompt
229 - # Prepare input tensors.
230 - if is_prompt:
231 - (input_tokens, input_positions, attn_metadata, seq_lens,
232 - multi_modal_kwargs
233 - ) = self._prepare_prompt(seq_group_metadata_list)
234 - else:
235 - (input_tokens, input_positions,
236 - attn_metadata) = self._prepare_decode(seq_group_metadata_list)
237 - seq_lens = []
238 - sampling_metadata = SamplingMetadata.prepare(
239 - seq_group_metadata_list,
240 - seq_lens,
241 - # subquery_lens is not needed if chunked prefill is not
242 - # supported. Since CPU worker doesn't support chunked prefill
243 - # just use seq_lens instead.
244 - seq_lens,
245 - self.device,
246 - pin_memory=False,
247 - generators=self.get_generators(finished_requests_ids))
248 - # Broadcast the metadata.
249 - metadata_dict = {
250 - "input_tokens": input_tokens,
251 - "input_positions": input_positions,
252 - "selected_token_indices":
253 - sampling_metadata.selected_token_indices,
254 - "multi_modal_kwargs": multi_modal_kwargs,
255 - }
256 - metadata_dict.update(attn_metadata.asdict_zerocopy())
257 - broadcast_tensor_dict(metadata_dict, src=0)
258 - else:
259 - metadata_dict = broadcast_tensor_dict(src=0)
260 - input_tokens = metadata_dict.pop("input_tokens")
261 - input_positions = metadata_dict.pop("input_positions")
262 - selected_token_indices = metadata_dict.pop(
263 - "selected_token_indices")
264 - multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
265 - attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
266 - sampling_metadata = SamplingMetadata(
267 - seq_groups=None,
268 - selected_token_indices=selected_token_indices,
269 - categorized_sample_indices=None,
270 - num_prompts=0,
271 - )
272 -
273 - return ModelInputForXPU(input_tokens=input_tokens,
274 - input_positions=input_positions,
275 - attn_metadata=attn_metadata,
276 - sampling_metadata=sampling_metadata,
277 - multi_modal_kwargs=multi_modal_kwargs)
474 + tensor_dict: Dict[str,
475 + Any]) -> ModelInputForXPUWithSamplingMetadata:
476 + return (
477 + ModelInputForXPUWithSamplingMetadata.from_broadcasted_tensor_dict(
478 + tensor_dict,
479 + attn_backend=self.attn_backend,
480 + ))
481
279 - def _prepare_decode(
482 + def _prepare_model_input_tensors(
483 self,
484 seq_group_metadata_list: List[SequenceGroupMetadata],
282 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
283 - assert len(seq_group_metadata_list) > 0
284 - input_tokens: List[int] = []
285 - input_positions: List[int] = []
286 - slot_mapping: List[int] = []
287 - seq_lens: List[int] = []
288 - block_tables: List[List[int]] = []
289 -
485 + finished_requests_ids: Optional[List[str]] = None
486 + ) -> ModelInputForXPUWithSamplingMetadata:
487 + """Helper method to prepare the model input based on a given sequence
488 + group. Prepares metadata needed for the base model forward pass but not
489 + metadata for possible additional steps, e.g., sampling.
490 +
491 + """
492 + builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
493 for seq_group_metadata in seq_group_metadata_list:
291 - assert not seq_group_metadata.is_prompt
292 - assert seq_group_metadata.token_chunk_size == 1
494 + builder.add_seq_group(seq_group_metadata)
495
294 - seq_ids = list(seq_group_metadata.seq_data.keys())
496 + return builder.build() # type: ignore
497
296 - for seq_id in seq_ids:
297 - seq_data = seq_group_metadata.seq_data[seq_id]
298 - generation_token = seq_data.get_last_token_id()
299 - input_tokens.append(generation_token)
300 -
301 - seq_len = seq_data.get_len()
302 - position = seq_len - 1
303 - input_positions.append(position)
304 -
305 - seq_len = seq_len if self.sliding_window is None else min(
306 - seq_len, self.sliding_window)
307 - seq_lens.append(seq_len)
308 -
309 - block_table = seq_group_metadata.block_tables[seq_id]
310 - block_number = block_table[position // self.block_size]
311 - block_offset = position % self.block_size
312 - slot = block_number * self.block_size + block_offset
313 - slot_mapping.append(slot)
314 -
315 - if self.sliding_window is not None:
316 - sliding_window_blocks = (self.sliding_window //
317 - self.block_size)
318 - block_table = block_table[-sliding_window_blocks:]
319 - block_tables.append(block_table)
320 -
321 - max_decode_seq_len = max(seq_lens)
322 -
323 - input_tokens = torch.tensor(input_tokens,
324 - dtype=torch.long,
325 - device=self.device)
326 - input_positions = torch.tensor(input_positions,
327 - dtype=torch.long,
328 - device=self.device)
329 - slot_mapping = torch.tensor(slot_mapping,
330 - dtype=torch.long,
331 - device=self.device)
332 - seq_lens_tensor = torch.tensor(seq_lens,
333 - dtype=torch.int,
334 - device=self.device)
335 -
336 - block_tables = make_tensor_with_pad(
337 - block_tables,
338 - pad=0,
339 - dtype=torch.int,
340 - device=self.device,
341 - )
342 -
343 - attn_metadata = self.attn_backend.make_metadata(
344 - is_prompt=False,
345 - slot_mapping=slot_mapping,
346 - seq_lens=seq_lens,
347 - seqlen_q=None,
348 - max_seqlen=None,
349 - seq_lens_tensor=seq_lens_tensor,
350 - max_decode_seq_len=max_decode_seq_len,
351 - num_prefill_tokens=0,
352 - num_decode_tokens=len(input_tokens),
353 - num_prefills=0,
354 - block_tables=block_tables,
355 - )
356 - return (
357 - input_tokens,
358 - input_positions,
359 - attn_metadata,
360 - )
498 + def prepare_model_input(
499 + self,
500 + seq_group_metadata_list: List[SequenceGroupMetadata],
501 + virtual_engine: int = 0,
502 + finished_requests_ids: Optional[List[str]] = None
503 + ) -> ModelInputForXPUWithSamplingMetadata:
504 + """Prepare the model input based on a given sequence group, including
505 + metadata for the sampling step.
506 +
507 + """
508 + model_input = self._prepare_model_input_tensors(
509 + seq_group_metadata_list, finished_requests_ids)
510 + # Sampling metadata is only required for the final pp group
511 + generators = self.get_generators(finished_requests_ids)
512 + sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
513 + model_input.seq_lens,
514 + model_input.query_lens,
515 + self.device,
516 + pin_memory=False,
517 + generators=generators)
518 +
519 + return dataclasses.replace(model_input,
520 + sampling_metadata=sampling_metadata,
521 + virtual_engine=virtual_engine)
522
523 @torch.inference_mode()
524 def execute_model(
525 self,
365 - model_input: ModelInputForXPU,
526 + model_input: ModelInputForXPUWithSamplingMetadata,
527 kv_caches: List[torch.Tensor],
528 intermediate_tensors: Optional[IntermediateTensors] = None,
529 num_steps: int = 1,
@@ -372,20 +533,21 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
533 "XPUModelRunner does not support multi-step execution.")
534
535 model_executable = self.model
375 - execute_model_kwargs = {
376 - "input_ids":
377 - model_input.input_tokens,
378 - "positions":
379 - model_input.input_positions,
380 - "kv_caches":
381 - kv_caches,
382 - "attn_metadata":
383 - model_input.attn_metadata,
536 + if (self.observability_config is not None
537 + and self.observability_config.collect_model_forward_time):
538 + model_forward_start_time = time.time()
539 +
540 + hidden_states = model_executable(
541 + input_ids=model_input.input_tokens,
542 + positions=model_input.input_positions,
543 + kv_caches=kv_caches,
544 + attn_metadata=model_input.attn_metadata,
545 + intermediate_tensors=intermediate_tensors,
546 **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
385 - device=self.device),
386 - }
387 -
388 - hidden_states = model_executable(**execute_model_kwargs)
547 + device=self.device))
548 + if (self.observability_config is not None
549 + and self.observability_config.collect_model_forward_time):
550 + model_forward_end_time = time.time()
551
552 # Compute the logits.
553 logits = self.model.compute_logits(hidden_states,
@@ -396,109 +558,19 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
558 return []
559
560 # Sample the next token.
399 - output = self.model.sample(
561 + output: SamplerOutput = self.model.sample(
562 logits=logits,
563 sampling_metadata=model_input.sampling_metadata,
564 )
403 - return [output]
404 -
405 - def _prepare_prompt(
406 - self,
407 - seq_group_metadata_list: List[SequenceGroupMetadata],
408 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
409 - BatchedTensorInputs]:
410 - assert len(seq_group_metadata_list) > 0
411 - input_tokens: List[int] = []
412 - input_positions: List[int] = []
413 - slot_mapping: List[int] = []
414 - seq_lens: List[int] = []
415 - multi_modal_inputs_list: List[MultiModalInputs] = []
416 -
417 - for seq_group_metadata in seq_group_metadata_list:
418 - assert seq_group_metadata.is_prompt
419 - seq_ids = list(seq_group_metadata.seq_data.keys())
420 - assert len(seq_ids) == 1
421 - seq_id = seq_ids[0]
422 -
423 - seq_data = seq_group_metadata.seq_data[seq_id]
424 - prompt_tokens = seq_data.get_token_ids()
425 - computed_len = seq_data.get_num_computed_tokens()
426 - seq_len = len(prompt_tokens)
565 + if (self.observability_config is not None
566 + and self.observability_config.collect_model_forward_time
567 + and output is not None):
568 + model_forward_time = (model_forward_end_time -
569 + model_forward_start_time)
570 + # If there are multiple workers, we are still tracking the latency
571 + # from the start time of the driver worker to the end time of the
572 + # driver worker. The model forward time will then end up covering
573 + # the communication time as well.
574 + output.model_forward_time = model_forward_time
575
428 - seq_lens.append(seq_len) # Prompt token num
429 - input_tokens.extend(prompt_tokens) # Token ids
430 -
431 - # Token position ids
432 - # NOTE(woosuk): Here we assume that the first token in the prompt
433 - # is always the first token in the sequence.
434 - input_positions.extend(list(range(computed_len, seq_len)))
435 -
436 - mm_data = seq_group_metadata.multi_modal_data
437 - if mm_data:
438 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
439 - multi_modal_inputs_list.append(mm_kwargs)
440 -
441 - if seq_group_metadata.block_tables is None:
442 - # During memory profiling, the block tables are not initialized
443 - # yet. In this case, we just use a dummy slot mapping.
444 - slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
445 - continue
446 -
447 - # Compute the slot mapping.
448 - block_table = seq_group_metadata.block_tables[seq_id]
449 - # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
450 - # where start_idx is max(0, seq_len - sliding_window).
451 - # For example, if the prompt len is 10, sliding window is 8, and
452 - # block size is 4, the first two tokens are masked and the slot
453 - # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
454 - start_idx = 0
455 - if self.sliding_window is not None:
456 - start_idx = max(0, seq_len - self.sliding_window)
457 -
458 - for i in range(computed_len, seq_len):
459 - if i < start_idx:
460 - slot_mapping.append(_PAD_SLOT_ID)
461 - continue
462 -
463 - block_number = block_table[i //
464 - self.block_size] # type: ignore
465 - block_offset = i % self.block_size # type: ignore
466 - slot = block_number * self.block_size + block_offset
467 - slot_mapping.append(slot)
468 -
469 - num_prompt_tokens = len(input_tokens)
470 -
471 - input_tokens = torch.tensor(input_tokens,
472 - dtype=torch.long,
473 - device=self.device) # type: ignore
474 - input_positions = torch.tensor(input_positions,
475 - dtype=torch.long,
476 - device=self.device) # type: ignore
477 - slot_mapping = torch.tensor(slot_mapping,
478 - dtype=torch.long,
479 - device=self.device) # type: ignore
480 -
481 - max_seqlen = max(seq_lens)
482 - tmp = [0]
483 - tmp.extend(seq_lens)
484 - seqlen = torch.tensor(tmp)
485 - seqlen_q = torch.cumsum(seqlen, dim=0).to(device=self.device)
486 -
487 - attn_metadata = self.attn_backend.make_metadata(
488 - is_prompt=True,
489 - slot_mapping=slot_mapping,
490 - seq_lens=seq_lens,
491 - seqlen_q=seqlen_q,
492 - max_seqlen=max_seqlen,
493 - seq_lens_tensor=None,
494 - max_decode_seq_len=None,
495 - num_prefills=len(seq_lens),
496 - num_prefill_tokens=num_prompt_tokens,
497 - num_decode_tokens=0,
498 - block_tables=torch.tensor([], device=self.device, dtype=torch.int),
499 - )
500 -
501 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
502 -
503 - return (input_tokens, input_positions, attn_metadata, seq_lens,
504 - multi_modal_kwargs)
576 + return [output]
vllm/worker/xpu_worker.py
+3 -6
@@ -9,8 +9,8 @@ import torch
9 import torch.distributed
10
11 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 - ModelConfig, MultiModalConfig, ObservabilityConfig,
13 - ParallelConfig, PromptAdapterConfig, SchedulerConfig,
12 + ModelConfig, ObservabilityConfig, ParallelConfig,
13 + PromptAdapterConfig, SchedulerConfig,
14 SpeculativeConfig)
15 from vllm.distributed import (ensure_model_parallel_initialized,
16 init_distributed_environment)
@@ -46,7 +46,6 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
46 rank: int,
47 distributed_init_method: str,
48 lora_config: Optional[LoRAConfig] = None,
49 - multimodal_config: Optional[MultiModalConfig] = None,
49 speculative_config: Optional[SpeculativeConfig] = None,
50 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
51 is_driver_worker: bool = False,
@@ -73,8 +72,6 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
72 assert rank % parallel_config.tensor_parallel_size == 0, \
73 "Driver worker should be rank 0 of tensor parallel group."
74
76 - self.multimodal_config = multimodal_config
77 -
75 self.model_runner = XPUModelRunner( # type: ignore
76 model_config,
77 parallel_config,
@@ -85,7 +82,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
82 lora_config=self.lora_config,
83 kv_cache_dtype=self.cache_config.cache_dtype,
84 is_driver_worker=is_driver_worker,
88 - multimodal_config=multimodal_config,
85 + observability_config=self.observability_config,
86 )
87 # Uninitialized cache engine. Will be initialized by
88 # initialize_cache.