A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Hardware][Intel GPU] refactor xpu_model_runner for tp (#7712)
Kunshang Ji committed
Aug 23, 2024 at 11:06 UTC
fc5ebbd1d3453461ea6e00a78faf87c41d1aa625
3 files changed
+370
-650
vllm/executor/ray_xpu_executor.py
+17
-366
@@ -1,386 +1,37 @@
1
import asyncio
2
-import os
3
-from collections import defaultdict
4
-from itertools import islice, repeat
5
-from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
6
- Tuple, Union)
2
+from typing import List, Optional
3
4
import vllm.envs as envs
9
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, PromptAdapterConfig,
11
- SchedulerConfig, SpeculativeConfig)
12
-from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13
- DistributedGPUExecutor, DistributedGPUExecutorAsync)
14
-from vllm.executor.ray_utils import RayWorkerWrapper, ray
5
+from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
6
+from vllm.executor.xpu_executor import XPUExecutor
7
from vllm.logger import init_logger
16
-from vllm.lora.request import LoRARequest
17
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
18
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
19
- make_async)
20
-
21
-if ray is not None:
22
- from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
23
-
24
-if TYPE_CHECKING:
25
- from ray.util.placement_group import PlacementGroup
8
+from vllm.utils import get_vllm_instance_id, make_async
9
10
logger = init_logger(__name__)
11
29
-# If the env var is set, it uses the Ray's compiled DAG API
30
-# which optimizes the control plane overhead.
31
-# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
32
-USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
33
-
34
-
35
-class RayXPUExecutor(DistributedGPUExecutor):
36
-
37
- uses_ray: bool = True
38
-
39
- def __init__(
40
- self,
41
- model_config: ModelConfig,
42
- cache_config: CacheConfig,
43
- parallel_config: ParallelConfig,
44
- scheduler_config: SchedulerConfig,
45
- device_config: DeviceConfig,
46
- load_config: LoadConfig,
47
- lora_config: Optional[LoRAConfig],
48
- prompt_adapter_config: Optional[PromptAdapterConfig],
49
- speculative_config: Optional[SpeculativeConfig],
50
- ) -> None:
51
- assert device_config.device_type == "xpu"
52
- assert (not speculative_config
53
- ), "Speculative decoding not yet supported for XPU backend"
54
-
55
- self.model_config = model_config
56
- self.cache_config = cache_config
57
- self.load_config = load_config
58
- self.lora_config = lora_config
59
- self.parallel_config = parallel_config
60
- self.scheduler_config = scheduler_config
61
- self.device_config = device_config
62
- self.prompt_adapter_config = prompt_adapter_config
63
-
64
- placement_group = self.parallel_config.placement_group
65
-
66
- # Disable Ray usage stats collection.
67
- ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
68
- if ray_usage != "1":
69
- os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
70
-
71
- # Create the parallel GPU workers.
72
- self._init_workers_ray(placement_group)
73
-
74
- self.forward_dag = None
75
- if USE_RAY_COMPILED_DAG:
76
- self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
77
-
78
- # This is non-None when the execute model loop is running
79
- # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
80
- self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
81
- # Updated by implementations that require additional args to be passed
82
- # to the _run_workers execute_model call
83
- self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
84
-
85
- def _init_executor(self) -> None:
86
- pass
87
-
88
- def determine_num_available_blocks(self) -> Tuple[int, int]:
89
- """Determine the number of available KV blocks.
12
91
- This invokes `determine_num_available_blocks` on each worker and takes
92
- the min of the results, guaranteeing that the selected cache sizes are
93
- compatible with all workers.
94
-
95
- Returns:
96
- - Tuple[num_gpu_blocks, num_cpu_blocks]
97
- """
98
- # Get the maximum number of blocks that can be allocated on GPU and CPU.
99
- num_blocks = self._run_workers("determine_num_available_blocks", )
100
-
101
- # Since we use a shared centralized controller, we take the minimum
102
- # number of blocks across all workers to make sure all the memory
103
- # operators can be applied to all workers.
104
- num_gpu_blocks = min(b[0] for b in num_blocks)
105
- num_cpu_blocks = min(b[1] for b in num_blocks)
106
-
107
- return num_gpu_blocks, num_cpu_blocks
108
-
109
- def _get_worker_wrapper_args(self) -> Dict[str, Any]:
110
- return dict(
111
- worker_module_name="vllm.worker.xpu_worker",
112
- worker_class_name="XPUWorker",
113
- trust_remote_code=self.model_config.trust_remote_code,
114
- )
115
-
116
- def _init_workers_ray(self, placement_group: "PlacementGroup",
117
- **ray_remote_kwargs):
118
- if self.parallel_config.tensor_parallel_size == 1:
119
- # For single GPU case, we use a ray worker with constrained memory.
120
- num_gpus = self.cache_config.gpu_memory_utilization
121
- else:
122
- # Otherwise, the ray workers are allocated with a full GPU.
123
- num_gpus = 1
124
-
125
- # The driver dummy worker does not actually use any resources.
126
- # It holds the resource for the driver worker.
127
- self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
128
- # The remaining workers are the actual ray actors.
129
- self.workers: List[RayWorkerWrapper] = []
130
-
131
- # Create the workers.
132
- driver_ip = get_ip()
133
- worker_wrapper_kwargs = self._get_worker_wrapper_args()
134
- for bundle_id, bundle in enumerate(placement_group.bundle_specs):
135
- if not bundle.get("GPU", 0):
136
- continue
137
- scheduling_strategy = PlacementGroupSchedulingStrategy(
138
- placement_group=placement_group,
139
- placement_group_capture_child_tasks=True,
140
- placement_group_bundle_index=bundle_id,
141
- )
142
- worker = ray.remote(
143
- num_cpus=0,
144
- num_gpus=num_gpus,
145
- scheduling_strategy=scheduling_strategy,
146
- **ray_remote_kwargs,
147
- )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
148
-
149
- worker_ip = ray.get(worker.get_node_ip.remote())
150
- if worker_ip == driver_ip and self.driver_dummy_worker is None:
151
- # If the worker is on the same node as the driver, we use it
152
- # as the resource holder for the driver process.
153
- self.driver_dummy_worker = worker
154
- self.driver_worker = RayWorkerWrapper(**worker_wrapper_kwargs)
155
- else:
156
- # Else, added to the list of workers.
157
- self.workers.append(worker)
158
- if self.driver_dummy_worker is None:
159
- raise ValueError(
160
- "Ray does not allocate any GPUs on the driver node. Consider "
161
- "adjusting the Ray placement group or running the driver on a "
162
- "GPU node.")
13
+class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
14
15
+ def _get_env_vars_to_be_updated(self):
16
# Get the set of GPU IDs used on each node.
17
worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
18
use_dummy_driver=True)
19
168
- node_workers = defaultdict(list)
169
- node_gpus = defaultdict(list)
170
-
171
- for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
172
- node_workers[node_id].append(i)
173
- node_gpus[node_id].extend(gpu_ids)
174
- for node_id, gpu_ids in node_gpus.items():
175
- node_gpus[node_id] = sorted(gpu_ids)
176
-
177
- # TODO: add env var for xpu
178
-
179
- distributed_init_method = get_distributed_init_method(
180
- driver_ip, get_open_port())
181
-
182
- def collect_arg_helper_func(**kwargs):
183
- # avoid writing `{"name": value}` manually
184
- return kwargs
185
-
186
- init_worker_all_kwargs = []
187
-
188
- # Initialize the actual workers inside worker wrapper.
189
- for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids, ):
190
- local_rank = node_workers[node_id].index(rank)
191
- init_worker_all_kwargs.append(
192
- collect_arg_helper_func(
193
- model_config=self.model_config,
194
- parallel_config=self.parallel_config,
195
- scheduler_config=self.scheduler_config,
196
- device_config=self.device_config,
197
- cache_config=self.cache_config,
198
- load_config=self.load_config,
199
- local_rank=local_rank,
200
- rank=rank,
201
- distributed_init_method=distributed_init_method,
202
- lora_config=self.lora_config,
203
- is_driver_worker=rank == 0,
204
- ))
205
- self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
20
+ VLLM_INSTANCE_ID = get_vllm_instance_id()
21
207
- self._run_workers("init_device")
208
- self._run_workers(
209
- "load_model",
210
- max_concurrent_workers=self.parallel_config.
211
- max_parallel_loading_workers,
212
- )
22
+ # Set environment variables for the driver and workers.
23
+ all_args_to_update_environment_variables = [({
24
+ "VLLM_INSTANCE_ID":
25
+ VLLM_INSTANCE_ID,
26
+ "VLLM_TRACE_FUNCTION":
27
+ str(envs.VLLM_TRACE_FUNCTION),
28
+ }, ) for (_, _) in worker_node_and_gpu_ids]
29
+ return all_args_to_update_environment_variables
30
214
- def initialize_cache(self, num_gpu_blocks: int,
215
- num_cpu_blocks: int) -> None:
216
- """Initialize the KV cache in all workers.
217
- """
31
219
- # NOTE: We log here to avoid multiple logs when number of workers is
220
- # greater than one. We could log in the engine, but not all executors
221
- # have GPUs.
222
- logger.info("# GPU blocks: %d, "
223
- "# CPU blocks: %d", num_gpu_blocks, num_cpu_blocks)
224
-
225
- self.cache_config.num_gpu_blocks = num_gpu_blocks
226
- self.cache_config.num_cpu_blocks = num_cpu_blocks
227
-
228
- self._run_workers("initialize_cache",
229
- num_gpu_blocks=num_gpu_blocks,
230
- num_cpu_blocks=num_cpu_blocks)
231
-
232
- def _driver_execute_model(
233
- self,
234
- execute_model_req: Optional[ExecuteModelRequest] = None
235
- ) -> List[SamplerOutput]:
236
- """Run execute_model in the driver worker.
237
-
238
- Passing None will cause the driver to stop the model execution
239
- loop running in each of the remote workers.
240
- """
241
- return self.driver_worker.execute_method("execute_model",
242
- execute_model_req)
243
-
244
- def add_lora(self, lora_request: LoRARequest) -> bool:
245
- assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
246
- return self._run_workers(
247
- "add_lora",
248
- lora_request=lora_request,
249
- )
250
-
251
- def remove_lora(self, lora_id: int) -> bool:
252
- assert lora_id > 0, "lora_id must be greater than 0."
253
- return self._run_workers(
254
- "remove_lora",
255
- lora_id=lora_id,
256
- )
257
-
258
- def list_loras(self) -> Set[int]:
259
- return self._run_workers("list_loras")
260
-
261
- def _run_workers(
262
- self,
263
- method: str,
264
- *args,
265
- async_run_remote_workers_only: bool = False,
266
- all_args: Optional[List[Tuple[Any, ...]]] = None,
267
- all_kwargs: Optional[List[Dict[str, Any]]] = None,
268
- use_dummy_driver: bool = False,
269
- max_concurrent_workers: Optional[int] = None,
270
- **kwargs,
271
- ) -> Any:
272
- """Runs the given method on all workers. Can be used in the following
273
- ways:
274
-
275
- - args/kwargs: All workers share the same args/kwargs
276
- - args/kwargs and driver_args/driver_kwargs: Driver worker has
277
- different args
278
- - all_args/all_kwargs: args/kwargs for each worker are specified
279
- individually
280
- """
281
-
282
- if max_concurrent_workers:
283
- raise NotImplementedError(
284
- "max_concurrent_workers is not supported yet.")
285
-
286
- count = len(self.workers)
287
- all_worker_args = repeat(args, count) if all_args is None \
288
- else islice(all_args, 1, None)
289
- all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
290
- else islice(all_kwargs, 1, None)
291
-
292
- # Start the ray workers first.
293
- ray_worker_outputs = [
294
- worker.execute_method.remote(method, *worker_args, **worker_kwargs)
295
- for (worker, worker_args, worker_kwargs
296
- ) in zip(self.workers, all_worker_args, all_worker_kwargs)
297
- ]
298
-
299
- if async_run_remote_workers_only:
300
- # Just return futures
301
- return ray_worker_outputs
302
-
303
- driver_worker_output = []
304
- driver_args = args if all_args is None else all_args[0]
305
- driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
306
- # Start the driver worker after all the ray workers.
307
- if not use_dummy_driver:
308
- driver_worker_output = self.driver_worker.execute_method(
309
- method, *driver_args, **driver_kwargs)
310
- else:
311
- assert self.driver_dummy_worker is not None
312
- driver_worker_output = ray.get(
313
- self.driver_dummy_worker.execute_method.remote(
314
- method, *driver_args, **driver_kwargs))
315
- # Get the results of the ray workers.
316
- if self.workers:
317
- ray_worker_outputs = ray.get(ray_worker_outputs)
318
-
319
- return driver_worker_output + ray_worker_outputs
320
-
321
- def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
322
- """Wait for futures returned from _run_workers() with
323
- async_run_remote_workers_only to complete."""
324
- ray.get(parallel_worker_tasks)
325
-
326
- def _compiled_ray_dag(self, enable_asyncio: bool):
327
- import pkg_resources
328
- from packaging import version
329
-
330
- required_version = version.parse("2.32")
331
- current_version = version.parse(
332
- pkg_resources.get_distribution("ray").version)
333
- if current_version < required_version:
334
- raise ValueError(f"Ray version {required_version} or greater is "
335
- f"required, but found {current_version}")
336
-
337
- from ray.dag import InputNode, MultiOutputNode
338
- assert self.parallel_config.use_ray
339
-
340
- # Right now, compiled DAG requires at least 1 arg. We send
341
- # a dummy value for now. It will be fixed soon.
342
- with InputNode() as input_data:
343
- forward_dag = MultiOutputNode([
344
- worker.execute_model_compiled_dag_remote.
345
- bind( # type: ignore[attr-defined]
346
- input_data) for worker in self.workers
347
- ])
348
- return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
349
-
350
- def check_health(self) -> None:
351
- """Raises an error if engine is unhealthy."""
352
- self._check_if_any_actor_is_dead()
353
-
354
- def _check_if_any_actor_is_dead(self):
355
- if not self.workers:
356
- return
357
-
358
- dead_actors = []
359
- for actor in self.workers:
360
- actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
361
- if actor_state["State"] == "DEAD":
362
- dead_actors.append(actor)
363
- if dead_actors:
364
- raise RuntimeError("At least one Worker is dead. "
365
- f"Dead Workers: {dead_actors}. ")
366
-
367
-
368
-class RayXPUExecutorAsync(RayXPUExecutor, DistributedGPUExecutorAsync):
32
+class RayXPUExecutorAsync(RayXPUExecutor, RayGPUExecutorAsync):
33
34
def __init__(self, *args, **kwargs):
35
super().__init__(*args, **kwargs)
36
self.driver_exec_method = make_async(self.driver_worker.execute_method)
373
-
374
- async def _driver_execute_model_async(
375
- self,
376
- execute_model_req: Optional[ExecuteModelRequest] = None
377
- ) -> List[SamplerOutput]:
378
- return await self.driver_exec_method("execute_model",
379
- execute_model_req)
380
-
381
- async def _start_worker_execution_loop(self):
382
- coros = [
383
- worker.execute_method.remote("start_worker_execution_loop")
384
- for worker in self.workers
385
- ]
386
- return await asyncio.gather(*coros)
37
+ self.pp_locks: Optional[List[asyncio.Lock]] = None
vllm/worker/xpu_model_runner.py
+350
-278
@@ -1,14 +1,17 @@
1
+import dataclasses
2
+import time
3
+import weakref
4
from dataclasses import dataclass
2
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
5
+from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6
+ TypeVar)
7
8
import torch
9
import torch.nn as nn
10
11
from vllm.attention import get_attn_backend
12
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
- ModelConfig, MultiModalConfig, ParallelConfig,
13
+ ModelConfig, ObservabilityConfig, ParallelConfig,
14
PromptAdapterConfig, SchedulerConfig)
11
-from vllm.distributed import broadcast_tensor_dict
15
from vllm.inputs import INPUT_REGISTRY, InputRegistry
16
from vllm.logger import init_logger
17
from vllm.model_executor.model_loader import get_model
@@ -20,7 +23,7 @@ from vllm.sequence import (IntermediateTensors, SamplerOutput,
23
from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
24
from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
25
from vllm.worker.model_runner_base import (
23
- ModelRunnerBase, ModelRunnerInputBase,
26
+ ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
27
_add_attn_metadata_broadcastable_dict,
28
_add_sampling_metadata_broadcastable_dict,
29
_init_attn_metadata_from_tensor_dict,
@@ -37,6 +40,8 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
40
_BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
41
]
42
43
+TModelInputForXPU = TypeVar('TModelInputForXPU', bound="ModelInputForXPU")
44
+
45
46
@dataclass(frozen=True)
47
class ModelInputForXPU(ModelRunnerInputBase):
@@ -46,11 +51,40 @@ class ModelInputForXPU(ModelRunnerInputBase):
51
input_tokens: Optional[torch.Tensor] = None
52
input_positions: Optional[torch.Tensor] = None
53
attn_metadata: Optional["AttentionMetadata"] = None
49
- sampling_metadata: Optional["SamplingMetadata"] = None
54
multi_modal_kwargs: Optional[BatchedTensorInputs] = None
55
+ virtual_engine: Optional[int] = None
56
+ seq_lens: Optional[List[int]] = None
57
+ query_lens: Optional[List[int]] = None
58
+
59
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
60
+ tensor_dict = {
61
+ "input_tokens": self.input_tokens,
62
+ "input_positions": self.input_positions,
63
+ }
64
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
65
+
66
+ return tensor_dict
67
+
68
+ @classmethod
69
+ def from_broadcasted_tensor_dict(
70
+ cls: Type[TModelInputForXPU],
71
+ tensor_dict: Dict[str, Any],
72
+ attn_backend: Optional["AttentionBackend"] = None,
73
+ ) -> TModelInputForXPU:
74
+ if attn_backend is not None:
75
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
76
+ attn_backend, tensor_dict)
77
+ return cls(**tensor_dict)
78
+
79
+
80
+@dataclass(frozen=True)
81
+class ModelInputForXPUWithSamplingMetadata(ModelInputForXPU):
82
+ """
83
+ Used by the ModelRunner.
84
+ """
85
+ sampling_metadata: Optional["SamplingMetadata"] = None
86
52
- def as_broadcastable_tensor_dict(
53
- self) -> Dict[str, Union[int, torch.Tensor]]:
87
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
88
tensor_dict = {
89
"input_tokens": self.input_tokens,
90
"input_positions": self.input_positions,
@@ -62,10 +96,10 @@ class ModelInputForXPU(ModelRunnerInputBase):
96
97
@classmethod
98
def from_broadcasted_tensor_dict(
65
- cls: Type["ModelInputForXPU"],
99
+ cls,
100
tensor_dict: Dict[str, Any],
101
attn_backend: Optional["AttentionBackend"] = None,
68
- ) -> "ModelInputForXPU":
102
+ ) -> "ModelInputForXPUWithSamplingMetadata":
103
tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
104
if attn_backend is not None:
105
tensor_dict = _init_attn_metadata_from_tensor_dict(
@@ -73,7 +107,230 @@ class ModelInputForXPU(ModelRunnerInputBase):
107
return cls(**tensor_dict)
108
109
76
-class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
110
+class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
111
+
112
+ def __init__(self,
113
+ runner: "XPUModelRunner",
114
+ finished_requests_ids: Optional[List[str]] = None) -> None:
115
+ super().__init__()
116
+ self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
117
+ self.runner = runner
118
+ self.model_input_cls = self.runner._model_input_cls
119
+ self.attn_backend = self.runner.attn_backend
120
+ self.sliding_window = self.runner.sliding_window
121
+ self.block_size = self.runner.block_size
122
+ self.device = self.runner.device
123
+
124
+ def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
125
+ self.seq_group_metadata_list.append(seq_group_metadata)
126
+
127
+ def build(self) -> ModelInputForXPU:
128
+ is_prompt = self.seq_group_metadata_list[0].is_prompt
129
+ # Prepare input tensors.
130
+ if is_prompt:
131
+ (input_tokens, input_positions, attn_metadata, seq_lens,
132
+ multi_modal_kwargs) = self._prepare_prompt(
133
+ self.seq_group_metadata_list)
134
+ else:
135
+ (input_tokens, input_positions,
136
+ attn_metadata) = self._prepare_decode(
137
+ self.seq_group_metadata_list)
138
+ seq_lens = []
139
+ multi_modal_kwargs = None
140
+
141
+ return self.model_input_cls(
142
+ input_tokens=input_tokens,
143
+ input_positions=input_positions,
144
+ attn_metadata=attn_metadata,
145
+ multi_modal_kwargs=multi_modal_kwargs,
146
+ seq_lens=seq_lens,
147
+ query_lens=seq_lens,
148
+ )
149
+
150
+ def _prepare_prompt(
151
+ self,
152
+ seq_group_metadata_list: List[SequenceGroupMetadata],
153
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
154
+ BatchedTensorInputs]:
155
+ assert len(seq_group_metadata_list) > 0
156
+ input_tokens: List[int] = []
157
+ input_positions: List[int] = []
158
+ slot_mapping: List[int] = []
159
+ seq_lens: List[int] = []
160
+ multi_modal_inputs_list: List[MultiModalInputs] = []
161
+
162
+ for seq_group_metadata in seq_group_metadata_list:
163
+ assert seq_group_metadata.is_prompt
164
+ seq_ids = list(seq_group_metadata.seq_data.keys())
165
+ assert len(seq_ids) == 1
166
+ seq_id = seq_ids[0]
167
+
168
+ seq_data = seq_group_metadata.seq_data[seq_id]
169
+ prompt_tokens = seq_data.get_token_ids()
170
+ computed_len = seq_data.get_num_computed_tokens()
171
+ seq_len = len(prompt_tokens)
172
+
173
+ seq_lens.append(seq_len) # Prompt token num
174
+ input_tokens.extend(prompt_tokens) # Token ids
175
+
176
+ # Token position ids
177
+ # NOTE(woosuk): Here we assume that the first token in the prompt
178
+ # is always the first token in the sequence.
179
+ input_positions.extend(list(range(computed_len, seq_len)))
180
+
181
+ if seq_group_metadata.block_tables is None:
182
+ # During memory profiling, the block tables are not initialized
183
+ # yet. In this case, we just use a dummy slot mapping.
184
+ slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
185
+ continue
186
+
187
+ # Compute the slot mapping.
188
+ block_table = seq_group_metadata.block_tables[seq_id]
189
+ # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
190
+ # where start_idx is max(0, seq_len - sliding_window).
191
+ # For example, if the prompt len is 10, sliding window is 8, and
192
+ # block size is 4, the first two tokens are masked and the slot
193
+ # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
194
+ start_idx = 0
195
+ if self.sliding_window is not None:
196
+ start_idx = max(0, seq_len - self.sliding_window)
197
+
198
+ for i in range(computed_len, seq_len):
199
+ if i < start_idx:
200
+ slot_mapping.append(_PAD_SLOT_ID)
201
+ continue
202
+
203
+ block_number = block_table[i //
204
+ self.block_size] # type: ignore
205
+ block_offset = i % self.block_size # type: ignore
206
+ slot = block_number * self.block_size + block_offset
207
+ slot_mapping.append(slot)
208
+
209
+ num_prompt_tokens = len(input_tokens)
210
+
211
+ input_tokens = torch.tensor(input_tokens,
212
+ dtype=torch.long,
213
+ device=self.device) # type: ignore
214
+ input_positions = torch.tensor(input_positions,
215
+ dtype=torch.long,
216
+ device=self.device) # type: ignore
217
+ slot_mapping = torch.tensor(slot_mapping,
218
+ dtype=torch.long,
219
+ device=self.device) # type: ignore
220
+
221
+ max_seqlen = max(seq_lens)
222
+ tmp = [0]
223
+ tmp.extend(seq_lens)
224
+ seqlen = torch.tensor(tmp)
225
+ seqlen_q = torch.cumsum(seqlen, dim=0).to(device=self.device)
226
+
227
+ attn_metadata = self.attn_backend.make_metadata(
228
+ is_prompt=True,
229
+ slot_mapping=slot_mapping,
230
+ seq_lens=seq_lens,
231
+ seqlen_q=seqlen_q,
232
+ max_seqlen=max_seqlen,
233
+ seq_lens_tensor=torch.tensor([]),
234
+ max_decode_seq_len=0,
235
+ num_prefills=len(seq_lens),
236
+ num_prefill_tokens=num_prompt_tokens,
237
+ num_decode_tokens=0,
238
+ block_tables=torch.tensor([], device=self.device, dtype=torch.int),
239
+ )
240
+
241
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
242
+
243
+ return (input_tokens, input_positions, attn_metadata, seq_lens,
244
+ multi_modal_kwargs)
245
+
246
+ def _prepare_decode(
247
+ self,
248
+ seq_group_metadata_list: List[SequenceGroupMetadata],
249
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
250
+ assert len(seq_group_metadata_list) > 0
251
+ input_tokens: List[int] = []
252
+ input_positions: List[int] = []
253
+ slot_mapping: List[int] = []
254
+ seq_lens: List[int] = []
255
+ block_tables: List[List[int]] = []
256
+
257
+ for seq_group_metadata in seq_group_metadata_list:
258
+ assert not seq_group_metadata.is_prompt
259
+ assert seq_group_metadata.token_chunk_size == 1
260
+
261
+ seq_ids = list(seq_group_metadata.seq_data.keys())
262
+
263
+ for seq_id in seq_ids:
264
+ seq_data = seq_group_metadata.seq_data[seq_id]
265
+ generation_token = seq_data.get_last_token_id()
266
+ input_tokens.append(generation_token)
267
+
268
+ seq_len = seq_data.get_len()
269
+ position = seq_len - 1
270
+ input_positions.append(position)
271
+
272
+ seq_len = seq_len if self.sliding_window is None else min(
273
+ seq_len, self.sliding_window)
274
+ seq_lens.append(seq_len)
275
+
276
+ block_table = seq_group_metadata.block_tables[seq_id]
277
+ block_number = block_table[position // self.block_size]
278
+ block_offset = position % self.block_size
279
+ slot = block_number * self.block_size + block_offset
280
+ slot_mapping.append(slot)
281
+
282
+ if self.sliding_window is not None:
283
+ sliding_window_blocks = (self.sliding_window //
284
+ self.block_size)
285
+ block_table = block_table[-sliding_window_blocks:]
286
+ block_tables.append(block_table)
287
+
288
+ max_decode_seq_len = max(seq_lens)
289
+
290
+ input_tokens = torch.tensor(input_tokens,
291
+ dtype=torch.long,
292
+ device=self.device)
293
+ input_positions = torch.tensor(input_positions,
294
+ dtype=torch.long,
295
+ device=self.device)
296
+ slot_mapping = torch.tensor(slot_mapping,
297
+ dtype=torch.long,
298
+ device=self.device)
299
+ seq_lens_tensor = torch.tensor(seq_lens,
300
+ dtype=torch.int,
301
+ device=self.device)
302
+
303
+ block_tables = make_tensor_with_pad(
304
+ block_tables,
305
+ pad=0,
306
+ dtype=torch.int,
307
+ device=self.device,
308
+ )
309
+
310
+ attn_metadata = self.attn_backend.make_metadata(
311
+ is_prompt=False,
312
+ slot_mapping=slot_mapping,
313
+ seq_lens=seq_lens,
314
+ seqlen_q=torch.tensor([]),
315
+ max_seqlen=0,
316
+ seq_lens_tensor=seq_lens_tensor,
317
+ max_decode_seq_len=max_decode_seq_len,
318
+ num_prefill_tokens=0,
319
+ num_decode_tokens=len(input_tokens),
320
+ num_prefills=0,
321
+ block_tables=block_tables,
322
+ )
323
+ return (
324
+ input_tokens,
325
+ input_positions,
326
+ attn_metadata,
327
+ )
328
+
329
+
330
+class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
331
+ _model_input_cls: Type[ModelInputForXPUWithSamplingMetadata] = (
332
+ ModelInputForXPUWithSamplingMetadata)
333
+ _builder_cls: Type[ModelInputForXPUBuilder] = ModelInputForXPUBuilder
334
335
def __init__(
336
self,
@@ -84,30 +341,32 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
341
cache_config: CacheConfig,
342
load_config: LoadConfig,
343
lora_config: Optional[LoRAConfig],
87
- multimodal_config: Optional[MultiModalConfig],
344
kv_cache_dtype: Optional[str] = "auto",
89
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
345
is_driver_worker: bool = False,
346
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
347
+ return_hidden_states: bool = False,
348
+ observability_config: Optional[ObservabilityConfig] = None,
349
input_registry: InputRegistry = INPUT_REGISTRY,
350
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
93
- *args,
94
- **kwargs,
351
):
352
self.model_config = model_config
353
self.parallel_config = parallel_config
354
self.scheduler_config = scheduler_config
355
+ self.device_config = device_config
356
+ self.cache_config = cache_config
357
self.lora_config = lora_config
358
self.load_config = load_config
101
- self.cache_config = cache_config
102
- self.prompt_adapter_config = prompt_adapter_config
103
- self.multimodal_config = multimodal_config
359
self.is_driver_worker = is_driver_worker
360
+ self.prompt_adapter_config = prompt_adapter_config
361
+ self.observability_config = observability_config
362
+ if self.observability_config is not None:
363
+ print(f"observability_config is {self.observability_config}")
364
+ self.return_hidden_states = return_hidden_states
365
106
- self.sliding_window = model_config.get_sliding_window()
107
- self.device_config = device_config
366
self.device = self.device_config.device
367
368
self.kv_cache_dtype = kv_cache_dtype
369
+ self.sliding_window = model_config.get_sliding_window()
370
self.block_size = cache_config.block_size
371
372
self.attn_backend = get_attn_backend(
@@ -203,166 +462,68 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
462
# Run the model with the dummy inputs.
463
num_layers = self.model_config.get_num_layers(self.parallel_config)
464
kv_caches = [None] * num_layers
206
- model_input = self.prepare_model_input(seqs)
465
+ finished_requests_ids = [seq.request_id for seq in seqs]
466
+ model_input = self.prepare_model_input(
467
+ seqs, finished_requests_ids=finished_requests_ids)
468
self.execute_model(model_input, kv_caches)
469
torch.xpu.synchronize()
470
return
471
472
def make_model_input_from_broadcasted_tensor_dict(
212
- self, tensor_dict: Dict[str, Any]) -> ModelInputForXPU:
213
- return (ModelInputForXPU.from_broadcasted_tensor_dict(
214
- tensor_dict,
215
- attn_backend=self.attn_backend,
216
- ))
217
-
218
- def prepare_model_input(
473
self,
220
- seq_group_metadata_list: List[SequenceGroupMetadata],
221
- virtual_engine: int = 0,
222
- finished_requests_ids: Optional[List[str]] = None
223
- ) -> ModelInputForXPU:
224
- multi_modal_kwargs = None
225
- if self.is_driver_worker:
226
- # NOTE: We assume that all sequences in the group are all prompts or
227
- # all decodes.
228
- is_prompt = seq_group_metadata_list[0].is_prompt
229
- # Prepare input tensors.
230
- if is_prompt:
231
- (input_tokens, input_positions, attn_metadata, seq_lens,
232
- multi_modal_kwargs
233
- ) = self._prepare_prompt(seq_group_metadata_list)
234
- else:
235
- (input_tokens, input_positions,
236
- attn_metadata) = self._prepare_decode(seq_group_metadata_list)
237
- seq_lens = []
238
- sampling_metadata = SamplingMetadata.prepare(
239
- seq_group_metadata_list,
240
- seq_lens,
241
- # subquery_lens is not needed if chunked prefill is not
242
- # supported. Since CPU worker doesn't support chunked prefill
243
- # just use seq_lens instead.
244
- seq_lens,
245
- self.device,
246
- pin_memory=False,
247
- generators=self.get_generators(finished_requests_ids))
248
- # Broadcast the metadata.
249
- metadata_dict = {
250
- "input_tokens": input_tokens,
251
- "input_positions": input_positions,
252
- "selected_token_indices":
253
- sampling_metadata.selected_token_indices,
254
- "multi_modal_kwargs": multi_modal_kwargs,
255
- }
256
- metadata_dict.update(attn_metadata.asdict_zerocopy())
257
- broadcast_tensor_dict(metadata_dict, src=0)
258
- else:
259
- metadata_dict = broadcast_tensor_dict(src=0)
260
- input_tokens = metadata_dict.pop("input_tokens")
261
- input_positions = metadata_dict.pop("input_positions")
262
- selected_token_indices = metadata_dict.pop(
263
- "selected_token_indices")
264
- multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
265
- attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
266
- sampling_metadata = SamplingMetadata(
267
- seq_groups=None,
268
- selected_token_indices=selected_token_indices,
269
- categorized_sample_indices=None,
270
- num_prompts=0,
271
- )
272
-
273
- return ModelInputForXPU(input_tokens=input_tokens,
274
- input_positions=input_positions,
275
- attn_metadata=attn_metadata,
276
- sampling_metadata=sampling_metadata,
277
- multi_modal_kwargs=multi_modal_kwargs)
474
+ tensor_dict: Dict[str,
475
+ Any]) -> ModelInputForXPUWithSamplingMetadata:
476
+ return (
477
+ ModelInputForXPUWithSamplingMetadata.from_broadcasted_tensor_dict(
478
+ tensor_dict,
479
+ attn_backend=self.attn_backend,
480
+ ))
481
279
- def _prepare_decode(
482
+ def _prepare_model_input_tensors(
483
self,
484
seq_group_metadata_list: List[SequenceGroupMetadata],
282
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
283
- assert len(seq_group_metadata_list) > 0
284
- input_tokens: List[int] = []
285
- input_positions: List[int] = []
286
- slot_mapping: List[int] = []
287
- seq_lens: List[int] = []
288
- block_tables: List[List[int]] = []
289
-
485
+ finished_requests_ids: Optional[List[str]] = None
486
+ ) -> ModelInputForXPUWithSamplingMetadata:
487
+ """Helper method to prepare the model input based on a given sequence
488
+ group. Prepares metadata needed for the base model forward pass but not
489
+ metadata for possible additional steps, e.g., sampling.
490
+
491
+ """
492
+ builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
493
for seq_group_metadata in seq_group_metadata_list:
291
- assert not seq_group_metadata.is_prompt
292
- assert seq_group_metadata.token_chunk_size == 1
494
+ builder.add_seq_group(seq_group_metadata)
495
294
- seq_ids = list(seq_group_metadata.seq_data.keys())
496
+ return builder.build() # type: ignore
497
296
- for seq_id in seq_ids:
297
- seq_data = seq_group_metadata.seq_data[seq_id]
298
- generation_token = seq_data.get_last_token_id()
299
- input_tokens.append(generation_token)
300
-
301
- seq_len = seq_data.get_len()
302
- position = seq_len - 1
303
- input_positions.append(position)
304
-
305
- seq_len = seq_len if self.sliding_window is None else min(
306
- seq_len, self.sliding_window)
307
- seq_lens.append(seq_len)
308
-
309
- block_table = seq_group_metadata.block_tables[seq_id]
310
- block_number = block_table[position // self.block_size]
311
- block_offset = position % self.block_size
312
- slot = block_number * self.block_size + block_offset
313
- slot_mapping.append(slot)
314
-
315
- if self.sliding_window is not None:
316
- sliding_window_blocks = (self.sliding_window //
317
- self.block_size)
318
- block_table = block_table[-sliding_window_blocks:]
319
- block_tables.append(block_table)
320
-
321
- max_decode_seq_len = max(seq_lens)
322
-
323
- input_tokens = torch.tensor(input_tokens,
324
- dtype=torch.long,
325
- device=self.device)
326
- input_positions = torch.tensor(input_positions,
327
- dtype=torch.long,
328
- device=self.device)
329
- slot_mapping = torch.tensor(slot_mapping,
330
- dtype=torch.long,
331
- device=self.device)
332
- seq_lens_tensor = torch.tensor(seq_lens,
333
- dtype=torch.int,
334
- device=self.device)
335
-
336
- block_tables = make_tensor_with_pad(
337
- block_tables,
338
- pad=0,
339
- dtype=torch.int,
340
- device=self.device,
341
- )
342
-
343
- attn_metadata = self.attn_backend.make_metadata(
344
- is_prompt=False,
345
- slot_mapping=slot_mapping,
346
- seq_lens=seq_lens,
347
- seqlen_q=None,
348
- max_seqlen=None,
349
- seq_lens_tensor=seq_lens_tensor,
350
- max_decode_seq_len=max_decode_seq_len,
351
- num_prefill_tokens=0,
352
- num_decode_tokens=len(input_tokens),
353
- num_prefills=0,
354
- block_tables=block_tables,
355
- )
356
- return (
357
- input_tokens,
358
- input_positions,
359
- attn_metadata,
360
- )
498
+ def prepare_model_input(
499
+ self,
500
+ seq_group_metadata_list: List[SequenceGroupMetadata],
501
+ virtual_engine: int = 0,
502
+ finished_requests_ids: Optional[List[str]] = None
503
+ ) -> ModelInputForXPUWithSamplingMetadata:
504
+ """Prepare the model input based on a given sequence group, including
505
+ metadata for the sampling step.
506
+
507
+ """
508
+ model_input = self._prepare_model_input_tensors(
509
+ seq_group_metadata_list, finished_requests_ids)
510
+ # Sampling metadata is only required for the final pp group
511
+ generators = self.get_generators(finished_requests_ids)
512
+ sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
513
+ model_input.seq_lens,
514
+ model_input.query_lens,
515
+ self.device,
516
+ pin_memory=False,
517
+ generators=generators)
518
+
519
+ return dataclasses.replace(model_input,
520
+ sampling_metadata=sampling_metadata,
521
+ virtual_engine=virtual_engine)
522
523
@torch.inference_mode()
524
def execute_model(
525
self,
365
- model_input: ModelInputForXPU,
526
+ model_input: ModelInputForXPUWithSamplingMetadata,
527
kv_caches: List[torch.Tensor],
528
intermediate_tensors: Optional[IntermediateTensors] = None,
529
num_steps: int = 1,
@@ -372,20 +533,21 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
533
"XPUModelRunner does not support multi-step execution.")
534
535
model_executable = self.model
375
- execute_model_kwargs = {
376
- "input_ids":
377
- model_input.input_tokens,
378
- "positions":
379
- model_input.input_positions,
380
- "kv_caches":
381
- kv_caches,
382
- "attn_metadata":
383
- model_input.attn_metadata,
536
+ if (self.observability_config is not None
537
+ and self.observability_config.collect_model_forward_time):
538
+ model_forward_start_time = time.time()
539
+
540
+ hidden_states = model_executable(
541
+ input_ids=model_input.input_tokens,
542
+ positions=model_input.input_positions,
543
+ kv_caches=kv_caches,
544
+ attn_metadata=model_input.attn_metadata,
545
+ intermediate_tensors=intermediate_tensors,
546
**MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
385
- device=self.device),
386
- }
387
-
388
- hidden_states = model_executable(**execute_model_kwargs)
547
+ device=self.device))
548
+ if (self.observability_config is not None
549
+ and self.observability_config.collect_model_forward_time):
550
+ model_forward_end_time = time.time()
551
552
# Compute the logits.
553
logits = self.model.compute_logits(hidden_states,
@@ -396,109 +558,19 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
558
return []
559
560
# Sample the next token.
399
- output = self.model.sample(
561
+ output: SamplerOutput = self.model.sample(
562
logits=logits,
563
sampling_metadata=model_input.sampling_metadata,
564
)
403
- return [output]
404
-
405
- def _prepare_prompt(
406
- self,
407
- seq_group_metadata_list: List[SequenceGroupMetadata],
408
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
409
- BatchedTensorInputs]:
410
- assert len(seq_group_metadata_list) > 0
411
- input_tokens: List[int] = []
412
- input_positions: List[int] = []
413
- slot_mapping: List[int] = []
414
- seq_lens: List[int] = []
415
- multi_modal_inputs_list: List[MultiModalInputs] = []
416
-
417
- for seq_group_metadata in seq_group_metadata_list:
418
- assert seq_group_metadata.is_prompt
419
- seq_ids = list(seq_group_metadata.seq_data.keys())
420
- assert len(seq_ids) == 1
421
- seq_id = seq_ids[0]
422
-
423
- seq_data = seq_group_metadata.seq_data[seq_id]
424
- prompt_tokens = seq_data.get_token_ids()
425
- computed_len = seq_data.get_num_computed_tokens()
426
- seq_len = len(prompt_tokens)
565
+ if (self.observability_config is not None
566
+ and self.observability_config.collect_model_forward_time
567
+ and output is not None):
568
+ model_forward_time = (model_forward_end_time -
569
+ model_forward_start_time)
570
+ # If there are multiple workers, we are still tracking the latency
571
+ # from the start time of the driver worker to the end time of the
572
+ # driver worker. The model forward time will then end up covering
573
+ # the communication time as well.
574
+ output.model_forward_time = model_forward_time
575
428
- seq_lens.append(seq_len) # Prompt token num
429
- input_tokens.extend(prompt_tokens) # Token ids
430
-
431
- # Token position ids
432
- # NOTE(woosuk): Here we assume that the first token in the prompt
433
- # is always the first token in the sequence.
434
- input_positions.extend(list(range(computed_len, seq_len)))
435
-
436
- mm_data = seq_group_metadata.multi_modal_data
437
- if mm_data:
438
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
439
- multi_modal_inputs_list.append(mm_kwargs)
440
-
441
- if seq_group_metadata.block_tables is None:
442
- # During memory profiling, the block tables are not initialized
443
- # yet. In this case, we just use a dummy slot mapping.
444
- slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
445
- continue
446
-
447
- # Compute the slot mapping.
448
- block_table = seq_group_metadata.block_tables[seq_id]
449
- # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
450
- # where start_idx is max(0, seq_len - sliding_window).
451
- # For example, if the prompt len is 10, sliding window is 8, and
452
- # block size is 4, the first two tokens are masked and the slot
453
- # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
454
- start_idx = 0
455
- if self.sliding_window is not None:
456
- start_idx = max(0, seq_len - self.sliding_window)
457
-
458
- for i in range(computed_len, seq_len):
459
- if i < start_idx:
460
- slot_mapping.append(_PAD_SLOT_ID)
461
- continue
462
-
463
- block_number = block_table[i //
464
- self.block_size] # type: ignore
465
- block_offset = i % self.block_size # type: ignore
466
- slot = block_number * self.block_size + block_offset
467
- slot_mapping.append(slot)
468
-
469
- num_prompt_tokens = len(input_tokens)
470
-
471
- input_tokens = torch.tensor(input_tokens,
472
- dtype=torch.long,
473
- device=self.device) # type: ignore
474
- input_positions = torch.tensor(input_positions,
475
- dtype=torch.long,
476
- device=self.device) # type: ignore
477
- slot_mapping = torch.tensor(slot_mapping,
478
- dtype=torch.long,
479
- device=self.device) # type: ignore
480
-
481
- max_seqlen = max(seq_lens)
482
- tmp = [0]
483
- tmp.extend(seq_lens)
484
- seqlen = torch.tensor(tmp)
485
- seqlen_q = torch.cumsum(seqlen, dim=0).to(device=self.device)
486
-
487
- attn_metadata = self.attn_backend.make_metadata(
488
- is_prompt=True,
489
- slot_mapping=slot_mapping,
490
- seq_lens=seq_lens,
491
- seqlen_q=seqlen_q,
492
- max_seqlen=max_seqlen,
493
- seq_lens_tensor=None,
494
- max_decode_seq_len=None,
495
- num_prefills=len(seq_lens),
496
- num_prefill_tokens=num_prompt_tokens,
497
- num_decode_tokens=0,
498
- block_tables=torch.tensor([], device=self.device, dtype=torch.int),
499
- )
500
-
501
- multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
502
-
503
- return (input_tokens, input_positions, attn_metadata, seq_lens,
504
- multi_modal_kwargs)
576
+ return [output]
vllm/worker/xpu_worker.py
+3
-6
@@ -9,8 +9,8 @@ import torch
9
import torch.distributed
10
11
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
- ModelConfig, MultiModalConfig, ObservabilityConfig,
13
- ParallelConfig, PromptAdapterConfig, SchedulerConfig,
12
+ ModelConfig, ObservabilityConfig, ParallelConfig,
13
+ PromptAdapterConfig, SchedulerConfig,
14
SpeculativeConfig)
15
from vllm.distributed import (ensure_model_parallel_initialized,
16
init_distributed_environment)
@@ -46,7 +46,6 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
46
rank: int,
47
distributed_init_method: str,
48
lora_config: Optional[LoRAConfig] = None,
49
- multimodal_config: Optional[MultiModalConfig] = None,
49
speculative_config: Optional[SpeculativeConfig] = None,
50
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
51
is_driver_worker: bool = False,
@@ -73,8 +72,6 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
72
assert rank % parallel_config.tensor_parallel_size == 0, \
73
"Driver worker should be rank 0 of tensor parallel group."
74
76
- self.multimodal_config = multimodal_config
77
-
75
self.model_runner = XPUModelRunner( # type: ignore
76
model_config,
77
parallel_config,
@@ -85,7 +82,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
82
lora_config=self.lora_config,
83
kv_cache_dtype=self.cache_config.cache_dtype,
84
is_driver_worker=is_driver_worker,
88
- multimodal_config=multimodal_config,
85
+ observability_config=self.observability_config,
86
)
87
# Uninitialized cache engine. Will be initialized by
88
# initialize_cache.