@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e893795443
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[2/N] executor pass the complete config to worker/modelrunner (#9938)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Nick Hill <nhill@redhat.com>

youkaichao committed Nov 2, 2024 at 07:35 UTC e8937954434037ac787efa800f01d9d294185439
44 files changed +250 -580
tests/lora/test_long_context.py
+1 -7
@@ -138,13 +138,7 @@ def test_rotary_emb_replaced(dist_init):
138 enable_lora=True)
139 engine_config = engine_args.create_engine_config()
140 model_runner = ModelRunner(
141 - model_config=engine_config.model_config,
142 - parallel_config=engine_config.parallel_config,
143 - scheduler_config=engine_config.scheduler_config,
144 - device_config=engine_config.device_config,
145 - cache_config=engine_config.cache_config,
146 - load_config=engine_config.load_config,
147 - lora_config=engine_config.lora_config,
141 + vllm_config=engine_config,
142 is_driver_worker=True,
143 )
144 model_runner.load_model()
tests/lora/test_worker.py
+8 -4
@@ -4,7 +4,8 @@ import tempfile
4 from unittest.mock import patch
5
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 - ModelConfig, ParallelConfig, SchedulerConfig)
7 + ModelConfig, ParallelConfig, SchedulerConfig,
8 + VllmConfig)
9 from vllm.lora.models import LoRAMapping
10 from vllm.lora.request import LoRARequest
11 from vllm.worker.worker import Worker
@@ -12,7 +13,7 @@ from vllm.worker.worker import Worker
13
14 @patch.dict(os.environ, {"RANK": "0"})
15 def test_worker_apply_lora(sql_lora_files):
15 - worker = Worker(
16 + vllm_config = VllmConfig(
17 model_config=ModelConfig(
18 "meta-llama/Llama-2-7b-hf",
19 task="auto",
@@ -34,10 +35,13 @@ def test_worker_apply_lora(sql_lora_files):
35 gpu_memory_utilization=1.,
36 swap_space=0,
37 cache_dtype="auto"),
37 - local_rank=0,
38 - rank=0,
38 lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
39 max_loras=32),
40 + )
41 + worker = Worker(
42 + vllm_config=vllm_config,
43 + local_rank=0,
44 + rank=0,
45 distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
46 )
47 worker.init_device()
tests/spec_decode/utils.py
+1 -6
@@ -81,12 +81,7 @@ def create_worker(cls: Callable[..., T],
81 get_ip(), get_open_port())
82
83 worker = cls(
84 - model_config=engine_config.model_config,
85 - parallel_config=engine_config.parallel_config,
86 - scheduler_config=engine_config.scheduler_config,
87 - device_config=engine_config.device_config,
88 - cache_config=engine_config.cache_config,
89 - load_config=engine_config.load_config,
84 + vllm_config=engine_config,
85 local_rank=0,
86 rank=0,
87 distributed_init_method=distributed_init_method,
tests/worker/test_encoder_decoder_model_runner.py
+1 -8
@@ -19,14 +19,7 @@ def _create_model_runner(model: str, *args,
19 engine_args = EngineArgs(model, *args, **kwargs)
20 engine_config = engine_args.create_engine_config()
21 model_runner = EncoderDecoderModelRunner(
22 - model_config=engine_config.model_config,
23 - parallel_config=engine_config.parallel_config,
24 - scheduler_config=engine_config.scheduler_config,
25 - device_config=engine_config.device_config,
26 - cache_config=engine_config.cache_config,
27 - load_config=engine_config.load_config,
28 - lora_config=engine_config.lora_config,
29 - prompt_adapter_config=engine_config.prompt_adapter_config,
22 + vllm_config=engine_config,
23 is_driver_worker=True,
24 )
25 return model_runner
tests/worker/test_model_runner.py
+1 -9
@@ -16,15 +16,7 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
16 engine_args = EngineArgs(model, *args, **kwargs)
17 engine_config = engine_args.create_engine_config()
18 model_runner = ModelRunner(
19 - model_config=engine_config.model_config,
20 - parallel_config=engine_config.parallel_config,
21 - scheduler_config=engine_config.scheduler_config,
22 - device_config=engine_config.device_config,
23 - cache_config=engine_config.cache_config,
24 - load_config=engine_config.load_config,
25 - lora_config=engine_config.lora_config,
26 - prompt_adapter_config=engine_config.prompt_adapter_config,
27 - observability_config=engine_config.observability_config,
19 + vllm_config=engine_config,
20 is_driver_worker=True,
21 )
22 return model_runner
tests/worker/test_profile.py
+1 -6
@@ -24,12 +24,7 @@ def test_gpu_memory_profiling():
24 distributed_init_method = get_distributed_init_method(
25 get_ip(), get_open_port())
26 worker = Worker(
27 - model_config=engine_config.model_config,
28 - parallel_config=engine_config.parallel_config,
29 - scheduler_config=engine_config.scheduler_config,
30 - device_config=engine_config.device_config,
31 - cache_config=engine_config.cache_config,
32 - load_config=engine_config.load_config,
27 + vllm_config=engine_config,
28 local_rank=0,
29 rank=0,
30 distributed_init_method=distributed_init_method,
tests/worker/test_swap.py
+1 -6
@@ -19,12 +19,7 @@ def test_swap() -> None:
19 distributed_init_method = get_distributed_init_method(
20 get_ip(), get_open_port())
21 worker = Worker(
22 - model_config=engine_config.model_config,
23 - parallel_config=engine_config.parallel_config,
24 - scheduler_config=engine_config.scheduler_config,
25 - device_config=engine_config.device_config,
26 - cache_config=engine_config.cache_config,
27 - load_config=engine_config.load_config,
22 + vllm_config=engine_config,
23 local_rank=0,
24 rank=0,
25 distributed_init_method=distributed_init_method,
vllm/config.py
+9 -15
@@ -1,6 +1,6 @@
1 import enum
2 import json
3 -from dataclasses import dataclass, field, fields
3 +from dataclasses import dataclass, field
4 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
5 Mapping, Optional, Set, Tuple, Type, Union)
6
@@ -1941,9 +1941,9 @@ class ObservabilityConfig:
1941 f"installed. Original error:\n{otel_import_error_traceback}")
1942
1943
1944 -@dataclass(frozen=True)
1945 -class EngineConfig:
1946 - """Dataclass which contains all engine-related configuration. This
1944 +@dataclass
1945 +class VllmConfig:
1946 + """Dataclass which contains all vllm-related configuration. This
1947 simplifies passing around the distinct configurations in the codebase.
1948 """
1949
@@ -1953,11 +1953,11 @@ class EngineConfig:
1953 scheduler_config: SchedulerConfig
1954 device_config: DeviceConfig
1955 load_config: LoadConfig
1956 - lora_config: Optional[LoRAConfig]
1957 - speculative_config: Optional[SpeculativeConfig]
1958 - decoding_config: Optional[DecodingConfig]
1959 - observability_config: Optional[ObservabilityConfig]
1960 - prompt_adapter_config: Optional[PromptAdapterConfig]
1956 + lora_config: Optional[LoRAConfig] = None
1957 + speculative_config: Optional[SpeculativeConfig] = None
1958 + decoding_config: Optional[DecodingConfig] = None
1959 + observability_config: Optional[ObservabilityConfig] = None
1960 + prompt_adapter_config: Optional[PromptAdapterConfig] = None
1961
1962 def __post_init__(self):
1963 """Verify configs are valid & consistent with each other.
@@ -1975,9 +1975,3 @@ class EngineConfig:
1975 if self.prompt_adapter_config:
1976 self.prompt_adapter_config.verify_with_model_config(
1977 self.model_config)
1978 -
1979 - def to_dict(self):
1980 - """Return the configs as a dictionary, for use in **kwargs.
1981 - """
1982 - return dict(
1983 - (field.name, getattr(self, field.name)) for field in fields(self))
vllm/engine/arg_utils.py
+7 -6
@@ -9,10 +9,11 @@ import torch
9
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12 - DeviceConfig, EngineConfig, LoadConfig, LoadFormat,
13 - LoRAConfig, ModelConfig, ObservabilityConfig,
14 - ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15 - SpeculativeConfig, TaskOption, TokenizerPoolConfig)
12 + DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13 + ModelConfig, ObservabilityConfig, ParallelConfig,
14 + PromptAdapterConfig, SchedulerConfig,
15 + SpeculativeConfig, TaskOption, TokenizerPoolConfig,
16 + VllmConfig)
17 from vllm.executor.executor_base import ExecutorBase
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -955,7 +956,7 @@ class EngineArgs:
956 ignore_patterns=self.ignore_patterns,
957 )
958
958 - def create_engine_config(self) -> EngineConfig:
959 + def create_engine_config(self) -> VllmConfig:
960 # gguf file needs a specific model loader and doesn't use hf_repo
961 if check_gguf_file(self.model):
962 self.quantization = self.load_format = "gguf"
@@ -1167,7 +1168,7 @@ class EngineArgs:
1168 or "all" in detailed_trace_modules,
1169 )
1170
1170 - return EngineConfig(
1171 + return VllmConfig(
1172 model_config=model_config,
1173 cache_config=cache_config,
1174 parallel_config=parallel_config,
vllm/engine/async_llm_engine.py
+4 -4
@@ -7,8 +7,8 @@ from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
7 from weakref import ReferenceType
8
9 import vllm.envs as envs
10 -from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
11 - ParallelConfig, SchedulerConfig)
10 +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11 + ParallelConfig, SchedulerConfig, VllmConfig)
12 from vllm.core.scheduler import SchedulerOutputs
13 from vllm.engine.arg_utils import AsyncEngineArgs
14 from vllm.engine.async_timeout import asyncio_timeout
@@ -604,7 +604,7 @@ class AsyncLLMEngine(EngineClient):
604
605 @classmethod
606 def _get_executor_cls(
607 - cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]:
607 + cls, engine_config: VllmConfig) -> Type[ExecutorAsyncBase]:
608 distributed_executor_backend = (
609 engine_config.parallel_config.distributed_executor_backend)
610 if isinstance(distributed_executor_backend, type):
@@ -663,7 +663,7 @@ class AsyncLLMEngine(EngineClient):
663 def from_engine_args(
664 cls,
665 engine_args: AsyncEngineArgs,
666 - engine_config: Optional[EngineConfig] = None,
666 + engine_config: Optional[VllmConfig] = None,
667 start_engine_loop: bool = True,
668 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
669 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
vllm/engine/llm_engine.py
+5 -4
@@ -13,8 +13,9 @@ import torch
13 from typing_extensions import TypeIs, TypeVar
14
15 import vllm.envs as envs
16 -from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
17 - ObservabilityConfig, ParallelConfig, SchedulerConfig)
16 +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
17 + ObservabilityConfig, ParallelConfig, SchedulerConfig,
18 + VllmConfig)
19 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
20 SchedulerOutputs)
21 from vllm.engine.arg_utils import EngineArgs
@@ -219,7 +220,7 @@ class LLMEngine:
220
221 def __init__(
222 self,
222 - vllm_config: EngineConfig,
223 + vllm_config: VllmConfig,
224 executor_class: Type[ExecutorBase],
225 log_stats: bool,
226 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -500,7 +501,7 @@ class LLMEngine:
501
502 @classmethod
503 def _get_executor_cls(cls,
503 - engine_config: EngineConfig) -> Type[ExecutorBase]:
504 + engine_config: VllmConfig) -> Type[ExecutorBase]:
505 distributed_executor_backend = (
506 engine_config.parallel_config.distributed_executor_backend)
507 # Initialize the cluster and specify the executor class.
vllm/engine/multiprocessing/client.py
+2 -2
@@ -13,7 +13,7 @@ from zmq import Frame # type: ignore[attr-defined]
13 from zmq.asyncio import Socket
14
15 from vllm import PoolingParams
16 -from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16 +from vllm.config import DecodingConfig, ModelConfig, VllmConfig
17 from vllm.core.scheduler import SchedulerOutputs
18 from vllm.engine.arg_utils import AsyncEngineArgs
19 # yapf conflicts with isort for this block
@@ -78,7 +78,7 @@ class MQLLMEngineClient(EngineClient):
78 every N seconds, confirming the engine is healthy
79 """
80
81 - def __init__(self, ipc_path: str, engine_config: EngineConfig,
81 + def __init__(self, ipc_path: str, engine_config: VllmConfig,
82 engine_pid: int):
83 self.context = zmq.asyncio.Context()
84 self._errored_with: Optional[BaseException] = None
vllm/executor/cpu_executor.py
+1 -8
@@ -138,18 +138,11 @@ class CPUExecutor(ExecutorBase):
138 assert self.distributed_init_method is not None
139
140 kwargs = dict(
141 - model_config=self.model_config,
142 - parallel_config=self.parallel_config,
143 - scheduler_config=self.scheduler_config,
144 - device_config=self.device_config,
145 - cache_config=self.cache_config,
146 - load_config=self.load_config,
141 + vllm_config=self.vllm_config,
142 local_rank=local_rank,
143 rank=rank,
144 distributed_init_method=self.distributed_init_method,
150 - lora_config=self.lora_config,
145 kv_cache_dtype=self.cache_config.cache_dtype,
152 - prompt_adapter_config=self.prompt_adapter_config,
146 is_driver_worker=rank == 0,
147 )
148 wrapper.init_worker(**kwargs)
vllm/executor/executor_base.py
+2 -2
@@ -1,7 +1,7 @@
1 from abc import ABC, abstractmethod
2 from typing import List, Optional, Set, Tuple
3
4 -from vllm.config import EngineConfig
4 +from vllm.config import VllmConfig
5 from vllm.lora.request import LoRARequest
6 from vllm.model_executor.layers.sampler import SamplerOutput
7 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -20,7 +20,7 @@ class ExecutorBase(ABC):
20
21 def __init__(
22 self,
23 - vllm_config: EngineConfig,
23 + vllm_config: VllmConfig,
24 ) -> None:
25 self.vllm_config = vllm_config
26 self.model_config = vllm_config.model_config
vllm/executor/gpu_executor.py
+1 -10
@@ -49,21 +49,12 @@ class GPUExecutor(ExecutorBase):
49 distributed_init_method = get_distributed_init_method(
50 get_ip(), get_open_port())
51 return dict(
52 - model_config=self.model_config,
53 - parallel_config=self.parallel_config,
54 - scheduler_config=self.scheduler_config,
55 - device_config=self.device_config,
56 - cache_config=self.cache_config,
57 - load_config=self.load_config,
52 + vllm_config=self.vllm_config,
53 local_rank=local_rank,
54 rank=rank,
55 distributed_init_method=distributed_init_method,
61 - lora_config=self.lora_config,
62 - speculative_config=self.speculative_config,
63 - prompt_adapter_config=self.prompt_adapter_config,
56 is_driver_worker=(not self.parallel_config)
57 or (rank % self.parallel_config.tensor_parallel_size == 0),
66 - observability_config=self.observability_config,
58 )
59
60 def _get_worker_module_and_class(
vllm/executor/neuron_executor.py
+1 -5
@@ -29,11 +29,7 @@ class NeuronExecutor(ExecutorBase):
29 distributed_init_method = get_distributed_init_method(
30 get_ip(), get_open_port())
31 self.driver_worker = NeuronWorker(
32 - model_config=self.model_config,
33 - parallel_config=self.parallel_config,
34 - scheduler_config=self.scheduler_config,
35 - device_config=self.device_config,
36 - cache_config=self.cache_config,
32 + vllm_config=self.vllm_config,
33 local_rank=0,
34 rank=0,
35 distributed_init_method=distributed_init_method)
vllm/executor/openvino_executor.py
+1 -7
@@ -48,16 +48,10 @@ class OpenVINOExecutor(ExecutorBase):
48 get_ip(), get_open_port())
49 self.driver_worker = OpenVINOWorker(
50 ov_core=self.ov_core,
51 - model_config=self.model_config,
52 - parallel_config=self.parallel_config,
53 - scheduler_config=self.scheduler_config,
54 - device_config=self.device_config,
55 - cache_config=self.cache_config,
56 - load_config=self.load_config,
51 + vllm_config=self.vllm_config,
52 local_rank=0,
53 rank=0,
54 distributed_init_method=distributed_init_method,
60 - lora_config=self.lora_config,
55 kv_cache_dtype=self.cache_config.cache_dtype,
56 is_driver_worker=True,
57 )
vllm/executor/tpu_executor.py
+1 -6
@@ -44,12 +44,7 @@ class TPUExecutor(ExecutorBase):
44 distributed_init_method = get_distributed_init_method(
45 get_ip(), get_open_port())
46 return dict(
47 - model_config=self.model_config,
48 - parallel_config=self.parallel_config,
49 - scheduler_config=self.scheduler_config,
50 - device_config=self.device_config,
51 - cache_config=self.cache_config,
52 - load_config=self.load_config,
47 + vllm_config=self.vllm_config,
48 local_rank=local_rank,
49 rank=rank,
50 distributed_init_method=distributed_init_method,
vllm/spec_decode/draft_model_runner.py
+3 -33
@@ -17,9 +17,6 @@ except (ModuleNotFoundError, ImportError) as err:
17 "Draft model speculative decoding currently only supports"
18 "CUDA and ROCm flash attention backend.") from err
19
20 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
21 - ModelConfig, ObservabilityConfig, ParallelConfig,
22 - PromptAdapterConfig, SchedulerConfig)
20 from vllm.logger import init_logger
21 from vllm.multimodal import MultiModalInputs
22 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
@@ -49,40 +46,13 @@ class TP1DraftModelRunner(ModelRunner):
46 any broadcasting inside execute_model).
47 """
48
52 - def __init__(
53 - self,
54 - model_config: ModelConfig,
55 - parallel_config: ParallelConfig,
56 - scheduler_config: SchedulerConfig,
57 - device_config: DeviceConfig,
58 - cache_config: CacheConfig,
59 - load_config: LoadConfig,
60 - lora_config: Optional[LoRAConfig],
61 - kv_cache_dtype: Optional[str] = "auto",
62 - is_driver_worker: bool = False,
63 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
64 - return_hidden_states: bool = False,
65 - observability_config: Optional[ObservabilityConfig] = None,
66 - ):
67 - if return_hidden_states:
49 + def __init__(self, *args, **kwargs):
50 + if kwargs.get("return_hidden_states"):
51 raise ValueError(
52 "return_hidden_states is not supported for TP1DraftModelRunner."
53 )
54
72 - super().__init__(
73 - model_config=model_config,
74 - parallel_config=parallel_config,
75 - scheduler_config=scheduler_config,
76 - device_config=device_config,
77 - cache_config=cache_config,
78 - load_config=load_config,
79 - lora_config=lora_config,
80 - kv_cache_dtype=kv_cache_dtype,
81 - is_driver_worker=is_driver_worker,
82 - prompt_adapter_config=prompt_adapter_config,
83 - return_hidden_states=return_hidden_states,
84 - observability_config=observability_config,
85 - )
55 + super().__init__(*args, **kwargs)
56
57 def _update_sampling_metadata(self, sampling_metadata, num_seqs,
58 num_queries):
vllm/spec_decode/ngram_worker.py
+1 -1
@@ -21,7 +21,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
21 def __init__(self, *args, **kwargs):
22 # Get local_rank/vocab_size from kwargs attribute
23 self.local_rank = kwargs["local_rank"]
24 - self.vocab_size = kwargs["model_config"].get_vocab_size()
24 + self.vocab_size = kwargs["vllm_config"].model_config.get_vocab_size()
25
26 # Lazy initialization list.
27 self._proposer: Top1Proposer
vllm/spec_decode/spec_decode_worker.py
+18 -17
@@ -1,10 +1,11 @@
1 +import copy
2 from collections import defaultdict
3 from functools import cached_property
4 from typing import Any, Dict, List, Optional, Set, Tuple, Type
5
6 import torch
7
7 -from vllm.config import ParallelConfig, SpeculativeConfig
8 +from vllm.config import ParallelConfig, SpeculativeConfig, VllmConfig
9 from vllm.distributed.communication_op import broadcast_tensor_dict
10 from vllm.logger import init_logger
11 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
@@ -45,8 +46,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
46 """Helper method that is the entrypoint for Executors which use
47 WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config.
48 """
48 - assert "speculative_config" in kwargs
49 - speculative_config: SpeculativeConfig = kwargs.get("speculative_config")
49 + vllm_config: VllmConfig = kwargs.get("vllm_config")
50 + speculative_config: SpeculativeConfig = vllm_config.speculative_config
51 assert speculative_config is not None
52
53 draft_worker_kwargs = kwargs.copy()
@@ -58,14 +59,16 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
59 target_worker.model_runner.disable_logprobs =\
60 speculative_config.disable_logprobs
61
62 + draft_worker_config = copy.deepcopy(vllm_config)
63 + draft_worker_config.model_config = speculative_config.draft_model_config
64 + draft_worker_config.parallel_config = speculative_config.draft_parallel_config # noqa
65 + # TODO allow draft-model specific load config.
66 +
67 # Override draft-model specific worker args.
68 draft_worker_kwargs.update(
63 - model_config=speculative_config.draft_model_config,
64 - parallel_config=speculative_config.draft_parallel_config,
69 + vllm_config=draft_worker_config,
70 ngram_prompt_lookup_max=speculative_config.ngram_prompt_lookup_max,
71 ngram_prompt_lookup_min=speculative_config.ngram_prompt_lookup_min,
67 - # TODO allow draft-model specific load config.
68 - #load_config=load_config,
72 )
73
74 spec_decode_worker = SpecDecodeWorker.create_worker(
@@ -134,29 +137,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
137 draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
138 ngram_prompt_lookup_min = (
139 draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
140 + draft_model_config = draft_worker_kwargs["vllm_config"].model_config
141 + draft_parallel_config: ParallelConfig = draft_worker_kwargs[
142 + 'vllm_config'].parallel_config
143 if ngram_prompt_lookup_max > 0:
144 proposer_worker = NGramWorker(**draft_worker_kwargs)
145 proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
146 ngram_prompt_lookup_max)
147 else:
142 - draft_parallel_config: ParallelConfig = draft_worker_kwargs[
143 - 'parallel_config']
148 draft_tp = draft_parallel_config.tensor_parallel_size
149 target_tp = scorer_worker.parallel_config.tensor_parallel_size
150
147 - if draft_worker_kwargs[
148 - "model_config"].hf_config.model_type == "mlp_speculator":
151 + if draft_model_config.hf_config.model_type == "mlp_speculator":
152 proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
150 - elif draft_worker_kwargs[
151 - "model_config"].hf_config.model_type == "medusa":
153 + elif draft_model_config.hf_config.model_type == "medusa":
154 proposer_worker = MedusaWorker(**draft_worker_kwargs)
155 else:
156 if draft_tp == 1:
157 draft_worker_kwargs[
158 "model_runner_cls"] = TP1DraftModelRunner
159 else:
158 - if draft_worker_kwargs[
159 - "model_config"].hf_config.model_type == "eagle":
160 + if draft_model_config.hf_config.model_type == "eagle":
161 raise NotImplementedError(
162 "EAGLE does not support TP > 1 yet")
163
@@ -190,8 +191,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
191 "[Speculative Decoding] Disabling MQA scorer as the "
192 "MQA is only available with flash attn backend.")
193
193 - if "model_config" in draft_worker_kwargs and \
194 - draft_worker_kwargs["model_config"].max_model_len < \
194 + if draft_model_config and \
195 + draft_model_config.max_model_len < \
196 scorer_worker.model_config.max_model_len:
197 disable_mqa_scorer = True
198 logger.info(
vllm/spec_decode/target_model_runner.py
+9 -25
@@ -1,8 +1,6 @@
1 from typing import List, Optional
2
3 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4 - ModelConfig, ObservabilityConfig, ParallelConfig,
5 - PromptAdapterConfig, SchedulerConfig)
3 +from vllm.config import VllmConfig
4 from vllm.sequence import SequenceGroupMetadata
5 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
6 ModelRunner)
@@ -20,35 +18,21 @@ class TargetModelRunner(ModelRunner):
18 requested or not.
19 """
20
23 - def __init__(self,
24 - model_config: ModelConfig,
25 - parallel_config: ParallelConfig,
26 - scheduler_config: SchedulerConfig,
27 - device_config: DeviceConfig,
28 - cache_config: CacheConfig,
29 - load_config: LoadConfig,
30 - lora_config: Optional[LoRAConfig],
31 - kv_cache_dtype: Optional[str] = "auto",
32 - is_driver_worker: bool = False,
33 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34 - return_hidden_states: bool = False,
35 - observability_config: Optional[ObservabilityConfig] = None):
21 + def __init__(
22 + self,
23 + vllm_config: VllmConfig,
24 + kv_cache_dtype: Optional[str] = "auto",
25 + is_driver_worker: bool = False,
26 + return_hidden_states: bool = False,
27 + ):
28 # An internal boolean member variable to indicate if token log
29 # probabilities are needed or not.
30 self.disable_logprobs = True
31 super().__init__(
40 - model_config=model_config,
41 - parallel_config=parallel_config,
42 - scheduler_config=scheduler_config,
43 - device_config=device_config,
44 - cache_config=cache_config,
45 - load_config=load_config,
46 - lora_config=lora_config,
32 + vllm_config=vllm_config,
33 kv_cache_dtype=kv_cache_dtype,
34 is_driver_worker=is_driver_worker,
49 - prompt_adapter_config=prompt_adapter_config,
35 return_hidden_states=return_hidden_states,
51 - observability_config=observability_config,
36 )
37
38 def prepare_model_input(
vllm/v1/engine/llm_engine.py
+5 -4
@@ -2,8 +2,9 @@ import time
2 from typing import (Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type,
3 Union)
4
5 -from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
6 - ObservabilityConfig, ParallelConfig, SchedulerConfig)
5 +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
6 + ObservabilityConfig, ParallelConfig, SchedulerConfig,
7 + VllmConfig)
8 from vllm.engine.arg_utils import EngineArgs
9 from vllm.engine.metrics_types import StatLoggerBase
10 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
@@ -32,7 +33,7 @@ class LLMEngine:
33
34 def __init__(
35 self,
35 - vllm_config: EngineConfig,
36 + vllm_config: VllmConfig,
37 executor_class: Type[GPUExecutor],
38 log_stats: bool,
39 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -477,7 +478,7 @@ class LLMEngine:
478 return self.lora_config
479
480 @classmethod
480 - def _get_executor_cls(cls, engine_config: EngineConfig):
481 + def _get_executor_cls(cls, engine_config: VllmConfig):
482 return GPUExecutor
483
484 def is_tracing_enabled(self) -> bool:
vllm/v1/executor/gpu_executor.py
+1 -10
@@ -56,19 +56,10 @@ class GPUExecutor:
56 distributed_init_method = get_distributed_init_method(
57 get_ip(), get_open_port())
58 return Worker(
59 - model_config=self.model_config,
60 - parallel_config=self.parallel_config,
61 - scheduler_config=self.scheduler_config,
62 - device_config=self.device_config,
63 - cache_config=self.cache_config,
64 - load_config=self.load_config,
59 + vllm_config=self.vllm_config,
60 local_rank=local_rank,
61 rank=rank,
62 distributed_init_method=distributed_init_method,
68 - lora_config=self.lora_config,
69 - speculative_config=self.speculative_config,
70 - prompt_adapter_config=self.prompt_adapter_config,
71 - observability_config=self.observability_config,
63 )
64
65 def determine_num_available_blocks(self) -> Tuple[int, int]:
vllm/v1/worker/gpu_model_runner.py
+19 -22
@@ -7,9 +7,7 @@ import torch
7 import torch.distributed
8 import torch.nn as nn
9
10 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 - ModelConfig, ObservabilityConfig, ParallelConfig,
12 - PromptAdapterConfig, SchedulerConfig)
10 +from vllm.config import VllmConfig
11 from vllm.forward_context import set_forward_context
12 from vllm.logger import init_logger
13 from vllm.model_executor.model_loader import get_model
@@ -33,26 +31,25 @@ class GPUModelRunner:
31
32 def __init__(
33 self,
36 - model_config: ModelConfig,
37 - parallel_config: ParallelConfig,
38 - scheduler_config: SchedulerConfig,
39 - device_config: DeviceConfig,
40 - cache_config: CacheConfig,
41 - load_config: LoadConfig,
42 - lora_config: Optional[LoRAConfig] = None,
43 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
44 - observability_config: Optional[ObservabilityConfig] = None,
34 + vllm_config: VllmConfig,
35 ):
46 - self.model_config = model_config
47 - self.parallel_config = parallel_config
48 - self.scheduler_config = scheduler_config
49 - self.device_config = device_config
50 - self.cache_config = cache_config
51 - self.lora_config = lora_config
52 - self.load_config = load_config
53 - self.prompt_adapter_config = prompt_adapter_config
54 - self.observability_config = observability_config
55 -
36 + # TODO: use ModelRunnerBase.__init__(self, vllm_config=vllm_config)
37 + self.vllm_config = vllm_config
38 + self.model_config = vllm_config.model_config
39 + self.cache_config = vllm_config.cache_config
40 + self.lora_config = vllm_config.lora_config
41 + self.load_config = vllm_config.load_config
42 + self.parallel_config = vllm_config.parallel_config
43 + self.scheduler_config = vllm_config.scheduler_config
44 + self.device_config = vllm_config.device_config
45 + self.speculative_config = vllm_config.speculative_config
46 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
47 + self.observability_config = vllm_config.observability_config
48 +
49 + model_config = self.model_config
50 + cache_config = self.cache_config
51 + scheduler_config = self.scheduler_config
52 + parallel_config = self.parallel_config
53 self.device = self.device_config.device
54 self.pin_memory = is_pin_memory_available()
55 self.dtype = self.model_config.dtype
vllm/v1/worker/gpu_worker.py
+17 -33
@@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
6 import torch
7 import torch.distributed
8
9 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ObservabilityConfig, ParallelConfig,
11 - PromptAdapterConfig, SchedulerConfig,
12 - SpeculativeConfig)
9 +from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
10 from vllm.distributed import (ensure_model_parallel_initialized,
11 init_distributed_environment,
12 set_custom_all_reduce)
@@ -30,48 +27,35 @@ class Worker:
27
28 def __init__(
29 self,
33 - model_config: ModelConfig,
34 - parallel_config: ParallelConfig,
35 - scheduler_config: SchedulerConfig,
36 - device_config: DeviceConfig,
37 - cache_config: CacheConfig,
38 - load_config: LoadConfig,
30 + vllm_config: VllmConfig,
31 local_rank: int,
32 rank: int,
33 distributed_init_method: str,
42 - speculative_config: Optional[SpeculativeConfig] = None,
43 - lora_config: Optional[LoRAConfig] = None,
44 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
45 - observability_config: Optional[ObservabilityConfig] = None,
34 ):
47 - self.model_config = model_config
48 - self.parallel_config = parallel_config
49 - self.scheduler_config = scheduler_config
50 - self.device_config = device_config
51 - self.cache_config = cache_config
52 - self.load_config = load_config
35 +
36 + # TODO: use WorkerBase.__init__(self, vllm_config=vllm_config)
37 + self.vllm_config = vllm_config
38 + self.model_config = vllm_config.model_config
39 + self.cache_config = vllm_config.cache_config
40 + self.lora_config = vllm_config.lora_config
41 + self.load_config = vllm_config.load_config
42 + self.parallel_config = vllm_config.parallel_config
43 + self.scheduler_config = vllm_config.scheduler_config
44 + self.device_config = vllm_config.device_config
45 + self.speculative_config = vllm_config.speculative_config
46 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
47 + self.observability_config = vllm_config.observability_config
48 +
49 self.local_rank = local_rank
50 self.rank = rank
51 self.distributed_init_method = distributed_init_method
56 - self.lora_config = lora_config
57 - self.speculative_config = speculative_config
58 - self.prompt_adapter_config = prompt_adapter_config
59 - self.observability_config = observability_config
52
53 if self.model_config.trust_remote_code:
54 # note: lazy import to avoid importing torch before initializing
55 from vllm.utils import init_cached_hf_modules
56 init_cached_hf_modules()
57
66 - self.model_runner = GPUModelRunner(
67 - model_config,
68 - parallel_config,
69 - scheduler_config,
70 - device_config,
71 - cache_config,
72 - load_config,
73 - lora_config=lora_config,
74 - )
58 + self.model_runner = GPUModelRunner(vllm_config)
59
60 def initialize(self):
61 if self.device_config.device.type == "cuda":
vllm/worker/cpu_model_runner.py
+6 -19
@@ -8,9 +8,7 @@ import torch
8 from torch import nn
9
10 from vllm.attention import AttentionMetadata, get_attn_backend
11 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 - ModelConfig, ParallelConfig, PromptAdapterConfig,
13 - SchedulerConfig)
11 +from vllm.config import VllmConfig
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
@@ -412,29 +410,18 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
410
411 def __init__(
412 self,
415 - model_config: ModelConfig,
416 - parallel_config: ParallelConfig,
417 - scheduler_config: SchedulerConfig,
418 - device_config: DeviceConfig,
419 - cache_config: CacheConfig,
420 - load_config: LoadConfig,
421 - lora_config: Optional[LoRAConfig],
413 + vllm_config: VllmConfig,
414 kv_cache_dtype: Optional[str] = "auto",
423 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
415 is_driver_worker: bool = False,
416 *args,
417 **kwargs,
418 ):
428 - self.model_config = model_config
429 - self.parallel_config = parallel_config
430 - self.scheduler_config = scheduler_config
419 + ModelRunnerBase.__init__(self, vllm_config)
420 # Currently, CPU worker doesn't support chunked prefill.
421 assert self.scheduler_config.chunked_prefill_enabled is False
433 - self.device_config = device_config
434 - self.cache_config = cache_config
435 - self.lora_config = lora_config
436 - self.prompt_adapter_config = prompt_adapter_config
437 - self.load_config = load_config
422 + model_config = self.model_config
423 + cache_config = self.cache_config
424 +
425 self.is_driver_worker = is_driver_worker
426
427 self.device = self.device_config.device
vllm/worker/cpu_worker.py
+9 -28
@@ -6,9 +6,8 @@ import torch.distributed
6
7 import vllm.envs as envs
8 from vllm.attention import get_attn_backend
9 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, PromptAdapterConfig,
11 - SchedulerConfig)
9 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10 + ParallelConfig, VllmConfig)
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment)
13 from vllm.logger import init_logger
@@ -18,7 +17,8 @@ from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17 from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18 from vllm.worker.cpu_model_runner import CPUModelRunner
19 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21 - LoraNotSupportedWorkerBase, WorkerInput)
20 + LoraNotSupportedWorkerBase, WorkerBase,
21 + WorkerInput)
22
23 logger = init_logger(__name__)
24
@@ -121,31 +121,19 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
121
122 def __init__(
123 self,
124 - model_config: ModelConfig,
125 - parallel_config: ParallelConfig,
126 - scheduler_config: SchedulerConfig,
127 - device_config: DeviceConfig,
128 - cache_config: CacheConfig,
129 - load_config: LoadConfig,
124 + vllm_config: VllmConfig,
125 local_rank: int,
126 rank: int,
127 distributed_init_method: str,
133 - lora_config: Optional[LoRAConfig] = None,
128 kv_cache_dtype: Optional[str] = "auto",
135 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
129 is_driver_worker: bool = False,
130 ) -> None:
138 - self.model_config = model_config
139 - self.parallel_config = parallel_config
140 - self.scheduler_config = scheduler_config
141 - self.device_config = device_config
142 - self.cache_config = cache_config
143 - self.load_config = load_config
131 + WorkerBase.__init__(self, vllm_config=vllm_config)
132 +
133 self.local_rank = local_rank
134 self.rank = rank
135 self.distributed_init_method = distributed_init_method
147 - self.lora_config = lora_config
148 - self.prompt_adapter_config = prompt_adapter_config
136 +
137 self.is_driver_worker = is_driver_worker
138 if self.is_driver_worker:
139 assert self.rank == 0, "The driver worker must have rank 0."
@@ -166,15 +154,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
154 if self._is_encoder_decoder_model():
155 ModelRunnerClass = CPUEncoderDecoderModelRunner
156 self.model_runner: CPUModelRunner = ModelRunnerClass(
169 - model_config,
170 - parallel_config,
171 - scheduler_config,
172 - device_config,
173 - cache_config,
174 - load_config=self.load_config,
175 - lora_config=self.lora_config,
157 + vllm_config=vllm_config,
158 kv_cache_dtype=kv_cache_dtype,
177 - prompt_adapter_config=self.prompt_adapter_config,
159 is_driver_worker=is_driver_worker)
160 # Uninitialized cache engine. Will be initialized by
161 # initialize_cache.
vllm/worker/embedding_model_runner.py
+4 -22
@@ -3,9 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union
3
4 import torch
5
6 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 - ModelConfig, ObservabilityConfig, ParallelConfig,
8 - PromptAdapterConfig, SchedulerConfig)
6 +from vllm.config import VllmConfig
7 from vllm.distributed import get_pp_group
8 from vllm.forward_context import set_forward_context
9 from vllm.logger import init_logger
@@ -36,29 +34,13 @@ class EmbeddingModelRunner(
34
35 def __init__(
36 self,
39 - model_config: ModelConfig,
40 - parallel_config: ParallelConfig,
41 - scheduler_config: SchedulerConfig,
42 - device_config: DeviceConfig,
43 - cache_config: CacheConfig,
44 - load_config: LoadConfig,
45 - lora_config: Optional[LoRAConfig],
37 + vllm_config: VllmConfig,
38 kv_cache_dtype: Optional[str] = "auto",
39 is_driver_worker: bool = False,
48 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
49 - observability_config: Optional[ObservabilityConfig] = None,
40 ):
51 - super().__init__(model_config,
52 - parallel_config,
53 - scheduler_config,
54 - device_config,
55 - cache_config,
56 - load_config,
57 - lora_config=lora_config,
41 + super().__init__(vllm_config=vllm_config,
42 kv_cache_dtype=kv_cache_dtype,
59 - is_driver_worker=is_driver_worker,
60 - prompt_adapter_config=prompt_adapter_config,
61 - observability_config=observability_config)
43 + is_driver_worker=is_driver_worker)
44
45 @torch.inference_mode()
46 def execute_model(
vllm/worker/enc_dec_model_runner.py
+5 -20
@@ -11,9 +11,7 @@ from vllm.attention.backends.utils import PAD_SLOT_ID
11 from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
12 get_global_forced_attn_backend,
13 global_force_attn_backend)
14 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 - ModelConfig, ObservabilityConfig, ParallelConfig,
16 - PromptAdapterConfig, SchedulerConfig)
14 +from vllm.config import ModelConfig, VllmConfig
15 from vllm.forward_context import set_forward_context
16 from vllm.inputs import INPUT_REGISTRY, InputRegistry
17 from vllm.logger import init_logger
@@ -85,17 +83,9 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
83
84 def __init__(
85 self,
88 - model_config: ModelConfig,
89 - parallel_config: ParallelConfig,
90 - scheduler_config: SchedulerConfig,
91 - device_config: DeviceConfig,
92 - cache_config: CacheConfig,
93 - load_config: LoadConfig,
94 - lora_config: Optional[LoRAConfig],
86 + vllm_config: VllmConfig,
87 kv_cache_dtype: Optional[str] = "auto",
88 is_driver_worker: bool = False,
97 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
98 - observability_config: Optional[ObservabilityConfig] = None,
89 input_registry: InputRegistry = INPUT_REGISTRY,
90 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
91 ):
@@ -107,15 +97,10 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
97 models) but these arguments are present here for compatibility with
98 the base-class constructor.
99 '''
110 - self._maybe_force_supported_attention_backend(model_config)
100 + self._maybe_force_supported_attention_backend(vllm_config.model_config)
101 +
102 super().__init__(
112 - model_config,
113 - parallel_config,
114 - scheduler_config,
115 - device_config,
116 - cache_config,
117 - load_config,
118 - lora_config=None,
103 + vllm_config=vllm_config,
104 kv_cache_dtype=kv_cache_dtype,
105 is_driver_worker=is_driver_worker,
106 )
vllm/worker/model_runner.py
+7 -21
@@ -20,9 +20,7 @@ from vllm.attention.backends.abstract import AttentionState
20 from vllm.attention.backends.utils import CommonAttentionState
21 from vllm.compilation.compile_context import set_compile_context
22 from vllm.compilation.levels import CompilationLevel
23 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
24 - ModelConfig, ObservabilityConfig, ParallelConfig,
25 - PromptAdapterConfig, SchedulerConfig)
23 +from vllm.config import VllmConfig
24 from vllm.core.scheduler import SchedulerOutputs
25 from vllm.distributed import get_pp_group
26 from vllm.distributed.parallel_state import graph_capture
@@ -955,32 +953,20 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
953
954 def __init__(
955 self,
958 - model_config: ModelConfig,
959 - parallel_config: ParallelConfig,
960 - scheduler_config: SchedulerConfig,
961 - device_config: DeviceConfig,
962 - cache_config: CacheConfig,
963 - load_config: LoadConfig,
964 - lora_config: Optional[LoRAConfig],
956 + vllm_config: VllmConfig,
957 kv_cache_dtype: Optional[str] = "auto",
958 is_driver_worker: bool = False,
967 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
959 return_hidden_states: bool = False,
969 - observability_config: Optional[ObservabilityConfig] = None,
960 input_registry: InputRegistry = INPUT_REGISTRY,
961 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
962 ):
973 - self.model_config = model_config
974 - self.parallel_config = parallel_config
975 - self.scheduler_config = scheduler_config
976 - self.device_config = device_config
977 - self.cache_config = cache_config
978 - self.lora_config = lora_config
979 - self.load_config = load_config
963 +
964 + ModelRunnerBase.__init__(self, vllm_config)
965 + model_config = self.model_config
966 + cache_config = self.cache_config
967 +
968 self.is_driver_worker = is_driver_worker
981 - self.prompt_adapter_config = prompt_adapter_config
969 self.return_hidden_states = return_hidden_states
983 - self.observability_config = observability_config
970
971 self.device = self.device_config.device
972 self.pin_memory = is_pin_memory_available()
vllm/worker/model_runner_base.py
+17
@@ -9,6 +9,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
9 import torch
10 from torch import is_tensor
11
12 +from vllm.config import VllmConfig
13 from vllm.logger import init_logger
14 from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.platforms import current_platform
@@ -220,6 +221,22 @@ class ModelRunnerBase(ABC, Generic[T]):
221 ModelRunnerInputBase subclass.
222 """
223
224 + def __init__(
225 + self,
226 + vllm_config: VllmConfig,
227 + ) -> None:
228 + self.vllm_config = vllm_config
229 + self.model_config = vllm_config.model_config
230 + self.cache_config = vllm_config.cache_config
231 + self.lora_config = vllm_config.lora_config
232 + self.load_config = vllm_config.load_config
233 + self.parallel_config = vllm_config.parallel_config
234 + self.scheduler_config = vllm_config.scheduler_config
235 + self.device_config = vllm_config.device_config
236 + self.speculative_config = vllm_config.speculative_config
237 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
238 + self.observability_config = vllm_config.observability_config
239 +
240 # Map of request_id -> generator used for seeded random sampling
241 generators: Dict[str, torch.Generator] = {}
242
vllm/worker/multi_step_model_runner.py
+1
@@ -304,6 +304,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
304 # mypy: enable-error-code=type-var
305
306 def __init__(self, base_model_runner: GPUModelRunnerBase, *args, **kwargs):
307 +
308 super().__init__(*args, **kwargs)
309
310 # Check attention backend support.
vllm/worker/multi_step_worker.py
+1 -9
@@ -27,17 +27,9 @@ class MultiStepWorker(Worker):
27 # for multi-step model, wrap the model runner with MultiStepModelRunner
28 self.model_runner = MultiStepModelRunner(
29 base_model_runner,
30 - base_model_runner.model_config,
31 - base_model_runner.parallel_config,
32 - base_model_runner.scheduler_config,
33 - base_model_runner.device_config,
34 - base_model_runner.cache_config,
35 - load_config=base_model_runner.load_config,
36 - lora_config=self.lora_config,
30 + vllm_config=base_model_runner.vllm_config,
31 kv_cache_dtype=self.cache_config.cache_dtype,
32 is_driver_worker=base_model_runner.is_driver_worker,
39 - prompt_adapter_config=base_model_runner.prompt_adapter_config,
40 - observability_config=base_model_runner.observability_config,
33 )
34
35 pipeline_parallel_size = self.parallel_config.pipeline_parallel_size
vllm/worker/neuron_model_runner.py
+4 -12
@@ -7,8 +7,7 @@ import torch
7 from torch import nn
8 from transformers_neuronx.config import GenerationConfig
9
10 -from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
11 - SchedulerConfig)
10 +from vllm.config import VllmConfig
11 from vllm.logger import init_logger
12 from vllm.model_executor import SamplingMetadata
13 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -57,20 +56,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
56
57 def __init__(
58 self,
60 - model_config: ModelConfig,
61 - parallel_config: ParallelConfig,
62 - scheduler_config: SchedulerConfig,
63 - device_config: DeviceConfig,
59 + vllm_config: VllmConfig,
60 ):
65 - self.model_config = model_config
66 - self.parallel_config = parallel_config
67 - self.scheduler_config = scheduler_config
68 -
61 + ModelRunnerBase.__init__(self, vllm_config)
62 + model_config = self.model_config
63 if model_config is not None and model_config.get_sliding_window():
64 logger.warning("Sliding window is not supported on Neuron. "
65 "The model will run without sliding window.")
72 - self.device_config = (device_config
73 - if device_config is not None else DeviceConfig())
66 self.device = self.device_config.device
67 self.pin_memory = is_pin_memory_available()
68
vllm/worker/neuron_worker.py
+6 -14
@@ -4,15 +4,15 @@ from typing import List, Optional, Tuple
4 import torch
5 import torch.distributed
6
7 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig)
7 +from vllm.config import VllmConfig
8 from vllm.distributed import (ensure_model_parallel_initialized,
9 init_distributed_environment)
10 from vllm.model_executor import set_random_seed
11 from vllm.sequence import ExecuteModelRequest
12 from vllm.worker.neuron_model_runner import NeuronModelRunner
13 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
15 - LoraNotSupportedWorkerBase, WorkerInput)
14 + LoraNotSupportedWorkerBase, WorkerBase,
15 + WorkerInput)
16
17
18 class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
@@ -21,20 +21,12 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
21
22 def __init__(
23 self,
24 - model_config: ModelConfig,
25 - parallel_config: ParallelConfig,
26 - scheduler_config: SchedulerConfig,
27 - device_config: DeviceConfig,
28 - cache_config: CacheConfig,
24 + vllm_config: VllmConfig,
25 local_rank: int,
26 rank: int,
27 distributed_init_method: str,
28 ) -> None:
33 - self.model_config = model_config
34 - self.parallel_config = parallel_config
35 - self.scheduler_config = scheduler_config
36 - self.device_config = device_config
37 - self.cache_config = cache_config
29 + WorkerBase.__init__(self, vllm_config=vllm_config)
30 self.local_rank = local_rank
31 self.rank = rank
32 self.distributed_init_method = distributed_init_method
@@ -44,7 +36,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
36 init_cached_hf_modules()
37
38 self.model_runner: NeuronModelRunner = NeuronModelRunner(
47 - model_config, parallel_config, scheduler_config, device_config)
39 + vllm_config=vllm_config)
40 self.is_driver_worker = True
41
42 def init_device(self) -> None:
vllm/worker/openvino_model_runner.py
+13 -20
@@ -7,9 +7,7 @@ from torch import nn
7
8 from vllm.attention import get_attn_backend
9 from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
10 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 - ModelConfig, MultiModalConfig, ParallelConfig,
12 - SchedulerConfig)
10 +from vllm.config import VllmConfig
11 from vllm.logger import init_logger
12 from vllm.model_executor import SamplingMetadata
13 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -17,6 +15,7 @@ from vllm.model_executor.model_loader.openvino import get_model
15 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16 MultiModalInputs, MultiModalPlaceholderMap)
17 from vllm.sequence import SequenceGroupMetadata
18 +from vllm.worker.model_runner_base import ModelRunnerBase
19
20 logger = init_logger(__name__)
21
@@ -39,33 +38,21 @@ class ModelInput(NamedTuple):
38 multi_modal_kwargs={})
39
40
42 -class OpenVINOModelRunner:
41 +class OpenVINOModelRunner(ModelRunnerBase):
42
43 def __init__(
44 self,
45 ov_core: ov.Core,
47 - model_config: ModelConfig,
48 - parallel_config: ParallelConfig,
49 - scheduler_config: SchedulerConfig,
50 - device_config: DeviceConfig,
51 - cache_config: CacheConfig,
52 - load_config: LoadConfig,
53 - lora_config: Optional[LoRAConfig],
54 - multimodal_config: Optional[MultiModalConfig],
46 + vllm_config: VllmConfig,
47 kv_cache_dtype: Optional[str] = "auto",
48 is_driver_worker: bool = False,
49 *args,
50 **kwargs,
51 ):
52 self.ov_core = ov_core
61 - self.model_config = model_config
62 - self.parallel_config = parallel_config
63 - self.scheduler_config = scheduler_config
64 - self.device_config = device_config
65 - self.cache_config = cache_config
66 - self.lora_config = lora_config
67 - self.multimodal_config = multimodal_config
68 - self.load_config = load_config
53 + ModelRunnerBase.__init__(self, vllm_config=vllm_config)
54 + cache_config = self.cache_config
55 + model_config = self.model_config
56 self.is_driver_worker = is_driver_worker
57
58 self.device = self.device_config.device
@@ -369,3 +356,9 @@ class OpenVINOModelRunner:
356 sampling_metadata=sampling_metadata,
357 )
358 return output
359 +
360 + def prepare_model_input(self, *args, **kwargs):
361 + raise NotImplementedError
362 +
363 + def make_model_input_from_broadcasted_tensor_dict(self, *args, **kwargs):
364 + raise NotImplementedError
vllm/worker/openvino_worker.py
+6 -28
@@ -7,9 +7,8 @@ import torch.distributed
7
8 import vllm.envs as envs
9 from vllm.attention import get_attn_backend
10 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 - ModelConfig, MultiModalConfig, ParallelConfig,
12 - SchedulerConfig)
10 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
11 + ParallelConfig, VllmConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 init_distributed_environment)
@@ -22,7 +21,7 @@ from vllm.platforms import current_platform
21 from vllm.sampling_params import SamplingParams
22 from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
23 from vllm.worker.openvino_model_runner import OpenVINOModelRunner
25 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
24 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
25
26 logger = init_logger(__name__)
27
@@ -212,33 +211,19 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
211 def __init__(
212 self,
213 ov_core: ov.Core,
215 - model_config: ModelConfig,
216 - parallel_config: ParallelConfig,
217 - scheduler_config: SchedulerConfig,
218 - device_config: DeviceConfig,
219 - cache_config: CacheConfig,
220 - load_config: LoadConfig,
214 + vllm_config: VllmConfig,
215 local_rank: int,
216 rank: int,
217 distributed_init_method: str,
224 - lora_config: Optional[LoRAConfig] = None,
225 - multimodal_config: Optional[MultiModalConfig] = None,
218 kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined,
219 is_driver_worker: bool = False,
220 ) -> None:
221 self.ov_core = ov_core
230 - self.model_config = model_config
231 - self.parallel_config = parallel_config
222 + WorkerBase.__init__(self, vllm_config)
223 self.parallel_config.rank = rank
233 - self.scheduler_config = scheduler_config
234 - self.device_config = device_config
235 - self.cache_config = cache_config
236 - self.load_config = load_config
224 self.local_rank = local_rank
225 self.rank = rank
226 self.distributed_init_method = distributed_init_method
240 - self.lora_config = lora_config
241 - self.multimodal_config = multimodal_config
227 self.is_driver_worker = is_driver_worker
228 if self.is_driver_worker:
229 assert self.rank == 0, "The driver worker must have rank 0."
@@ -250,14 +235,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
235 init_cached_hf_modules()
236 self.model_runner = OpenVINOModelRunner(
237 self.ov_core,
253 - model_config,
254 - parallel_config,
255 - scheduler_config,
256 - device_config,
257 - cache_config,
258 - load_config=self.load_config,
259 - lora_config=self.lora_config,
260 - multimodal_config=self.multimodal_config,
238 + vllm_config=self.vllm_config,
239 kv_cache_dtype=kv_cache_dtype,
240 is_driver_worker=is_driver_worker,
241 )
vllm/worker/tpu_model_runner.py
+3 -14
@@ -12,8 +12,7 @@ import torch_xla.runtime as xr
12
13 from vllm.attention import AttentionMetadata, get_attn_backend
14 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
15 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
16 - ParallelConfig, SchedulerConfig)
15 +from vllm.config import VllmConfig
16 from vllm.logger import init_logger
17 from vllm.model_executor.layers.sampler import SamplerOutput
18 from vllm.model_executor.model_loader import get_model
@@ -90,20 +89,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
89
90 def __init__(
91 self,
93 - model_config: ModelConfig,
94 - parallel_config: ParallelConfig,
95 - scheduler_config: SchedulerConfig,
96 - device_config: DeviceConfig,
97 - cache_config: CacheConfig,
98 - load_config: LoadConfig,
92 + vllm_config: VllmConfig,
93 is_driver_worker: bool = False,
94 ):
101 - self.model_config = model_config
102 - self.parallel_config = parallel_config
103 - self.scheduler_config = scheduler_config
104 - self.device_config = device_config
105 - self.cache_config = cache_config
106 - self.load_config = load_config
95 + ModelRunnerBase.__init__(self, vllm_config=vllm_config)
96 self.is_driver_worker = is_driver_worker
97
98 self.block_size = self.cache_config.block_size
vllm/worker/tpu_worker.py
+6 -22
@@ -6,8 +6,7 @@ import torch_xla.core.xla_model as xm
6 import torch_xla.runtime as xr
7
8 import vllm.envs as envs
9 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
10 - ParallelConfig, SchedulerConfig)
9 +from vllm.config import VllmConfig
10 from vllm.distributed import (ensure_model_parallel_initialized,
11 init_distributed_environment)
12 from vllm.logger import init_logger
@@ -16,7 +15,8 @@ from vllm.sequence import ExecuteModelRequest
15 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
16 from vllm.worker.tpu_model_runner import TPUModelRunner
17 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
19 - LoraNotSupportedWorkerBase, WorkerInput)
18 + LoraNotSupportedWorkerBase, WorkerBase,
19 + WorkerInput)
20
21 logger = init_logger(__name__)
22
@@ -25,24 +25,14 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
25
26 def __init__(
27 self,
28 - model_config: ModelConfig,
29 - parallel_config: ParallelConfig,
30 - scheduler_config: SchedulerConfig,
31 - device_config: DeviceConfig,
32 - cache_config: CacheConfig,
33 - load_config: LoadConfig,
28 + vllm_config: VllmConfig,
29 local_rank: int,
30 rank: int,
31 distributed_init_method: str,
32 is_driver_worker: bool,
33 ) -> None:
39 - self.model_config = model_config
40 - self.parallel_config = parallel_config
34 + WorkerBase.__init__(self, vllm_config=vllm_config)
35 self.parallel_config.rank = rank
42 - self.scheduler_config = scheduler_config
43 - self.device_config = device_config
44 - self.cache_config = cache_config
45 - self.load_config = load_config
36 self.local_rank = local_rank
37 self.rank = rank
38 self.distributed_init_method = distributed_init_method
@@ -56,13 +46,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
46 self.cache_config.cache_dtype]
47
48 self.model_runner: TPUModelRunner = TPUModelRunner(
59 - model_config,
60 - parallel_config,
61 - scheduler_config,
62 - device_config,
63 - cache_config,
64 - load_config,
65 - is_driver_worker=is_driver_worker)
49 + vllm_config=vllm_config, is_driver_worker=is_driver_worker)
50
51 def init_device(self) -> None:
52 os.environ["PJRT_DEVICE"] = "TPU"
vllm/worker/worker.py
+10 -35
@@ -7,10 +7,7 @@ import torch
7 import torch.distributed
8
9 import vllm.envs as envs
10 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 - ModelConfig, ObservabilityConfig, ParallelConfig,
12 - PromptAdapterConfig, SchedulerConfig,
13 - SpeculativeConfig)
10 +from vllm.config import ParallelConfig, VllmConfig
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment,
13 set_custom_all_reduce)
@@ -27,7 +24,8 @@ from vllm.worker.cache_engine import CacheEngine
24 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
25 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
26 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
30 -from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
27 +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
28 + WorkerInput)
29
30 logger = init_logger(__name__)
31
@@ -42,46 +40,31 @@ class Worker(LocalOrDistributedWorkerBase):
40
41 def __init__(
42 self,
45 - model_config: ModelConfig,
46 - parallel_config: ParallelConfig,
47 - scheduler_config: SchedulerConfig,
48 - device_config: DeviceConfig,
49 - cache_config: CacheConfig,
50 - load_config: LoadConfig,
43 + vllm_config: VllmConfig,
44 local_rank: int,
45 rank: int,
46 distributed_init_method: str,
54 - lora_config: Optional[LoRAConfig] = None,
55 - speculative_config: Optional[SpeculativeConfig] = None,
56 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
47 is_driver_worker: bool = False,
48 model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
59 - observability_config: Optional[ObservabilityConfig] = None,
49 ) -> None:
61 - self.model_config = model_config
62 - self.parallel_config = parallel_config
50 + WorkerBase.__init__(self, vllm_config)
51 self.parallel_config.rank = rank
64 - self.scheduler_config = scheduler_config
65 - self.device_config = device_config
66 - self.cache_config = cache_config
52 self.local_rank = local_rank
53 self.rank = rank
54 self.distributed_init_method = distributed_init_method
70 - self.lora_config = lora_config
71 - self.load_config = load_config
72 - self.prompt_adapter_config = prompt_adapter_config
55 self.is_driver_worker = is_driver_worker
74 - if parallel_config and is_driver_worker:
75 - assert rank % parallel_config.tensor_parallel_size == 0, \
56 + if is_driver_worker:
57 + assert rank % self.parallel_config.tensor_parallel_size == 0, \
58 "Driver worker should be rank 0 of tensor parallel group."
59 if self.model_config.trust_remote_code:
60 # note: lazy import to avoid importing torch before initializing
61 from vllm.utils import init_cached_hf_modules
62 init_cached_hf_modules()
81 - self.observability_config = observability_config
63
64 # Return hidden states from target model if the draft model is an
65 # mlp_speculator
66 + speculative_config = self.speculative_config
67 + model_config = self.model_config
68 speculative_args = {} if speculative_config is None \
69 or (speculative_config.draft_model_config.model ==
70 model_config.model) \
@@ -97,17 +80,9 @@ class Worker(LocalOrDistributedWorkerBase):
80 elif self._is_encoder_decoder_model():
81 ModelRunnerClass = EncoderDecoderModelRunner
82 self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
100 - model_config,
101 - parallel_config,
102 - scheduler_config,
103 - device_config,
104 - cache_config,
105 - load_config=load_config,
106 - lora_config=self.lora_config,
83 + vllm_config=self.vllm_config,
84 kv_cache_dtype=self.cache_config.cache_dtype,
85 is_driver_worker=is_driver_worker,
109 - prompt_adapter_config=prompt_adapter_config,
110 - observability_config=observability_config,
86 **speculative_args,
87 )
88 # Uninitialized cache engine. Will be initialized by
vllm/worker/worker_base.py
+17 -1
@@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
7
8 import torch
9
10 -from vllm.config import ObservabilityConfig
10 +from vllm.config import ObservabilityConfig, VllmConfig
11 from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
@@ -29,6 +29,22 @@ class WorkerBase(ABC):
29 communicate request metadata to other workers.
30 """
31
32 + def __init__(
33 + self,
34 + vllm_config: VllmConfig,
35 + ) -> None:
36 + self.vllm_config = vllm_config
37 + self.model_config = vllm_config.model_config
38 + self.cache_config = vllm_config.cache_config
39 + self.lora_config = vllm_config.lora_config
40 + self.load_config = vllm_config.load_config
41 + self.parallel_config = vllm_config.parallel_config
42 + self.scheduler_config = vllm_config.scheduler_config
43 + self.device_config = vllm_config.device_config
44 + self.speculative_config = vllm_config.speculative_config
45 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
46 + self.observability_config = vllm_config.observability_config
47 +
48 @abstractmethod
49 def init_device(self) -> None:
50 """Initialize device state, such as loading the model or other on-device
vllm/worker/xpu_model_runner.py
+6 -23
@@ -10,9 +10,7 @@ import torch
10 import torch.nn as nn
11
12 from vllm.attention import get_attn_backend
13 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14 - ModelConfig, ObservabilityConfig, ParallelConfig,
15 - PromptAdapterConfig, SchedulerConfig)
13 +from vllm.config import VllmConfig
14 from vllm.distributed import get_pp_group
15 from vllm.inputs import INPUT_REGISTRY, InputRegistry
16 from vllm.logger import init_logger
@@ -363,33 +361,18 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
361
362 def __init__(
363 self,
366 - model_config: ModelConfig,
367 - parallel_config: ParallelConfig,
368 - scheduler_config: SchedulerConfig,
369 - device_config: DeviceConfig,
370 - cache_config: CacheConfig,
371 - load_config: LoadConfig,
372 - lora_config: Optional[LoRAConfig],
364 + vllm_config: VllmConfig,
365 kv_cache_dtype: Optional[str] = "auto",
366 is_driver_worker: bool = False,
375 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
367 return_hidden_states: bool = False,
377 - observability_config: Optional[ObservabilityConfig] = None,
368 input_registry: InputRegistry = INPUT_REGISTRY,
369 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
370 ):
381 - self.model_config = model_config
382 - self.parallel_config = parallel_config
383 - self.scheduler_config = scheduler_config
384 - self.device_config = device_config
385 - self.cache_config = cache_config
386 - self.lora_config = lora_config
387 - self.load_config = load_config
371 +
372 + ModelRunnerBase.__init__(self, vllm_config=vllm_config)
373 + model_config = self.model_config
374 + cache_config = self.cache_config
375 self.is_driver_worker = is_driver_worker
389 - self.prompt_adapter_config = prompt_adapter_config
390 - self.observability_config = observability_config
391 - if self.observability_config is not None:
392 - print(f"observability_config is {self.observability_config}")
376 self.return_hidden_states = return_hidden_states
377
378 self.device = self.device_config.device
vllm/worker/xpu_worker.py
+8 -32
@@ -8,10 +8,7 @@ import oneccl_bindings_for_pytorch # noqa: F401
8 import torch
9 import torch.distributed
10
11 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 - ModelConfig, ObservabilityConfig, ParallelConfig,
13 - PromptAdapterConfig, SchedulerConfig,
14 - SpeculativeConfig)
11 +from vllm.config import VllmConfig
12 from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment)
14 from vllm.logger import init_logger
@@ -19,7 +16,7 @@ from vllm.model_executor import set_random_seed
16 from vllm.platforms import current_platform
17 from vllm.worker.cache_engine import CacheEngine
18 from vllm.worker.worker import Worker
22 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
20 from vllm.worker.xpu_model_runner import XPUModelRunner
21
22 logger = init_logger(__name__)
@@ -36,53 +33,32 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
33
34 def __init__(
35 self,
39 - model_config: ModelConfig,
40 - parallel_config: ParallelConfig,
41 - scheduler_config: SchedulerConfig,
42 - device_config: DeviceConfig,
43 - cache_config: CacheConfig,
44 - load_config: LoadConfig,
36 + vllm_config: VllmConfig,
37 local_rank: int,
38 rank: int,
39 distributed_init_method: str,
48 - lora_config: Optional[LoRAConfig] = None,
49 - speculative_config: Optional[SpeculativeConfig] = None,
50 - prompt_adapter_config: Optional[PromptAdapterConfig] = None,
40 is_driver_worker: bool = False,
52 - observability_config: Optional[ObservabilityConfig] = None,
41 ) -> None:
42 + WorkerBase.__init__(self, vllm_config=vllm_config)
43 + device_config = self.device_config
44 + parallel_config = self.parallel_config
45 assert device_config.device_type == "xpu"
46 assert current_platform.is_xpu()
47
57 - self.model_config = model_config
58 - self.parallel_config = parallel_config
48 self.parallel_config.rank = rank
60 - self.scheduler_config = scheduler_config
61 - self.device_config = device_config
62 - self.cache_config = cache_config
63 - self.load_config = load_config
49 +
50 self.local_rank = local_rank
51 self.rank = rank
52 self.distributed_init_method = distributed_init_method
67 - self.lora_config = lora_config
68 - self.prompt_adapter_config = prompt_adapter_config
53 self.is_driver_worker = is_driver_worker
70 - self.observability_config = observability_config
54 if parallel_config and is_driver_worker:
55 assert rank % parallel_config.tensor_parallel_size == 0, \
56 "Driver worker should be rank 0 of tensor parallel group."
57
58 self.model_runner = XPUModelRunner( # type: ignore
76 - model_config,
77 - parallel_config,
78 - scheduler_config,
79 - device_config,
80 - cache_config,
81 - load_config=self.load_config,
82 - lora_config=self.lora_config,
59 + vllm_config=vllm_config,
60 kv_cache_dtype=self.cache_config.cache_dtype,
61 is_driver_worker=is_driver_worker,
85 - observability_config=self.observability_config,
62 )
63 # Uninitialized cache engine. Will be initialized by
64 # initialize_cache.