A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[2/N] executor pass the complete config to worker/modelrunner (#9938)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Nick Hill <nhill@redhat.com>
youkaichao committed
Nov 2, 2024 at 07:35 UTC
e8937954434037ac787efa800f01d9d294185439
44 files changed
+250
-580
tests/lora/test_long_context.py
+1
-7
@@ -138,13 +138,7 @@ def test_rotary_emb_replaced(dist_init):
138
enable_lora=True)
139
engine_config = engine_args.create_engine_config()
140
model_runner = ModelRunner(
141
- model_config=engine_config.model_config,
142
- parallel_config=engine_config.parallel_config,
143
- scheduler_config=engine_config.scheduler_config,
144
- device_config=engine_config.device_config,
145
- cache_config=engine_config.cache_config,
146
- load_config=engine_config.load_config,
147
- lora_config=engine_config.lora_config,
141
+ vllm_config=engine_config,
142
is_driver_worker=True,
143
)
144
model_runner.load_model()
tests/lora/test_worker.py
+8
-4
@@ -4,7 +4,8 @@ import tempfile
4
from unittest.mock import patch
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
- ModelConfig, ParallelConfig, SchedulerConfig)
7
+ ModelConfig, ParallelConfig, SchedulerConfig,
8
+ VllmConfig)
9
from vllm.lora.models import LoRAMapping
10
from vllm.lora.request import LoRARequest
11
from vllm.worker.worker import Worker
@@ -12,7 +13,7 @@ from vllm.worker.worker import Worker
13
14
@patch.dict(os.environ, {"RANK": "0"})
15
def test_worker_apply_lora(sql_lora_files):
15
- worker = Worker(
16
+ vllm_config = VllmConfig(
17
model_config=ModelConfig(
18
"meta-llama/Llama-2-7b-hf",
19
task="auto",
@@ -34,10 +35,13 @@ def test_worker_apply_lora(sql_lora_files):
35
gpu_memory_utilization=1.,
36
swap_space=0,
37
cache_dtype="auto"),
37
- local_rank=0,
38
- rank=0,
38
lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
39
max_loras=32),
40
+ )
41
+ worker = Worker(
42
+ vllm_config=vllm_config,
43
+ local_rank=0,
44
+ rank=0,
45
distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
46
)
47
worker.init_device()
tests/spec_decode/utils.py
+1
-6
@@ -81,12 +81,7 @@ def create_worker(cls: Callable[..., T],
81
get_ip(), get_open_port())
82
83
worker = cls(
84
- model_config=engine_config.model_config,
85
- parallel_config=engine_config.parallel_config,
86
- scheduler_config=engine_config.scheduler_config,
87
- device_config=engine_config.device_config,
88
- cache_config=engine_config.cache_config,
89
- load_config=engine_config.load_config,
84
+ vllm_config=engine_config,
85
local_rank=0,
86
rank=0,
87
distributed_init_method=distributed_init_method,
tests/worker/test_encoder_decoder_model_runner.py
+1
-8
@@ -19,14 +19,7 @@ def _create_model_runner(model: str, *args,
19
engine_args = EngineArgs(model, *args, **kwargs)
20
engine_config = engine_args.create_engine_config()
21
model_runner = EncoderDecoderModelRunner(
22
- model_config=engine_config.model_config,
23
- parallel_config=engine_config.parallel_config,
24
- scheduler_config=engine_config.scheduler_config,
25
- device_config=engine_config.device_config,
26
- cache_config=engine_config.cache_config,
27
- load_config=engine_config.load_config,
28
- lora_config=engine_config.lora_config,
29
- prompt_adapter_config=engine_config.prompt_adapter_config,
22
+ vllm_config=engine_config,
23
is_driver_worker=True,
24
)
25
return model_runner
tests/worker/test_model_runner.py
+1
-9
@@ -16,15 +16,7 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
16
engine_args = EngineArgs(model, *args, **kwargs)
17
engine_config = engine_args.create_engine_config()
18
model_runner = ModelRunner(
19
- model_config=engine_config.model_config,
20
- parallel_config=engine_config.parallel_config,
21
- scheduler_config=engine_config.scheduler_config,
22
- device_config=engine_config.device_config,
23
- cache_config=engine_config.cache_config,
24
- load_config=engine_config.load_config,
25
- lora_config=engine_config.lora_config,
26
- prompt_adapter_config=engine_config.prompt_adapter_config,
27
- observability_config=engine_config.observability_config,
19
+ vllm_config=engine_config,
20
is_driver_worker=True,
21
)
22
return model_runner
tests/worker/test_profile.py
+1
-6
@@ -24,12 +24,7 @@ def test_gpu_memory_profiling():
24
distributed_init_method = get_distributed_init_method(
25
get_ip(), get_open_port())
26
worker = Worker(
27
- model_config=engine_config.model_config,
28
- parallel_config=engine_config.parallel_config,
29
- scheduler_config=engine_config.scheduler_config,
30
- device_config=engine_config.device_config,
31
- cache_config=engine_config.cache_config,
32
- load_config=engine_config.load_config,
27
+ vllm_config=engine_config,
28
local_rank=0,
29
rank=0,
30
distributed_init_method=distributed_init_method,
tests/worker/test_swap.py
+1
-6
@@ -19,12 +19,7 @@ def test_swap() -> None:
19
distributed_init_method = get_distributed_init_method(
20
get_ip(), get_open_port())
21
worker = Worker(
22
- model_config=engine_config.model_config,
23
- parallel_config=engine_config.parallel_config,
24
- scheduler_config=engine_config.scheduler_config,
25
- device_config=engine_config.device_config,
26
- cache_config=engine_config.cache_config,
27
- load_config=engine_config.load_config,
22
+ vllm_config=engine_config,
23
local_rank=0,
24
rank=0,
25
distributed_init_method=distributed_init_method,
vllm/config.py
+9
-15
@@ -1,6 +1,6 @@
1
import enum
2
import json
3
-from dataclasses import dataclass, field, fields
3
+from dataclasses import dataclass, field
4
from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
5
Mapping, Optional, Set, Tuple, Type, Union)
6
@@ -1941,9 +1941,9 @@ class ObservabilityConfig:
1941
f"installed. Original error:\n{otel_import_error_traceback}")
1942
1943
1944
-@dataclass(frozen=True)
1945
-class EngineConfig:
1946
- """Dataclass which contains all engine-related configuration. This
1944
+@dataclass
1945
+class VllmConfig:
1946
+ """Dataclass which contains all vllm-related configuration. This
1947
simplifies passing around the distinct configurations in the codebase.
1948
"""
1949
@@ -1953,11 +1953,11 @@ class EngineConfig:
1953
scheduler_config: SchedulerConfig
1954
device_config: DeviceConfig
1955
load_config: LoadConfig
1956
- lora_config: Optional[LoRAConfig]
1957
- speculative_config: Optional[SpeculativeConfig]
1958
- decoding_config: Optional[DecodingConfig]
1959
- observability_config: Optional[ObservabilityConfig]
1960
- prompt_adapter_config: Optional[PromptAdapterConfig]
1956
+ lora_config: Optional[LoRAConfig] = None
1957
+ speculative_config: Optional[SpeculativeConfig] = None
1958
+ decoding_config: Optional[DecodingConfig] = None
1959
+ observability_config: Optional[ObservabilityConfig] = None
1960
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None
1961
1962
def __post_init__(self):
1963
"""Verify configs are valid & consistent with each other.
@@ -1975,9 +1975,3 @@ class EngineConfig:
1975
if self.prompt_adapter_config:
1976
self.prompt_adapter_config.verify_with_model_config(
1977
self.model_config)
1978
-
1979
- def to_dict(self):
1980
- """Return the configs as a dictionary, for use in **kwargs.
1981
- """
1982
- return dict(
1983
- (field.name, getattr(self, field.name)) for field in fields(self))
vllm/engine/arg_utils.py
+7
-6
@@ -9,10 +9,11 @@ import torch
9
10
import vllm.envs as envs
11
from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12
- DeviceConfig, EngineConfig, LoadConfig, LoadFormat,
13
- LoRAConfig, ModelConfig, ObservabilityConfig,
14
- ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15
- SpeculativeConfig, TaskOption, TokenizerPoolConfig)
12
+ DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13
+ ModelConfig, ObservabilityConfig, ParallelConfig,
14
+ PromptAdapterConfig, SchedulerConfig,
15
+ SpeculativeConfig, TaskOption, TokenizerPoolConfig,
16
+ VllmConfig)
17
from vllm.executor.executor_base import ExecutorBase
18
from vllm.logger import init_logger
19
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -955,7 +956,7 @@ class EngineArgs:
956
ignore_patterns=self.ignore_patterns,
957
)
958
958
- def create_engine_config(self) -> EngineConfig:
959
+ def create_engine_config(self) -> VllmConfig:
960
# gguf file needs a specific model loader and doesn't use hf_repo
961
if check_gguf_file(self.model):
962
self.quantization = self.load_format = "gguf"
@@ -1167,7 +1168,7 @@ class EngineArgs:
1168
or "all" in detailed_trace_modules,
1169
)
1170
1170
- return EngineConfig(
1171
+ return VllmConfig(
1172
model_config=model_config,
1173
cache_config=cache_config,
1174
parallel_config=parallel_config,
vllm/engine/async_llm_engine.py
+4
-4
@@ -7,8 +7,8 @@ from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
7
from weakref import ReferenceType
8
9
import vllm.envs as envs
10
-from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
11
- ParallelConfig, SchedulerConfig)
10
+from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11
+ ParallelConfig, SchedulerConfig, VllmConfig)
12
from vllm.core.scheduler import SchedulerOutputs
13
from vllm.engine.arg_utils import AsyncEngineArgs
14
from vllm.engine.async_timeout import asyncio_timeout
@@ -604,7 +604,7 @@ class AsyncLLMEngine(EngineClient):
604
605
@classmethod
606
def _get_executor_cls(
607
- cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]:
607
+ cls, engine_config: VllmConfig) -> Type[ExecutorAsyncBase]:
608
distributed_executor_backend = (
609
engine_config.parallel_config.distributed_executor_backend)
610
if isinstance(distributed_executor_backend, type):
@@ -663,7 +663,7 @@ class AsyncLLMEngine(EngineClient):
663
def from_engine_args(
664
cls,
665
engine_args: AsyncEngineArgs,
666
- engine_config: Optional[EngineConfig] = None,
666
+ engine_config: Optional[VllmConfig] = None,
667
start_engine_loop: bool = True,
668
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
669
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
vllm/engine/llm_engine.py
+5
-4
@@ -13,8 +13,9 @@ import torch
13
from typing_extensions import TypeIs, TypeVar
14
15
import vllm.envs as envs
16
-from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
17
- ObservabilityConfig, ParallelConfig, SchedulerConfig)
16
+from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
17
+ ObservabilityConfig, ParallelConfig, SchedulerConfig,
18
+ VllmConfig)
19
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
20
SchedulerOutputs)
21
from vllm.engine.arg_utils import EngineArgs
@@ -219,7 +220,7 @@ class LLMEngine:
220
221
def __init__(
222
self,
222
- vllm_config: EngineConfig,
223
+ vllm_config: VllmConfig,
224
executor_class: Type[ExecutorBase],
225
log_stats: bool,
226
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -500,7 +501,7 @@ class LLMEngine:
501
502
@classmethod
503
def _get_executor_cls(cls,
503
- engine_config: EngineConfig) -> Type[ExecutorBase]:
504
+ engine_config: VllmConfig) -> Type[ExecutorBase]:
505
distributed_executor_backend = (
506
engine_config.parallel_config.distributed_executor_backend)
507
# Initialize the cluster and specify the executor class.
vllm/engine/multiprocessing/client.py
+2
-2
@@ -13,7 +13,7 @@ from zmq import Frame # type: ignore[attr-defined]
13
from zmq.asyncio import Socket
14
15
from vllm import PoolingParams
16
-from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16
+from vllm.config import DecodingConfig, ModelConfig, VllmConfig
17
from vllm.core.scheduler import SchedulerOutputs
18
from vllm.engine.arg_utils import AsyncEngineArgs
19
# yapf conflicts with isort for this block
@@ -78,7 +78,7 @@ class MQLLMEngineClient(EngineClient):
78
every N seconds, confirming the engine is healthy
79
"""
80
81
- def __init__(self, ipc_path: str, engine_config: EngineConfig,
81
+ def __init__(self, ipc_path: str, engine_config: VllmConfig,
82
engine_pid: int):
83
self.context = zmq.asyncio.Context()
84
self._errored_with: Optional[BaseException] = None
vllm/executor/cpu_executor.py
+1
-8
@@ -138,18 +138,11 @@ class CPUExecutor(ExecutorBase):
138
assert self.distributed_init_method is not None
139
140
kwargs = dict(
141
- model_config=self.model_config,
142
- parallel_config=self.parallel_config,
143
- scheduler_config=self.scheduler_config,
144
- device_config=self.device_config,
145
- cache_config=self.cache_config,
146
- load_config=self.load_config,
141
+ vllm_config=self.vllm_config,
142
local_rank=local_rank,
143
rank=rank,
144
distributed_init_method=self.distributed_init_method,
150
- lora_config=self.lora_config,
145
kv_cache_dtype=self.cache_config.cache_dtype,
152
- prompt_adapter_config=self.prompt_adapter_config,
146
is_driver_worker=rank == 0,
147
)
148
wrapper.init_worker(**kwargs)
vllm/executor/executor_base.py
+2
-2
@@ -1,7 +1,7 @@
1
from abc import ABC, abstractmethod
2
from typing import List, Optional, Set, Tuple
3
4
-from vllm.config import EngineConfig
4
+from vllm.config import VllmConfig
5
from vllm.lora.request import LoRARequest
6
from vllm.model_executor.layers.sampler import SamplerOutput
7
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -20,7 +20,7 @@ class ExecutorBase(ABC):
20
21
def __init__(
22
self,
23
- vllm_config: EngineConfig,
23
+ vllm_config: VllmConfig,
24
) -> None:
25
self.vllm_config = vllm_config
26
self.model_config = vllm_config.model_config
vllm/executor/gpu_executor.py
+1
-10
@@ -49,21 +49,12 @@ class GPUExecutor(ExecutorBase):
49
distributed_init_method = get_distributed_init_method(
50
get_ip(), get_open_port())
51
return dict(
52
- model_config=self.model_config,
53
- parallel_config=self.parallel_config,
54
- scheduler_config=self.scheduler_config,
55
- device_config=self.device_config,
56
- cache_config=self.cache_config,
57
- load_config=self.load_config,
52
+ vllm_config=self.vllm_config,
53
local_rank=local_rank,
54
rank=rank,
55
distributed_init_method=distributed_init_method,
61
- lora_config=self.lora_config,
62
- speculative_config=self.speculative_config,
63
- prompt_adapter_config=self.prompt_adapter_config,
56
is_driver_worker=(not self.parallel_config)
57
or (rank % self.parallel_config.tensor_parallel_size == 0),
66
- observability_config=self.observability_config,
58
)
59
60
def _get_worker_module_and_class(
vllm/executor/neuron_executor.py
+1
-5
@@ -29,11 +29,7 @@ class NeuronExecutor(ExecutorBase):
29
distributed_init_method = get_distributed_init_method(
30
get_ip(), get_open_port())
31
self.driver_worker = NeuronWorker(
32
- model_config=self.model_config,
33
- parallel_config=self.parallel_config,
34
- scheduler_config=self.scheduler_config,
35
- device_config=self.device_config,
36
- cache_config=self.cache_config,
32
+ vllm_config=self.vllm_config,
33
local_rank=0,
34
rank=0,
35
distributed_init_method=distributed_init_method)
vllm/executor/openvino_executor.py
+1
-7
@@ -48,16 +48,10 @@ class OpenVINOExecutor(ExecutorBase):
48
get_ip(), get_open_port())
49
self.driver_worker = OpenVINOWorker(
50
ov_core=self.ov_core,
51
- model_config=self.model_config,
52
- parallel_config=self.parallel_config,
53
- scheduler_config=self.scheduler_config,
54
- device_config=self.device_config,
55
- cache_config=self.cache_config,
56
- load_config=self.load_config,
51
+ vllm_config=self.vllm_config,
52
local_rank=0,
53
rank=0,
54
distributed_init_method=distributed_init_method,
60
- lora_config=self.lora_config,
55
kv_cache_dtype=self.cache_config.cache_dtype,
56
is_driver_worker=True,
57
)
vllm/executor/tpu_executor.py
+1
-6
@@ -44,12 +44,7 @@ class TPUExecutor(ExecutorBase):
44
distributed_init_method = get_distributed_init_method(
45
get_ip(), get_open_port())
46
return dict(
47
- model_config=self.model_config,
48
- parallel_config=self.parallel_config,
49
- scheduler_config=self.scheduler_config,
50
- device_config=self.device_config,
51
- cache_config=self.cache_config,
52
- load_config=self.load_config,
47
+ vllm_config=self.vllm_config,
48
local_rank=local_rank,
49
rank=rank,
50
distributed_init_method=distributed_init_method,
vllm/spec_decode/draft_model_runner.py
+3
-33
@@ -17,9 +17,6 @@ except (ModuleNotFoundError, ImportError) as err:
17
"Draft model speculative decoding currently only supports"
18
"CUDA and ROCm flash attention backend.") from err
19
20
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
21
- ModelConfig, ObservabilityConfig, ParallelConfig,
22
- PromptAdapterConfig, SchedulerConfig)
20
from vllm.logger import init_logger
21
from vllm.multimodal import MultiModalInputs
22
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
@@ -49,40 +46,13 @@ class TP1DraftModelRunner(ModelRunner):
46
any broadcasting inside execute_model).
47
"""
48
52
- def __init__(
53
- self,
54
- model_config: ModelConfig,
55
- parallel_config: ParallelConfig,
56
- scheduler_config: SchedulerConfig,
57
- device_config: DeviceConfig,
58
- cache_config: CacheConfig,
59
- load_config: LoadConfig,
60
- lora_config: Optional[LoRAConfig],
61
- kv_cache_dtype: Optional[str] = "auto",
62
- is_driver_worker: bool = False,
63
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
64
- return_hidden_states: bool = False,
65
- observability_config: Optional[ObservabilityConfig] = None,
66
- ):
67
- if return_hidden_states:
49
+ def __init__(self, *args, **kwargs):
50
+ if kwargs.get("return_hidden_states"):
51
raise ValueError(
52
"return_hidden_states is not supported for TP1DraftModelRunner."
53
)
54
72
- super().__init__(
73
- model_config=model_config,
74
- parallel_config=parallel_config,
75
- scheduler_config=scheduler_config,
76
- device_config=device_config,
77
- cache_config=cache_config,
78
- load_config=load_config,
79
- lora_config=lora_config,
80
- kv_cache_dtype=kv_cache_dtype,
81
- is_driver_worker=is_driver_worker,
82
- prompt_adapter_config=prompt_adapter_config,
83
- return_hidden_states=return_hidden_states,
84
- observability_config=observability_config,
85
- )
55
+ super().__init__(*args, **kwargs)
56
57
def _update_sampling_metadata(self, sampling_metadata, num_seqs,
58
num_queries):
vllm/spec_decode/ngram_worker.py
+1
-1
@@ -21,7 +21,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
21
def __init__(self, *args, **kwargs):
22
# Get local_rank/vocab_size from kwargs attribute
23
self.local_rank = kwargs["local_rank"]
24
- self.vocab_size = kwargs["model_config"].get_vocab_size()
24
+ self.vocab_size = kwargs["vllm_config"].model_config.get_vocab_size()
25
26
# Lazy initialization list.
27
self._proposer: Top1Proposer
vllm/spec_decode/spec_decode_worker.py
+18
-17
@@ -1,10 +1,11 @@
1
+import copy
2
from collections import defaultdict
3
from functools import cached_property
4
from typing import Any, Dict, List, Optional, Set, Tuple, Type
5
6
import torch
7
7
-from vllm.config import ParallelConfig, SpeculativeConfig
8
+from vllm.config import ParallelConfig, SpeculativeConfig, VllmConfig
9
from vllm.distributed.communication_op import broadcast_tensor_dict
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
@@ -45,8 +46,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
46
"""Helper method that is the entrypoint for Executors which use
47
WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config.
48
"""
48
- assert "speculative_config" in kwargs
49
- speculative_config: SpeculativeConfig = kwargs.get("speculative_config")
49
+ vllm_config: VllmConfig = kwargs.get("vllm_config")
50
+ speculative_config: SpeculativeConfig = vllm_config.speculative_config
51
assert speculative_config is not None
52
53
draft_worker_kwargs = kwargs.copy()
@@ -58,14 +59,16 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
59
target_worker.model_runner.disable_logprobs =\
60
speculative_config.disable_logprobs
61
62
+ draft_worker_config = copy.deepcopy(vllm_config)
63
+ draft_worker_config.model_config = speculative_config.draft_model_config
64
+ draft_worker_config.parallel_config = speculative_config.draft_parallel_config # noqa
65
+ # TODO allow draft-model specific load config.
66
+
67
# Override draft-model specific worker args.
68
draft_worker_kwargs.update(
63
- model_config=speculative_config.draft_model_config,
64
- parallel_config=speculative_config.draft_parallel_config,
69
+ vllm_config=draft_worker_config,
70
ngram_prompt_lookup_max=speculative_config.ngram_prompt_lookup_max,
71
ngram_prompt_lookup_min=speculative_config.ngram_prompt_lookup_min,
67
- # TODO allow draft-model specific load config.
68
- #load_config=load_config,
72
)
73
74
spec_decode_worker = SpecDecodeWorker.create_worker(
@@ -134,29 +137,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
137
draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
138
ngram_prompt_lookup_min = (
139
draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
140
+ draft_model_config = draft_worker_kwargs["vllm_config"].model_config
141
+ draft_parallel_config: ParallelConfig = draft_worker_kwargs[
142
+ 'vllm_config'].parallel_config
143
if ngram_prompt_lookup_max > 0:
144
proposer_worker = NGramWorker(**draft_worker_kwargs)
145
proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
146
ngram_prompt_lookup_max)
147
else:
142
- draft_parallel_config: ParallelConfig = draft_worker_kwargs[
143
- 'parallel_config']
148
draft_tp = draft_parallel_config.tensor_parallel_size
149
target_tp = scorer_worker.parallel_config.tensor_parallel_size
150
147
- if draft_worker_kwargs[
148
- "model_config"].hf_config.model_type == "mlp_speculator":
151
+ if draft_model_config.hf_config.model_type == "mlp_speculator":
152
proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
150
- elif draft_worker_kwargs[
151
- "model_config"].hf_config.model_type == "medusa":
153
+ elif draft_model_config.hf_config.model_type == "medusa":
154
proposer_worker = MedusaWorker(**draft_worker_kwargs)
155
else:
156
if draft_tp == 1:
157
draft_worker_kwargs[
158
"model_runner_cls"] = TP1DraftModelRunner
159
else:
158
- if draft_worker_kwargs[
159
- "model_config"].hf_config.model_type == "eagle":
160
+ if draft_model_config.hf_config.model_type == "eagle":
161
raise NotImplementedError(
162
"EAGLE does not support TP > 1 yet")
163
@@ -190,8 +191,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
191
"[Speculative Decoding] Disabling MQA scorer as the "
192
"MQA is only available with flash attn backend.")
193
193
- if "model_config" in draft_worker_kwargs and \
194
- draft_worker_kwargs["model_config"].max_model_len < \
194
+ if draft_model_config and \
195
+ draft_model_config.max_model_len < \
196
scorer_worker.model_config.max_model_len:
197
disable_mqa_scorer = True
198
logger.info(
vllm/spec_decode/target_model_runner.py
+9
-25
@@ -1,8 +1,6 @@
1
from typing import List, Optional
2
3
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4
- ModelConfig, ObservabilityConfig, ParallelConfig,
5
- PromptAdapterConfig, SchedulerConfig)
3
+from vllm.config import VllmConfig
4
from vllm.sequence import SequenceGroupMetadata
5
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
6
ModelRunner)
@@ -20,35 +18,21 @@ class TargetModelRunner(ModelRunner):
18
requested or not.
19
"""
20
23
- def __init__(self,
24
- model_config: ModelConfig,
25
- parallel_config: ParallelConfig,
26
- scheduler_config: SchedulerConfig,
27
- device_config: DeviceConfig,
28
- cache_config: CacheConfig,
29
- load_config: LoadConfig,
30
- lora_config: Optional[LoRAConfig],
31
- kv_cache_dtype: Optional[str] = "auto",
32
- is_driver_worker: bool = False,
33
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34
- return_hidden_states: bool = False,
35
- observability_config: Optional[ObservabilityConfig] = None):
21
+ def __init__(
22
+ self,
23
+ vllm_config: VllmConfig,
24
+ kv_cache_dtype: Optional[str] = "auto",
25
+ is_driver_worker: bool = False,
26
+ return_hidden_states: bool = False,
27
+ ):
28
# An internal boolean member variable to indicate if token log
29
# probabilities are needed or not.
30
self.disable_logprobs = True
31
super().__init__(
40
- model_config=model_config,
41
- parallel_config=parallel_config,
42
- scheduler_config=scheduler_config,
43
- device_config=device_config,
44
- cache_config=cache_config,
45
- load_config=load_config,
46
- lora_config=lora_config,
32
+ vllm_config=vllm_config,
33
kv_cache_dtype=kv_cache_dtype,
34
is_driver_worker=is_driver_worker,
49
- prompt_adapter_config=prompt_adapter_config,
35
return_hidden_states=return_hidden_states,
51
- observability_config=observability_config,
36
)
37
38
def prepare_model_input(
vllm/v1/engine/llm_engine.py
+5
-4
@@ -2,8 +2,9 @@ import time
2
from typing import (Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type,
3
Union)
4
5
-from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
6
- ObservabilityConfig, ParallelConfig, SchedulerConfig)
5
+from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
6
+ ObservabilityConfig, ParallelConfig, SchedulerConfig,
7
+ VllmConfig)
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.engine.metrics_types import StatLoggerBase
10
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
@@ -32,7 +33,7 @@ class LLMEngine:
33
34
def __init__(
35
self,
35
- vllm_config: EngineConfig,
36
+ vllm_config: VllmConfig,
37
executor_class: Type[GPUExecutor],
38
log_stats: bool,
39
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -477,7 +478,7 @@ class LLMEngine:
478
return self.lora_config
479
480
@classmethod
480
- def _get_executor_cls(cls, engine_config: EngineConfig):
481
+ def _get_executor_cls(cls, engine_config: VllmConfig):
482
return GPUExecutor
483
484
def is_tracing_enabled(self) -> bool:
vllm/v1/executor/gpu_executor.py
+1
-10
@@ -56,19 +56,10 @@ class GPUExecutor:
56
distributed_init_method = get_distributed_init_method(
57
get_ip(), get_open_port())
58
return Worker(
59
- model_config=self.model_config,
60
- parallel_config=self.parallel_config,
61
- scheduler_config=self.scheduler_config,
62
- device_config=self.device_config,
63
- cache_config=self.cache_config,
64
- load_config=self.load_config,
59
+ vllm_config=self.vllm_config,
60
local_rank=local_rank,
61
rank=rank,
62
distributed_init_method=distributed_init_method,
68
- lora_config=self.lora_config,
69
- speculative_config=self.speculative_config,
70
- prompt_adapter_config=self.prompt_adapter_config,
71
- observability_config=self.observability_config,
63
)
64
65
def determine_num_available_blocks(self) -> Tuple[int, int]:
vllm/v1/worker/gpu_model_runner.py
+19
-22
@@ -7,9 +7,7 @@ import torch
7
import torch.distributed
8
import torch.nn as nn
9
10
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
- ModelConfig, ObservabilityConfig, ParallelConfig,
12
- PromptAdapterConfig, SchedulerConfig)
10
+from vllm.config import VllmConfig
11
from vllm.forward_context import set_forward_context
12
from vllm.logger import init_logger
13
from vllm.model_executor.model_loader import get_model
@@ -33,26 +31,25 @@ class GPUModelRunner:
31
32
def __init__(
33
self,
36
- model_config: ModelConfig,
37
- parallel_config: ParallelConfig,
38
- scheduler_config: SchedulerConfig,
39
- device_config: DeviceConfig,
40
- cache_config: CacheConfig,
41
- load_config: LoadConfig,
42
- lora_config: Optional[LoRAConfig] = None,
43
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
44
- observability_config: Optional[ObservabilityConfig] = None,
34
+ vllm_config: VllmConfig,
35
):
46
- self.model_config = model_config
47
- self.parallel_config = parallel_config
48
- self.scheduler_config = scheduler_config
49
- self.device_config = device_config
50
- self.cache_config = cache_config
51
- self.lora_config = lora_config
52
- self.load_config = load_config
53
- self.prompt_adapter_config = prompt_adapter_config
54
- self.observability_config = observability_config
55
-
36
+ # TODO: use ModelRunnerBase.__init__(self, vllm_config=vllm_config)
37
+ self.vllm_config = vllm_config
38
+ self.model_config = vllm_config.model_config
39
+ self.cache_config = vllm_config.cache_config
40
+ self.lora_config = vllm_config.lora_config
41
+ self.load_config = vllm_config.load_config
42
+ self.parallel_config = vllm_config.parallel_config
43
+ self.scheduler_config = vllm_config.scheduler_config
44
+ self.device_config = vllm_config.device_config
45
+ self.speculative_config = vllm_config.speculative_config
46
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
47
+ self.observability_config = vllm_config.observability_config
48
+
49
+ model_config = self.model_config
50
+ cache_config = self.cache_config
51
+ scheduler_config = self.scheduler_config
52
+ parallel_config = self.parallel_config
53
self.device = self.device_config.device
54
self.pin_memory = is_pin_memory_available()
55
self.dtype = self.model_config.dtype
vllm/v1/worker/gpu_worker.py
+17
-33
@@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
6
import torch
7
import torch.distributed
8
9
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ObservabilityConfig, ParallelConfig,
11
- PromptAdapterConfig, SchedulerConfig,
12
- SpeculativeConfig)
9
+from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
10
from vllm.distributed import (ensure_model_parallel_initialized,
11
init_distributed_environment,
12
set_custom_all_reduce)
@@ -30,48 +27,35 @@ class Worker:
27
28
def __init__(
29
self,
33
- model_config: ModelConfig,
34
- parallel_config: ParallelConfig,
35
- scheduler_config: SchedulerConfig,
36
- device_config: DeviceConfig,
37
- cache_config: CacheConfig,
38
- load_config: LoadConfig,
30
+ vllm_config: VllmConfig,
31
local_rank: int,
32
rank: int,
33
distributed_init_method: str,
42
- speculative_config: Optional[SpeculativeConfig] = None,
43
- lora_config: Optional[LoRAConfig] = None,
44
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
45
- observability_config: Optional[ObservabilityConfig] = None,
34
):
47
- self.model_config = model_config
48
- self.parallel_config = parallel_config
49
- self.scheduler_config = scheduler_config
50
- self.device_config = device_config
51
- self.cache_config = cache_config
52
- self.load_config = load_config
35
+
36
+ # TODO: use WorkerBase.__init__(self, vllm_config=vllm_config)
37
+ self.vllm_config = vllm_config
38
+ self.model_config = vllm_config.model_config
39
+ self.cache_config = vllm_config.cache_config
40
+ self.lora_config = vllm_config.lora_config
41
+ self.load_config = vllm_config.load_config
42
+ self.parallel_config = vllm_config.parallel_config
43
+ self.scheduler_config = vllm_config.scheduler_config
44
+ self.device_config = vllm_config.device_config
45
+ self.speculative_config = vllm_config.speculative_config
46
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
47
+ self.observability_config = vllm_config.observability_config
48
+
49
self.local_rank = local_rank
50
self.rank = rank
51
self.distributed_init_method = distributed_init_method
56
- self.lora_config = lora_config
57
- self.speculative_config = speculative_config
58
- self.prompt_adapter_config = prompt_adapter_config
59
- self.observability_config = observability_config
52
53
if self.model_config.trust_remote_code:
54
# note: lazy import to avoid importing torch before initializing
55
from vllm.utils import init_cached_hf_modules
56
init_cached_hf_modules()
57
66
- self.model_runner = GPUModelRunner(
67
- model_config,
68
- parallel_config,
69
- scheduler_config,
70
- device_config,
71
- cache_config,
72
- load_config,
73
- lora_config=lora_config,
74
- )
58
+ self.model_runner = GPUModelRunner(vllm_config)
59
60
def initialize(self):
61
if self.device_config.device.type == "cuda":
vllm/worker/cpu_model_runner.py
+6
-19
@@ -8,9 +8,7 @@ import torch
8
from torch import nn
9
10
from vllm.attention import AttentionMetadata, get_attn_backend
11
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
- ModelConfig, ParallelConfig, PromptAdapterConfig,
13
- SchedulerConfig)
11
+from vllm.config import VllmConfig
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
@@ -412,29 +410,18 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
410
411
def __init__(
412
self,
415
- model_config: ModelConfig,
416
- parallel_config: ParallelConfig,
417
- scheduler_config: SchedulerConfig,
418
- device_config: DeviceConfig,
419
- cache_config: CacheConfig,
420
- load_config: LoadConfig,
421
- lora_config: Optional[LoRAConfig],
413
+ vllm_config: VllmConfig,
414
kv_cache_dtype: Optional[str] = "auto",
423
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
415
is_driver_worker: bool = False,
416
*args,
417
**kwargs,
418
):
428
- self.model_config = model_config
429
- self.parallel_config = parallel_config
430
- self.scheduler_config = scheduler_config
419
+ ModelRunnerBase.__init__(self, vllm_config)
420
# Currently, CPU worker doesn't support chunked prefill.
421
assert self.scheduler_config.chunked_prefill_enabled is False
433
- self.device_config = device_config
434
- self.cache_config = cache_config
435
- self.lora_config = lora_config
436
- self.prompt_adapter_config = prompt_adapter_config
437
- self.load_config = load_config
422
+ model_config = self.model_config
423
+ cache_config = self.cache_config
424
+
425
self.is_driver_worker = is_driver_worker
426
427
self.device = self.device_config.device
vllm/worker/cpu_worker.py
+9
-28
@@ -6,9 +6,8 @@ import torch.distributed
6
7
import vllm.envs as envs
8
from vllm.attention import get_attn_backend
9
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, PromptAdapterConfig,
11
- SchedulerConfig)
9
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10
+ ParallelConfig, VllmConfig)
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment)
13
from vllm.logger import init_logger
@@ -18,7 +17,8 @@ from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17
from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18
from vllm.worker.cpu_model_runner import CPUModelRunner
19
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21
- LoraNotSupportedWorkerBase, WorkerInput)
20
+ LoraNotSupportedWorkerBase, WorkerBase,
21
+ WorkerInput)
22
23
logger = init_logger(__name__)
24
@@ -121,31 +121,19 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
121
122
def __init__(
123
self,
124
- model_config: ModelConfig,
125
- parallel_config: ParallelConfig,
126
- scheduler_config: SchedulerConfig,
127
- device_config: DeviceConfig,
128
- cache_config: CacheConfig,
129
- load_config: LoadConfig,
124
+ vllm_config: VllmConfig,
125
local_rank: int,
126
rank: int,
127
distributed_init_method: str,
133
- lora_config: Optional[LoRAConfig] = None,
128
kv_cache_dtype: Optional[str] = "auto",
135
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
129
is_driver_worker: bool = False,
130
) -> None:
138
- self.model_config = model_config
139
- self.parallel_config = parallel_config
140
- self.scheduler_config = scheduler_config
141
- self.device_config = device_config
142
- self.cache_config = cache_config
143
- self.load_config = load_config
131
+ WorkerBase.__init__(self, vllm_config=vllm_config)
132
+
133
self.local_rank = local_rank
134
self.rank = rank
135
self.distributed_init_method = distributed_init_method
147
- self.lora_config = lora_config
148
- self.prompt_adapter_config = prompt_adapter_config
136
+
137
self.is_driver_worker = is_driver_worker
138
if self.is_driver_worker:
139
assert self.rank == 0, "The driver worker must have rank 0."
@@ -166,15 +154,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
154
if self._is_encoder_decoder_model():
155
ModelRunnerClass = CPUEncoderDecoderModelRunner
156
self.model_runner: CPUModelRunner = ModelRunnerClass(
169
- model_config,
170
- parallel_config,
171
- scheduler_config,
172
- device_config,
173
- cache_config,
174
- load_config=self.load_config,
175
- lora_config=self.lora_config,
157
+ vllm_config=vllm_config,
158
kv_cache_dtype=kv_cache_dtype,
177
- prompt_adapter_config=self.prompt_adapter_config,
159
is_driver_worker=is_driver_worker)
160
# Uninitialized cache engine. Will be initialized by
161
# initialize_cache.
vllm/worker/embedding_model_runner.py
+4
-22
@@ -3,9 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
6
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
- ModelConfig, ObservabilityConfig, ParallelConfig,
8
- PromptAdapterConfig, SchedulerConfig)
6
+from vllm.config import VllmConfig
7
from vllm.distributed import get_pp_group
8
from vllm.forward_context import set_forward_context
9
from vllm.logger import init_logger
@@ -36,29 +34,13 @@ class EmbeddingModelRunner(
34
35
def __init__(
36
self,
39
- model_config: ModelConfig,
40
- parallel_config: ParallelConfig,
41
- scheduler_config: SchedulerConfig,
42
- device_config: DeviceConfig,
43
- cache_config: CacheConfig,
44
- load_config: LoadConfig,
45
- lora_config: Optional[LoRAConfig],
37
+ vllm_config: VllmConfig,
38
kv_cache_dtype: Optional[str] = "auto",
39
is_driver_worker: bool = False,
48
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
49
- observability_config: Optional[ObservabilityConfig] = None,
40
):
51
- super().__init__(model_config,
52
- parallel_config,
53
- scheduler_config,
54
- device_config,
55
- cache_config,
56
- load_config,
57
- lora_config=lora_config,
41
+ super().__init__(vllm_config=vllm_config,
42
kv_cache_dtype=kv_cache_dtype,
59
- is_driver_worker=is_driver_worker,
60
- prompt_adapter_config=prompt_adapter_config,
61
- observability_config=observability_config)
43
+ is_driver_worker=is_driver_worker)
44
45
@torch.inference_mode()
46
def execute_model(
vllm/worker/enc_dec_model_runner.py
+5
-20
@@ -11,9 +11,7 @@ from vllm.attention.backends.utils import PAD_SLOT_ID
11
from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
12
get_global_forced_attn_backend,
13
global_force_attn_backend)
14
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
- ModelConfig, ObservabilityConfig, ParallelConfig,
16
- PromptAdapterConfig, SchedulerConfig)
14
+from vllm.config import ModelConfig, VllmConfig
15
from vllm.forward_context import set_forward_context
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry
17
from vllm.logger import init_logger
@@ -85,17 +83,9 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
83
84
def __init__(
85
self,
88
- model_config: ModelConfig,
89
- parallel_config: ParallelConfig,
90
- scheduler_config: SchedulerConfig,
91
- device_config: DeviceConfig,
92
- cache_config: CacheConfig,
93
- load_config: LoadConfig,
94
- lora_config: Optional[LoRAConfig],
86
+ vllm_config: VllmConfig,
87
kv_cache_dtype: Optional[str] = "auto",
88
is_driver_worker: bool = False,
97
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
98
- observability_config: Optional[ObservabilityConfig] = None,
89
input_registry: InputRegistry = INPUT_REGISTRY,
90
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
91
):
@@ -107,15 +97,10 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
97
models) but these arguments are present here for compatibility with
98
the base-class constructor.
99
'''
110
- self._maybe_force_supported_attention_backend(model_config)
100
+ self._maybe_force_supported_attention_backend(vllm_config.model_config)
101
+
102
super().__init__(
112
- model_config,
113
- parallel_config,
114
- scheduler_config,
115
- device_config,
116
- cache_config,
117
- load_config,
118
- lora_config=None,
103
+ vllm_config=vllm_config,
104
kv_cache_dtype=kv_cache_dtype,
105
is_driver_worker=is_driver_worker,
106
)
vllm/worker/model_runner.py
+7
-21
@@ -20,9 +20,7 @@ from vllm.attention.backends.abstract import AttentionState
20
from vllm.attention.backends.utils import CommonAttentionState
21
from vllm.compilation.compile_context import set_compile_context
22
from vllm.compilation.levels import CompilationLevel
23
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
24
- ModelConfig, ObservabilityConfig, ParallelConfig,
25
- PromptAdapterConfig, SchedulerConfig)
23
+from vllm.config import VllmConfig
24
from vllm.core.scheduler import SchedulerOutputs
25
from vllm.distributed import get_pp_group
26
from vllm.distributed.parallel_state import graph_capture
@@ -955,32 +953,20 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
953
954
def __init__(
955
self,
958
- model_config: ModelConfig,
959
- parallel_config: ParallelConfig,
960
- scheduler_config: SchedulerConfig,
961
- device_config: DeviceConfig,
962
- cache_config: CacheConfig,
963
- load_config: LoadConfig,
964
- lora_config: Optional[LoRAConfig],
956
+ vllm_config: VllmConfig,
957
kv_cache_dtype: Optional[str] = "auto",
958
is_driver_worker: bool = False,
967
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
959
return_hidden_states: bool = False,
969
- observability_config: Optional[ObservabilityConfig] = None,
960
input_registry: InputRegistry = INPUT_REGISTRY,
961
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
962
):
973
- self.model_config = model_config
974
- self.parallel_config = parallel_config
975
- self.scheduler_config = scheduler_config
976
- self.device_config = device_config
977
- self.cache_config = cache_config
978
- self.lora_config = lora_config
979
- self.load_config = load_config
963
+
964
+ ModelRunnerBase.__init__(self, vllm_config)
965
+ model_config = self.model_config
966
+ cache_config = self.cache_config
967
+
968
self.is_driver_worker = is_driver_worker
981
- self.prompt_adapter_config = prompt_adapter_config
969
self.return_hidden_states = return_hidden_states
983
- self.observability_config = observability_config
970
971
self.device = self.device_config.device
972
self.pin_memory = is_pin_memory_available()
vllm/worker/model_runner_base.py
+17
@@ -9,6 +9,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
9
import torch
10
from torch import is_tensor
11
12
+from vllm.config import VllmConfig
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.platforms import current_platform
@@ -220,6 +221,22 @@ class ModelRunnerBase(ABC, Generic[T]):
221
ModelRunnerInputBase subclass.
222
"""
223
224
+ def __init__(
225
+ self,
226
+ vllm_config: VllmConfig,
227
+ ) -> None:
228
+ self.vllm_config = vllm_config
229
+ self.model_config = vllm_config.model_config
230
+ self.cache_config = vllm_config.cache_config
231
+ self.lora_config = vllm_config.lora_config
232
+ self.load_config = vllm_config.load_config
233
+ self.parallel_config = vllm_config.parallel_config
234
+ self.scheduler_config = vllm_config.scheduler_config
235
+ self.device_config = vllm_config.device_config
236
+ self.speculative_config = vllm_config.speculative_config
237
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
238
+ self.observability_config = vllm_config.observability_config
239
+
240
# Map of request_id -> generator used for seeded random sampling
241
generators: Dict[str, torch.Generator] = {}
242
vllm/worker/multi_step_model_runner.py
+1
@@ -304,6 +304,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
304
# mypy: enable-error-code=type-var
305
306
def __init__(self, base_model_runner: GPUModelRunnerBase, *args, **kwargs):
307
+
308
super().__init__(*args, **kwargs)
309
310
# Check attention backend support.
vllm/worker/multi_step_worker.py
+1
-9
@@ -27,17 +27,9 @@ class MultiStepWorker(Worker):
27
# for multi-step model, wrap the model runner with MultiStepModelRunner
28
self.model_runner = MultiStepModelRunner(
29
base_model_runner,
30
- base_model_runner.model_config,
31
- base_model_runner.parallel_config,
32
- base_model_runner.scheduler_config,
33
- base_model_runner.device_config,
34
- base_model_runner.cache_config,
35
- load_config=base_model_runner.load_config,
36
- lora_config=self.lora_config,
30
+ vllm_config=base_model_runner.vllm_config,
31
kv_cache_dtype=self.cache_config.cache_dtype,
32
is_driver_worker=base_model_runner.is_driver_worker,
39
- prompt_adapter_config=base_model_runner.prompt_adapter_config,
40
- observability_config=base_model_runner.observability_config,
33
)
34
35
pipeline_parallel_size = self.parallel_config.pipeline_parallel_size
vllm/worker/neuron_model_runner.py
+4
-12
@@ -7,8 +7,7 @@ import torch
7
from torch import nn
8
from transformers_neuronx.config import GenerationConfig
9
10
-from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
11
- SchedulerConfig)
10
+from vllm.config import VllmConfig
11
from vllm.logger import init_logger
12
from vllm.model_executor import SamplingMetadata
13
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -57,20 +56,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
56
57
def __init__(
58
self,
60
- model_config: ModelConfig,
61
- parallel_config: ParallelConfig,
62
- scheduler_config: SchedulerConfig,
63
- device_config: DeviceConfig,
59
+ vllm_config: VllmConfig,
60
):
65
- self.model_config = model_config
66
- self.parallel_config = parallel_config
67
- self.scheduler_config = scheduler_config
68
-
61
+ ModelRunnerBase.__init__(self, vllm_config)
62
+ model_config = self.model_config
63
if model_config is not None and model_config.get_sliding_window():
64
logger.warning("Sliding window is not supported on Neuron. "
65
"The model will run without sliding window.")
72
- self.device_config = (device_config
73
- if device_config is not None else DeviceConfig())
66
self.device = self.device_config.device
67
self.pin_memory = is_pin_memory_available()
68
vllm/worker/neuron_worker.py
+6
-14
@@ -4,15 +4,15 @@ from typing import List, Optional, Tuple
4
import torch
5
import torch.distributed
6
7
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig)
7
+from vllm.config import VllmConfig
8
from vllm.distributed import (ensure_model_parallel_initialized,
9
init_distributed_environment)
10
from vllm.model_executor import set_random_seed
11
from vllm.sequence import ExecuteModelRequest
12
from vllm.worker.neuron_model_runner import NeuronModelRunner
13
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
15
- LoraNotSupportedWorkerBase, WorkerInput)
14
+ LoraNotSupportedWorkerBase, WorkerBase,
15
+ WorkerInput)
16
17
18
class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
@@ -21,20 +21,12 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
21
22
def __init__(
23
self,
24
- model_config: ModelConfig,
25
- parallel_config: ParallelConfig,
26
- scheduler_config: SchedulerConfig,
27
- device_config: DeviceConfig,
28
- cache_config: CacheConfig,
24
+ vllm_config: VllmConfig,
25
local_rank: int,
26
rank: int,
27
distributed_init_method: str,
28
) -> None:
33
- self.model_config = model_config
34
- self.parallel_config = parallel_config
35
- self.scheduler_config = scheduler_config
36
- self.device_config = device_config
37
- self.cache_config = cache_config
29
+ WorkerBase.__init__(self, vllm_config=vllm_config)
30
self.local_rank = local_rank
31
self.rank = rank
32
self.distributed_init_method = distributed_init_method
@@ -44,7 +36,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
36
init_cached_hf_modules()
37
38
self.model_runner: NeuronModelRunner = NeuronModelRunner(
47
- model_config, parallel_config, scheduler_config, device_config)
39
+ vllm_config=vllm_config)
40
self.is_driver_worker = True
41
42
def init_device(self) -> None:
vllm/worker/openvino_model_runner.py
+13
-20
@@ -7,9 +7,7 @@ from torch import nn
7
8
from vllm.attention import get_attn_backend
9
from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
10
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
- ModelConfig, MultiModalConfig, ParallelConfig,
12
- SchedulerConfig)
10
+from vllm.config import VllmConfig
11
from vllm.logger import init_logger
12
from vllm.model_executor import SamplingMetadata
13
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -17,6 +15,7 @@ from vllm.model_executor.model_loader.openvino import get_model
15
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16
MultiModalInputs, MultiModalPlaceholderMap)
17
from vllm.sequence import SequenceGroupMetadata
18
+from vllm.worker.model_runner_base import ModelRunnerBase
19
20
logger = init_logger(__name__)
21
@@ -39,33 +38,21 @@ class ModelInput(NamedTuple):
38
multi_modal_kwargs={})
39
40
42
-class OpenVINOModelRunner:
41
+class OpenVINOModelRunner(ModelRunnerBase):
42
43
def __init__(
44
self,
45
ov_core: ov.Core,
47
- model_config: ModelConfig,
48
- parallel_config: ParallelConfig,
49
- scheduler_config: SchedulerConfig,
50
- device_config: DeviceConfig,
51
- cache_config: CacheConfig,
52
- load_config: LoadConfig,
53
- lora_config: Optional[LoRAConfig],
54
- multimodal_config: Optional[MultiModalConfig],
46
+ vllm_config: VllmConfig,
47
kv_cache_dtype: Optional[str] = "auto",
48
is_driver_worker: bool = False,
49
*args,
50
**kwargs,
51
):
52
self.ov_core = ov_core
61
- self.model_config = model_config
62
- self.parallel_config = parallel_config
63
- self.scheduler_config = scheduler_config
64
- self.device_config = device_config
65
- self.cache_config = cache_config
66
- self.lora_config = lora_config
67
- self.multimodal_config = multimodal_config
68
- self.load_config = load_config
53
+ ModelRunnerBase.__init__(self, vllm_config=vllm_config)
54
+ cache_config = self.cache_config
55
+ model_config = self.model_config
56
self.is_driver_worker = is_driver_worker
57
58
self.device = self.device_config.device
@@ -369,3 +356,9 @@ class OpenVINOModelRunner:
356
sampling_metadata=sampling_metadata,
357
)
358
return output
359
+
360
+ def prepare_model_input(self, *args, **kwargs):
361
+ raise NotImplementedError
362
+
363
+ def make_model_input_from_broadcasted_tensor_dict(self, *args, **kwargs):
364
+ raise NotImplementedError
vllm/worker/openvino_worker.py
+6
-28
@@ -7,9 +7,8 @@ import torch.distributed
7
8
import vllm.envs as envs
9
from vllm.attention import get_attn_backend
10
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
- ModelConfig, MultiModalConfig, ParallelConfig,
12
- SchedulerConfig)
10
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
11
+ ParallelConfig, VllmConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
init_distributed_environment)
@@ -22,7 +21,7 @@ from vllm.platforms import current_platform
21
from vllm.sampling_params import SamplingParams
22
from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
23
from vllm.worker.openvino_model_runner import OpenVINOModelRunner
25
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
24
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
25
26
logger = init_logger(__name__)
27
@@ -212,33 +211,19 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
211
def __init__(
212
self,
213
ov_core: ov.Core,
215
- model_config: ModelConfig,
216
- parallel_config: ParallelConfig,
217
- scheduler_config: SchedulerConfig,
218
- device_config: DeviceConfig,
219
- cache_config: CacheConfig,
220
- load_config: LoadConfig,
214
+ vllm_config: VllmConfig,
215
local_rank: int,
216
rank: int,
217
distributed_init_method: str,
224
- lora_config: Optional[LoRAConfig] = None,
225
- multimodal_config: Optional[MultiModalConfig] = None,
218
kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined,
219
is_driver_worker: bool = False,
220
) -> None:
221
self.ov_core = ov_core
230
- self.model_config = model_config
231
- self.parallel_config = parallel_config
222
+ WorkerBase.__init__(self, vllm_config)
223
self.parallel_config.rank = rank
233
- self.scheduler_config = scheduler_config
234
- self.device_config = device_config
235
- self.cache_config = cache_config
236
- self.load_config = load_config
224
self.local_rank = local_rank
225
self.rank = rank
226
self.distributed_init_method = distributed_init_method
240
- self.lora_config = lora_config
241
- self.multimodal_config = multimodal_config
227
self.is_driver_worker = is_driver_worker
228
if self.is_driver_worker:
229
assert self.rank == 0, "The driver worker must have rank 0."
@@ -250,14 +235,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
235
init_cached_hf_modules()
236
self.model_runner = OpenVINOModelRunner(
237
self.ov_core,
253
- model_config,
254
- parallel_config,
255
- scheduler_config,
256
- device_config,
257
- cache_config,
258
- load_config=self.load_config,
259
- lora_config=self.lora_config,
260
- multimodal_config=self.multimodal_config,
238
+ vllm_config=self.vllm_config,
239
kv_cache_dtype=kv_cache_dtype,
240
is_driver_worker=is_driver_worker,
241
)
vllm/worker/tpu_model_runner.py
+3
-14
@@ -12,8 +12,7 @@ import torch_xla.runtime as xr
12
13
from vllm.attention import AttentionMetadata, get_attn_backend
14
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
15
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
16
- ParallelConfig, SchedulerConfig)
15
+from vllm.config import VllmConfig
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.sampler import SamplerOutput
18
from vllm.model_executor.model_loader import get_model
@@ -90,20 +89,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
89
90
def __init__(
91
self,
93
- model_config: ModelConfig,
94
- parallel_config: ParallelConfig,
95
- scheduler_config: SchedulerConfig,
96
- device_config: DeviceConfig,
97
- cache_config: CacheConfig,
98
- load_config: LoadConfig,
92
+ vllm_config: VllmConfig,
93
is_driver_worker: bool = False,
94
):
101
- self.model_config = model_config
102
- self.parallel_config = parallel_config
103
- self.scheduler_config = scheduler_config
104
- self.device_config = device_config
105
- self.cache_config = cache_config
106
- self.load_config = load_config
95
+ ModelRunnerBase.__init__(self, vllm_config=vllm_config)
96
self.is_driver_worker = is_driver_worker
97
98
self.block_size = self.cache_config.block_size
vllm/worker/tpu_worker.py
+6
-22
@@ -6,8 +6,7 @@ import torch_xla.core.xla_model as xm
6
import torch_xla.runtime as xr
7
8
import vllm.envs as envs
9
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
10
- ParallelConfig, SchedulerConfig)
9
+from vllm.config import VllmConfig
10
from vllm.distributed import (ensure_model_parallel_initialized,
11
init_distributed_environment)
12
from vllm.logger import init_logger
@@ -16,7 +15,8 @@ from vllm.sequence import ExecuteModelRequest
15
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
16
from vllm.worker.tpu_model_runner import TPUModelRunner
17
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
19
- LoraNotSupportedWorkerBase, WorkerInput)
18
+ LoraNotSupportedWorkerBase, WorkerBase,
19
+ WorkerInput)
20
21
logger = init_logger(__name__)
22
@@ -25,24 +25,14 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
25
26
def __init__(
27
self,
28
- model_config: ModelConfig,
29
- parallel_config: ParallelConfig,
30
- scheduler_config: SchedulerConfig,
31
- device_config: DeviceConfig,
32
- cache_config: CacheConfig,
33
- load_config: LoadConfig,
28
+ vllm_config: VllmConfig,
29
local_rank: int,
30
rank: int,
31
distributed_init_method: str,
32
is_driver_worker: bool,
33
) -> None:
39
- self.model_config = model_config
40
- self.parallel_config = parallel_config
34
+ WorkerBase.__init__(self, vllm_config=vllm_config)
35
self.parallel_config.rank = rank
42
- self.scheduler_config = scheduler_config
43
- self.device_config = device_config
44
- self.cache_config = cache_config
45
- self.load_config = load_config
36
self.local_rank = local_rank
37
self.rank = rank
38
self.distributed_init_method = distributed_init_method
@@ -56,13 +46,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
46
self.cache_config.cache_dtype]
47
48
self.model_runner: TPUModelRunner = TPUModelRunner(
59
- model_config,
60
- parallel_config,
61
- scheduler_config,
62
- device_config,
63
- cache_config,
64
- load_config,
65
- is_driver_worker=is_driver_worker)
49
+ vllm_config=vllm_config, is_driver_worker=is_driver_worker)
50
51
def init_device(self) -> None:
52
os.environ["PJRT_DEVICE"] = "TPU"
vllm/worker/worker.py
+10
-35
@@ -7,10 +7,7 @@ import torch
7
import torch.distributed
8
9
import vllm.envs as envs
10
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
- ModelConfig, ObservabilityConfig, ParallelConfig,
12
- PromptAdapterConfig, SchedulerConfig,
13
- SpeculativeConfig)
10
+from vllm.config import ParallelConfig, VllmConfig
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment,
13
set_custom_all_reduce)
@@ -27,7 +24,8 @@ from vllm.worker.cache_engine import CacheEngine
24
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
25
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
26
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
30
-from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
27
+from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
28
+ WorkerInput)
29
30
logger = init_logger(__name__)
31
@@ -42,46 +40,31 @@ class Worker(LocalOrDistributedWorkerBase):
40
41
def __init__(
42
self,
45
- model_config: ModelConfig,
46
- parallel_config: ParallelConfig,
47
- scheduler_config: SchedulerConfig,
48
- device_config: DeviceConfig,
49
- cache_config: CacheConfig,
50
- load_config: LoadConfig,
43
+ vllm_config: VllmConfig,
44
local_rank: int,
45
rank: int,
46
distributed_init_method: str,
54
- lora_config: Optional[LoRAConfig] = None,
55
- speculative_config: Optional[SpeculativeConfig] = None,
56
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
47
is_driver_worker: bool = False,
48
model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
59
- observability_config: Optional[ObservabilityConfig] = None,
49
) -> None:
61
- self.model_config = model_config
62
- self.parallel_config = parallel_config
50
+ WorkerBase.__init__(self, vllm_config)
51
self.parallel_config.rank = rank
64
- self.scheduler_config = scheduler_config
65
- self.device_config = device_config
66
- self.cache_config = cache_config
52
self.local_rank = local_rank
53
self.rank = rank
54
self.distributed_init_method = distributed_init_method
70
- self.lora_config = lora_config
71
- self.load_config = load_config
72
- self.prompt_adapter_config = prompt_adapter_config
55
self.is_driver_worker = is_driver_worker
74
- if parallel_config and is_driver_worker:
75
- assert rank % parallel_config.tensor_parallel_size == 0, \
56
+ if is_driver_worker:
57
+ assert rank % self.parallel_config.tensor_parallel_size == 0, \
58
"Driver worker should be rank 0 of tensor parallel group."
59
if self.model_config.trust_remote_code:
60
# note: lazy import to avoid importing torch before initializing
61
from vllm.utils import init_cached_hf_modules
62
init_cached_hf_modules()
81
- self.observability_config = observability_config
63
64
# Return hidden states from target model if the draft model is an
65
# mlp_speculator
66
+ speculative_config = self.speculative_config
67
+ model_config = self.model_config
68
speculative_args = {} if speculative_config is None \
69
or (speculative_config.draft_model_config.model ==
70
model_config.model) \
@@ -97,17 +80,9 @@ class Worker(LocalOrDistributedWorkerBase):
80
elif self._is_encoder_decoder_model():
81
ModelRunnerClass = EncoderDecoderModelRunner
82
self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
100
- model_config,
101
- parallel_config,
102
- scheduler_config,
103
- device_config,
104
- cache_config,
105
- load_config=load_config,
106
- lora_config=self.lora_config,
83
+ vllm_config=self.vllm_config,
84
kv_cache_dtype=self.cache_config.cache_dtype,
85
is_driver_worker=is_driver_worker,
109
- prompt_adapter_config=prompt_adapter_config,
110
- observability_config=observability_config,
86
**speculative_args,
87
)
88
# Uninitialized cache engine. Will be initialized by
vllm/worker/worker_base.py
+17
-1
@@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
7
8
import torch
9
10
-from vllm.config import ObservabilityConfig
10
+from vllm.config import ObservabilityConfig, VllmConfig
11
from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
@@ -29,6 +29,22 @@ class WorkerBase(ABC):
29
communicate request metadata to other workers.
30
"""
31
32
+ def __init__(
33
+ self,
34
+ vllm_config: VllmConfig,
35
+ ) -> None:
36
+ self.vllm_config = vllm_config
37
+ self.model_config = vllm_config.model_config
38
+ self.cache_config = vllm_config.cache_config
39
+ self.lora_config = vllm_config.lora_config
40
+ self.load_config = vllm_config.load_config
41
+ self.parallel_config = vllm_config.parallel_config
42
+ self.scheduler_config = vllm_config.scheduler_config
43
+ self.device_config = vllm_config.device_config
44
+ self.speculative_config = vllm_config.speculative_config
45
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
46
+ self.observability_config = vllm_config.observability_config
47
+
48
@abstractmethod
49
def init_device(self) -> None:
50
"""Initialize device state, such as loading the model or other on-device
vllm/worker/xpu_model_runner.py
+6
-23
@@ -10,9 +10,7 @@ import torch
10
import torch.nn as nn
11
12
from vllm.attention import get_attn_backend
13
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14
- ModelConfig, ObservabilityConfig, ParallelConfig,
15
- PromptAdapterConfig, SchedulerConfig)
13
+from vllm.config import VllmConfig
14
from vllm.distributed import get_pp_group
15
from vllm.inputs import INPUT_REGISTRY, InputRegistry
16
from vllm.logger import init_logger
@@ -363,33 +361,18 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
361
362
def __init__(
363
self,
366
- model_config: ModelConfig,
367
- parallel_config: ParallelConfig,
368
- scheduler_config: SchedulerConfig,
369
- device_config: DeviceConfig,
370
- cache_config: CacheConfig,
371
- load_config: LoadConfig,
372
- lora_config: Optional[LoRAConfig],
364
+ vllm_config: VllmConfig,
365
kv_cache_dtype: Optional[str] = "auto",
366
is_driver_worker: bool = False,
375
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
367
return_hidden_states: bool = False,
377
- observability_config: Optional[ObservabilityConfig] = None,
368
input_registry: InputRegistry = INPUT_REGISTRY,
369
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
370
):
381
- self.model_config = model_config
382
- self.parallel_config = parallel_config
383
- self.scheduler_config = scheduler_config
384
- self.device_config = device_config
385
- self.cache_config = cache_config
386
- self.lora_config = lora_config
387
- self.load_config = load_config
371
+
372
+ ModelRunnerBase.__init__(self, vllm_config=vllm_config)
373
+ model_config = self.model_config
374
+ cache_config = self.cache_config
375
self.is_driver_worker = is_driver_worker
389
- self.prompt_adapter_config = prompt_adapter_config
390
- self.observability_config = observability_config
391
- if self.observability_config is not None:
392
- print(f"observability_config is {self.observability_config}")
376
self.return_hidden_states = return_hidden_states
377
378
self.device = self.device_config.device
vllm/worker/xpu_worker.py
+8
-32
@@ -8,10 +8,7 @@ import oneccl_bindings_for_pytorch # noqa: F401
8
import torch
9
import torch.distributed
10
11
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
- ModelConfig, ObservabilityConfig, ParallelConfig,
13
- PromptAdapterConfig, SchedulerConfig,
14
- SpeculativeConfig)
11
+from vllm.config import VllmConfig
12
from vllm.distributed import (ensure_model_parallel_initialized,
13
init_distributed_environment)
14
from vllm.logger import init_logger
@@ -19,7 +16,7 @@ from vllm.model_executor import set_random_seed
16
from vllm.platforms import current_platform
17
from vllm.worker.cache_engine import CacheEngine
18
from vllm.worker.worker import Worker
22
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
20
from vllm.worker.xpu_model_runner import XPUModelRunner
21
22
logger = init_logger(__name__)
@@ -36,53 +33,32 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
33
34
def __init__(
35
self,
39
- model_config: ModelConfig,
40
- parallel_config: ParallelConfig,
41
- scheduler_config: SchedulerConfig,
42
- device_config: DeviceConfig,
43
- cache_config: CacheConfig,
44
- load_config: LoadConfig,
36
+ vllm_config: VllmConfig,
37
local_rank: int,
38
rank: int,
39
distributed_init_method: str,
48
- lora_config: Optional[LoRAConfig] = None,
49
- speculative_config: Optional[SpeculativeConfig] = None,
50
- prompt_adapter_config: Optional[PromptAdapterConfig] = None,
40
is_driver_worker: bool = False,
52
- observability_config: Optional[ObservabilityConfig] = None,
41
) -> None:
42
+ WorkerBase.__init__(self, vllm_config=vllm_config)
43
+ device_config = self.device_config
44
+ parallel_config = self.parallel_config
45
assert device_config.device_type == "xpu"
46
assert current_platform.is_xpu()
47
57
- self.model_config = model_config
58
- self.parallel_config = parallel_config
48
self.parallel_config.rank = rank
60
- self.scheduler_config = scheduler_config
61
- self.device_config = device_config
62
- self.cache_config = cache_config
63
- self.load_config = load_config
49
+
50
self.local_rank = local_rank
51
self.rank = rank
52
self.distributed_init_method = distributed_init_method
67
- self.lora_config = lora_config
68
- self.prompt_adapter_config = prompt_adapter_config
53
self.is_driver_worker = is_driver_worker
70
- self.observability_config = observability_config
54
if parallel_config and is_driver_worker:
55
assert rank % parallel_config.tensor_parallel_size == 0, \
56
"Driver worker should be rank 0 of tensor parallel group."
57
58
self.model_runner = XPUModelRunner( # type: ignore
76
- model_config,
77
- parallel_config,
78
- scheduler_config,
79
- device_config,
80
- cache_config,
81
- load_config=self.load_config,
82
- lora_config=self.lora_config,
59
+ vllm_config=vllm_config,
60
kv_cache_dtype=self.cache_config.cache_dtype,
61
is_driver_worker=is_driver_worker,
85
- observability_config=self.observability_config,
62
)
63
# Uninitialized cache engine. Will be initialized by
64
# initialize_cache.