A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[platforms] absorb worker cls difference into platforms folder (#10555)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Nick Hill <nhill@redhat.com>
youkaichao committed
Nov 21, 2024 at 21:00 UTC
a111d0151ffed94582bec65635979e04e5b63676
21 files changed
+273
-283
vllm/config.py
+117
-121
@@ -926,56 +926,56 @@ class LoadConfig:
926
f"{rocm_supported_load_format}")
927
928
929
+@dataclass
930
class ParallelConfig:
930
- """Configuration for the distributed execution.
931
+ """Configuration for the distributed execution."""
932
932
- Args:
933
- pipeline_parallel_size: Number of pipeline parallel groups.
934
- tensor_parallel_size: Number of tensor parallel groups.
935
- worker_use_ray: Deprecated, use distributed_executor_backend instead.
936
- max_parallel_loading_workers: Maximum number of multiple batches
937
- when load model sequentially. To avoid RAM OOM when using tensor
938
- parallel and large models.
939
- disable_custom_all_reduce: Disable the custom all-reduce kernel and
940
- fall back to NCCL.
941
- tokenizer_pool_config: Config for the tokenizer pool.
942
- If None, will use synchronous tokenization.
943
- ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
944
- https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
945
- placement_group: ray distributed model workers placement group.
946
- distributed_executor_backend: Backend to use for distributed model
947
- workers, either "ray" or "mp" (multiprocessing). If the product
948
- of pipeline_parallel_size and tensor_parallel_size is less than
949
- or equal to the number of GPUs available, "mp" will be used to
950
- keep processing on a single host. Otherwise, this will default
951
- to "ray" if Ray is installed and fail otherwise. Note that tpu
952
- and hpu only support Ray for distributed inference.
953
- """
933
+ pipeline_parallel_size: int = 1 # Number of pipeline parallel groups.
934
+ tensor_parallel_size: int = 1 # Number of tensor parallel groups.
935
955
- def __init__(
956
- self,
957
- pipeline_parallel_size: int,
958
- tensor_parallel_size: int,
959
- worker_use_ray: Optional[bool] = None,
960
- max_parallel_loading_workers: Optional[int] = None,
961
- disable_custom_all_reduce: bool = False,
962
- tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
963
- ray_workers_use_nsight: bool = False,
964
- placement_group: Optional["PlacementGroup"] = None,
965
- distributed_executor_backend: Optional[Union[
966
- str, Type["ExecutorBase"]]] = None,
967
- ) -> None:
968
- self.pipeline_parallel_size = pipeline_parallel_size
969
- self.tensor_parallel_size = tensor_parallel_size
970
- self.distributed_executor_backend = distributed_executor_backend
971
- self.max_parallel_loading_workers = max_parallel_loading_workers
972
- self.disable_custom_all_reduce = disable_custom_all_reduce
973
- self.tokenizer_pool_config = tokenizer_pool_config
974
- self.ray_workers_use_nsight = ray_workers_use_nsight
975
- self.placement_group = placement_group
976
- self.world_size = pipeline_parallel_size * self.tensor_parallel_size
977
-
978
- if worker_use_ray:
936
+ # Deprecated, use distributed_executor_backend instead.
937
+ worker_use_ray: Optional[bool] = None
938
+
939
+ # Maximum number of multiple batches
940
+ # when load model sequentially. To avoid RAM OOM when using tensor
941
+ # parallel and large models.
942
+ max_parallel_loading_workers: Optional[int] = None
943
+
944
+ # Disable the custom all-reduce kernel and fall back to NCCL.
945
+ disable_custom_all_reduce: bool = False
946
+
947
+ # Config for the tokenizer pool. If None, will use synchronous tokenization.
948
+ tokenizer_pool_config: Optional[TokenizerPoolConfig] = None
949
+
950
+ # Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
951
+ ray_workers_use_nsight: bool = False
952
+
953
+ # ray distributed model workers placement group.
954
+ placement_group: Optional["PlacementGroup"] = None
955
+
956
+ # Backend to use for distributed model
957
+ # workers, either "ray" or "mp" (multiprocessing). If the product
958
+ # of pipeline_parallel_size and tensor_parallel_size is less than
959
+ # or equal to the number of GPUs available, "mp" will be used to
960
+ # keep processing on a single host. Otherwise, this will default
961
+ # to "ray" if Ray is installed and fail otherwise. Note that tpu
962
+ # and hpu only support Ray for distributed inference.
963
+ distributed_executor_backend: Optional[Union[str,
964
+ Type["ExecutorBase"]]] = None
965
+
966
+ # the full name of the worker class to use. If "auto", the worker class
967
+ # will be determined based on the platform.
968
+ worker_cls: str = "auto"
969
+
970
+ world_size: int = field(init=False)
971
+
972
+ rank: int = 0
973
+
974
+ def __post_init__(self) -> None:
975
+ self.world_size = self.pipeline_parallel_size * \
976
+ self.tensor_parallel_size
977
+
978
+ if self.worker_use_ray:
979
if self.distributed_executor_backend is None:
980
self.distributed_executor_backend = "ray"
981
elif not self.use_ray:
@@ -1026,7 +1026,6 @@ class ParallelConfig:
1026
backend)
1027
1028
self._verify_args()
1029
- self.rank: int = 0
1029
1030
@property
1031
def use_ray(self) -> bool:
@@ -1059,100 +1058,97 @@ class ParallelConfig:
1058
"run with Ray.")
1059
1060
1061
+@dataclass
1062
class SchedulerConfig:
1063
- """Scheduler configuration.
1063
+ """Scheduler configuration."""
1064
1065
- Args:
1066
- task: The task to use the model for.
1067
- max_num_batched_tokens: Maximum number of tokens to be processed in
1068
- a single iteration.
1069
- max_num_seqs: Maximum number of sequences to be processed in a single
1070
- iteration.
1071
- max_model_len: Maximum length of a sequence (including prompt
1072
- and generated text).
1073
- num_lookahead_slots: The number of slots to allocate per sequence per
1074
- step, beyond the known token ids. This is used in speculative
1075
- decoding to store KV activations of tokens which may or may not be
1076
- accepted.
1077
- delay_factor: Apply a delay (of delay factor multiplied by previous
1078
- prompt latency) before scheduling next prompt.
1079
- enable_chunked_prefill: If True, prefill requests can be chunked based
1080
- on the remaining max_num_batched_tokens.
1081
- preemption_mode: Whether to perform preemption by swapping or
1082
- recomputation. If not specified, we determine the mode as follows:
1083
- We use recomputation by default since it incurs lower overhead than
1084
- swapping. However, when the sequence group has multiple sequences
1085
- (e.g., beam search), recomputation is not currently supported. In
1086
- such a case, we use swapping instead.
1087
- send_delta_data: Private API. If used, scheduler sends delta data to
1088
- workers instead of an entire data. It should be enabled only
1089
- when SPMD worker architecture is enabled. I.e.,
1090
- VLLM_USE_RAY_SPMD_WORKER=1
1091
- policy: The scheduling policy to use. "fcfs" (default) or "priority".
1092
- """
1065
+ task: str = "generate" # The task to use the model for.
1066
+
1067
+ # Maximum number of tokens to be processed in a single iteration.
1068
+ max_num_batched_tokens: int = field(default=None) # type: ignore
1069
+
1070
+ # Maximum number of sequences to be processed in a single iteration.
1071
+ max_num_seqs: int = 128
1072
+
1073
+ # Maximum length of a sequence (including prompt and generated text).
1074
+ max_model_len: int = 8192
1075
+
1076
+ # The number of slots to allocate per sequence per
1077
+ # step, beyond the known token ids. This is used in speculative
1078
+ # decoding to store KV activations of tokens which may or may not be
1079
+ # accepted.
1080
+ num_lookahead_slots: int = 0
1081
+
1082
+ # Apply a delay (of delay factor multiplied by previous
1083
+ # prompt latency) before scheduling next prompt.
1084
+ delay_factor: float = 0.0
1085
+
1086
+ # If True, prefill requests can be chunked based
1087
+ # on the remaining max_num_batched_tokens.
1088
+ enable_chunked_prefill: bool = False
1089
+
1090
+ is_multimodal_model: bool = False
1091
1094
- def __init__(self,
1095
- task: _Task,
1096
- max_num_batched_tokens: Optional[int],
1097
- max_num_seqs: int,
1098
- max_model_len: int,
1099
- num_lookahead_slots: int = 0,
1100
- delay_factor: float = 0.0,
1101
- enable_chunked_prefill: bool = False,
1102
- is_multimodal_model: bool = False,
1103
- preemption_mode: Optional[str] = None,
1104
- num_scheduler_steps: int = 1,
1105
- multi_step_stream_outputs: bool = False,
1106
- send_delta_data: bool = False,
1107
- policy: str = "fcfs") -> None:
1108
- if max_num_batched_tokens is None:
1109
- if enable_chunked_prefill:
1110
- if num_scheduler_steps > 1:
1092
+ # Whether to perform preemption by swapping or
1093
+ # recomputation. If not specified, we determine the mode as follows:
1094
+ # We use recomputation by default since it incurs lower overhead than
1095
+ # swapping. However, when the sequence group has multiple sequences
1096
+ # (e.g., beam search), recomputation is not currently supported. In
1097
+ # such a case, we use swapping instead.
1098
+ preemption_mode: Optional[str] = None
1099
+
1100
+ num_scheduler_steps: int = 1
1101
+
1102
+ multi_step_stream_outputs: bool = False
1103
+
1104
+ # Private API. If used, scheduler sends delta data to
1105
+ # workers instead of an entire data. It should be enabled only
1106
+ # when SPMD worker architecture is enabled. I.e.,
1107
+ # VLLM_USE_RAY_SPMD_WORKER=1
1108
+ send_delta_data: bool = False
1109
+
1110
+ # The scheduling policy to use. "fcfs" (default) or "priority".
1111
+ policy: str = "fcfs"
1112
+
1113
+ chunked_prefill_enabled: bool = field(init=False)
1114
+
1115
+ def __post_init__(self) -> None:
1116
+ if self.max_num_batched_tokens is None:
1117
+ if self.enable_chunked_prefill:
1118
+ if self.num_scheduler_steps > 1:
1119
# Multi-step Chunked-Prefill doesn't allow prompt-chunking
1120
# for now. Have max_num_batched_tokens set to max_model_len
1121
# so we don't reject sequences on account of a short
1122
# max_num_batched_tokens.
1115
- max_num_batched_tokens = max(max_model_len, 2048)
1123
+ self.max_num_batched_tokens = max(self.max_model_len, 2048)
1124
else:
1125
# It is the values that have the best balance between ITL
1126
# and TTFT on A100. Note it is not optimized for throughput.
1119
- max_num_batched_tokens = 512
1127
+ self.max_num_batched_tokens = 512
1128
else:
1129
# If max_model_len is too short, use 2048 as the default value
1130
# for higher throughput.
1123
- max_num_batched_tokens = max(max_model_len, 2048)
1131
+ self.max_num_batched_tokens = max(self.max_model_len, 2048)
1132
1125
- if task == "embedding":
1133
+ if self.task == "embedding":
1134
# For embedding, choose specific value for higher throughput
1127
- max_num_batched_tokens = max(
1128
- max_num_batched_tokens,
1135
+ self.max_num_batched_tokens = max(
1136
+ self.max_num_batched_tokens,
1137
_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS,
1138
)
1131
- if is_multimodal_model:
1139
+ if self.is_multimodal_model:
1140
# The value needs to be at least the number of multimodal tokens
1133
- max_num_batched_tokens = max(
1134
- max_num_batched_tokens,
1141
+ self.max_num_batched_tokens = max(
1142
+ self.max_num_batched_tokens,
1143
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
1144
)
1145
1138
- self.max_num_batched_tokens = max_num_batched_tokens
1139
-
1140
- if enable_chunked_prefill:
1146
+ if self.enable_chunked_prefill:
1147
logger.info(
1148
"Chunked prefill is enabled with max_num_batched_tokens=%d.",
1149
self.max_num_batched_tokens)
1150
1145
- self.task: Final = task
1146
- self.max_num_seqs = max_num_seqs
1147
- self.max_model_len = max_model_len
1148
- self.num_lookahead_slots = num_lookahead_slots
1149
- self.delay_factor = delay_factor
1150
- self.chunked_prefill_enabled = enable_chunked_prefill
1151
- self.preemption_mode = preemption_mode
1152
- self.num_scheduler_steps = num_scheduler_steps
1153
- self.multi_step_stream_outputs = multi_step_stream_outputs
1154
- self.send_delta_data = send_delta_data
1155
- self.policy = policy
1151
+ self.chunked_prefill_enabled = self.enable_chunked_prefill
1152
self._verify_args()
1153
1154
def _verify_args(self) -> None:
@@ -2293,10 +2289,10 @@ class VllmConfig:
2289
2290
model_config: ModelConfig = field(default=None, init=True) # type: ignore
2291
cache_config: CacheConfig = field(default=None, init=True) # type: ignore
2296
- parallel_config: ParallelConfig = field(default=None,
2297
- init=True) # type: ignore
2298
- scheduler_config: SchedulerConfig = field(default=None,
2299
- init=True) # type: ignore
2292
+ parallel_config: ParallelConfig = field(default_factory=ParallelConfig,
2293
+ init=True)
2294
+ scheduler_config: SchedulerConfig = field(default_factory=SchedulerConfig,
2295
+ init=True)
2296
device_config: DeviceConfig = field(default=None,
2297
init=True) # type: ignore
2298
load_config: LoadConfig = field(default=None, init=True) # type: ignore
vllm/engine/arg_utils.py
+10
-1
@@ -191,6 +191,7 @@ class EngineArgs:
191
override_neuron_config: Optional[Dict[str, Any]] = None
192
override_pooler_config: Optional[PoolerConfig] = None
193
compilation_config: Optional[CompilationConfig] = None
194
+ worker_cls: str = "auto"
195
196
def __post_init__(self):
197
if not self.tokenizer:
@@ -887,6 +888,12 @@ class EngineArgs:
888
'compilers, using -O without space is also '
889
'supported. -O3 is equivalent to -O 3.')
890
891
+ parser.add_argument(
892
+ '--worker-cls',
893
+ type=str,
894
+ default="auto",
895
+ help='The worker class to use for distributed execution.')
896
+
897
return parser
898
899
@classmethod
@@ -999,7 +1006,9 @@ class EngineArgs:
1006
self.tokenizer_pool_extra_config,
1007
),
1008
ray_workers_use_nsight=self.ray_workers_use_nsight,
1002
- distributed_executor_backend=self.distributed_executor_backend)
1009
+ distributed_executor_backend=self.distributed_executor_backend,
1010
+ worker_cls=self.worker_cls,
1011
+ )
1012
1013
max_model_len = model_config.max_model_len
1014
use_long_context = max_model_len > 32768
vllm/executor/cpu_executor.py
+1
-6
@@ -115,13 +115,8 @@ class CPUExecutor(ExecutorBase):
115
local_rank: int = 0,
116
rank: int = 0,
117
):
118
- worker_module_name = "vllm.worker.cpu_worker"
119
- worker_class_name = "CPUWorker"
118
121
- wrapper = WorkerWrapperBase(
122
- worker_module_name=worker_module_name,
123
- worker_class_name=worker_class_name,
124
- )
119
+ wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
120
121
assert self.distributed_init_method is not None
122
vllm/executor/gpu_executor.py
+6
-43
@@ -1,4 +1,4 @@
1
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
1
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
2
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
@@ -8,19 +8,14 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
8
from vllm.sequence import ExecuteModelRequest, PoolerOutput
9
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10
make_async)
11
-from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase
11
+from vllm.worker.worker_base import WorkerWrapperBase
12
13
logger = init_logger(__name__)
14
15
16
-def create_worker(worker_module_name: str, worker_class_name: str,
17
- worker_class_fn: Optional[Callable[[], Type[WorkerBase]]],
18
- **kwargs):
19
- wrapper = WorkerWrapperBase(
20
- worker_module_name=worker_module_name,
21
- worker_class_name=worker_class_name,
22
- worker_class_fn=worker_class_fn,
23
- )
16
+def create_worker(**kwargs):
17
+ vllm_config = kwargs.get("vllm_config")
18
+ wrapper = WorkerWrapperBase(vllm_config=vllm_config)
19
wrapper.init_worker(**kwargs)
20
return wrapper.worker
21
@@ -57,43 +52,11 @@ class GPUExecutor(ExecutorBase):
52
or (rank % self.parallel_config.tensor_parallel_size == 0),
53
)
54
60
- def _get_worker_module_and_class(
61
- self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
62
- worker_class_fn = None
63
- if self.scheduler_config.is_multi_step:
64
- worker_module_name = "vllm.worker.multi_step_worker"
65
- worker_class_name = "MultiStepWorker"
66
- elif self.speculative_config:
67
- worker_module_name = "vllm.spec_decode.spec_decode_worker"
68
- worker_class_name = "create_spec_worker"
69
- else:
70
- worker_module_name = "vllm.worker.worker"
71
- worker_class_name = "Worker"
72
- return (worker_module_name, worker_class_name, worker_class_fn)
73
-
74
- def _get_create_worker_kwargs(
75
- self,
76
- local_rank: int = 0,
77
- rank: int = 0,
78
- distributed_init_method: Optional[str] = None) -> Dict:
79
- worker_kwargs = self._get_worker_kwargs(local_rank, rank,
80
- distributed_init_method)
81
-
82
- (worker_module_name, worker_class_name,
83
- worker_class_fn) = self._get_worker_module_and_class()
84
- worker_kwargs.update(
85
- worker_module_name=worker_module_name,
86
- worker_class_name=worker_class_name,
87
- worker_class_fn=worker_class_fn,
88
- )
89
-
90
- return worker_kwargs
91
-
55
def _create_worker(self,
56
local_rank: int = 0,
57
rank: int = 0,
58
distributed_init_method: Optional[str] = None):
96
- return create_worker(**self._get_create_worker_kwargs(
59
+ return create_worker(**self._get_worker_kwargs(
60
local_rank=local_rank,
61
rank=rank,
62
distributed_init_method=distributed_init_method))
vllm/executor/hpu_executor.py
+1
-4
@@ -48,10 +48,7 @@ class HPUExecutor(ExecutorBase):
48
local_rank: int = 0,
49
rank: int = 0,
50
distributed_init_method: Optional[str] = None):
51
- wrapper = WorkerWrapperBase(
52
- worker_module_name="vllm.worker.hpu_worker",
53
- worker_class_name="HPUWorker",
54
- )
51
+ wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
52
wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
53
distributed_init_method))
54
return wrapper.worker
vllm/executor/multiproc_gpu_executor.py
+1
-1
@@ -90,7 +90,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
90
result_handler,
91
partial(
92
create_worker,
93
- **self._get_create_worker_kwargs(
93
+ **self._get_worker_kwargs(
94
rank=rank,
95
local_rank=rank,
96
distributed_init_method=distributed_init_method,
vllm/executor/neuron_executor.py
+3
-2
@@ -7,6 +7,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput
7
from vllm.sequence import ExecuteModelRequest
8
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9
make_async)
10
+from vllm.worker.worker_base import WorkerWrapperBase
11
12
logger = init_logger(__name__)
13
@@ -25,10 +26,10 @@ class NeuronExecutor(ExecutorBase):
26
self._init_worker()
27
28
def _init_worker(self):
28
- from vllm.worker.neuron_worker import NeuronWorker
29
+ wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
30
distributed_init_method = get_distributed_init_method(
31
get_ip(), get_open_port())
31
- self.driver_worker = NeuronWorker(
32
+ self.driver_worker = wrapper.init_worker(
33
vllm_config=self.vllm_config,
34
local_rank=0,
35
rank=0,
vllm/executor/openvino_executor.py
+3
-5
@@ -14,6 +14,7 @@ from vllm.platforms import current_platform
14
from vllm.sequence import ExecuteModelRequest
15
from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
16
get_open_port, make_async)
17
+from vllm.worker.worker_base import WorkerWrapperBase
18
19
logger = init_logger(__name__)
20
@@ -38,15 +39,12 @@ class OpenVINOExecutor(ExecutorBase):
39
self._init_worker()
40
41
def _init_worker(self):
41
- from vllm.worker.openvino_worker import OpenVINOWorker
42
43
- assert (
44
- self.parallel_config.world_size == 1
45
- ), "OpenVINOExecutor only supports single CPU socket currently."
43
+ wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
44
45
distributed_init_method = get_distributed_init_method(
46
get_ip(), get_open_port())
49
- self.driver_worker = OpenVINOWorker(
47
+ self.driver_worker = wrapper.init_worker(
48
ov_core=self.ov_core,
49
vllm_config=self.vllm_config,
50
local_rank=0,
vllm/executor/ray_gpu_executor.py
+2
-14
@@ -91,17 +91,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
91
92
return ray_remote_kwargs
93
94
- def _get_worker_wrapper_args(self) -> Dict[str, Any]:
95
- (worker_module_name, worker_class_name,
96
- worker_class_fn) = self._get_worker_module_and_class()
97
-
98
- return dict(
99
- worker_module_name=worker_module_name,
100
- worker_class_name=worker_class_name,
101
- worker_class_fn=worker_class_fn,
102
- trust_remote_code=self.model_config.trust_remote_code,
103
- )
104
-
94
# child class could overwrite this to return actual env vars.
95
def _get_env_vars_to_be_updated(self):
96
return self._env_vars_for_all_workers
@@ -135,7 +124,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
124
125
# Create the workers.
126
driver_ip = get_ip()
138
- worker_wrapper_kwargs = self._get_worker_wrapper_args()
127
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
128
if not bundle.get("GPU", 0):
129
continue
@@ -150,7 +138,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
138
num_gpus=num_gpus,
139
scheduling_strategy=scheduling_strategy,
140
**ray_remote_kwargs,
153
- )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
141
+ )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
142
143
if self.use_ray_spmd_worker:
144
self.workers.append(worker)
@@ -161,7 +149,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
149
# as the resource holder for the driver process.
150
self.driver_dummy_worker = worker
151
self.driver_worker = RayWorkerWrapper(
164
- **worker_wrapper_kwargs)
152
+ vllm_config=self.vllm_config)
153
else:
154
# Else, added to the list of workers.
155
self.workers.append(worker)
vllm/executor/ray_hpu_executor.py
+3
-33
@@ -2,8 +2,7 @@ import asyncio
2
import os
3
from collections import defaultdict
4
from itertools import islice, repeat
5
-from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
6
- Type)
5
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6
7
import msgspec
8
@@ -18,7 +17,6 @@ from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18
get_ip, get_open_port, get_vllm_instance_id,
19
make_async)
21
-from vllm.worker.worker_base import WorkerBase
20
21
if ray is not None:
22
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -81,33 +79,6 @@ class RayHPUExecutor(DistributedGPUExecutor):
79
def finish_measurements(self):
80
self._run_workers("finish_measurements")
81
84
- def _get_worker_module_and_class(
85
- self
86
- ) -> Tuple[str, str, Optional[Callable[[],
87
- Type[WorkerBase]]]]: # noqa: F821
88
- worker_class_fn = None
89
- if self.scheduler_config.is_multi_step:
90
- raise NotImplementedError(
91
- "Multi-step execution is not implemented for HPU")
92
- elif self.speculative_config:
93
- raise NotImplementedError(
94
- "Speculative decoding is not implemented for HPU")
95
- else:
96
- worker_module_name = "vllm.worker.hpu_worker"
97
- worker_class_name = "HPUWorker"
98
- return (worker_module_name, worker_class_name, worker_class_fn)
99
-
100
- def _get_worker_wrapper_args(self) -> Dict[str, Any]:
101
- (worker_module_name, worker_class_name,
102
- worker_class_fn) = self._get_worker_module_and_class()
103
-
104
- return dict(
105
- worker_module_name=worker_module_name,
106
- worker_class_name=worker_class_name,
107
- worker_class_fn=worker_class_fn,
108
- trust_remote_code=self.model_config.trust_remote_code,
109
- )
110
-
82
def _init_workers_ray(self, placement_group: "PlacementGroup",
83
**ray_remote_kwargs):
84
# Otherwise, the ray workers are allocated with a full GPU.
@@ -128,7 +99,6 @@ class RayHPUExecutor(DistributedGPUExecutor):
99
100
# Create the workers.
101
driver_ip = get_ip()
131
- worker_wrapper_kwargs = self._get_worker_wrapper_args()
102
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
103
if not bundle.get("HPU", 0):
104
continue
@@ -144,7 +114,7 @@ class RayHPUExecutor(DistributedGPUExecutor):
114
resources={'HPU': num_gpus},
115
scheduling_strategy=scheduling_strategy,
116
**ray_remote_kwargs,
147
- )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
117
+ )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
118
119
if self.use_ray_spmd_worker:
120
self.workers.append(worker)
@@ -155,7 +125,7 @@ class RayHPUExecutor(DistributedGPUExecutor):
125
# as the resource holder for the driver process.
126
self.driver_dummy_worker = worker
127
self.driver_worker = RayWorkerWrapper(
158
- **worker_wrapper_kwargs)
128
+ vllm_config=self.vllm_config)
129
else:
130
# Else, added to the list of workers.
131
self.workers.append(worker)
vllm/executor/ray_tpu_executor.py
+2
-17
@@ -69,14 +69,6 @@ class RayTPUExecutor(TPUExecutor):
69
placement_group_bundle_index=bundle_id,
70
)
71
72
- assert self.speculative_config is None
73
- if self.scheduler_config.is_multi_step:
74
- worker_module_name = "vllm.worker.multi_step_tpu_worker"
75
- worker_class_name = "MultiStepTPUWorker"
76
- else:
77
- worker_module_name = "vllm.worker.tpu_worker"
78
- worker_class_name = "TPUWorker"
79
-
72
# GKE does not fetch environment information from metadata server
73
# and instead sets these from within the Ray process. Therefore we
74
# need to override the Ray environment variables manually.
@@ -95,11 +87,7 @@ class RayTPUExecutor(TPUExecutor):
87
resources={"TPU": 1},
88
scheduling_strategy=scheduling_strategy,
89
**ray_remote_kwargs,
98
- )(RayWorkerWrapper).remote(
99
- worker_module_name=worker_module_name,
100
- worker_class_name=worker_class_name,
101
- trust_remote_code=self.model_config.trust_remote_code,
102
- )
90
+ )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
91
if override_env:
92
worker.override_env_vars.remote(override_env)
93
@@ -109,10 +97,7 @@ class RayTPUExecutor(TPUExecutor):
97
# as the resource holder for the driver process.
98
self.driver_dummy_worker = worker
99
self.driver_worker = RayWorkerWrapper(
112
- worker_module_name=worker_module_name,
113
- worker_class_name=worker_class_name,
114
- trust_remote_code=self.model_config.trust_remote_code,
115
- )
100
+ vllm_config=self.vllm_config)
101
else:
102
# Else, added to the list of workers.
103
self.workers.append(worker)
vllm/executor/xpu_executor.py
+1
-13
@@ -1,4 +1,4 @@
1
-from typing import Callable, List, Optional, Tuple, Type, Union
1
+from typing import List, Optional, Union
2
3
from vllm.executor.executor_base import ExecutorAsyncBase
4
from vllm.executor.gpu_executor import GPUExecutor
@@ -6,7 +6,6 @@ from vllm.logger import init_logger
6
from vllm.model_executor.layers.sampler import SamplerOutput
7
from vllm.sequence import ExecuteModelRequest, PoolerOutput
8
from vllm.utils import make_async
9
-from vllm.worker.worker_base import WorkerBase
9
10
logger = init_logger(__name__)
11
@@ -22,17 +21,6 @@ class XPUExecutor(GPUExecutor):
21
22
GPUExecutor._init_executor(self)
23
25
- def _get_worker_module_and_class(
26
- self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
27
- worker_class_fn = None
28
- if self.speculative_config is not None:
29
- raise NotImplementedError(
30
- "XPU does not support speculative decoding")
31
- else:
32
- worker_module_name = "vllm.worker.xpu_worker"
33
- worker_class_name = "XPUWorker"
34
- return (worker_module_name, worker_class_name, worker_class_fn)
35
-
24
def execute_model(
25
self, execute_model_req: ExecuteModelRequest
26
) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
vllm/platforms/cpu.py
+2
@@ -84,3 +84,5 @@ class CpuPlatform(Platform):
84
"distributed executor backend."),
85
parallel_config.distributed_executor_backend)
86
parallel_config.distributed_executor_backend = "mp"
87
+ if parallel_config.worker_cls == "auto":
88
+ parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
vllm/platforms/cuda.py
+20
-1
@@ -4,7 +4,7 @@ pynvml. However, it should not initialize cuda context.
4
5
import os
6
from functools import lru_cache, wraps
7
-from typing import Callable, List, Tuple, TypeVar
7
+from typing import TYPE_CHECKING, Callable, List, Tuple, TypeVar
8
9
import pynvml
10
import torch
@@ -16,6 +16,11 @@ from vllm.logger import init_logger
16
17
from .interface import DeviceCapability, Platform, PlatformEnum
18
19
+if TYPE_CHECKING:
20
+ from vllm.config import VllmConfig
21
+else:
22
+ VllmConfig = None
23
+
24
logger = init_logger(__name__)
25
26
_P = ParamSpec("_P")
@@ -157,3 +162,17 @@ class CudaPlatform(Platform):
162
" machine has no NVLink equipped.")
163
return False
164
return True
165
+
166
+ @classmethod
167
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
168
+ parallel_config = vllm_config.parallel_config
169
+ scheduler_config = vllm_config.scheduler_config
170
+ if parallel_config.worker_cls == "auto":
171
+ if scheduler_config.is_multi_step:
172
+ parallel_config.worker_cls = \
173
+ "vllm.worker.multi_step_worker.MultiStepWorker"
174
+ elif vllm_config.speculative_config:
175
+ parallel_config.worker_cls = \
176
+ "vllm.spec_decode.spec_decode_worker.create_spec_worker"
177
+ else:
178
+ parallel_config.worker_cls = "vllm.worker.worker.Worker"
vllm/platforms/hpu.py
+23
@@ -1,7 +1,14 @@
1
+from typing import TYPE_CHECKING
2
+
3
import torch
4
5
from .interface import Platform, PlatformEnum, _Backend
6
7
+if TYPE_CHECKING:
8
+ from vllm.config import VllmConfig
9
+else:
10
+ VllmConfig = None
11
+
12
13
class HpuPlatform(Platform):
14
_enum = PlatformEnum.HPU
@@ -14,3 +21,19 @@ class HpuPlatform(Platform):
21
@staticmethod
22
def inference_mode():
23
return torch.no_grad()
24
+
25
+ @classmethod
26
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
27
+
28
+ scheduler_config = vllm_config.scheduler_config
29
+ if scheduler_config.is_multi_step:
30
+ raise NotImplementedError(
31
+ "Multi-step execution is not implemented for HPU")
32
+
33
+ if vllm_config.speculative_config is not None:
34
+ raise NotImplementedError(
35
+ "Speculative decoding is not implemented for HPU")
36
+
37
+ parallel_config = vllm_config.parallel_config
38
+ if parallel_config.worker_cls == "auto":
39
+ parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"
vllm/platforms/neuron.py
+14
@@ -1,5 +1,12 @@
1
+from typing import TYPE_CHECKING
2
+
3
from .interface import Platform, PlatformEnum
4
5
+if TYPE_CHECKING:
6
+ from vllm.config import VllmConfig
7
+else:
8
+ VllmConfig = None
9
+
10
11
class NeuronPlatform(Platform):
12
_enum = PlatformEnum.NEURON
@@ -8,3 +15,10 @@ class NeuronPlatform(Platform):
15
@classmethod
16
def get_device_name(cls, device_id: int = 0) -> str:
17
return "neuron"
18
+
19
+ @classmethod
20
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
21
+ parallel_config = vllm_config.parallel_config
22
+ if parallel_config.worker_cls == "auto":
23
+ parallel_config.worker_cls = \
24
+ "vllm.worker.neuron_worker.NeuronWorker"
vllm/platforms/openvino.py
+18
@@ -1,3 +1,5 @@
1
+from typing import TYPE_CHECKING
2
+
3
import torch
4
5
import vllm.envs as envs
@@ -5,6 +7,11 @@ from vllm.logger import init_logger
7
8
from .interface import Platform, PlatformEnum, _Backend
9
10
+if TYPE_CHECKING:
11
+ from vllm.config import VllmConfig
12
+else:
13
+ VllmConfig = None
14
+
15
logger = init_logger(__name__)
16
17
@@ -38,3 +45,14 @@ class OpenVinoPlatform(Platform):
45
def is_pin_memory_available(self) -> bool:
46
logger.warning("Pin memory is not supported on OpenViNO.")
47
return False
48
+
49
+ @classmethod
50
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
51
+ parallel_config = vllm_config.parallel_config
52
+ assert (
53
+ parallel_config.world_size == 1
54
+ ), "OpenVINOExecutor only supports single CPU socket currently."
55
+
56
+ if parallel_config.worker_cls == "auto":
57
+ parallel_config.worker_cls = \
58
+ "vllm.worker.openvino_worker.OpenVINOWorker"
vllm/platforms/rocm.py
+20
@@ -1,5 +1,6 @@
1
import os
2
from functools import lru_cache
3
+from typing import TYPE_CHECKING
4
5
import torch
6
@@ -7,6 +8,11 @@ from vllm.logger import init_logger
8
9
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
10
11
+if TYPE_CHECKING:
12
+ from vllm.config import VllmConfig
13
+else:
14
+ VllmConfig = None
15
+
16
logger = init_logger(__name__)
17
18
try:
@@ -58,3 +64,17 @@ class RocmPlatform(Platform):
64
def get_device_total_memory(cls, device_id: int = 0) -> int:
65
device_props = torch.cuda.get_device_properties(device_id)
66
return device_props.total_memory
67
+
68
+ @classmethod
69
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
70
+ parallel_config = vllm_config.parallel_config
71
+ scheduler_config = vllm_config.scheduler_config
72
+ if parallel_config.worker_cls == "auto":
73
+ if scheduler_config.is_multi_step:
74
+ parallel_config.worker_cls = \
75
+ "vllm.worker.multi_step_worker.MultiStepWorker"
76
+ elif vllm_config.speculative_config:
77
+ parallel_config.worker_cls = \
78
+ "vllm.spec_decode.spec_decode_worker.create_spec_worker"
79
+ else:
80
+ parallel_config.worker_cls = "vllm.worker.worker.Worker"
vllm/platforms/tpu.py
+12
@@ -48,3 +48,15 @@ class TpuPlatform(Platform):
48
49
if compilation_config.backend == "":
50
compilation_config.backend = "openxla"
51
+
52
+ assert vllm_config.speculative_config is None, \
53
+ "TPU does not support speculative decoding"
54
+
55
+ parallel_config = vllm_config.parallel_config
56
+ scheduler_config = vllm_config.scheduler_config
57
+ if parallel_config.worker_cls == "auto":
58
+ if scheduler_config.is_multi_step:
59
+ parallel_config.worker_cls = \
60
+ "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
61
+ else:
62
+ parallel_config.worker_cls = "vllm.worker.tpu_worker.TPUWorker"
vllm/platforms/xpu.py
+6
@@ -57,6 +57,10 @@ class XPUPlatform(Platform):
57
"mode.")
58
model_config.enforce_eager = True
59
60
+ if vllm_config.speculative_config is not None:
61
+ raise NotImplementedError(
62
+ "XPU does not support speculative decoding")
63
+
64
# check and update parallel config
65
parallel_config = vllm_config.parallel_config
66
if (parallel_config.distributed_executor_backend is not None
@@ -66,3 +70,5 @@ class XPUPlatform(Platform):
70
" executor backend.",
71
parallel_config.distributed_executor_backend)
72
parallel_config.distributed_executor_backend = "ray"
73
+ if parallel_config.worker_cls == "auto":
74
+ parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
vllm/worker/worker_base.py
+8
-22
@@ -1,9 +1,8 @@
1
import dataclasses
2
-import importlib
2
import os
3
import time
4
from abc import ABC, abstractmethod
6
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
5
+from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7
import torch
8
@@ -15,7 +14,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput
14
from vllm.platforms import current_platform
15
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
16
from vllm.utils import (enable_trace_function_call_for_thread,
18
- update_environment_variables)
17
+ resolve_obj_by_qualname, update_environment_variables)
18
from vllm.worker.model_runner_base import (BroadcastableModelInput,
19
ModelRunnerBase,
20
ModelRunnerInputBase)
@@ -411,23 +410,14 @@ class WorkerWrapperBase:
410
We first instantiate the WorkerWrapper, which remembers the worker module
411
and class name. Then, when we call `update_environment_variables`, and the
412
real initialization happens in `init_worker`.
414
-
415
- If worker_class_fn is specified, it will be executed to get the worker
416
- class.
417
- Otherwise, the worker class will be obtained by dynamically importing it
418
- using worker_module_name and worker_class_name.
413
"""
414
415
def __init__(
416
self,
423
- worker_module_name: str,
424
- worker_class_name: str,
425
- trust_remote_code: bool = False,
426
- worker_class_fn: Optional[Callable[[],
427
- Type[WorkerBase]]] = None) -> None:
428
- self.worker_module_name = worker_module_name
429
- self.worker_class_name = worker_class_name
430
- self.worker_class_fn = worker_class_fn
417
+ vllm_config: VllmConfig,
418
+ ) -> None:
419
+ self.vllm_config = vllm_config
420
+ trust_remote_code = vllm_config.model_config.trust_remote_code
421
self.worker: Optional[WorkerBase] = None
422
if trust_remote_code:
423
# note: lazy import to avoid importing torch before initializing
@@ -456,12 +446,8 @@ class WorkerWrapperBase:
446
from vllm.plugins import load_general_plugins
447
load_general_plugins()
448
459
- if self.worker_class_fn:
460
- worker_class = self.worker_class_fn()
461
- else:
462
- mod = importlib.import_module(self.worker_module_name)
463
- worker_class = getattr(mod, self.worker_class_name)
464
-
449
+ worker_class = resolve_obj_by_qualname(
450
+ self.vllm_config.parallel_config.worker_cls)
451
self.worker = worker_class(*args, **kwargs)
452
assert self.worker is not None
453