@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a111d0151f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[platforms] absorb worker cls difference into platforms folder (#10555)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Nick Hill <nhill@redhat.com>

youkaichao committed Nov 21, 2024 at 21:00 UTC a111d0151ffed94582bec65635979e04e5b63676
21 files changed +273 -283
vllm/config.py
+117 -121
@@ -926,56 +926,56 @@ class LoadConfig:
926 f"{rocm_supported_load_format}")
927
928
929 +@dataclass
930 class ParallelConfig:
930 - """Configuration for the distributed execution.
931 + """Configuration for the distributed execution."""
932
932 - Args:
933 - pipeline_parallel_size: Number of pipeline parallel groups.
934 - tensor_parallel_size: Number of tensor parallel groups.
935 - worker_use_ray: Deprecated, use distributed_executor_backend instead.
936 - max_parallel_loading_workers: Maximum number of multiple batches
937 - when load model sequentially. To avoid RAM OOM when using tensor
938 - parallel and large models.
939 - disable_custom_all_reduce: Disable the custom all-reduce kernel and
940 - fall back to NCCL.
941 - tokenizer_pool_config: Config for the tokenizer pool.
942 - If None, will use synchronous tokenization.
943 - ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
944 - https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
945 - placement_group: ray distributed model workers placement group.
946 - distributed_executor_backend: Backend to use for distributed model
947 - workers, either "ray" or "mp" (multiprocessing). If the product
948 - of pipeline_parallel_size and tensor_parallel_size is less than
949 - or equal to the number of GPUs available, "mp" will be used to
950 - keep processing on a single host. Otherwise, this will default
951 - to "ray" if Ray is installed and fail otherwise. Note that tpu
952 - and hpu only support Ray for distributed inference.
953 - """
933 + pipeline_parallel_size: int = 1 # Number of pipeline parallel groups.
934 + tensor_parallel_size: int = 1 # Number of tensor parallel groups.
935
955 - def __init__(
956 - self,
957 - pipeline_parallel_size: int,
958 - tensor_parallel_size: int,
959 - worker_use_ray: Optional[bool] = None,
960 - max_parallel_loading_workers: Optional[int] = None,
961 - disable_custom_all_reduce: bool = False,
962 - tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
963 - ray_workers_use_nsight: bool = False,
964 - placement_group: Optional["PlacementGroup"] = None,
965 - distributed_executor_backend: Optional[Union[
966 - str, Type["ExecutorBase"]]] = None,
967 - ) -> None:
968 - self.pipeline_parallel_size = pipeline_parallel_size
969 - self.tensor_parallel_size = tensor_parallel_size
970 - self.distributed_executor_backend = distributed_executor_backend
971 - self.max_parallel_loading_workers = max_parallel_loading_workers
972 - self.disable_custom_all_reduce = disable_custom_all_reduce
973 - self.tokenizer_pool_config = tokenizer_pool_config
974 - self.ray_workers_use_nsight = ray_workers_use_nsight
975 - self.placement_group = placement_group
976 - self.world_size = pipeline_parallel_size * self.tensor_parallel_size
977 -
978 - if worker_use_ray:
936 + # Deprecated, use distributed_executor_backend instead.
937 + worker_use_ray: Optional[bool] = None
938 +
939 + # Maximum number of multiple batches
940 + # when load model sequentially. To avoid RAM OOM when using tensor
941 + # parallel and large models.
942 + max_parallel_loading_workers: Optional[int] = None
943 +
944 + # Disable the custom all-reduce kernel and fall back to NCCL.
945 + disable_custom_all_reduce: bool = False
946 +
947 + # Config for the tokenizer pool. If None, will use synchronous tokenization.
948 + tokenizer_pool_config: Optional[TokenizerPoolConfig] = None
949 +
950 + # Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
951 + ray_workers_use_nsight: bool = False
952 +
953 + # ray distributed model workers placement group.
954 + placement_group: Optional["PlacementGroup"] = None
955 +
956 + # Backend to use for distributed model
957 + # workers, either "ray" or "mp" (multiprocessing). If the product
958 + # of pipeline_parallel_size and tensor_parallel_size is less than
959 + # or equal to the number of GPUs available, "mp" will be used to
960 + # keep processing on a single host. Otherwise, this will default
961 + # to "ray" if Ray is installed and fail otherwise. Note that tpu
962 + # and hpu only support Ray for distributed inference.
963 + distributed_executor_backend: Optional[Union[str,
964 + Type["ExecutorBase"]]] = None
965 +
966 + # the full name of the worker class to use. If "auto", the worker class
967 + # will be determined based on the platform.
968 + worker_cls: str = "auto"
969 +
970 + world_size: int = field(init=False)
971 +
972 + rank: int = 0
973 +
974 + def __post_init__(self) -> None:
975 + self.world_size = self.pipeline_parallel_size * \
976 + self.tensor_parallel_size
977 +
978 + if self.worker_use_ray:
979 if self.distributed_executor_backend is None:
980 self.distributed_executor_backend = "ray"
981 elif not self.use_ray:
@@ -1026,7 +1026,6 @@ class ParallelConfig:
1026 backend)
1027
1028 self._verify_args()
1029 - self.rank: int = 0
1029
1030 @property
1031 def use_ray(self) -> bool:
@@ -1059,100 +1058,97 @@ class ParallelConfig:
1058 "run with Ray.")
1059
1060
1061 +@dataclass
1062 class SchedulerConfig:
1063 - """Scheduler configuration.
1063 + """Scheduler configuration."""
1064
1065 - Args:
1066 - task: The task to use the model for.
1067 - max_num_batched_tokens: Maximum number of tokens to be processed in
1068 - a single iteration.
1069 - max_num_seqs: Maximum number of sequences to be processed in a single
1070 - iteration.
1071 - max_model_len: Maximum length of a sequence (including prompt
1072 - and generated text).
1073 - num_lookahead_slots: The number of slots to allocate per sequence per
1074 - step, beyond the known token ids. This is used in speculative
1075 - decoding to store KV activations of tokens which may or may not be
1076 - accepted.
1077 - delay_factor: Apply a delay (of delay factor multiplied by previous
1078 - prompt latency) before scheduling next prompt.
1079 - enable_chunked_prefill: If True, prefill requests can be chunked based
1080 - on the remaining max_num_batched_tokens.
1081 - preemption_mode: Whether to perform preemption by swapping or
1082 - recomputation. If not specified, we determine the mode as follows:
1083 - We use recomputation by default since it incurs lower overhead than
1084 - swapping. However, when the sequence group has multiple sequences
1085 - (e.g., beam search), recomputation is not currently supported. In
1086 - such a case, we use swapping instead.
1087 - send_delta_data: Private API. If used, scheduler sends delta data to
1088 - workers instead of an entire data. It should be enabled only
1089 - when SPMD worker architecture is enabled. I.e.,
1090 - VLLM_USE_RAY_SPMD_WORKER=1
1091 - policy: The scheduling policy to use. "fcfs" (default) or "priority".
1092 - """
1065 + task: str = "generate" # The task to use the model for.
1066 +
1067 + # Maximum number of tokens to be processed in a single iteration.
1068 + max_num_batched_tokens: int = field(default=None) # type: ignore
1069 +
1070 + # Maximum number of sequences to be processed in a single iteration.
1071 + max_num_seqs: int = 128
1072 +
1073 + # Maximum length of a sequence (including prompt and generated text).
1074 + max_model_len: int = 8192
1075 +
1076 + # The number of slots to allocate per sequence per
1077 + # step, beyond the known token ids. This is used in speculative
1078 + # decoding to store KV activations of tokens which may or may not be
1079 + # accepted.
1080 + num_lookahead_slots: int = 0
1081 +
1082 + # Apply a delay (of delay factor multiplied by previous
1083 + # prompt latency) before scheduling next prompt.
1084 + delay_factor: float = 0.0
1085 +
1086 + # If True, prefill requests can be chunked based
1087 + # on the remaining max_num_batched_tokens.
1088 + enable_chunked_prefill: bool = False
1089 +
1090 + is_multimodal_model: bool = False
1091
1094 - def __init__(self,
1095 - task: _Task,
1096 - max_num_batched_tokens: Optional[int],
1097 - max_num_seqs: int,
1098 - max_model_len: int,
1099 - num_lookahead_slots: int = 0,
1100 - delay_factor: float = 0.0,
1101 - enable_chunked_prefill: bool = False,
1102 - is_multimodal_model: bool = False,
1103 - preemption_mode: Optional[str] = None,
1104 - num_scheduler_steps: int = 1,
1105 - multi_step_stream_outputs: bool = False,
1106 - send_delta_data: bool = False,
1107 - policy: str = "fcfs") -> None:
1108 - if max_num_batched_tokens is None:
1109 - if enable_chunked_prefill:
1110 - if num_scheduler_steps > 1:
1092 + # Whether to perform preemption by swapping or
1093 + # recomputation. If not specified, we determine the mode as follows:
1094 + # We use recomputation by default since it incurs lower overhead than
1095 + # swapping. However, when the sequence group has multiple sequences
1096 + # (e.g., beam search), recomputation is not currently supported. In
1097 + # such a case, we use swapping instead.
1098 + preemption_mode: Optional[str] = None
1099 +
1100 + num_scheduler_steps: int = 1
1101 +
1102 + multi_step_stream_outputs: bool = False
1103 +
1104 + # Private API. If used, scheduler sends delta data to
1105 + # workers instead of an entire data. It should be enabled only
1106 + # when SPMD worker architecture is enabled. I.e.,
1107 + # VLLM_USE_RAY_SPMD_WORKER=1
1108 + send_delta_data: bool = False
1109 +
1110 + # The scheduling policy to use. "fcfs" (default) or "priority".
1111 + policy: str = "fcfs"
1112 +
1113 + chunked_prefill_enabled: bool = field(init=False)
1114 +
1115 + def __post_init__(self) -> None:
1116 + if self.max_num_batched_tokens is None:
1117 + if self.enable_chunked_prefill:
1118 + if self.num_scheduler_steps > 1:
1119 # Multi-step Chunked-Prefill doesn't allow prompt-chunking
1120 # for now. Have max_num_batched_tokens set to max_model_len
1121 # so we don't reject sequences on account of a short
1122 # max_num_batched_tokens.
1115 - max_num_batched_tokens = max(max_model_len, 2048)
1123 + self.max_num_batched_tokens = max(self.max_model_len, 2048)
1124 else:
1125 # It is the values that have the best balance between ITL
1126 # and TTFT on A100. Note it is not optimized for throughput.
1119 - max_num_batched_tokens = 512
1127 + self.max_num_batched_tokens = 512
1128 else:
1129 # If max_model_len is too short, use 2048 as the default value
1130 # for higher throughput.
1123 - max_num_batched_tokens = max(max_model_len, 2048)
1131 + self.max_num_batched_tokens = max(self.max_model_len, 2048)
1132
1125 - if task == "embedding":
1133 + if self.task == "embedding":
1134 # For embedding, choose specific value for higher throughput
1127 - max_num_batched_tokens = max(
1128 - max_num_batched_tokens,
1135 + self.max_num_batched_tokens = max(
1136 + self.max_num_batched_tokens,
1137 _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS,
1138 )
1131 - if is_multimodal_model:
1139 + if self.is_multimodal_model:
1140 # The value needs to be at least the number of multimodal tokens
1133 - max_num_batched_tokens = max(
1134 - max_num_batched_tokens,
1141 + self.max_num_batched_tokens = max(
1142 + self.max_num_batched_tokens,
1143 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
1144 )
1145
1138 - self.max_num_batched_tokens = max_num_batched_tokens
1139 -
1140 - if enable_chunked_prefill:
1146 + if self.enable_chunked_prefill:
1147 logger.info(
1148 "Chunked prefill is enabled with max_num_batched_tokens=%d.",
1149 self.max_num_batched_tokens)
1150
1145 - self.task: Final = task
1146 - self.max_num_seqs = max_num_seqs
1147 - self.max_model_len = max_model_len
1148 - self.num_lookahead_slots = num_lookahead_slots
1149 - self.delay_factor = delay_factor
1150 - self.chunked_prefill_enabled = enable_chunked_prefill
1151 - self.preemption_mode = preemption_mode
1152 - self.num_scheduler_steps = num_scheduler_steps
1153 - self.multi_step_stream_outputs = multi_step_stream_outputs
1154 - self.send_delta_data = send_delta_data
1155 - self.policy = policy
1151 + self.chunked_prefill_enabled = self.enable_chunked_prefill
1152 self._verify_args()
1153
1154 def _verify_args(self) -> None:
@@ -2293,10 +2289,10 @@ class VllmConfig:
2289
2290 model_config: ModelConfig = field(default=None, init=True) # type: ignore
2291 cache_config: CacheConfig = field(default=None, init=True) # type: ignore
2296 - parallel_config: ParallelConfig = field(default=None,
2297 - init=True) # type: ignore
2298 - scheduler_config: SchedulerConfig = field(default=None,
2299 - init=True) # type: ignore
2292 + parallel_config: ParallelConfig = field(default_factory=ParallelConfig,
2293 + init=True)
2294 + scheduler_config: SchedulerConfig = field(default_factory=SchedulerConfig,
2295 + init=True)
2296 device_config: DeviceConfig = field(default=None,
2297 init=True) # type: ignore
2298 load_config: LoadConfig = field(default=None, init=True) # type: ignore
vllm/engine/arg_utils.py
+10 -1
@@ -191,6 +191,7 @@ class EngineArgs:
191 override_neuron_config: Optional[Dict[str, Any]] = None
192 override_pooler_config: Optional[PoolerConfig] = None
193 compilation_config: Optional[CompilationConfig] = None
194 + worker_cls: str = "auto"
195
196 def __post_init__(self):
197 if not self.tokenizer:
@@ -887,6 +888,12 @@ class EngineArgs:
888 'compilers, using -O without space is also '
889 'supported. -O3 is equivalent to -O 3.')
890
891 + parser.add_argument(
892 + '--worker-cls',
893 + type=str,
894 + default="auto",
895 + help='The worker class to use for distributed execution.')
896 +
897 return parser
898
899 @classmethod
@@ -999,7 +1006,9 @@ class EngineArgs:
1006 self.tokenizer_pool_extra_config,
1007 ),
1008 ray_workers_use_nsight=self.ray_workers_use_nsight,
1002 - distributed_executor_backend=self.distributed_executor_backend)
1009 + distributed_executor_backend=self.distributed_executor_backend,
1010 + worker_cls=self.worker_cls,
1011 + )
1012
1013 max_model_len = model_config.max_model_len
1014 use_long_context = max_model_len > 32768
vllm/executor/cpu_executor.py
+1 -6
@@ -115,13 +115,8 @@ class CPUExecutor(ExecutorBase):
115 local_rank: int = 0,
116 rank: int = 0,
117 ):
118 - worker_module_name = "vllm.worker.cpu_worker"
119 - worker_class_name = "CPUWorker"
118
121 - wrapper = WorkerWrapperBase(
122 - worker_module_name=worker_module_name,
123 - worker_class_name=worker_class_name,
124 - )
119 + wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
120
121 assert self.distributed_init_method is not None
122
vllm/executor/gpu_executor.py
+6 -43
@@ -1,4 +1,4 @@
1 -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
1 +from typing import Any, Dict, List, Optional, Set, Tuple, Union
2
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
@@ -8,19 +8,14 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
8 from vllm.sequence import ExecuteModelRequest, PoolerOutput
9 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10 make_async)
11 -from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase
11 +from vllm.worker.worker_base import WorkerWrapperBase
12
13 logger = init_logger(__name__)
14
15
16 -def create_worker(worker_module_name: str, worker_class_name: str,
17 - worker_class_fn: Optional[Callable[[], Type[WorkerBase]]],
18 - **kwargs):
19 - wrapper = WorkerWrapperBase(
20 - worker_module_name=worker_module_name,
21 - worker_class_name=worker_class_name,
22 - worker_class_fn=worker_class_fn,
23 - )
16 +def create_worker(**kwargs):
17 + vllm_config = kwargs.get("vllm_config")
18 + wrapper = WorkerWrapperBase(vllm_config=vllm_config)
19 wrapper.init_worker(**kwargs)
20 return wrapper.worker
21
@@ -57,43 +52,11 @@ class GPUExecutor(ExecutorBase):
52 or (rank % self.parallel_config.tensor_parallel_size == 0),
53 )
54
60 - def _get_worker_module_and_class(
61 - self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
62 - worker_class_fn = None
63 - if self.scheduler_config.is_multi_step:
64 - worker_module_name = "vllm.worker.multi_step_worker"
65 - worker_class_name = "MultiStepWorker"
66 - elif self.speculative_config:
67 - worker_module_name = "vllm.spec_decode.spec_decode_worker"
68 - worker_class_name = "create_spec_worker"
69 - else:
70 - worker_module_name = "vllm.worker.worker"
71 - worker_class_name = "Worker"
72 - return (worker_module_name, worker_class_name, worker_class_fn)
73 -
74 - def _get_create_worker_kwargs(
75 - self,
76 - local_rank: int = 0,
77 - rank: int = 0,
78 - distributed_init_method: Optional[str] = None) -> Dict:
79 - worker_kwargs = self._get_worker_kwargs(local_rank, rank,
80 - distributed_init_method)
81 -
82 - (worker_module_name, worker_class_name,
83 - worker_class_fn) = self._get_worker_module_and_class()
84 - worker_kwargs.update(
85 - worker_module_name=worker_module_name,
86 - worker_class_name=worker_class_name,
87 - worker_class_fn=worker_class_fn,
88 - )
89 -
90 - return worker_kwargs
91 -
55 def _create_worker(self,
56 local_rank: int = 0,
57 rank: int = 0,
58 distributed_init_method: Optional[str] = None):
96 - return create_worker(**self._get_create_worker_kwargs(
59 + return create_worker(**self._get_worker_kwargs(
60 local_rank=local_rank,
61 rank=rank,
62 distributed_init_method=distributed_init_method))
vllm/executor/hpu_executor.py
+1 -4
@@ -48,10 +48,7 @@ class HPUExecutor(ExecutorBase):
48 local_rank: int = 0,
49 rank: int = 0,
50 distributed_init_method: Optional[str] = None):
51 - wrapper = WorkerWrapperBase(
52 - worker_module_name="vllm.worker.hpu_worker",
53 - worker_class_name="HPUWorker",
54 - )
51 + wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
52 wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
53 distributed_init_method))
54 return wrapper.worker
vllm/executor/multiproc_gpu_executor.py
+1 -1
@@ -90,7 +90,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
90 result_handler,
91 partial(
92 create_worker,
93 - **self._get_create_worker_kwargs(
93 + **self._get_worker_kwargs(
94 rank=rank,
95 local_rank=rank,
96 distributed_init_method=distributed_init_method,
vllm/executor/neuron_executor.py
+3 -2
@@ -7,6 +7,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput
7 from vllm.sequence import ExecuteModelRequest
8 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9 make_async)
10 +from vllm.worker.worker_base import WorkerWrapperBase
11
12 logger = init_logger(__name__)
13
@@ -25,10 +26,10 @@ class NeuronExecutor(ExecutorBase):
26 self._init_worker()
27
28 def _init_worker(self):
28 - from vllm.worker.neuron_worker import NeuronWorker
29 + wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
30 distributed_init_method = get_distributed_init_method(
31 get_ip(), get_open_port())
31 - self.driver_worker = NeuronWorker(
32 + self.driver_worker = wrapper.init_worker(
33 vllm_config=self.vllm_config,
34 local_rank=0,
35 rank=0,
vllm/executor/openvino_executor.py
+3 -5
@@ -14,6 +14,7 @@ from vllm.platforms import current_platform
14 from vllm.sequence import ExecuteModelRequest
15 from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
16 get_open_port, make_async)
17 +from vllm.worker.worker_base import WorkerWrapperBase
18
19 logger = init_logger(__name__)
20
@@ -38,15 +39,12 @@ class OpenVINOExecutor(ExecutorBase):
39 self._init_worker()
40
41 def _init_worker(self):
41 - from vllm.worker.openvino_worker import OpenVINOWorker
42
43 - assert (
44 - self.parallel_config.world_size == 1
45 - ), "OpenVINOExecutor only supports single CPU socket currently."
43 + wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
44
45 distributed_init_method = get_distributed_init_method(
46 get_ip(), get_open_port())
49 - self.driver_worker = OpenVINOWorker(
47 + self.driver_worker = wrapper.init_worker(
48 ov_core=self.ov_core,
49 vllm_config=self.vllm_config,
50 local_rank=0,
vllm/executor/ray_gpu_executor.py
+2 -14
@@ -91,17 +91,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
91
92 return ray_remote_kwargs
93
94 - def _get_worker_wrapper_args(self) -> Dict[str, Any]:
95 - (worker_module_name, worker_class_name,
96 - worker_class_fn) = self._get_worker_module_and_class()
97 -
98 - return dict(
99 - worker_module_name=worker_module_name,
100 - worker_class_name=worker_class_name,
101 - worker_class_fn=worker_class_fn,
102 - trust_remote_code=self.model_config.trust_remote_code,
103 - )
104 -
94 # child class could overwrite this to return actual env vars.
95 def _get_env_vars_to_be_updated(self):
96 return self._env_vars_for_all_workers
@@ -135,7 +124,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
124
125 # Create the workers.
126 driver_ip = get_ip()
138 - worker_wrapper_kwargs = self._get_worker_wrapper_args()
127 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
128 if not bundle.get("GPU", 0):
129 continue
@@ -150,7 +138,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
138 num_gpus=num_gpus,
139 scheduling_strategy=scheduling_strategy,
140 **ray_remote_kwargs,
153 - )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
141 + )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
142
143 if self.use_ray_spmd_worker:
144 self.workers.append(worker)
@@ -161,7 +149,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
149 # as the resource holder for the driver process.
150 self.driver_dummy_worker = worker
151 self.driver_worker = RayWorkerWrapper(
164 - **worker_wrapper_kwargs)
152 + vllm_config=self.vllm_config)
153 else:
154 # Else, added to the list of workers.
155 self.workers.append(worker)
vllm/executor/ray_hpu_executor.py
+3 -33
@@ -2,8 +2,7 @@ import asyncio
2 import os
3 from collections import defaultdict
4 from itertools import islice, repeat
5 -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
6 - Type)
5 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6
7 import msgspec
8
@@ -18,7 +17,6 @@ from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18 get_ip, get_open_port, get_vllm_instance_id,
19 make_async)
21 -from vllm.worker.worker_base import WorkerBase
20
21 if ray is not None:
22 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -81,33 +79,6 @@ class RayHPUExecutor(DistributedGPUExecutor):
79 def finish_measurements(self):
80 self._run_workers("finish_measurements")
81
84 - def _get_worker_module_and_class(
85 - self
86 - ) -> Tuple[str, str, Optional[Callable[[],
87 - Type[WorkerBase]]]]: # noqa: F821
88 - worker_class_fn = None
89 - if self.scheduler_config.is_multi_step:
90 - raise NotImplementedError(
91 - "Multi-step execution is not implemented for HPU")
92 - elif self.speculative_config:
93 - raise NotImplementedError(
94 - "Speculative decoding is not implemented for HPU")
95 - else:
96 - worker_module_name = "vllm.worker.hpu_worker"
97 - worker_class_name = "HPUWorker"
98 - return (worker_module_name, worker_class_name, worker_class_fn)
99 -
100 - def _get_worker_wrapper_args(self) -> Dict[str, Any]:
101 - (worker_module_name, worker_class_name,
102 - worker_class_fn) = self._get_worker_module_and_class()
103 -
104 - return dict(
105 - worker_module_name=worker_module_name,
106 - worker_class_name=worker_class_name,
107 - worker_class_fn=worker_class_fn,
108 - trust_remote_code=self.model_config.trust_remote_code,
109 - )
110 -
82 def _init_workers_ray(self, placement_group: "PlacementGroup",
83 **ray_remote_kwargs):
84 # Otherwise, the ray workers are allocated with a full GPU.
@@ -128,7 +99,6 @@ class RayHPUExecutor(DistributedGPUExecutor):
99
100 # Create the workers.
101 driver_ip = get_ip()
131 - worker_wrapper_kwargs = self._get_worker_wrapper_args()
102 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
103 if not bundle.get("HPU", 0):
104 continue
@@ -144,7 +114,7 @@ class RayHPUExecutor(DistributedGPUExecutor):
114 resources={'HPU': num_gpus},
115 scheduling_strategy=scheduling_strategy,
116 **ray_remote_kwargs,
147 - )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
117 + )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
118
119 if self.use_ray_spmd_worker:
120 self.workers.append(worker)
@@ -155,7 +125,7 @@ class RayHPUExecutor(DistributedGPUExecutor):
125 # as the resource holder for the driver process.
126 self.driver_dummy_worker = worker
127 self.driver_worker = RayWorkerWrapper(
158 - **worker_wrapper_kwargs)
128 + vllm_config=self.vllm_config)
129 else:
130 # Else, added to the list of workers.
131 self.workers.append(worker)
vllm/executor/ray_tpu_executor.py
+2 -17
@@ -69,14 +69,6 @@ class RayTPUExecutor(TPUExecutor):
69 placement_group_bundle_index=bundle_id,
70 )
71
72 - assert self.speculative_config is None
73 - if self.scheduler_config.is_multi_step:
74 - worker_module_name = "vllm.worker.multi_step_tpu_worker"
75 - worker_class_name = "MultiStepTPUWorker"
76 - else:
77 - worker_module_name = "vllm.worker.tpu_worker"
78 - worker_class_name = "TPUWorker"
79 -
72 # GKE does not fetch environment information from metadata server
73 # and instead sets these from within the Ray process. Therefore we
74 # need to override the Ray environment variables manually.
@@ -95,11 +87,7 @@ class RayTPUExecutor(TPUExecutor):
87 resources={"TPU": 1},
88 scheduling_strategy=scheduling_strategy,
89 **ray_remote_kwargs,
98 - )(RayWorkerWrapper).remote(
99 - worker_module_name=worker_module_name,
100 - worker_class_name=worker_class_name,
101 - trust_remote_code=self.model_config.trust_remote_code,
102 - )
90 + )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
91 if override_env:
92 worker.override_env_vars.remote(override_env)
93
@@ -109,10 +97,7 @@ class RayTPUExecutor(TPUExecutor):
97 # as the resource holder for the driver process.
98 self.driver_dummy_worker = worker
99 self.driver_worker = RayWorkerWrapper(
112 - worker_module_name=worker_module_name,
113 - worker_class_name=worker_class_name,
114 - trust_remote_code=self.model_config.trust_remote_code,
115 - )
100 + vllm_config=self.vllm_config)
101 else:
102 # Else, added to the list of workers.
103 self.workers.append(worker)
vllm/executor/xpu_executor.py
+1 -13
@@ -1,4 +1,4 @@
1 -from typing import Callable, List, Optional, Tuple, Type, Union
1 +from typing import List, Optional, Union
2
3 from vllm.executor.executor_base import ExecutorAsyncBase
4 from vllm.executor.gpu_executor import GPUExecutor
@@ -6,7 +6,6 @@ from vllm.logger import init_logger
6 from vllm.model_executor.layers.sampler import SamplerOutput
7 from vllm.sequence import ExecuteModelRequest, PoolerOutput
8 from vllm.utils import make_async
9 -from vllm.worker.worker_base import WorkerBase
9
10 logger = init_logger(__name__)
11
@@ -22,17 +21,6 @@ class XPUExecutor(GPUExecutor):
21
22 GPUExecutor._init_executor(self)
23
25 - def _get_worker_module_and_class(
26 - self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
27 - worker_class_fn = None
28 - if self.speculative_config is not None:
29 - raise NotImplementedError(
30 - "XPU does not support speculative decoding")
31 - else:
32 - worker_module_name = "vllm.worker.xpu_worker"
33 - worker_class_name = "XPUWorker"
34 - return (worker_module_name, worker_class_name, worker_class_fn)
35 -
24 def execute_model(
25 self, execute_model_req: ExecuteModelRequest
26 ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
vllm/platforms/cpu.py
+2
@@ -84,3 +84,5 @@ class CpuPlatform(Platform):
84 "distributed executor backend."),
85 parallel_config.distributed_executor_backend)
86 parallel_config.distributed_executor_backend = "mp"
87 + if parallel_config.worker_cls == "auto":
88 + parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
vllm/platforms/cuda.py
+20 -1
@@ -4,7 +4,7 @@ pynvml. However, it should not initialize cuda context.
4
5 import os
6 from functools import lru_cache, wraps
7 -from typing import Callable, List, Tuple, TypeVar
7 +from typing import TYPE_CHECKING, Callable, List, Tuple, TypeVar
8
9 import pynvml
10 import torch
@@ -16,6 +16,11 @@ from vllm.logger import init_logger
16
17 from .interface import DeviceCapability, Platform, PlatformEnum
18
19 +if TYPE_CHECKING:
20 + from vllm.config import VllmConfig
21 +else:
22 + VllmConfig = None
23 +
24 logger = init_logger(__name__)
25
26 _P = ParamSpec("_P")
@@ -157,3 +162,17 @@ class CudaPlatform(Platform):
162 " machine has no NVLink equipped.")
163 return False
164 return True
165 +
166 + @classmethod
167 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
168 + parallel_config = vllm_config.parallel_config
169 + scheduler_config = vllm_config.scheduler_config
170 + if parallel_config.worker_cls == "auto":
171 + if scheduler_config.is_multi_step:
172 + parallel_config.worker_cls = \
173 + "vllm.worker.multi_step_worker.MultiStepWorker"
174 + elif vllm_config.speculative_config:
175 + parallel_config.worker_cls = \
176 + "vllm.spec_decode.spec_decode_worker.create_spec_worker"
177 + else:
178 + parallel_config.worker_cls = "vllm.worker.worker.Worker"
vllm/platforms/hpu.py
+23
@@ -1,7 +1,14 @@
1 +from typing import TYPE_CHECKING
2 +
3 import torch
4
5 from .interface import Platform, PlatformEnum, _Backend
6
7 +if TYPE_CHECKING:
8 + from vllm.config import VllmConfig
9 +else:
10 + VllmConfig = None
11 +
12
13 class HpuPlatform(Platform):
14 _enum = PlatformEnum.HPU
@@ -14,3 +21,19 @@ class HpuPlatform(Platform):
21 @staticmethod
22 def inference_mode():
23 return torch.no_grad()
24 +
25 + @classmethod
26 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
27 +
28 + scheduler_config = vllm_config.scheduler_config
29 + if scheduler_config.is_multi_step:
30 + raise NotImplementedError(
31 + "Multi-step execution is not implemented for HPU")
32 +
33 + if vllm_config.speculative_config is not None:
34 + raise NotImplementedError(
35 + "Speculative decoding is not implemented for HPU")
36 +
37 + parallel_config = vllm_config.parallel_config
38 + if parallel_config.worker_cls == "auto":
39 + parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"
vllm/platforms/neuron.py
+14
@@ -1,5 +1,12 @@
1 +from typing import TYPE_CHECKING
2 +
3 from .interface import Platform, PlatformEnum
4
5 +if TYPE_CHECKING:
6 + from vllm.config import VllmConfig
7 +else:
8 + VllmConfig = None
9 +
10
11 class NeuronPlatform(Platform):
12 _enum = PlatformEnum.NEURON
@@ -8,3 +15,10 @@ class NeuronPlatform(Platform):
15 @classmethod
16 def get_device_name(cls, device_id: int = 0) -> str:
17 return "neuron"
18 +
19 + @classmethod
20 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
21 + parallel_config = vllm_config.parallel_config
22 + if parallel_config.worker_cls == "auto":
23 + parallel_config.worker_cls = \
24 + "vllm.worker.neuron_worker.NeuronWorker"
vllm/platforms/openvino.py
+18
@@ -1,3 +1,5 @@
1 +from typing import TYPE_CHECKING
2 +
3 import torch
4
5 import vllm.envs as envs
@@ -5,6 +7,11 @@ from vllm.logger import init_logger
7
8 from .interface import Platform, PlatformEnum, _Backend
9
10 +if TYPE_CHECKING:
11 + from vllm.config import VllmConfig
12 +else:
13 + VllmConfig = None
14 +
15 logger = init_logger(__name__)
16
17
@@ -38,3 +45,14 @@ class OpenVinoPlatform(Platform):
45 def is_pin_memory_available(self) -> bool:
46 logger.warning("Pin memory is not supported on OpenViNO.")
47 return False
48 +
49 + @classmethod
50 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
51 + parallel_config = vllm_config.parallel_config
52 + assert (
53 + parallel_config.world_size == 1
54 + ), "OpenVINOExecutor only supports single CPU socket currently."
55 +
56 + if parallel_config.worker_cls == "auto":
57 + parallel_config.worker_cls = \
58 + "vllm.worker.openvino_worker.OpenVINOWorker"
vllm/platforms/rocm.py
+20
@@ -1,5 +1,6 @@
1 import os
2 from functools import lru_cache
3 +from typing import TYPE_CHECKING
4
5 import torch
6
@@ -7,6 +8,11 @@ from vllm.logger import init_logger
8
9 from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
10
11 +if TYPE_CHECKING:
12 + from vllm.config import VllmConfig
13 +else:
14 + VllmConfig = None
15 +
16 logger = init_logger(__name__)
17
18 try:
@@ -58,3 +64,17 @@ class RocmPlatform(Platform):
64 def get_device_total_memory(cls, device_id: int = 0) -> int:
65 device_props = torch.cuda.get_device_properties(device_id)
66 return device_props.total_memory
67 +
68 + @classmethod
69 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
70 + parallel_config = vllm_config.parallel_config
71 + scheduler_config = vllm_config.scheduler_config
72 + if parallel_config.worker_cls == "auto":
73 + if scheduler_config.is_multi_step:
74 + parallel_config.worker_cls = \
75 + "vllm.worker.multi_step_worker.MultiStepWorker"
76 + elif vllm_config.speculative_config:
77 + parallel_config.worker_cls = \
78 + "vllm.spec_decode.spec_decode_worker.create_spec_worker"
79 + else:
80 + parallel_config.worker_cls = "vllm.worker.worker.Worker"
vllm/platforms/tpu.py
+12
@@ -48,3 +48,15 @@ class TpuPlatform(Platform):
48
49 if compilation_config.backend == "":
50 compilation_config.backend = "openxla"
51 +
52 + assert vllm_config.speculative_config is None, \
53 + "TPU does not support speculative decoding"
54 +
55 + parallel_config = vllm_config.parallel_config
56 + scheduler_config = vllm_config.scheduler_config
57 + if parallel_config.worker_cls == "auto":
58 + if scheduler_config.is_multi_step:
59 + parallel_config.worker_cls = \
60 + "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
61 + else:
62 + parallel_config.worker_cls = "vllm.worker.tpu_worker.TPUWorker"
vllm/platforms/xpu.py
+6
@@ -57,6 +57,10 @@ class XPUPlatform(Platform):
57 "mode.")
58 model_config.enforce_eager = True
59
60 + if vllm_config.speculative_config is not None:
61 + raise NotImplementedError(
62 + "XPU does not support speculative decoding")
63 +
64 # check and update parallel config
65 parallel_config = vllm_config.parallel_config
66 if (parallel_config.distributed_executor_backend is not None
@@ -66,3 +70,5 @@ class XPUPlatform(Platform):
70 " executor backend.",
71 parallel_config.distributed_executor_backend)
72 parallel_config.distributed_executor_backend = "ray"
73 + if parallel_config.worker_cls == "auto":
74 + parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
vllm/worker/worker_base.py
+8 -22
@@ -1,9 +1,8 @@
1 import dataclasses
2 -import importlib
2 import os
3 import time
4 from abc import ABC, abstractmethod
6 -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
5 +from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7 import torch
8
@@ -15,7 +14,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.platforms import current_platform
15 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
16 from vllm.utils import (enable_trace_function_call_for_thread,
18 - update_environment_variables)
17 + resolve_obj_by_qualname, update_environment_variables)
18 from vllm.worker.model_runner_base import (BroadcastableModelInput,
19 ModelRunnerBase,
20 ModelRunnerInputBase)
@@ -411,23 +410,14 @@ class WorkerWrapperBase:
410 We first instantiate the WorkerWrapper, which remembers the worker module
411 and class name. Then, when we call `update_environment_variables`, and the
412 real initialization happens in `init_worker`.
414 -
415 - If worker_class_fn is specified, it will be executed to get the worker
416 - class.
417 - Otherwise, the worker class will be obtained by dynamically importing it
418 - using worker_module_name and worker_class_name.
413 """
414
415 def __init__(
416 self,
423 - worker_module_name: str,
424 - worker_class_name: str,
425 - trust_remote_code: bool = False,
426 - worker_class_fn: Optional[Callable[[],
427 - Type[WorkerBase]]] = None) -> None:
428 - self.worker_module_name = worker_module_name
429 - self.worker_class_name = worker_class_name
430 - self.worker_class_fn = worker_class_fn
417 + vllm_config: VllmConfig,
418 + ) -> None:
419 + self.vllm_config = vllm_config
420 + trust_remote_code = vllm_config.model_config.trust_remote_code
421 self.worker: Optional[WorkerBase] = None
422 if trust_remote_code:
423 # note: lazy import to avoid importing torch before initializing
@@ -456,12 +446,8 @@ class WorkerWrapperBase:
446 from vllm.plugins import load_general_plugins
447 load_general_plugins()
448
459 - if self.worker_class_fn:
460 - worker_class = self.worker_class_fn()
461 - else:
462 - mod = importlib.import_module(self.worker_module_name)
463 - worker_class = getattr(mod, self.worker_class_name)
464 -
449 + worker_class = resolve_obj_by_qualname(
450 + self.vllm_config.parallel_config.worker_cls)
451 self.worker = worker_class(*args, **kwargs)
452 assert self.worker is not None
453