@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d0e3d3724
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] clean up executor class hierarchy between v1 and v0 (#12171)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 18, 2025 at 14:35 UTC 6d0e3d372446cde48b387d4d3530e25fc6e06320
6 files changed +61 -798
vllm/executor/executor_base.py
-10
@@ -79,16 +79,6 @@ class ExecutorBase(ABC):
79 b = min([r[1] for r in results])
80 return a, b
81
82 - def initialize(self, num_gpu_blocks: int) -> None:
83 - """
84 - Initialize the KV caches and begin the model execution loop of the
85 - underlying workers.
86 - For V1 compatibility.
87 - """
88 - logger.info("# GPU blocks: %d", num_gpu_blocks)
89 - self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
90 - self.collective_rpc("compile_or_warm_up_model")
91 -
82 def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
83 """Initialize the KV cache by invoking the underlying worker.
84 """
vllm/v1/executor/abstract.py
+58 -29
@@ -1,63 +1,92 @@
1 -from abc import ABC, abstractmethod
1 from typing import Type
2
3 from vllm.config import VllmConfig
4 +from vllm.executor.executor_base import ExecutorBase
5 +from vllm.executor.ray_distributed_executor import ( # noqa
6 + RayDistributedExecutor as RayDistributedExecutorV0)
7 +from vllm.executor.uniproc_executor import ( # noqa
8 + ExecutorWithExternalLauncher as ExecutorWithExternalLauncherV0)
9 +from vllm.executor.uniproc_executor import ( # noqa
10 + UniProcExecutor as UniProcExecutorV0)
11 from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
12 from vllm.v1.outputs import ModelRunnerOutput
13
14
9 -class Executor(ABC):
10 - """Abstract class for executors."""
15 +class Executor(ExecutorBase):
16 + """
17 + Abstract class for v1 executors, mainly define some methods for v1.
18 + For methods shared by v0 and v1, define them in ExecutorBase"""
19
20 @staticmethod
21 def get_class(vllm_config: VllmConfig) -> Type["Executor"]:
22 executor_class: Type[Executor]
23 + parallel_config = vllm_config.parallel_config
24 distributed_executor_backend = (
16 - vllm_config.parallel_config.distributed_executor_backend)
25 + parallel_config.distributed_executor_backend)
26 + if distributed_executor_backend is None:
27 + # If the user does not specify the distributed executor backend,
28 + # we will choose the backend based on the world size.
29 + if parallel_config.world_size > 1:
30 + distributed_executor_backend = "mp"
31 + else:
32 + distributed_executor_backend = "uni"
33 +
34 if distributed_executor_backend == "ray":
18 - from vllm.executor.ray_distributed_executor import ( # noqa
19 - RayDistributedExecutor)
35 executor_class = RayDistributedExecutor
36 elif distributed_executor_backend == "mp":
37 from vllm.v1.executor.multiproc_executor import MultiprocExecutor
38 executor_class = MultiprocExecutor
39 + elif distributed_executor_backend == "uni":
40 + executor_class = UniProcExecutor
41 + elif distributed_executor_backend == "external_launcher":
42 + # TODO: make v1 scheduling deterministic
43 + # to support external launcher
44 + executor_class = ExecutorWithExternalLauncher
45 else:
25 - assert (distributed_executor_backend is None)
26 - from vllm.v1.executor.uniproc_executor import UniprocExecutor
27 - executor_class = UniprocExecutor
46 + raise ValueError("Unknown distributed executor backend: "
47 + f"{distributed_executor_backend}")
48 return executor_class
49
30 - @abstractmethod
31 - def __init__(self, vllm_config: VllmConfig) -> None:
32 - raise NotImplementedError
33 -
34 - @abstractmethod
50 def initialize(self, kv_cache_config: KVCacheConfig) -> None:
36 - raise NotImplementedError
51 + """
52 + Initialize the KV caches and begin the model execution loop of the
53 + underlying workers.
54 + """
55 + self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
56 + self.collective_rpc("compile_or_warm_up_model")
57
38 - @abstractmethod
58 def determine_available_memory(self) -> int: # in bytes
40 - raise NotImplementedError
59 + output = self.collective_rpc("determine_available_memory")
60 + # Since we use a shared centralized controller, we take the minimum
61 + # memory size across all workers to make sure all the memory
62 + # operators can be applied to all workers.
63 + return min(output)
64
42 - @abstractmethod
65 def get_kv_cache_spec(self) -> KVCacheSpec:
44 - raise NotImplementedError
66 + output = self.collective_rpc("get_kv_cache_spec")
67 + for x in output:
68 + assert x == output[0]
69 + return output[0]
70
46 - @abstractmethod
71 def execute_model(
72 self,
73 scheduler_output,
74 ) -> ModelRunnerOutput:
51 - raise NotImplementedError
75 + output = self.collective_rpc("execute_model",
76 + args=(scheduler_output, ))
77 + return output[0]
78
53 - @abstractmethod
79 def profile(self, is_start: bool = True):
55 - raise NotImplementedError
80 + self.collective_rpc("profile", args=(is_start, ))
81 +
82 +
83 +class UniProcExecutor(UniProcExecutorV0, Executor):
84 + pass
85 +
86 +
87 +class ExecutorWithExternalLauncher(ExecutorWithExternalLauncherV0, Executor):
88 + pass
89
57 - @abstractmethod
58 - def shutdown(self):
59 - pass
90
61 - @abstractmethod
62 - def check_health(self) -> None:
63 - raise NotImplementedError
91 +class RayDistributedExecutor(RayDistributedExecutorV0, Executor):
92 + pass
vllm/v1/executor/multiproc_executor.py
+3 -47
@@ -25,8 +25,6 @@ from vllm.logger import init_logger
25 from vllm.utils import (get_distributed_init_method, get_mp_context,
26 get_open_port, get_open_zmq_ipc_path, zmq_socket_ctx)
27 from vllm.v1.executor.abstract import Executor
28 -from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
29 -from vllm.v1.outputs import ModelRunnerOutput
28 from vllm.worker.worker_base import WorkerWrapperBase
29
30 logger = init_logger(__name__)
@@ -37,7 +35,7 @@ POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
35
36 class MultiprocExecutor(Executor):
37
40 - def __init__(self, vllm_config: VllmConfig) -> None:
38 + def _init_executor(self) -> None:
39 # Call self.shutdown at exit to clean up
40 # and ensure workers will be terminated.
41 self._finalizer = weakref.finalize(self, self.shutdown)
@@ -55,9 +53,6 @@ class MultiprocExecutor(Executor):
53
54 signal.signal(signal.SIGUSR1, sigusr1_handler)
55
58 - self.vllm_config = vllm_config
59 - self.parallel_config = vllm_config.parallel_config
60 -
56 self.world_size = self.parallel_config.world_size
57 tensor_parallel_size = self.parallel_config.tensor_parallel_size
58 assert self.world_size == tensor_parallel_size, (
@@ -82,7 +77,8 @@ class MultiprocExecutor(Executor):
77 # Create workers
78 self.workers: List[WorkerProcHandle] = []
79 for rank in range(self.world_size):
85 - worker = WorkerProc.make_worker_process(vllm_config, rank, rank,
80 + worker = WorkerProc.make_worker_process(self.vllm_config, rank,
81 + rank,
82 distributed_init_method,
83 scheduler_output_handle)
84 self.workers.append(worker)
@@ -93,34 +89,6 @@ class MultiprocExecutor(Executor):
89 for w in self.workers:
90 w.worker_response_mq.wait_until_ready()
91
96 - def initialize(self, kv_cache_config: KVCacheConfig) -> None:
97 - """
98 - Initialize the KV caches and begin the model execution loop of the
99 - underlying workers.
100 - """
101 - self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
102 - self.collective_rpc("compile_or_warm_up_model")
103 -
104 - def determine_available_memory(self) -> int:
105 - """
106 - Determine the available memory (in bytes) for KV cache by invoking the
107 - underlying worker.
108 - """
109 - memory_sizes = self.collective_rpc("determine_available_memory")
110 -
111 - # Since we use a shared centralized controller, we take the minimum
112 - # memory size across all workers to make sure all the memory
113 - # operators can be applied to all workers.
114 - return min(memory_sizes)
115 -
116 - def get_kv_cache_spec(self) -> KVCacheSpec:
117 - """
118 - Get all kv cache needed by the model by invoking the underlying worker.
119 - """
120 - kv_cache_specs = self.collective_rpc("get_kv_cache_spec")
121 - assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
122 - return kv_cache_specs[0]
123 -
92 def collective_rpc(self,
93 method: Union[str, Callable],
94 timeout: Optional[float] = None,
@@ -172,18 +140,6 @@ class MultiprocExecutor(Executor):
140 # Re-raise any other exceptions
141 raise e
142
175 - def execute_model(
176 - self,
177 - scheduler_output,
178 - ) -> ModelRunnerOutput:
179 - model_output = self.collective_rpc("execute_model",
180 - args=(scheduler_output, ))[0]
181 - return model_output
182 -
183 - def profile(self, is_start: bool = True):
184 - self.collective_rpc("profile", args=(is_start, ))
185 - return
186 -
143 def _ensure_worker_termination(self):
144 """Ensure that all worker processes are terminated. Assumes workers have
145 received termination requests. Waits for processing, then sends
vllm/v1/executor/ray_executor.py deleted
-344
@@ -1,344 +0,0 @@
1 -import os
2 -from collections import defaultdict
3 -from itertools import islice, repeat
4 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
5 -
6 -import vllm.envs as envs
7 -from vllm.config import VllmConfig
8 -from vllm.logger import init_logger
9 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10 -from vllm.v1.executor.abstract import Executor
11 -from vllm.v1.executor.ray_utils import (RayWorkerWrapper,
12 - initialize_ray_cluster, ray)
13 -from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
14 -from vllm.v1.outputs import ModelRunnerOutput
15 -
16 -if ray is not None:
17 - from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
18 -
19 -if TYPE_CHECKING:
20 - from ray.util.placement_group import PlacementGroup
21 -
22 -logger = init_logger(__name__)
23 -
24 -
25 -class RayExecutor(Executor):
26 -
27 - def __init__(self, vllm_config: VllmConfig) -> None:
28 - self.vllm_config = vllm_config
29 - self.parallel_config = vllm_config.parallel_config
30 - self.model_config = vllm_config.model_config
31 - self.forward_dag: Optional[ray.dag.CompiledDAG] = None
32 -
33 - # Disable Ray usage stats collection.
34 - ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
35 - if ray_usage != "1":
36 - os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
37 -
38 - initialize_ray_cluster(self.parallel_config)
39 - placement_group = self.parallel_config.placement_group
40 -
41 - # Create the parallel GPU workers.
42 - self._init_workers_ray(placement_group)
43 -
44 - def _init_workers_ray(self, placement_group: "PlacementGroup",
45 - **ray_remote_kwargs):
46 - # A list of workers to run a model.
47 - self.workers: List[RayWorkerWrapper] = []
48 - if self.parallel_config.ray_workers_use_nsight:
49 - ray_remote_kwargs = self._configure_ray_workers_use_nsight(
50 - ray_remote_kwargs)
51 -
52 - # Create the workers.
53 - driver_ip = get_ip()
54 - for bundle_id, bundle in enumerate(placement_group.bundle_specs):
55 - if not bundle.get("GPU", 0):
56 - # Skip bundles that don't have GPUs,
57 - # as each worker needs one GPU.
58 - continue
59 - scheduling_strategy = PlacementGroupSchedulingStrategy(
60 - placement_group=placement_group,
61 - placement_group_capture_child_tasks=True,
62 - placement_group_bundle_index=bundle_id,
63 - )
64 -
65 - worker = ray.remote(
66 - num_cpus=0,
67 - num_gpus=1,
68 - scheduling_strategy=scheduling_strategy,
69 - **ray_remote_kwargs,
70 - )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
71 - self.workers.append(worker)
72 -
73 - logger.debug("workers: %s", self.workers)
74 - worker_ips = [
75 - ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
76 - for worker in self.workers
77 - ]
78 - ip_counts: Dict[str, int] = {}
79 - for ip in worker_ips:
80 - ip_counts[ip] = ip_counts.get(ip, 0) + 1
81 -
82 - worker_to_ip = dict(zip(self.workers, worker_ips))
83 -
84 - def sort_by_driver_then_worker_ip(worker):
85 - """
86 - Sort the workers based on 3 properties:
87 - 1. If the worker is on the same node as the driver (vllm engine),
88 - it should be placed first.
89 - 2. Then, if the worker is on a node with fewer workers, it should
90 - be placed first.
91 - 3. Finally, if the work is on a node with smaller IP address, it
92 - should be placed first. This is simply a tiebreaker to make
93 - sure the workers are sorted in a deterministic way.
94 - """
95 - ip = worker_to_ip[worker]
96 - return (ip != driver_ip, ip_counts[ip], ip)
97 -
98 - # After sorting, the workers on the same node will be
99 - # close to each other, and the workers on the driver
100 - # node will be placed first.
101 - self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
102 -
103 - # Get the set of GPU IDs used on each node.
104 - worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids")
105 -
106 - node_workers = defaultdict(list) # node id -> list of worker ranks
107 - node_gpus = defaultdict(list) # node id -> list of gpu ids
108 -
109 - for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
110 - node_workers[node_id].append(i)
111 - # `gpu_ids` can be a list of strings or integers.
112 - # convert them to integers for consistency.
113 - # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
114 - # string sorting is not sufficient.
115 - # see https://github.com/vllm-project/vllm/issues/5590
116 - gpu_ids = [int(x) for x in gpu_ids]
117 - node_gpus[node_id].extend(gpu_ids)
118 -
119 - for node_id, gpu_ids in node_gpus.items():
120 - node_gpus[node_id] = sorted(gpu_ids)
121 -
122 - all_ips = set(worker_ips)
123 - n_ips = len(all_ips)
124 - n_nodes = len(node_workers)
125 -
126 - if n_nodes != n_ips:
127 - raise RuntimeError(
128 - f"Every node should have a unique IP address. Got {n_nodes}"
129 - f" nodes with node ids {list(node_workers.keys())} and "
130 - f"{n_ips} unique IP addresses {all_ips}. Please check your"
131 - " network configuration. If you set `VLLM_HOST_IP` or "
132 - "`HOST_IP` environment variable, make sure it is unique for"
133 - " each node.")
134 -
135 - # Set environment variables for the driver and workers.
136 - all_args_to_update_environment_variables = [({
137 - "CUDA_VISIBLE_DEVICES":
138 - ",".join(map(str, node_gpus[node_id])),
139 - "VLLM_TRACE_FUNCTION":
140 - str(envs.VLLM_TRACE_FUNCTION),
141 - "VLLM_USE_V1":
142 - str(int(envs.VLLM_USE_V1)),
143 - **({
144 - "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
145 - } if envs.VLLM_ATTENTION_BACKEND is not None else {})
146 - }, ) for (node_id, _) in worker_node_and_gpu_ids]
147 -
148 - self._env_vars_for_all_workers = (
149 - all_args_to_update_environment_variables)
150 -
151 - self._run_workers("update_environment_variables",
152 - all_args=self._get_env_vars_to_be_updated())
153 -
154 - if len(node_gpus) == 1:
155 - # in single node case, we don't need to get the IP address.
156 - # the loopback address is sufficient
157 - # NOTE: a node may have several IP addresses, one for each
158 - # network interface. `get_ip()` might return any of them,
159 - # while they might not work for communication inside the node
160 - # if the network setup is complicated. Using the loopback address
161 - # solves this issue, as it always works for communication inside
162 - # the node.
163 - driver_ip = "127.0.0.1"
164 - distributed_init_method = get_distributed_init_method(
165 - driver_ip, get_open_port())
166 -
167 - # Initialize the actual workers inside worker wrapper.
168 - init_worker_all_kwargs = [
169 - self._get_worker_kwargs(
170 - local_rank=node_workers[node_id].index(rank),
171 - rank=rank,
172 - distributed_init_method=distributed_init_method,
173 - ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
174 - ]
175 - self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
176 - self._run_workers("initialize")
177 - self._run_workers("load_model")
178 -
179 - def _configure_ray_workers_use_nsight(self,
180 - ray_remote_kwargs) -> Dict[str, Any]:
181 - # If nsight profiling is enabled, we need to set the profiling
182 - # configuration for the ray workers as runtime env.
183 - runtime_env = ray_remote_kwargs.setdefault("runtime_env", {})
184 - runtime_env.update({
185 - "nsight": {
186 - "t": "cuda,cudnn,cublas",
187 - "o": "'worker_process_%p'",
188 - "cuda-graph-trace": "node",
189 - }
190 - })
191 -
192 - return ray_remote_kwargs
193 -
194 - def _get_env_vars_to_be_updated(self):
195 - return self._env_vars_for_all_workers
196 -
197 - def _get_worker_kwargs(
198 - self,
199 - local_rank: int = 0,
200 - rank: int = 0,
201 - distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
202 - """
203 - Return worker init args for a given rank.
204 - """
205 - if distributed_init_method is None:
206 - distributed_init_method = get_distributed_init_method(
207 - get_ip(), get_open_port())
208 - return dict(
209 - vllm_config=self.vllm_config,
210 - local_rank=local_rank,
211 - rank=rank,
212 - distributed_init_method=distributed_init_method,
213 - )
214 -
215 - def determine_available_memory(self) -> int:
216 - """
217 - Determine the available GPU memory in bytes.
218 -
219 - This invokes `determine_available_memory` on each worker and takes
220 - the min of the results, guaranteeing that the selected cache sizes are
221 - compatible with all workers.
222 - """
223 -
224 - memory_sizes = self._run_workers("determine_available_memory")
225 -
226 - # Since we use a shared centralized controller, we take the minimum
227 - # memory size across all workers to make sure all the memory
228 - # operators can be applied to all workers.
229 - return min(memory_sizes)
230 -
231 - def initialize(self, kv_cache_config: KVCacheConfig) -> None:
232 - """
233 - Initialize the KV cache in all workers.
234 - """
235 - self._run_workers("initialize_cache", kv_cache_config)
236 - self._run_workers("compile_or_warm_up_model")
237 -
238 - def get_kv_cache_spec(self) -> KVCacheSpec:
239 - """
240 - Get all kv cache needed by the model
241 -
242 - This invokes `get_kv_cache_spec` on each worker and asserts that
243 - they are identical. The KVCacheSpec is then returned.
244 - """
245 - kv_cache_specs = self._run_workers("get_kv_cache_spec")
246 - assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
247 - return kv_cache_specs[0]
248 -
249 - def _run_workers(
250 - self,
251 - method: str,
252 - *args,
253 - all_args: Optional[List[Tuple[Any, ...]]] = None,
254 - all_kwargs: Optional[List[Dict[str, Any]]] = None,
255 - **kwargs,
256 - ) -> Any:
257 - """
258 - Runs the given method on all workers. Can be used in the following
259 - ways:
260 -
261 - Args:
262 - - args/kwargs: All workers share the same args/kwargs
263 - - all_args/all_kwargs: args/kwargs for each worker are specified
264 - individually
265 - """
266 - count = len(self.workers)
267 - all_worker_args = repeat(args, count) if all_args is None \
268 - else islice(all_args, 0, None)
269 - all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
270 - else islice(all_kwargs, 0, None)
271 -
272 - ray_worker_refs = [
273 - worker.execute_method.remote( # type: ignore[attr-defined]
274 - method, *worker_args, **worker_kwargs)
275 - for (worker, worker_args, worker_kwargs
276 - ) in zip(self.workers, all_worker_args, all_worker_kwargs)
277 - ]
278 - return ray.get(ray_worker_refs)
279 -
280 - def execute_model(
281 - self,
282 - scheduler_output,
283 - ) -> ModelRunnerOutput:
284 - if self.forward_dag is None:
285 - self.forward_dag = self._compiled_ray_dag()
286 - # Only the first worker (with rank 0) returns the execution result.
287 - # Others return None.
288 - output = ray.get(self.forward_dag.execute(scheduler_output))[0]
289 - return output
290 -
291 - def profile(self, is_start=True):
292 - raise NotImplementedError
293 -
294 - def shutdown(self):
295 - if hasattr(self, "forward_dag") and self.forward_dag is not None:
296 - self.forward_dag.teardown()
297 - import ray
298 - for worker in self.workers:
299 - ray.kill(worker)
300 - self.forward_dag = None
301 -
302 - def check_health(self) -> None:
303 - logger.debug("Called check_health.")
304 -
305 - def _check_ray_compiled_graph_installation(self):
306 - import pkg_resources
307 - from packaging import version
308 -
309 - required_version = version.parse("2.39")
310 - current_version = version.parse(
311 - pkg_resources.get_distribution("ray").version)
312 - if current_version < required_version:
313 - raise ValueError(f"Ray version {required_version} is "
314 - f"required, but found {current_version}")
315 -
316 - import importlib.util
317 - raycg = importlib.util.find_spec("ray.experimental.compiled_dag_ref")
318 - if raycg is None:
319 - raise ValueError("Ray Compiled Graph is not installed. "
320 - "Run `pip install ray[adag]` to install it.")
321 -
322 - cupy_spec = importlib.util.find_spec("cupy")
323 - if cupy_spec is None and envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL:
324 - raise ValueError(
325 - "cupy is not installed but required since "
326 - "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL is set."
327 - "Run `pip install ray[adag]` and check cupy installation.")
328 -
329 - def _compiled_ray_dag(self):
330 - assert self.parallel_config.use_ray
331 - self._check_ray_compiled_graph_installation()
332 - from ray.dag import InputNode, MultiOutputNode
333 -
334 - with InputNode() as input_batches:
335 - outputs = [
336 - worker.execute_model.bind( # type: ignore[attr-defined]
337 - input_batches) for worker in self.workers
338 - ]
339 - forward_dag = MultiOutputNode(outputs)
340 -
341 - return forward_dag.experimental_compile()
342 -
343 - def __del__(self):
344 - self.shutdown()
vllm/v1/executor/ray_utils.py deleted
-280
@@ -1,280 +0,0 @@
1 -import time
2 -from collections import defaultdict
3 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
4 -
5 -from vllm.config import ParallelConfig
6 -from vllm.logger import init_logger
7 -from vllm.platforms import current_platform
8 -from vllm.utils import get_ip
9 -from vllm.v1.outputs import ModelRunnerOutput
10 -from vllm.worker.worker_base import WorkerWrapperBase
11 -
12 -if TYPE_CHECKING:
13 - from vllm.v1.core.scheduler import SchedulerOutput
14 -
15 -logger = init_logger(__name__)
16 -PG_WAIT_TIMEOUT = 60
17 -
18 -try:
19 - import ray
20 - from ray.util import placement_group_table
21 - from ray.util.placement_group import PlacementGroup
22 - try:
23 - from ray._private.state import available_resources_per_node
24 - except ImportError:
25 - # Ray 2.9.x doesn't expose `available_resources_per_node`
26 - from ray._private.state import state as _state
27 - available_resources_per_node = _state._available_resources_per_node
28 -
29 - class RayWorkerWrapper(WorkerWrapperBase):
30 -
31 - def __init__(self, *args, **kwargs) -> None:
32 - super().__init__(*args, **kwargs)
33 - # Since the compiled DAG runs a main execution
34 - # in a different thread that calls cuda.set_device.
35 - # The flag indicates is set_device is called on
36 - # that thread. It will be removed soon.
37 - self.compiled_dag_cuda_device_set = False
38 -
39 - def get_node_ip(self) -> str:
40 - return get_ip()
41 -
42 - def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
43 - node_id = ray.get_runtime_context().get_node_id()
44 - device_key = current_platform.ray_device_key
45 - if not device_key:
46 - raise RuntimeError("current platform %s does not support ray.",
47 - current_platform.device_name)
48 - gpu_ids = ray.get_runtime_context().get_accelerator_ids(
49 - )[device_key]
50 - return node_id, gpu_ids
51 -
52 - def setup_device_if_necessary(self):
53 - # TODO(swang): This is needed right now because Ray CG executes
54 - # on a background thread, so we need to reset torch's current
55 - # device.
56 - # We can remove this API after it is fixed in compiled graph.
57 - import torch
58 - assert self.worker is not None, "Worker is not initialized"
59 - if not self.compiled_dag_cuda_device_set:
60 - torch.cuda.set_device(self.worker.device)
61 - self.compiled_dag_cuda_device_set = True
62 -
63 - def execute_model(
64 - self,
65 - scheduler_output: "SchedulerOutput",
66 - ) -> ModelRunnerOutput:
67 - self.setup_device_if_necessary()
68 - assert self.worker is not None, "Worker is not initialized"
69 - output = self.worker.model_runner.execute_model(scheduler_output)
70 - return output
71 -
72 - ray_import_err = None
73 -
74 -except ImportError as e:
75 - ray = None # type: ignore
76 - ray_import_err = e
77 - RayWorkerWrapper = None # type: ignore
78 -
79 -
80 -def ray_is_available() -> bool:
81 - """Returns True if Ray is available."""
82 - return ray is not None
83 -
84 -
85 -def assert_ray_available():
86 - """
87 - Raise an exception if Ray is not available.
88 - """
89 - if ray is None:
90 - raise ValueError("Failed to import Ray, please install Ray with "
91 - "`pip install ray`.") from ray_import_err
92 -
93 -
94 -def _verify_bundles(placement_group: "PlacementGroup",
95 - parallel_config: ParallelConfig, device_str: str):
96 - """
97 - Verify a given placement group has bundles located in the right place.
98 -
99 - There are 2 rules.
100 - - Warn if all tensor parallel workers cannot fit in a single node.
101 - - Fail if driver node is not included in a placement group.
102 -
103 - Args:
104 - placement_group: The placement group to verify.
105 - parallel_config: The parallel configuration.
106 - device_str: The required device.
107 - """
108 - assert ray.is_initialized(), (
109 - "Ray is not initialized although distributed-executor-backend is ray.")
110 - pg_data = placement_group_table(placement_group)
111 - # bundle_idx -> node_id
112 - bundle_to_node_ids = pg_data["bundles_to_node_id"]
113 - # bundle_idx -> bundle (e.g., {"GPU": 1})
114 - bundles = pg_data["bundles"]
115 - # node_id -> List of bundle (e.g., {"GPU": 1})
116 - node_id_to_bundle: Dict[str, List[Dict[str, float]]] = defaultdict(list)
117 -
118 - for bundle_idx, node_id in bundle_to_node_ids.items():
119 - node_id_to_bundle[node_id].append(bundles[bundle_idx])
120 - driver_node_id = ray.get_runtime_context().get_node_id()
121 -
122 - if driver_node_id not in node_id_to_bundle:
123 - raise RuntimeError(
124 - f"driver node id {driver_node_id} is not included in a placement "
125 - f"group {placement_group.id}. Node id -> bundles "
126 - f"{node_id_to_bundle}. "
127 - "You don't have enough GPUs available in a current node. Check "
128 - "`ray status` to see if you have available GPUs in a node "
129 - f"{driver_node_id} before starting an vLLM engine.")
130 -
131 - for node_id, bundles in node_id_to_bundle.items():
132 - if len(bundles) < parallel_config.tensor_parallel_size:
133 - logger.warning(
134 - "tensor_parallel_size=%d "
135 - "is bigger than a reserved number of %ss (%d "
136 - "%ss) in a node %s. Tensor parallel workers can be "
137 - "spread out to 2+ nodes which can degrade the performance "
138 - "unless you have fast interconnect across nodes, like "
139 - "Infiniband. To resolve this issue, make sure you have more "
140 - "than %d GPUs available at each node.",
141 - parallel_config.tensor_parallel_size, device_str, len(bundles),
142 - device_str, node_id, parallel_config.tensor_parallel_size)
143 -
144 -
145 -def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
146 - """Wait until a placement group is ready.
147 -
148 - It prints the informative log messages if the placement group is
149 - not created within time.
150 -
151 - """
152 - # Wait until PG is ready - this will block until all
153 - # requested resources are available, and will timeout
154 - # if they cannot be provisioned.
155 - placement_group_specs = current_placement_group.bundle_specs
156 -
157 - s = time.time()
158 - pg_ready_ref = current_placement_group.ready()
159 - wait_interval = 10
160 - while time.time() - s < PG_WAIT_TIMEOUT:
161 - ready, _ = ray.wait([pg_ready_ref], timeout=wait_interval)
162 - if len(ready) > 0:
163 - break
164 -
165 - # Exponential backoff for warning print.
166 - wait_interval *= 2
167 - logger.info(
168 - "Waiting for creating a placement group of specs for "
169 - "%d seconds. specs=%s. Check "
170 - "`ray status` to see if you have enough resources.",
171 - int(time.time() - s), placement_group_specs)
172 -
173 - try:
174 - ray.get(pg_ready_ref, timeout=0)
175 - except ray.exceptions.GetTimeoutError:
176 - raise ValueError(
177 - "Cannot provide a placement group of "
178 - f"{placement_group_specs=} within {PG_WAIT_TIMEOUT} seconds. See "
179 - "`ray status` to make sure the cluster has enough resources."
180 - ) from None
181 -
182 -
183 -def initialize_ray_cluster(
184 - parallel_config: ParallelConfig,
185 - ray_address: Optional[str] = None,
186 -):
187 - """Initialize the distributed cluster with Ray.
188 -
189 - it will connect to the Ray cluster and create a placement group
190 - for the workers, which includes the specification of the resources
191 - for each distributed worker.
192 -
193 - Args:
194 - parallel_config: The configurations for parallel execution.
195 - ray_address: The address of the Ray cluster. If None, uses
196 - the default Ray cluster address.
197 - """
198 - assert_ray_available()
199 -
200 - # Connect to a ray cluster.
201 - if current_platform.is_rocm() or current_platform.is_xpu():
202 - # Try to connect existing ray instance and create a new one if not found
203 - try:
204 - ray.init("auto")
205 - except ConnectionError:
206 - logger.warning(
207 - "No existing RAY instance detected. "
208 - "A new instance will be launched with current node resources.")
209 - ray.init(address=ray_address,
210 - ignore_reinit_error=True,
211 - num_gpus=parallel_config.world_size)
212 - else:
213 - ray.init(address=ray_address, ignore_reinit_error=True)
214 -
215 - if parallel_config.placement_group:
216 - # Placement group is already set.
217 - return
218 -
219 - device_str = current_platform.ray_device_key
220 - if not device_str:
221 - raise ValueError(
222 - f"current platform {current_platform.device_name} does not "
223 - "support ray.")
224 - # Create placement group for worker processes
225 - current_placement_group = ray.util.get_current_placement_group()
226 - if current_placement_group:
227 - # We are in a placement group
228 - bundles = current_placement_group.bundle_specs
229 - # Verify that we can use the placement group.
230 - device_bundles = 0
231 - for bundle in bundles:
232 - bundle_devices = bundle.get(device_str, 0)
233 - if bundle_devices > 1:
234 - raise ValueError(
235 - "Placement group bundle cannot have more than 1 "
236 - f"{device_str}.")
237 - if bundle_devices:
238 - device_bundles += 1
239 - if parallel_config.world_size > device_bundles:
240 - raise ValueError(
241 - f"The number of required {device_str}s exceeds the total "
242 - f"number of available {device_str}s in the placement group."
243 - f"Required number of devices: {parallel_config.world_size}. "
244 - f"Total number of devices: {device_bundles}.")
245 - else:
246 - num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
247 - if parallel_config.world_size > num_devices_in_cluster:
248 - raise ValueError(
249 - f"The number of required {device_str}s exceeds the total "
250 - f"number of available {device_str}s in the placement group.")
251 - # Create a new placement group
252 - placement_group_specs: List[Dict[str, float]] = ([{
253 - device_str: 1.0
254 - } for _ in range(parallel_config.world_size)])
255 -
256 - # vLLM engine is also a worker to execute model with an accelerator,
257 - # so it requires to have the device in a current node. Check if
258 - # the current node has at least one device.
259 - current_ip = get_ip()
260 - current_node_id = ray.get_runtime_context().get_node_id()
261 - current_node_resource = available_resources_per_node()[current_node_id]
262 - if current_node_resource.get(device_str, 0) < 1:
263 - raise ValueError(
264 - f"Current node has no {device_str} available. "
265 - f"{current_node_resource=}. vLLM engine cannot start without "
266 - f"{device_str}. Make sure you have at least 1 {device_str} "
267 - f"available in a node {current_node_id=} {current_ip=}.")
268 - # This way, at least bundle is required to be created in a current
269 - # node.
270 - placement_group_specs[0][f"node:{current_ip}"] = 0.001
271 -
272 - # By default, Ray packs resources as much as possible.
273 - current_placement_group = ray.util.placement_group(
274 - placement_group_specs, strategy="PACK")
275 - _wait_until_pg_ready(current_placement_group)
276 -
277 - assert current_placement_group is not None
278 - _verify_bundles(current_placement_group, parallel_config, device_str)
279 - # Set the placement group in the parallel config
280 - parallel_config.placement_group = current_placement_group
vllm/v1/executor/uniproc_executor.py deleted
-88
@@ -1,88 +0,0 @@
1 -import os
2 -from typing import Optional
3 -
4 -from vllm.config import VllmConfig
5 -from vllm.logger import init_logger
6 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7 -from vllm.v1.executor.abstract import Executor
8 -from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
9 -from vllm.v1.outputs import ModelRunnerOutput
10 -from vllm.v1.worker.gpu_worker import Worker
11 -
12 -logger = init_logger(__name__)
13 -
14 -
15 -class UniprocExecutor(Executor):
16 -
17 - def __init__(self, vllm_config: VllmConfig) -> None:
18 - self.vllm_config = vllm_config
19 - self.model_config = vllm_config.model_config
20 - self.cache_config = vllm_config.cache_config
21 - self.lora_config = vllm_config.lora_config
22 - self.load_config = vllm_config.load_config
23 - self.parallel_config = vllm_config.parallel_config
24 - self.scheduler_config = vllm_config.scheduler_config
25 - self.device_config = vllm_config.device_config
26 - self.speculative_config = vllm_config.speculative_config
27 - self.prompt_adapter_config = vllm_config.prompt_adapter_config
28 - self.observability_config = vllm_config.observability_config
29 -
30 - self.worker: Worker = self._create_worker()
31 - self.worker.init_device()
32 - self.worker.load_model()
33 -
34 - def _create_worker(
35 - self,
36 - local_rank: int = 0,
37 - rank: int = 0,
38 - distributed_init_method: Optional[str] = None) -> Worker:
39 - """Return worker init args for a given rank."""
40 - # see https://github.com/NVIDIA/nccl/issues/1234
41 - os.environ['NCCL_CUMEM_ENABLE'] = '0'
42 -
43 - if distributed_init_method is None:
44 - distributed_init_method = get_distributed_init_method(
45 - get_ip(), get_open_port())
46 - return Worker(
47 - vllm_config=self.vllm_config,
48 - local_rank=local_rank,
49 - rank=rank,
50 - distributed_init_method=distributed_init_method,
51 - )
52 -
53 - def determine_available_memory(self) -> int:
54 - """Determine the available memory (in bytes) for KV cache by invoking
55 - the underlying worker.
56 - """
57 - return self.worker.determine_available_memory()
58 -
59 - def get_kv_cache_spec(self) -> KVCacheSpec:
60 - """Get all kv cache needed by the model by invoking the underlying
61 - worker.
62 - """
63 - return self.worker.get_kv_cache_spec()
64 -
65 - def initialize(self, kv_cache_config: KVCacheConfig) -> None:
66 - """Initialize the KV cache by invoking the underlying worker.
67 - """
68 - self.worker.initialize_cache(kv_cache_config)
69 - self.worker.compile_or_warm_up_model()
70 -
71 - def execute_model(
72 - self,
73 - scheduler_output,
74 - ) -> ModelRunnerOutput:
75 - output = self.worker.execute_model(scheduler_output)
76 - assert output is not None
77 - return output
78 -
79 - def profile(self, is_start: bool = True):
80 - self.worker.profile(is_start)
81 -
82 - def shutdown(self):
83 - pass
84 -
85 - def check_health(self) -> None:
86 - # UniprocExecutor will always be healthy as long as
87 - # it's running.
88 - return