A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core] clean up executor class hierarchy between v1 and v0 (#12171)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 18, 2025 at 14:35 UTC
6d0e3d372446cde48b387d4d3530e25fc6e06320
6 files changed
+61
-798
vllm/executor/executor_base.py
-10
@@ -79,16 +79,6 @@ class ExecutorBase(ABC):
79
b = min([r[1] for r in results])
80
return a, b
81
82
- def initialize(self, num_gpu_blocks: int) -> None:
83
- """
84
- Initialize the KV caches and begin the model execution loop of the
85
- underlying workers.
86
- For V1 compatibility.
87
- """
88
- logger.info("# GPU blocks: %d", num_gpu_blocks)
89
- self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
90
- self.collective_rpc("compile_or_warm_up_model")
91
-
82
def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
83
"""Initialize the KV cache by invoking the underlying worker.
84
"""
vllm/v1/executor/abstract.py
+58
-29
@@ -1,63 +1,92 @@
1
-from abc import ABC, abstractmethod
1
from typing import Type
2
3
from vllm.config import VllmConfig
4
+from vllm.executor.executor_base import ExecutorBase
5
+from vllm.executor.ray_distributed_executor import ( # noqa
6
+ RayDistributedExecutor as RayDistributedExecutorV0)
7
+from vllm.executor.uniproc_executor import ( # noqa
8
+ ExecutorWithExternalLauncher as ExecutorWithExternalLauncherV0)
9
+from vllm.executor.uniproc_executor import ( # noqa
10
+ UniProcExecutor as UniProcExecutorV0)
11
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
12
from vllm.v1.outputs import ModelRunnerOutput
13
14
9
-class Executor(ABC):
10
- """Abstract class for executors."""
15
+class Executor(ExecutorBase):
16
+ """
17
+ Abstract class for v1 executors, mainly define some methods for v1.
18
+ For methods shared by v0 and v1, define them in ExecutorBase"""
19
20
@staticmethod
21
def get_class(vllm_config: VllmConfig) -> Type["Executor"]:
22
executor_class: Type[Executor]
23
+ parallel_config = vllm_config.parallel_config
24
distributed_executor_backend = (
16
- vllm_config.parallel_config.distributed_executor_backend)
25
+ parallel_config.distributed_executor_backend)
26
+ if distributed_executor_backend is None:
27
+ # If the user does not specify the distributed executor backend,
28
+ # we will choose the backend based on the world size.
29
+ if parallel_config.world_size > 1:
30
+ distributed_executor_backend = "mp"
31
+ else:
32
+ distributed_executor_backend = "uni"
33
+
34
if distributed_executor_backend == "ray":
18
- from vllm.executor.ray_distributed_executor import ( # noqa
19
- RayDistributedExecutor)
35
executor_class = RayDistributedExecutor
36
elif distributed_executor_backend == "mp":
37
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
38
executor_class = MultiprocExecutor
39
+ elif distributed_executor_backend == "uni":
40
+ executor_class = UniProcExecutor
41
+ elif distributed_executor_backend == "external_launcher":
42
+ # TODO: make v1 scheduling deterministic
43
+ # to support external launcher
44
+ executor_class = ExecutorWithExternalLauncher
45
else:
25
- assert (distributed_executor_backend is None)
26
- from vllm.v1.executor.uniproc_executor import UniprocExecutor
27
- executor_class = UniprocExecutor
46
+ raise ValueError("Unknown distributed executor backend: "
47
+ f"{distributed_executor_backend}")
48
return executor_class
49
30
- @abstractmethod
31
- def __init__(self, vllm_config: VllmConfig) -> None:
32
- raise NotImplementedError
33
-
34
- @abstractmethod
50
def initialize(self, kv_cache_config: KVCacheConfig) -> None:
36
- raise NotImplementedError
51
+ """
52
+ Initialize the KV caches and begin the model execution loop of the
53
+ underlying workers.
54
+ """
55
+ self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
56
+ self.collective_rpc("compile_or_warm_up_model")
57
38
- @abstractmethod
58
def determine_available_memory(self) -> int: # in bytes
40
- raise NotImplementedError
59
+ output = self.collective_rpc("determine_available_memory")
60
+ # Since we use a shared centralized controller, we take the minimum
61
+ # memory size across all workers to make sure all the memory
62
+ # operators can be applied to all workers.
63
+ return min(output)
64
42
- @abstractmethod
65
def get_kv_cache_spec(self) -> KVCacheSpec:
44
- raise NotImplementedError
66
+ output = self.collective_rpc("get_kv_cache_spec")
67
+ for x in output:
68
+ assert x == output[0]
69
+ return output[0]
70
46
- @abstractmethod
71
def execute_model(
72
self,
73
scheduler_output,
74
) -> ModelRunnerOutput:
51
- raise NotImplementedError
75
+ output = self.collective_rpc("execute_model",
76
+ args=(scheduler_output, ))
77
+ return output[0]
78
53
- @abstractmethod
79
def profile(self, is_start: bool = True):
55
- raise NotImplementedError
80
+ self.collective_rpc("profile", args=(is_start, ))
81
+
82
+
83
+class UniProcExecutor(UniProcExecutorV0, Executor):
84
+ pass
85
+
86
+
87
+class ExecutorWithExternalLauncher(ExecutorWithExternalLauncherV0, Executor):
88
+ pass
89
57
- @abstractmethod
58
- def shutdown(self):
59
- pass
90
61
- @abstractmethod
62
- def check_health(self) -> None:
63
- raise NotImplementedError
91
+class RayDistributedExecutor(RayDistributedExecutorV0, Executor):
92
+ pass
vllm/v1/executor/multiproc_executor.py
+3
-47
@@ -25,8 +25,6 @@ from vllm.logger import init_logger
25
from vllm.utils import (get_distributed_init_method, get_mp_context,
26
get_open_port, get_open_zmq_ipc_path, zmq_socket_ctx)
27
from vllm.v1.executor.abstract import Executor
28
-from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
29
-from vllm.v1.outputs import ModelRunnerOutput
28
from vllm.worker.worker_base import WorkerWrapperBase
29
30
logger = init_logger(__name__)
@@ -37,7 +35,7 @@ POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
35
36
class MultiprocExecutor(Executor):
37
40
- def __init__(self, vllm_config: VllmConfig) -> None:
38
+ def _init_executor(self) -> None:
39
# Call self.shutdown at exit to clean up
40
# and ensure workers will be terminated.
41
self._finalizer = weakref.finalize(self, self.shutdown)
@@ -55,9 +53,6 @@ class MultiprocExecutor(Executor):
53
54
signal.signal(signal.SIGUSR1, sigusr1_handler)
55
58
- self.vllm_config = vllm_config
59
- self.parallel_config = vllm_config.parallel_config
60
-
56
self.world_size = self.parallel_config.world_size
57
tensor_parallel_size = self.parallel_config.tensor_parallel_size
58
assert self.world_size == tensor_parallel_size, (
@@ -82,7 +77,8 @@ class MultiprocExecutor(Executor):
77
# Create workers
78
self.workers: List[WorkerProcHandle] = []
79
for rank in range(self.world_size):
85
- worker = WorkerProc.make_worker_process(vllm_config, rank, rank,
80
+ worker = WorkerProc.make_worker_process(self.vllm_config, rank,
81
+ rank,
82
distributed_init_method,
83
scheduler_output_handle)
84
self.workers.append(worker)
@@ -93,34 +89,6 @@ class MultiprocExecutor(Executor):
89
for w in self.workers:
90
w.worker_response_mq.wait_until_ready()
91
96
- def initialize(self, kv_cache_config: KVCacheConfig) -> None:
97
- """
98
- Initialize the KV caches and begin the model execution loop of the
99
- underlying workers.
100
- """
101
- self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
102
- self.collective_rpc("compile_or_warm_up_model")
103
-
104
- def determine_available_memory(self) -> int:
105
- """
106
- Determine the available memory (in bytes) for KV cache by invoking the
107
- underlying worker.
108
- """
109
- memory_sizes = self.collective_rpc("determine_available_memory")
110
-
111
- # Since we use a shared centralized controller, we take the minimum
112
- # memory size across all workers to make sure all the memory
113
- # operators can be applied to all workers.
114
- return min(memory_sizes)
115
-
116
- def get_kv_cache_spec(self) -> KVCacheSpec:
117
- """
118
- Get all kv cache needed by the model by invoking the underlying worker.
119
- """
120
- kv_cache_specs = self.collective_rpc("get_kv_cache_spec")
121
- assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
122
- return kv_cache_specs[0]
123
-
92
def collective_rpc(self,
93
method: Union[str, Callable],
94
timeout: Optional[float] = None,
@@ -172,18 +140,6 @@ class MultiprocExecutor(Executor):
140
# Re-raise any other exceptions
141
raise e
142
175
- def execute_model(
176
- self,
177
- scheduler_output,
178
- ) -> ModelRunnerOutput:
179
- model_output = self.collective_rpc("execute_model",
180
- args=(scheduler_output, ))[0]
181
- return model_output
182
-
183
- def profile(self, is_start: bool = True):
184
- self.collective_rpc("profile", args=(is_start, ))
185
- return
186
-
143
def _ensure_worker_termination(self):
144
"""Ensure that all worker processes are terminated. Assumes workers have
145
received termination requests. Waits for processing, then sends
vllm/v1/executor/ray_executor.py
deleted
-344
@@ -1,344 +0,0 @@
1
-import os
2
-from collections import defaultdict
3
-from itertools import islice, repeat
4
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
5
-
6
-import vllm.envs as envs
7
-from vllm.config import VllmConfig
8
-from vllm.logger import init_logger
9
-from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10
-from vllm.v1.executor.abstract import Executor
11
-from vllm.v1.executor.ray_utils import (RayWorkerWrapper,
12
- initialize_ray_cluster, ray)
13
-from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
14
-from vllm.v1.outputs import ModelRunnerOutput
15
-
16
-if ray is not None:
17
- from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
18
-
19
-if TYPE_CHECKING:
20
- from ray.util.placement_group import PlacementGroup
21
-
22
-logger = init_logger(__name__)
23
-
24
-
25
-class RayExecutor(Executor):
26
-
27
- def __init__(self, vllm_config: VllmConfig) -> None:
28
- self.vllm_config = vllm_config
29
- self.parallel_config = vllm_config.parallel_config
30
- self.model_config = vllm_config.model_config
31
- self.forward_dag: Optional[ray.dag.CompiledDAG] = None
32
-
33
- # Disable Ray usage stats collection.
34
- ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
35
- if ray_usage != "1":
36
- os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
37
-
38
- initialize_ray_cluster(self.parallel_config)
39
- placement_group = self.parallel_config.placement_group
40
-
41
- # Create the parallel GPU workers.
42
- self._init_workers_ray(placement_group)
43
-
44
- def _init_workers_ray(self, placement_group: "PlacementGroup",
45
- **ray_remote_kwargs):
46
- # A list of workers to run a model.
47
- self.workers: List[RayWorkerWrapper] = []
48
- if self.parallel_config.ray_workers_use_nsight:
49
- ray_remote_kwargs = self._configure_ray_workers_use_nsight(
50
- ray_remote_kwargs)
51
-
52
- # Create the workers.
53
- driver_ip = get_ip()
54
- for bundle_id, bundle in enumerate(placement_group.bundle_specs):
55
- if not bundle.get("GPU", 0):
56
- # Skip bundles that don't have GPUs,
57
- # as each worker needs one GPU.
58
- continue
59
- scheduling_strategy = PlacementGroupSchedulingStrategy(
60
- placement_group=placement_group,
61
- placement_group_capture_child_tasks=True,
62
- placement_group_bundle_index=bundle_id,
63
- )
64
-
65
- worker = ray.remote(
66
- num_cpus=0,
67
- num_gpus=1,
68
- scheduling_strategy=scheduling_strategy,
69
- **ray_remote_kwargs,
70
- )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
71
- self.workers.append(worker)
72
-
73
- logger.debug("workers: %s", self.workers)
74
- worker_ips = [
75
- ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
76
- for worker in self.workers
77
- ]
78
- ip_counts: Dict[str, int] = {}
79
- for ip in worker_ips:
80
- ip_counts[ip] = ip_counts.get(ip, 0) + 1
81
-
82
- worker_to_ip = dict(zip(self.workers, worker_ips))
83
-
84
- def sort_by_driver_then_worker_ip(worker):
85
- """
86
- Sort the workers based on 3 properties:
87
- 1. If the worker is on the same node as the driver (vllm engine),
88
- it should be placed first.
89
- 2. Then, if the worker is on a node with fewer workers, it should
90
- be placed first.
91
- 3. Finally, if the work is on a node with smaller IP address, it
92
- should be placed first. This is simply a tiebreaker to make
93
- sure the workers are sorted in a deterministic way.
94
- """
95
- ip = worker_to_ip[worker]
96
- return (ip != driver_ip, ip_counts[ip], ip)
97
-
98
- # After sorting, the workers on the same node will be
99
- # close to each other, and the workers on the driver
100
- # node will be placed first.
101
- self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
102
-
103
- # Get the set of GPU IDs used on each node.
104
- worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids")
105
-
106
- node_workers = defaultdict(list) # node id -> list of worker ranks
107
- node_gpus = defaultdict(list) # node id -> list of gpu ids
108
-
109
- for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
110
- node_workers[node_id].append(i)
111
- # `gpu_ids` can be a list of strings or integers.
112
- # convert them to integers for consistency.
113
- # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
114
- # string sorting is not sufficient.
115
- # see https://github.com/vllm-project/vllm/issues/5590
116
- gpu_ids = [int(x) for x in gpu_ids]
117
- node_gpus[node_id].extend(gpu_ids)
118
-
119
- for node_id, gpu_ids in node_gpus.items():
120
- node_gpus[node_id] = sorted(gpu_ids)
121
-
122
- all_ips = set(worker_ips)
123
- n_ips = len(all_ips)
124
- n_nodes = len(node_workers)
125
-
126
- if n_nodes != n_ips:
127
- raise RuntimeError(
128
- f"Every node should have a unique IP address. Got {n_nodes}"
129
- f" nodes with node ids {list(node_workers.keys())} and "
130
- f"{n_ips} unique IP addresses {all_ips}. Please check your"
131
- " network configuration. If you set `VLLM_HOST_IP` or "
132
- "`HOST_IP` environment variable, make sure it is unique for"
133
- " each node.")
134
-
135
- # Set environment variables for the driver and workers.
136
- all_args_to_update_environment_variables = [({
137
- "CUDA_VISIBLE_DEVICES":
138
- ",".join(map(str, node_gpus[node_id])),
139
- "VLLM_TRACE_FUNCTION":
140
- str(envs.VLLM_TRACE_FUNCTION),
141
- "VLLM_USE_V1":
142
- str(int(envs.VLLM_USE_V1)),
143
- **({
144
- "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
145
- } if envs.VLLM_ATTENTION_BACKEND is not None else {})
146
- }, ) for (node_id, _) in worker_node_and_gpu_ids]
147
-
148
- self._env_vars_for_all_workers = (
149
- all_args_to_update_environment_variables)
150
-
151
- self._run_workers("update_environment_variables",
152
- all_args=self._get_env_vars_to_be_updated())
153
-
154
- if len(node_gpus) == 1:
155
- # in single node case, we don't need to get the IP address.
156
- # the loopback address is sufficient
157
- # NOTE: a node may have several IP addresses, one for each
158
- # network interface. `get_ip()` might return any of them,
159
- # while they might not work for communication inside the node
160
- # if the network setup is complicated. Using the loopback address
161
- # solves this issue, as it always works for communication inside
162
- # the node.
163
- driver_ip = "127.0.0.1"
164
- distributed_init_method = get_distributed_init_method(
165
- driver_ip, get_open_port())
166
-
167
- # Initialize the actual workers inside worker wrapper.
168
- init_worker_all_kwargs = [
169
- self._get_worker_kwargs(
170
- local_rank=node_workers[node_id].index(rank),
171
- rank=rank,
172
- distributed_init_method=distributed_init_method,
173
- ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
174
- ]
175
- self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
176
- self._run_workers("initialize")
177
- self._run_workers("load_model")
178
-
179
- def _configure_ray_workers_use_nsight(self,
180
- ray_remote_kwargs) -> Dict[str, Any]:
181
- # If nsight profiling is enabled, we need to set the profiling
182
- # configuration for the ray workers as runtime env.
183
- runtime_env = ray_remote_kwargs.setdefault("runtime_env", {})
184
- runtime_env.update({
185
- "nsight": {
186
- "t": "cuda,cudnn,cublas",
187
- "o": "'worker_process_%p'",
188
- "cuda-graph-trace": "node",
189
- }
190
- })
191
-
192
- return ray_remote_kwargs
193
-
194
- def _get_env_vars_to_be_updated(self):
195
- return self._env_vars_for_all_workers
196
-
197
- def _get_worker_kwargs(
198
- self,
199
- local_rank: int = 0,
200
- rank: int = 0,
201
- distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
202
- """
203
- Return worker init args for a given rank.
204
- """
205
- if distributed_init_method is None:
206
- distributed_init_method = get_distributed_init_method(
207
- get_ip(), get_open_port())
208
- return dict(
209
- vllm_config=self.vllm_config,
210
- local_rank=local_rank,
211
- rank=rank,
212
- distributed_init_method=distributed_init_method,
213
- )
214
-
215
- def determine_available_memory(self) -> int:
216
- """
217
- Determine the available GPU memory in bytes.
218
-
219
- This invokes `determine_available_memory` on each worker and takes
220
- the min of the results, guaranteeing that the selected cache sizes are
221
- compatible with all workers.
222
- """
223
-
224
- memory_sizes = self._run_workers("determine_available_memory")
225
-
226
- # Since we use a shared centralized controller, we take the minimum
227
- # memory size across all workers to make sure all the memory
228
- # operators can be applied to all workers.
229
- return min(memory_sizes)
230
-
231
- def initialize(self, kv_cache_config: KVCacheConfig) -> None:
232
- """
233
- Initialize the KV cache in all workers.
234
- """
235
- self._run_workers("initialize_cache", kv_cache_config)
236
- self._run_workers("compile_or_warm_up_model")
237
-
238
- def get_kv_cache_spec(self) -> KVCacheSpec:
239
- """
240
- Get all kv cache needed by the model
241
-
242
- This invokes `get_kv_cache_spec` on each worker and asserts that
243
- they are identical. The KVCacheSpec is then returned.
244
- """
245
- kv_cache_specs = self._run_workers("get_kv_cache_spec")
246
- assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
247
- return kv_cache_specs[0]
248
-
249
- def _run_workers(
250
- self,
251
- method: str,
252
- *args,
253
- all_args: Optional[List[Tuple[Any, ...]]] = None,
254
- all_kwargs: Optional[List[Dict[str, Any]]] = None,
255
- **kwargs,
256
- ) -> Any:
257
- """
258
- Runs the given method on all workers. Can be used in the following
259
- ways:
260
-
261
- Args:
262
- - args/kwargs: All workers share the same args/kwargs
263
- - all_args/all_kwargs: args/kwargs for each worker are specified
264
- individually
265
- """
266
- count = len(self.workers)
267
- all_worker_args = repeat(args, count) if all_args is None \
268
- else islice(all_args, 0, None)
269
- all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
270
- else islice(all_kwargs, 0, None)
271
-
272
- ray_worker_refs = [
273
- worker.execute_method.remote( # type: ignore[attr-defined]
274
- method, *worker_args, **worker_kwargs)
275
- for (worker, worker_args, worker_kwargs
276
- ) in zip(self.workers, all_worker_args, all_worker_kwargs)
277
- ]
278
- return ray.get(ray_worker_refs)
279
-
280
- def execute_model(
281
- self,
282
- scheduler_output,
283
- ) -> ModelRunnerOutput:
284
- if self.forward_dag is None:
285
- self.forward_dag = self._compiled_ray_dag()
286
- # Only the first worker (with rank 0) returns the execution result.
287
- # Others return None.
288
- output = ray.get(self.forward_dag.execute(scheduler_output))[0]
289
- return output
290
-
291
- def profile(self, is_start=True):
292
- raise NotImplementedError
293
-
294
- def shutdown(self):
295
- if hasattr(self, "forward_dag") and self.forward_dag is not None:
296
- self.forward_dag.teardown()
297
- import ray
298
- for worker in self.workers:
299
- ray.kill(worker)
300
- self.forward_dag = None
301
-
302
- def check_health(self) -> None:
303
- logger.debug("Called check_health.")
304
-
305
- def _check_ray_compiled_graph_installation(self):
306
- import pkg_resources
307
- from packaging import version
308
-
309
- required_version = version.parse("2.39")
310
- current_version = version.parse(
311
- pkg_resources.get_distribution("ray").version)
312
- if current_version < required_version:
313
- raise ValueError(f"Ray version {required_version} is "
314
- f"required, but found {current_version}")
315
-
316
- import importlib.util
317
- raycg = importlib.util.find_spec("ray.experimental.compiled_dag_ref")
318
- if raycg is None:
319
- raise ValueError("Ray Compiled Graph is not installed. "
320
- "Run `pip install ray[adag]` to install it.")
321
-
322
- cupy_spec = importlib.util.find_spec("cupy")
323
- if cupy_spec is None and envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL:
324
- raise ValueError(
325
- "cupy is not installed but required since "
326
- "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL is set."
327
- "Run `pip install ray[adag]` and check cupy installation.")
328
-
329
- def _compiled_ray_dag(self):
330
- assert self.parallel_config.use_ray
331
- self._check_ray_compiled_graph_installation()
332
- from ray.dag import InputNode, MultiOutputNode
333
-
334
- with InputNode() as input_batches:
335
- outputs = [
336
- worker.execute_model.bind( # type: ignore[attr-defined]
337
- input_batches) for worker in self.workers
338
- ]
339
- forward_dag = MultiOutputNode(outputs)
340
-
341
- return forward_dag.experimental_compile()
342
-
343
- def __del__(self):
344
- self.shutdown()
vllm/v1/executor/ray_utils.py
deleted
-280
@@ -1,280 +0,0 @@
1
-import time
2
-from collections import defaultdict
3
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
4
-
5
-from vllm.config import ParallelConfig
6
-from vllm.logger import init_logger
7
-from vllm.platforms import current_platform
8
-from vllm.utils import get_ip
9
-from vllm.v1.outputs import ModelRunnerOutput
10
-from vllm.worker.worker_base import WorkerWrapperBase
11
-
12
-if TYPE_CHECKING:
13
- from vllm.v1.core.scheduler import SchedulerOutput
14
-
15
-logger = init_logger(__name__)
16
-PG_WAIT_TIMEOUT = 60
17
-
18
-try:
19
- import ray
20
- from ray.util import placement_group_table
21
- from ray.util.placement_group import PlacementGroup
22
- try:
23
- from ray._private.state import available_resources_per_node
24
- except ImportError:
25
- # Ray 2.9.x doesn't expose `available_resources_per_node`
26
- from ray._private.state import state as _state
27
- available_resources_per_node = _state._available_resources_per_node
28
-
29
- class RayWorkerWrapper(WorkerWrapperBase):
30
-
31
- def __init__(self, *args, **kwargs) -> None:
32
- super().__init__(*args, **kwargs)
33
- # Since the compiled DAG runs a main execution
34
- # in a different thread that calls cuda.set_device.
35
- # The flag indicates is set_device is called on
36
- # that thread. It will be removed soon.
37
- self.compiled_dag_cuda_device_set = False
38
-
39
- def get_node_ip(self) -> str:
40
- return get_ip()
41
-
42
- def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
43
- node_id = ray.get_runtime_context().get_node_id()
44
- device_key = current_platform.ray_device_key
45
- if not device_key:
46
- raise RuntimeError("current platform %s does not support ray.",
47
- current_platform.device_name)
48
- gpu_ids = ray.get_runtime_context().get_accelerator_ids(
49
- )[device_key]
50
- return node_id, gpu_ids
51
-
52
- def setup_device_if_necessary(self):
53
- # TODO(swang): This is needed right now because Ray CG executes
54
- # on a background thread, so we need to reset torch's current
55
- # device.
56
- # We can remove this API after it is fixed in compiled graph.
57
- import torch
58
- assert self.worker is not None, "Worker is not initialized"
59
- if not self.compiled_dag_cuda_device_set:
60
- torch.cuda.set_device(self.worker.device)
61
- self.compiled_dag_cuda_device_set = True
62
-
63
- def execute_model(
64
- self,
65
- scheduler_output: "SchedulerOutput",
66
- ) -> ModelRunnerOutput:
67
- self.setup_device_if_necessary()
68
- assert self.worker is not None, "Worker is not initialized"
69
- output = self.worker.model_runner.execute_model(scheduler_output)
70
- return output
71
-
72
- ray_import_err = None
73
-
74
-except ImportError as e:
75
- ray = None # type: ignore
76
- ray_import_err = e
77
- RayWorkerWrapper = None # type: ignore
78
-
79
-
80
-def ray_is_available() -> bool:
81
- """Returns True if Ray is available."""
82
- return ray is not None
83
-
84
-
85
-def assert_ray_available():
86
- """
87
- Raise an exception if Ray is not available.
88
- """
89
- if ray is None:
90
- raise ValueError("Failed to import Ray, please install Ray with "
91
- "`pip install ray`.") from ray_import_err
92
-
93
-
94
-def _verify_bundles(placement_group: "PlacementGroup",
95
- parallel_config: ParallelConfig, device_str: str):
96
- """
97
- Verify a given placement group has bundles located in the right place.
98
-
99
- There are 2 rules.
100
- - Warn if all tensor parallel workers cannot fit in a single node.
101
- - Fail if driver node is not included in a placement group.
102
-
103
- Args:
104
- placement_group: The placement group to verify.
105
- parallel_config: The parallel configuration.
106
- device_str: The required device.
107
- """
108
- assert ray.is_initialized(), (
109
- "Ray is not initialized although distributed-executor-backend is ray.")
110
- pg_data = placement_group_table(placement_group)
111
- # bundle_idx -> node_id
112
- bundle_to_node_ids = pg_data["bundles_to_node_id"]
113
- # bundle_idx -> bundle (e.g., {"GPU": 1})
114
- bundles = pg_data["bundles"]
115
- # node_id -> List of bundle (e.g., {"GPU": 1})
116
- node_id_to_bundle: Dict[str, List[Dict[str, float]]] = defaultdict(list)
117
-
118
- for bundle_idx, node_id in bundle_to_node_ids.items():
119
- node_id_to_bundle[node_id].append(bundles[bundle_idx])
120
- driver_node_id = ray.get_runtime_context().get_node_id()
121
-
122
- if driver_node_id not in node_id_to_bundle:
123
- raise RuntimeError(
124
- f"driver node id {driver_node_id} is not included in a placement "
125
- f"group {placement_group.id}. Node id -> bundles "
126
- f"{node_id_to_bundle}. "
127
- "You don't have enough GPUs available in a current node. Check "
128
- "`ray status` to see if you have available GPUs in a node "
129
- f"{driver_node_id} before starting an vLLM engine.")
130
-
131
- for node_id, bundles in node_id_to_bundle.items():
132
- if len(bundles) < parallel_config.tensor_parallel_size:
133
- logger.warning(
134
- "tensor_parallel_size=%d "
135
- "is bigger than a reserved number of %ss (%d "
136
- "%ss) in a node %s. Tensor parallel workers can be "
137
- "spread out to 2+ nodes which can degrade the performance "
138
- "unless you have fast interconnect across nodes, like "
139
- "Infiniband. To resolve this issue, make sure you have more "
140
- "than %d GPUs available at each node.",
141
- parallel_config.tensor_parallel_size, device_str, len(bundles),
142
- device_str, node_id, parallel_config.tensor_parallel_size)
143
-
144
-
145
-def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
146
- """Wait until a placement group is ready.
147
-
148
- It prints the informative log messages if the placement group is
149
- not created within time.
150
-
151
- """
152
- # Wait until PG is ready - this will block until all
153
- # requested resources are available, and will timeout
154
- # if they cannot be provisioned.
155
- placement_group_specs = current_placement_group.bundle_specs
156
-
157
- s = time.time()
158
- pg_ready_ref = current_placement_group.ready()
159
- wait_interval = 10
160
- while time.time() - s < PG_WAIT_TIMEOUT:
161
- ready, _ = ray.wait([pg_ready_ref], timeout=wait_interval)
162
- if len(ready) > 0:
163
- break
164
-
165
- # Exponential backoff for warning print.
166
- wait_interval *= 2
167
- logger.info(
168
- "Waiting for creating a placement group of specs for "
169
- "%d seconds. specs=%s. Check "
170
- "`ray status` to see if you have enough resources.",
171
- int(time.time() - s), placement_group_specs)
172
-
173
- try:
174
- ray.get(pg_ready_ref, timeout=0)
175
- except ray.exceptions.GetTimeoutError:
176
- raise ValueError(
177
- "Cannot provide a placement group of "
178
- f"{placement_group_specs=} within {PG_WAIT_TIMEOUT} seconds. See "
179
- "`ray status` to make sure the cluster has enough resources."
180
- ) from None
181
-
182
-
183
-def initialize_ray_cluster(
184
- parallel_config: ParallelConfig,
185
- ray_address: Optional[str] = None,
186
-):
187
- """Initialize the distributed cluster with Ray.
188
-
189
- it will connect to the Ray cluster and create a placement group
190
- for the workers, which includes the specification of the resources
191
- for each distributed worker.
192
-
193
- Args:
194
- parallel_config: The configurations for parallel execution.
195
- ray_address: The address of the Ray cluster. If None, uses
196
- the default Ray cluster address.
197
- """
198
- assert_ray_available()
199
-
200
- # Connect to a ray cluster.
201
- if current_platform.is_rocm() or current_platform.is_xpu():
202
- # Try to connect existing ray instance and create a new one if not found
203
- try:
204
- ray.init("auto")
205
- except ConnectionError:
206
- logger.warning(
207
- "No existing RAY instance detected. "
208
- "A new instance will be launched with current node resources.")
209
- ray.init(address=ray_address,
210
- ignore_reinit_error=True,
211
- num_gpus=parallel_config.world_size)
212
- else:
213
- ray.init(address=ray_address, ignore_reinit_error=True)
214
-
215
- if parallel_config.placement_group:
216
- # Placement group is already set.
217
- return
218
-
219
- device_str = current_platform.ray_device_key
220
- if not device_str:
221
- raise ValueError(
222
- f"current platform {current_platform.device_name} does not "
223
- "support ray.")
224
- # Create placement group for worker processes
225
- current_placement_group = ray.util.get_current_placement_group()
226
- if current_placement_group:
227
- # We are in a placement group
228
- bundles = current_placement_group.bundle_specs
229
- # Verify that we can use the placement group.
230
- device_bundles = 0
231
- for bundle in bundles:
232
- bundle_devices = bundle.get(device_str, 0)
233
- if bundle_devices > 1:
234
- raise ValueError(
235
- "Placement group bundle cannot have more than 1 "
236
- f"{device_str}.")
237
- if bundle_devices:
238
- device_bundles += 1
239
- if parallel_config.world_size > device_bundles:
240
- raise ValueError(
241
- f"The number of required {device_str}s exceeds the total "
242
- f"number of available {device_str}s in the placement group."
243
- f"Required number of devices: {parallel_config.world_size}. "
244
- f"Total number of devices: {device_bundles}.")
245
- else:
246
- num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
247
- if parallel_config.world_size > num_devices_in_cluster:
248
- raise ValueError(
249
- f"The number of required {device_str}s exceeds the total "
250
- f"number of available {device_str}s in the placement group.")
251
- # Create a new placement group
252
- placement_group_specs: List[Dict[str, float]] = ([{
253
- device_str: 1.0
254
- } for _ in range(parallel_config.world_size)])
255
-
256
- # vLLM engine is also a worker to execute model with an accelerator,
257
- # so it requires to have the device in a current node. Check if
258
- # the current node has at least one device.
259
- current_ip = get_ip()
260
- current_node_id = ray.get_runtime_context().get_node_id()
261
- current_node_resource = available_resources_per_node()[current_node_id]
262
- if current_node_resource.get(device_str, 0) < 1:
263
- raise ValueError(
264
- f"Current node has no {device_str} available. "
265
- f"{current_node_resource=}. vLLM engine cannot start without "
266
- f"{device_str}. Make sure you have at least 1 {device_str} "
267
- f"available in a node {current_node_id=} {current_ip=}.")
268
- # This way, at least bundle is required to be created in a current
269
- # node.
270
- placement_group_specs[0][f"node:{current_ip}"] = 0.001
271
-
272
- # By default, Ray packs resources as much as possible.
273
- current_placement_group = ray.util.placement_group(
274
- placement_group_specs, strategy="PACK")
275
- _wait_until_pg_ready(current_placement_group)
276
-
277
- assert current_placement_group is not None
278
- _verify_bundles(current_placement_group, parallel_config, device_str)
279
- # Set the placement group in the parallel config
280
- parallel_config.placement_group = current_placement_group
vllm/v1/executor/uniproc_executor.py
deleted
-88
@@ -1,88 +0,0 @@
1
-import os
2
-from typing import Optional
3
-
4
-from vllm.config import VllmConfig
5
-from vllm.logger import init_logger
6
-from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7
-from vllm.v1.executor.abstract import Executor
8
-from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
9
-from vllm.v1.outputs import ModelRunnerOutput
10
-from vllm.v1.worker.gpu_worker import Worker
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class UniprocExecutor(Executor):
16
-
17
- def __init__(self, vllm_config: VllmConfig) -> None:
18
- self.vllm_config = vllm_config
19
- self.model_config = vllm_config.model_config
20
- self.cache_config = vllm_config.cache_config
21
- self.lora_config = vllm_config.lora_config
22
- self.load_config = vllm_config.load_config
23
- self.parallel_config = vllm_config.parallel_config
24
- self.scheduler_config = vllm_config.scheduler_config
25
- self.device_config = vllm_config.device_config
26
- self.speculative_config = vllm_config.speculative_config
27
- self.prompt_adapter_config = vllm_config.prompt_adapter_config
28
- self.observability_config = vllm_config.observability_config
29
-
30
- self.worker: Worker = self._create_worker()
31
- self.worker.init_device()
32
- self.worker.load_model()
33
-
34
- def _create_worker(
35
- self,
36
- local_rank: int = 0,
37
- rank: int = 0,
38
- distributed_init_method: Optional[str] = None) -> Worker:
39
- """Return worker init args for a given rank."""
40
- # see https://github.com/NVIDIA/nccl/issues/1234
41
- os.environ['NCCL_CUMEM_ENABLE'] = '0'
42
-
43
- if distributed_init_method is None:
44
- distributed_init_method = get_distributed_init_method(
45
- get_ip(), get_open_port())
46
- return Worker(
47
- vllm_config=self.vllm_config,
48
- local_rank=local_rank,
49
- rank=rank,
50
- distributed_init_method=distributed_init_method,
51
- )
52
-
53
- def determine_available_memory(self) -> int:
54
- """Determine the available memory (in bytes) for KV cache by invoking
55
- the underlying worker.
56
- """
57
- return self.worker.determine_available_memory()
58
-
59
- def get_kv_cache_spec(self) -> KVCacheSpec:
60
- """Get all kv cache needed by the model by invoking the underlying
61
- worker.
62
- """
63
- return self.worker.get_kv_cache_spec()
64
-
65
- def initialize(self, kv_cache_config: KVCacheConfig) -> None:
66
- """Initialize the KV cache by invoking the underlying worker.
67
- """
68
- self.worker.initialize_cache(kv_cache_config)
69
- self.worker.compile_or_warm_up_model()
70
-
71
- def execute_model(
72
- self,
73
- scheduler_output,
74
- ) -> ModelRunnerOutput:
75
- output = self.worker.execute_model(scheduler_output)
76
- assert output is not None
77
- return output
78
-
79
- def profile(self, is_start: bool = True):
80
- self.worker.profile(is_start)
81
-
82
- def shutdown(self):
83
- pass
84
-
85
- def check_health(self) -> None:
86
- # UniprocExecutor will always be healthy as long as
87
- # it's running.
88
- return