@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a491d6f535
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] TP Ray executor (#11107)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com>

Rui Qiao committed Dec 23, 2024 at 15:00 UTC a491d6f535d96939d17e5290991dc975495c9580
5 files changed +617 -3
tests/basic_correctness/test_basic_correctness.py
+1 -1
@@ -130,7 +130,7 @@ def test_models_distributed(
130 # Import VLLM_USE_V1 dynamically to handle patching
131 from vllm.envs import VLLM_USE_V1
132 if VLLM_USE_V1 and distributed_executor_backend != "mp":
133 - pytest.skip(f"Skip {distributed_executor_backend} for V1")
133 + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
134
135 dtype = "half"
136 max_tokens = 5
vllm/v1/engine/llm_engine.py
+6 -1
@@ -21,6 +21,7 @@ from vllm.v1.engine.core_client import EngineCoreClient
21 from vllm.v1.engine.detokenizer import Detokenizer
22 from vllm.v1.engine.processor import Processor
23 from vllm.v1.executor.abstract import Executor
24 +from vllm.v1.executor.ray_utils import initialize_ray_cluster
25
26 logger = init_logger(__name__)
27
@@ -110,7 +111,11 @@ class LLMEngine:
111 executor_class: Type[Executor]
112 distributed_executor_backend = (
113 vllm_config.parallel_config.distributed_executor_backend)
113 - if distributed_executor_backend == "mp":
114 + if distributed_executor_backend == "ray":
115 + initialize_ray_cluster(vllm_config.parallel_config)
116 + from vllm.v1.executor.ray_executor import RayExecutor
117 + executor_class = RayExecutor
118 + elif distributed_executor_backend == "mp":
119 from vllm.v1.executor.multiproc_executor import MultiprocExecutor
120 executor_class = MultiprocExecutor
121 else:
vllm/v1/executor/ray_executor.py new
+339
@@ -0,0 +1,339 @@
1 +import os
2 +from collections import defaultdict
3 +from itertools import islice, repeat
4 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
5 +
6 +import vllm.envs as envs
7 +from vllm.config import VllmConfig
8 +from vllm.logger import init_logger
9 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10 +from vllm.v1.executor.abstract import Executor
11 +from vllm.v1.executor.ray_utils import RayWorkerWrapper, ray
12 +from vllm.v1.outputs import ModelRunnerOutput
13 +
14 +if ray is not None:
15 + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
16 +
17 +if TYPE_CHECKING:
18 + from ray.util.placement_group import PlacementGroup
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +class RayExecutor(Executor):
24 +
25 + def __init__(self, vllm_config: VllmConfig) -> None:
26 + self.vllm_config = vllm_config
27 + self.parallel_config = vllm_config.parallel_config
28 + self.model_config = vllm_config.model_config
29 + self.forward_dag: Optional[ray.dag.CompiledDAG] = None
30 +
31 + # Disable Ray usage stats collection.
32 + ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
33 + if ray_usage != "1":
34 + os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
35 +
36 + placement_group = self.parallel_config.placement_group
37 + # Create the parallel GPU workers.
38 + self._init_workers_ray(placement_group)
39 +
40 + def _init_workers_ray(self, placement_group: "PlacementGroup",
41 + **ray_remote_kwargs):
42 + # A list of workers to run a model.
43 + self.workers: List[RayWorkerWrapper] = []
44 + if self.parallel_config.ray_workers_use_nsight:
45 + ray_remote_kwargs = self._configure_ray_workers_use_nsight(
46 + ray_remote_kwargs)
47 +
48 + # Create the workers.
49 + driver_ip = get_ip()
50 + for bundle_id, bundle in enumerate(placement_group.bundle_specs):
51 + if not bundle.get("GPU", 0):
52 + # Skip bundles that don't have GPUs,
53 + # as each worker needs one GPU.
54 + continue
55 + scheduling_strategy = PlacementGroupSchedulingStrategy(
56 + placement_group=placement_group,
57 + placement_group_capture_child_tasks=True,
58 + placement_group_bundle_index=bundle_id,
59 + )
60 +
61 + worker = ray.remote(
62 + num_cpus=0,
63 + num_gpus=1,
64 + scheduling_strategy=scheduling_strategy,
65 + **ray_remote_kwargs,
66 + )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
67 + self.workers.append(worker)
68 +
69 + logger.debug("workers: %s", self.workers)
70 + worker_ips = [
71 + ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
72 + for worker in self.workers
73 + ]
74 + ip_counts: Dict[str, int] = {}
75 + for ip in worker_ips:
76 + ip_counts[ip] = ip_counts.get(ip, 0) + 1
77 +
78 + worker_to_ip = dict(zip(self.workers, worker_ips))
79 +
80 + def sort_by_driver_then_worker_ip(worker):
81 + """
82 + Sort the workers based on 3 properties:
83 + 1. If the worker is on the same node as the driver (vllm engine),
84 + it should be placed first.
85 + 2. Then, if the worker is on a node with fewer workers, it should
86 + be placed first.
87 + 3. Finally, if the work is on a node with smaller IP address, it
88 + should be placed first. This is simply a tiebreaker to make
89 + sure the workers are sorted in a deterministic way.
90 + """
91 + ip = worker_to_ip[worker]
92 + return (ip != driver_ip, ip_counts[ip], ip)
93 +
94 + # After sorting, the workers on the same node will be
95 + # close to each other, and the workers on the driver
96 + # node will be placed first.
97 + self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
98 +
99 + # Get the set of GPU IDs used on each node.
100 + worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids")
101 +
102 + node_workers = defaultdict(list) # node id -> list of worker ranks
103 + node_gpus = defaultdict(list) # node id -> list of gpu ids
104 +
105 + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
106 + node_workers[node_id].append(i)
107 + # `gpu_ids` can be a list of strings or integers.
108 + # convert them to integers for consistency.
109 + # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
110 + # string sorting is not sufficient.
111 + # see https://github.com/vllm-project/vllm/issues/5590
112 + gpu_ids = [int(x) for x in gpu_ids]
113 + node_gpus[node_id].extend(gpu_ids)
114 +
115 + for node_id, gpu_ids in node_gpus.items():
116 + node_gpus[node_id] = sorted(gpu_ids)
117 +
118 + all_ips = set(worker_ips)
119 + n_ips = len(all_ips)
120 + n_nodes = len(node_workers)
121 +
122 + if n_nodes != n_ips:
123 + raise RuntimeError(
124 + f"Every node should have a unique IP address. Got {n_nodes}"
125 + f" nodes with node ids {list(node_workers.keys())} and "
126 + f"{n_ips} unique IP addresses {all_ips}. Please check your"
127 + " network configuration. If you set `VLLM_HOST_IP` or "
128 + "`HOST_IP` environment variable, make sure it is unique for"
129 + " each node.")
130 +
131 + # Set environment variables for the driver and workers.
132 + all_args_to_update_environment_variables = [({
133 + "CUDA_VISIBLE_DEVICES":
134 + ",".join(map(str, node_gpus[node_id])),
135 + "VLLM_TRACE_FUNCTION":
136 + str(envs.VLLM_TRACE_FUNCTION),
137 + "VLLM_USE_V1":
138 + str(int(envs.VLLM_USE_V1)),
139 + **({
140 + "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
141 + } if envs.VLLM_ATTENTION_BACKEND is not None else {})
142 + }, ) for (node_id, _) in worker_node_and_gpu_ids]
143 +
144 + self._env_vars_for_all_workers = (
145 + all_args_to_update_environment_variables)
146 +
147 + self._run_workers("update_environment_variables",
148 + all_args=self._get_env_vars_to_be_updated())
149 +
150 + if len(node_gpus) == 1:
151 + # in single node case, we don't need to get the IP address.
152 + # the loopback address is sufficient
153 + # NOTE: a node may have several IP addresses, one for each
154 + # network interface. `get_ip()` might return any of them,
155 + # while they might not work for communication inside the node
156 + # if the network setup is complicated. Using the loopback address
157 + # solves this issue, as it always works for communication inside
158 + # the node.
159 + driver_ip = "127.0.0.1"
160 + distributed_init_method = get_distributed_init_method(
161 + driver_ip, get_open_port())
162 +
163 + # Initialize the actual workers inside worker wrapper.
164 + init_worker_all_kwargs = [
165 + self._get_worker_kwargs(
166 + local_rank=node_workers[node_id].index(rank),
167 + rank=rank,
168 + distributed_init_method=distributed_init_method,
169 + ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
170 + ]
171 + self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
172 + self._run_workers("initialize")
173 + self._run_workers("load_model")
174 +
175 + def _configure_ray_workers_use_nsight(self,
176 + ray_remote_kwargs) -> Dict[str, Any]:
177 + # If nsight profiling is enabled, we need to set the profiling
178 + # configuration for the ray workers as runtime env.
179 + runtime_env = ray_remote_kwargs.setdefault("runtime_env", {})
180 + runtime_env.update({
181 + "nsight": {
182 + "t": "cuda,cudnn,cublas",
183 + "o": "'worker_process_%p'",
184 + "cuda-graph-trace": "node",
185 + }
186 + })
187 +
188 + return ray_remote_kwargs
189 +
190 + def _get_env_vars_to_be_updated(self):
191 + return self._env_vars_for_all_workers
192 +
193 + def _get_worker_kwargs(
194 + self,
195 + local_rank: int = 0,
196 + rank: int = 0,
197 + distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
198 + """
199 + Return worker init args for a given rank.
200 + """
201 + if distributed_init_method is None:
202 + distributed_init_method = get_distributed_init_method(
203 + get_ip(), get_open_port())
204 + return dict(
205 + vllm_config=self.vllm_config,
206 + local_rank=local_rank,
207 + rank=rank,
208 + distributed_init_method=distributed_init_method,
209 + )
210 +
211 + def determine_num_available_blocks(self) -> Tuple[int, int]:
212 + """
213 + Determine the number of available KV blocks.
214 +
215 + This invokes `determine_num_available_blocks` on each worker and takes
216 + the min of the results, guaranteeing that the selected cache sizes are
217 + compatible with all workers.
218 +
219 + Returns:
220 + - tuple[num_gpu_blocks, num_cpu_blocks]
221 + """
222 + # Get the maximum number of blocks that can be allocated on GPU and CPU.
223 + num_blocks = self._run_workers("determine_num_available_blocks")
224 +
225 + # Since we use a shared centralized controller, we take the minimum
226 + # number of blocks across all workers to make sure all the memory
227 + # operators can be applied to all workers.
228 + num_gpu_blocks = min(b[0] for b in num_blocks)
229 + num_cpu_blocks = min(b[1] for b in num_blocks)
230 +
231 + return num_gpu_blocks, num_cpu_blocks
232 +
233 + def initialize(self, num_gpu_blocks: int) -> None:
234 + """
235 + Initialize the KV cache in all workers.
236 + """
237 + # NOTE: This is logged in the executor because there can be >1 worker
238 + # with other executors. We could log in the engine level, but work
239 + # remains to abstract away the device for non-GPU configurations.
240 + logger.info("# GPU blocks: %d", num_gpu_blocks)
241 + self._run_workers("initialize_cache", num_gpu_blocks)
242 + self._run_workers("compile_or_warm_up_model")
243 +
244 + def _run_workers(
245 + self,
246 + method: str,
247 + *args,
248 + all_args: Optional[List[Tuple[Any, ...]]] = None,
249 + all_kwargs: Optional[List[Dict[str, Any]]] = None,
250 + **kwargs,
251 + ) -> Any:
252 + """
253 + Runs the given method on all workers. Can be used in the following
254 + ways:
255 +
256 + Args:
257 + - args/kwargs: All workers share the same args/kwargs
258 + - all_args/all_kwargs: args/kwargs for each worker are specified
259 + individually
260 + """
261 + count = len(self.workers)
262 + all_worker_args = repeat(args, count) if all_args is None \
263 + else islice(all_args, 0, None)
264 + all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
265 + else islice(all_kwargs, 0, None)
266 +
267 + ray_worker_refs = [
268 + worker.execute_method.remote( # type: ignore[attr-defined]
269 + method, *worker_args, **worker_kwargs)
270 + for (worker, worker_args, worker_kwargs
271 + ) in zip(self.workers, all_worker_args, all_worker_kwargs)
272 + ]
273 + return ray.get(ray_worker_refs)
274 +
275 + def execute_model(
276 + self,
277 + scheduler_output,
278 + ) -> ModelRunnerOutput:
279 + if self.forward_dag is None:
280 + self.forward_dag = self._compiled_ray_dag()
281 + # Only the first worker (with rank 0) returns the execution result.
282 + # Others return None.
283 + output = ray.get(self.forward_dag.execute(scheduler_output))[0]
284 + return output
285 +
286 + def profile(self, is_start=True):
287 + raise NotImplementedError
288 +
289 + def shutdown(self):
290 + if hasattr(self, "forward_dag") and self.forward_dag is not None:
291 + self.forward_dag.teardown()
292 + import ray
293 + for worker in self.workers:
294 + ray.kill(worker)
295 + self.forward_dag = None
296 +
297 + def check_health(self) -> None:
298 + logger.debug("Called check_health.")
299 +
300 + def _check_ray_compiled_graph_installation(self):
301 + import pkg_resources
302 + from packaging import version
303 +
304 + required_version = version.parse("2.39")
305 + current_version = version.parse(
306 + pkg_resources.get_distribution("ray").version)
307 + if current_version < required_version:
308 + raise ValueError(f"Ray version {required_version} is "
309 + f"required, but found {current_version}")
310 +
311 + import importlib.util
312 + raycg = importlib.util.find_spec("ray.experimental.compiled_dag_ref")
313 + if raycg is None:
314 + raise ValueError("Ray Compiled Graph is not installed. "
315 + "Run `pip install ray[adag]` to install it.")
316 +
317 + cupy_spec = importlib.util.find_spec("cupy")
318 + if cupy_spec is None and envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL:
319 + raise ValueError(
320 + "cupy is not installed but required since "
321 + "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL is set."
322 + "Run `pip install ray[adag]` and check cupy installation.")
323 +
324 + def _compiled_ray_dag(self):
325 + assert self.parallel_config.use_ray
326 + self._check_ray_compiled_graph_installation()
327 + from ray.dag import InputNode, MultiOutputNode
328 +
329 + with InputNode() as input_batches:
330 + outputs = [
331 + worker.execute_model.bind( # type: ignore[attr-defined]
332 + input_batches) for worker in self.workers
333 + ]
334 + forward_dag = MultiOutputNode(outputs)
335 +
336 + return forward_dag.experimental_compile()
337 +
338 + def __del__(self):
339 + self.shutdown()
vllm/v1/executor/ray_utils.py new
+271
@@ -0,0 +1,271 @@
1 +import time
2 +from collections import defaultdict
3 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
4 +
5 +from vllm.config import ParallelConfig
6 +from vllm.logger import init_logger
7 +from vllm.platforms import current_platform
8 +from vllm.utils import get_ip
9 +from vllm.v1.outputs import ModelRunnerOutput
10 +from vllm.worker.worker_base import WorkerWrapperBase
11 +
12 +if TYPE_CHECKING:
13 + from vllm.v1.core.scheduler import SchedulerOutput
14 +
15 +logger = init_logger(__name__)
16 +PG_WAIT_TIMEOUT = 60
17 +
18 +try:
19 + import ray
20 + from ray.util import placement_group_table
21 + from ray.util.placement_group import PlacementGroup
22 + try:
23 + from ray._private.state import available_resources_per_node
24 + except ImportError:
25 + # Ray 2.9.x doesn't expose `available_resources_per_node`
26 + from ray._private.state import state as _state
27 + available_resources_per_node = _state._available_resources_per_node
28 +
29 + class RayWorkerWrapper(WorkerWrapperBase):
30 +
31 + def __init__(self, *args, **kwargs) -> None:
32 + super().__init__(*args, **kwargs)
33 + # Since the compiled DAG runs a main execution
34 + # in a different thread that calls cuda.set_device.
35 + # The flag indicates is set_device is called on
36 + # that thread. It will be removed soon.
37 + self.compiled_dag_cuda_device_set = False
38 +
39 + def get_node_ip(self) -> str:
40 + return get_ip()
41 +
42 + def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
43 + node_id = ray.get_runtime_context().get_node_id()
44 + gpu_ids = ray.get_gpu_ids()
45 + return node_id, gpu_ids
46 +
47 + def setup_device_if_necessary(self):
48 + # TODO(swang): This is needed right now because Ray CG executes
49 + # on a background thread, so we need to reset torch's current
50 + # device.
51 + # We can remove this API after it is fixed in compiled graph.
52 + import torch
53 + assert self.worker is not None, "Worker is not initialized"
54 + if not self.compiled_dag_cuda_device_set:
55 + torch.cuda.set_device(self.worker.device)
56 + self.compiled_dag_cuda_device_set = True
57 +
58 + def execute_model(
59 + self,
60 + scheduler_output: "SchedulerOutput",
61 + ) -> ModelRunnerOutput:
62 + self.setup_device_if_necessary()
63 + assert self.worker is not None, "Worker is not initialized"
64 + output = self.worker.model_runner.execute_model(scheduler_output)
65 + return output
66 +
67 + ray_import_err = None
68 +
69 +except ImportError as e:
70 + ray = None # type: ignore
71 + ray_import_err = e
72 + RayWorkerWrapper = None # type: ignore
73 +
74 +
75 +def ray_is_available() -> bool:
76 + """Returns True if Ray is available."""
77 + return ray is not None
78 +
79 +
80 +def assert_ray_available():
81 + """
82 + Raise an exception if Ray is not available.
83 + """
84 + if ray is None:
85 + raise ValueError("Failed to import Ray, please install Ray with "
86 + "`pip install ray`.") from ray_import_err
87 +
88 +
89 +def _verify_bundles(placement_group: "PlacementGroup",
90 + parallel_config: ParallelConfig, device_str: str):
91 + """
92 + Verify a given placement group has bundles located in the right place.
93 +
94 + There are 2 rules.
95 + - Warn if all tensor parallel workers cannot fit in a single node.
96 + - Fail if driver node is not included in a placement group.
97 +
98 + Args:
99 + placement_group: The placement group to verify.
100 + parallel_config: The parallel configuration.
101 + device_str: The required device.
102 + """
103 + assert ray.is_initialized(), (
104 + "Ray is not initialized although distributed-executor-backend is ray.")
105 + pg_data = placement_group_table(placement_group)
106 + # bundle_idx -> node_id
107 + bundle_to_node_ids = pg_data["bundles_to_node_id"]
108 + # bundle_idx -> bundle (e.g., {"GPU": 1})
109 + bundles = pg_data["bundles"]
110 + # node_id -> List of bundle (e.g., {"GPU": 1})
111 + node_id_to_bundle: Dict[str, List[Dict[str, float]]] = defaultdict(list)
112 +
113 + for bundle_idx, node_id in bundle_to_node_ids.items():
114 + node_id_to_bundle[node_id].append(bundles[bundle_idx])
115 + driver_node_id = ray.get_runtime_context().get_node_id()
116 +
117 + if driver_node_id not in node_id_to_bundle:
118 + raise RuntimeError(
119 + f"driver node id {driver_node_id} is not included in a placement "
120 + f"group {placement_group.id}. Node id -> bundles "
121 + f"{node_id_to_bundle}. "
122 + "You don't have enough GPUs available in a current node. Check "
123 + "`ray status` to see if you have available GPUs in a node "
124 + f"{driver_node_id} before starting an vLLM engine.")
125 +
126 + for node_id, bundles in node_id_to_bundle.items():
127 + if len(bundles) < parallel_config.tensor_parallel_size:
128 + logger.warning(
129 + "tensor_parallel_size=%d "
130 + "is bigger than a reserved number of %ss (%d "
131 + "%ss) in a node %s. Tensor parallel workers can be "
132 + "spread out to 2+ nodes which can degrade the performance "
133 + "unless you have fast interconnect across nodes, like "
134 + "Infiniband. To resolve this issue, make sure you have more "
135 + "than %d GPUs available at each node.",
136 + parallel_config.tensor_parallel_size, device_str, len(bundles),
137 + device_str, node_id, parallel_config.tensor_parallel_size)
138 +
139 +
140 +def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
141 + """Wait until a placement group is ready.
142 +
143 + It prints the informative log messages if the placement group is
144 + not created within time.
145 +
146 + """
147 + # Wait until PG is ready - this will block until all
148 + # requested resources are available, and will timeout
149 + # if they cannot be provisioned.
150 + placement_group_specs = current_placement_group.bundle_specs
151 +
152 + s = time.time()
153 + pg_ready_ref = current_placement_group.ready()
154 + wait_interval = 10
155 + while time.time() - s < PG_WAIT_TIMEOUT:
156 + ready, _ = ray.wait([pg_ready_ref], timeout=wait_interval)
157 + if len(ready) > 0:
158 + break
159 +
160 + # Exponential backoff for warning print.
161 + wait_interval *= 2
162 + logger.info(
163 + "Waiting for creating a placement group of specs for "
164 + "%d seconds. specs=%s. Check "
165 + "`ray status` to see if you have enough resources.",
166 + int(time.time() - s), placement_group_specs)
167 +
168 + try:
169 + ray.get(pg_ready_ref, timeout=0)
170 + except ray.exceptions.GetTimeoutError:
171 + raise ValueError(
172 + "Cannot provide a placement group of "
173 + f"{placement_group_specs=} within {PG_WAIT_TIMEOUT} seconds. See "
174 + "`ray status` to make sure the cluster has enough resources."
175 + ) from None
176 +
177 +
178 +def initialize_ray_cluster(
179 + parallel_config: ParallelConfig,
180 + ray_address: Optional[str] = None,
181 +):
182 + """Initialize the distributed cluster with Ray.
183 +
184 + it will connect to the Ray cluster and create a placement group
185 + for the workers, which includes the specification of the resources
186 + for each distributed worker.
187 +
188 + Args:
189 + parallel_config: The configurations for parallel execution.
190 + ray_address: The address of the Ray cluster. If None, uses
191 + the default Ray cluster address.
192 + """
193 + assert_ray_available()
194 +
195 + # Connect to a ray cluster.
196 + if current_platform.is_rocm() or current_platform.is_xpu():
197 + # Try to connect existing ray instance and create a new one if not found
198 + try:
199 + ray.init("auto")
200 + except ConnectionError:
201 + logger.warning(
202 + "No existing RAY instance detected. "
203 + "A new instance will be launched with current node resources.")
204 + ray.init(address=ray_address,
205 + ignore_reinit_error=True,
206 + num_gpus=parallel_config.world_size)
207 + else:
208 + ray.init(address=ray_address, ignore_reinit_error=True)
209 +
210 + if parallel_config.placement_group:
211 + # Placement group is already set.
212 + return
213 +
214 + device_str = "GPU" if not current_platform.is_tpu() else "TPU"
215 + # Create placement group for worker processes
216 + current_placement_group = ray.util.get_current_placement_group()
217 + if current_placement_group:
218 + # We are in a placement group
219 + bundles = current_placement_group.bundle_specs
220 + # Verify that we can use the placement group.
221 + device_bundles = 0
222 + for bundle in bundles:
223 + bundle_devices = bundle.get(device_str, 0)
224 + if bundle_devices > 1:
225 + raise ValueError(
226 + "Placement group bundle cannot have more than 1 "
227 + f"{device_str}.")
228 + if bundle_devices:
229 + device_bundles += 1
230 + if parallel_config.world_size > device_bundles:
231 + raise ValueError(
232 + f"The number of required {device_str}s exceeds the total "
233 + f"number of available {device_str}s in the placement group."
234 + f"Required number of devices: {parallel_config.world_size}. "
235 + f"Total number of devices: {device_bundles}.")
236 + else:
237 + num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
238 + if parallel_config.world_size > num_devices_in_cluster:
239 + raise ValueError(
240 + f"The number of required {device_str}s exceeds the total "
241 + f"number of available {device_str}s in the placement group.")
242 + # Create a new placement group
243 + placement_group_specs: List[Dict[str, float]] = ([{
244 + device_str: 1.0
245 + } for _ in range(parallel_config.world_size)])
246 +
247 + # vLLM engine is also a worker to execute model with an accelerator,
248 + # so it requires to have the device in a current node. Check if
249 + # the current node has at least one device.
250 + current_ip = get_ip()
251 + current_node_id = ray.get_runtime_context().get_node_id()
252 + current_node_resource = available_resources_per_node()[current_node_id]
253 + if current_node_resource.get(device_str, 0) < 1:
254 + raise ValueError(
255 + f"Current node has no {device_str} available. "
256 + f"{current_node_resource=}. vLLM engine cannot start without "
257 + f"{device_str}. Make sure you have at least 1 {device_str} "
258 + f"available in a node {current_node_id=} {current_ip=}.")
259 + # This way, at least bundle is required to be created in a current
260 + # node.
261 + placement_group_specs[0][f"node:{current_ip}"] = 0.001
262 +
263 + # By default, Ray packs resources as much as possible.
264 + current_placement_group = ray.util.placement_group(
265 + placement_group_specs, strategy="PACK")
266 + _wait_until_pg_ready(current_placement_group)
267 +
268 + assert current_placement_group is not None
269 + _verify_bundles(current_placement_group, parallel_config, device_str)
270 + # Set the placement group in the parallel config
271 + parallel_config.placement_group = current_placement_group
vllm/v1/worker/gpu_worker.py
-1
@@ -202,7 +202,6 @@ class Worker:
202 ) -> ModelRunnerOutput:
203 output = self.model_runner.execute_model(scheduler_output)
204 return output if self.rank == 0 else None
205 - return output
205
206 def profile(self, is_start: bool = True):
207 if self.profiler is None: