@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de5e6c44c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 16:00 UTC

[Feat][Executor] Introduce RayExecutorV2 (#36836)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>

Jeffrey Wang committed Apr 1, 2026 at 14:34 UTC de5e6c44c6fb8aebcc67ff57e85693f6951c8795
14 files changed +1603 -30
.buildkite/test_areas/distributed.yaml
+34
@@ -224,6 +224,20 @@ steps:
224 commands:
225 - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code"
226
227 +- label: MessageQueue TCP Multi-Node (2 GPUs)
228 + timeout_in_minutes: 10
229 + working_dir: "/vllm-workspace/tests"
230 + num_devices: 1
231 + num_nodes: 2
232 + no_plugin: true
233 + optional: true
234 + source_file_dependencies:
235 + - vllm/distributed/device_communicators/shm_broadcast.py
236 + - vllm/distributed/parallel_state.py
237 + - tests/distributed/test_mq_tcp_multinode.py
238 + commands:
239 + - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 1 $IMAGE_TAG "torchrun --nnodes 2 --nproc-per-node=1 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_mq_tcp_multinode.py" "torchrun --nnodes 2 --nproc-per-node=1 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_mq_tcp_multinode.py"
240 +
241 - label: Distributed NixlConnector PD accuracy (4 GPUs)
242 timeout_in_minutes: 30
243 working_dir: "/vllm-workspace/tests"
@@ -294,3 +308,23 @@ steps:
308 commands:
309 - pytest -v -s distributed/test_pp_cudagraph.py
310 - pytest -v -s distributed/test_pipeline_parallel.py
311 +
312 +- label: RayExecutorV2 (4 GPUs)
313 + timeout_in_minutes: 60
314 + working_dir: "/vllm-workspace/tests"
315 + num_devices: 4
316 + source_file_dependencies:
317 + - vllm/v1/executor/ray_executor_v2.py
318 + - vllm/v1/executor/abstract.py
319 + - vllm/v1/executor/multiproc_executor.py
320 + - tests/distributed/test_ray_v2_executor.py
321 + - tests/distributed/test_ray_v2_executor_e2e.py
322 + - tests/distributed/test_pipeline_parallel.py
323 + - tests/basic_correctness/test_basic_correctness.py
324 + commands:
325 + - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
326 + - export NCCL_CUMEM_HOST_ENABLE=0
327 + - pytest -v -s distributed/test_ray_v2_executor.py
328 + - pytest -v -s distributed/test_ray_v2_executor_e2e.py
329 + - pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
330 + - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
tests/distributed/conftest.py
+29
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import os
4 import random
5
6 import msgspec
@@ -166,3 +167,31 @@ class MockSubscriber:
167 self.sub.close()
168 for replay in self.replay_sockets:
169 replay.close()
170 +
171 +
172 +@pytest.fixture
173 +def enable_ray_v2_backend():
174 + """Set env vars for the Ray V2 executor backend and shut down Ray
175 + between tests."""
176 + import ray
177 +
178 + saved = {
179 + "VLLM_USE_RAY_V2_EXECUTOR_BACKEND": os.environ.get(
180 + "VLLM_USE_RAY_V2_EXECUTOR_BACKEND"
181 + ),
182 + "VLLM_ENABLE_V1_MULTIPROCESSING": os.environ.get(
183 + "VLLM_ENABLE_V1_MULTIPROCESSING"
184 + ),
185 + }
186 + os.environ["VLLM_USE_RAY_V2_EXECUTOR_BACKEND"] = "1"
187 + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
188 + if ray.is_initialized():
189 + ray.shutdown()
190 + try:
191 + yield
192 + finally:
193 + if ray.is_initialized():
194 + ray.shutdown()
195 + os.environ.update({k: v for k, v in saved.items() if v is not None})
196 + for key in (k for k, v in saved.items() if v is None):
197 + os.environ.pop(key, None)
tests/distributed/test_mq_tcp_multinode.py new
+119
@@ -0,0 +1,119 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Multi-node integration test for MessageQueue TCP fallback.
5 +
6 +Verifies that when writer and readers span separate nodes (Docker containers
7 +with isolated /dev/shm), `create_from_process_group` correctly detects
8 +cross-node ranks via `in_the_same_node_as()` and falls back to ZMQ TCP
9 +transport — and that data actually arrives.
10 +"""
11 +
12 +import numpy as np
13 +import torch.distributed as dist
14 +
15 +from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
16 +from vllm.distributed.parallel_state import in_the_same_node_as
17 +
18 +
19 +def main():
20 + dist.init_process_group(backend="gloo")
21 +
22 + rank = dist.get_rank()
23 + world_size = dist.get_world_size()
24 + assert world_size >= 2, (
25 + f"Need at least 2 ranks across nodes, got world_size={world_size}"
26 + )
27 +
28 + # Verify that in_the_same_node_as detects cross-node correctly
29 + status = in_the_same_node_as(dist.group.WORLD, source_rank=0)
30 + local_count = sum(status)
31 + print(
32 + f"[Rank {rank}] in_the_same_node_as(source=0): {status} "
33 + f"(local={local_count}/{world_size})"
34 + )
35 + # With 2 Docker containers (1 proc each), rank 0 and rank 1
36 + # should be on different nodes.
37 + assert local_count < world_size, (
38 + f"Expected cross-node ranks but all {world_size} ranks appear local."
39 + )
40 +
41 + # Create MessageQueue
42 + writer_rank = 0
43 + mq = MessageQueue.create_from_process_group(
44 + dist.group.WORLD,
45 + max_chunk_bytes=1024 * 1024, # 1 MiB
46 + max_chunks=10,
47 + writer_rank=writer_rank,
48 + )
49 +
50 + # Verify the transport path selection
51 + if rank == writer_rank:
52 + print(
53 + f"[Rank {rank}] Writer: n_local_reader={mq.n_local_reader}, "
54 + f"n_remote_reader={mq.n_remote_reader}"
55 + )
56 + assert mq.n_remote_reader > 0, (
57 + "Writer should have at least 1 remote (TCP) reader in a multi-node setup."
58 + )
59 + else:
60 + if status[rank]:
61 + assert mq._is_local_reader, (
62 + f"Rank {rank} is on the same node as writer but is not a local reader."
63 + )
64 + print(f"[Rank {rank}] Reader: local (shared memory)")
65 + else:
66 + assert mq._is_remote_reader, (
67 + f"Rank {rank} is on a different node but is not a remote (TCP) reader."
68 + )
69 + print(f"[Rank {rank}] Reader: remote (TCP)")
70 +
71 + # Test data transfer: simple objects
72 + dist.barrier()
73 + if rank == writer_rank:
74 + mq.enqueue("hello_from_node0")
75 + else:
76 + msg = mq.dequeue(timeout=10)
77 + assert msg == "hello_from_node0"
78 + dist.barrier()
79 + print(f"[Rank {rank}] Simple object test passed")
80 +
81 + # Test data transfer: numpy arrays
82 + np.random.seed(42)
83 + arrays = [
84 + np.random.randint(0, 100, size=np.random.randint(100, 5000)) for _ in range(100)
85 + ]
86 +
87 + dist.barrier()
88 + if rank == writer_rank:
89 + for arr in arrays:
90 + mq.enqueue(arr)
91 + else:
92 + for i, expected in enumerate(arrays):
93 + received = mq.dequeue(timeout=10)
94 + assert np.array_equal(expected, received), (
95 + f"Array mismatch at index {i}: "
96 + f"expected shape {expected.shape}, got shape {received.shape}"
97 + )
98 + dist.barrier()
99 + print(f"[Rank {rank}] Numpy array test passed")
100 +
101 + # Test data transfer: large payload (> max_chunk_bytes)
102 + dist.barrier()
103 + big_array = np.zeros(200_000, dtype=np.int64) # ~1.6 MiB > 1 MiB chunk
104 + if rank == writer_rank:
105 + mq.enqueue(big_array)
106 + else:
107 + received = mq.dequeue(timeout=10)
108 + assert np.array_equal(big_array, received)
109 + dist.barrier()
110 + print(f"[Rank {rank}] Large payload test passed")
111 +
112 + # Done -- cleanup
113 + dist.barrier()
114 + print(f"[Rank {rank}] All MessageQueue TCP multi-node tests passed!")
115 + dist.destroy_process_group()
116 +
117 +
118 +if __name__ == "__main__":
119 + main()
tests/distributed/test_ray_v2_executor.py new
+345
@@ -0,0 +1,345 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""
5 +Integration tests for RayExecutorV2 at the executor level.
6 +Validates executor initialization, placement group support, RPC calls,
7 +and distributed execution with various TP/PP configurations.
8 +"""
9 +
10 +import gc
11 +import threading
12 +from unittest.mock import patch
13 +
14 +import pytest
15 +import ray
16 +
17 +from vllm import LLM
18 +from vllm.config import VllmConfig
19 +from vllm.engine.arg_utils import EngineArgs
20 +from vllm.v1.executor.ray_executor_v2 import RayExecutorV2
21 +
22 +pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
23 +
24 +MODEL = "facebook/opt-125m"
25 +
26 +
27 +def create_vllm_config(
28 + tensor_parallel_size: int = 1,
29 + pipeline_parallel_size: int = 1,
30 + max_model_len: int = 256,
31 + gpu_memory_utilization: float = 0.3,
32 + placement_group=None,
33 +) -> VllmConfig:
34 + engine_args = EngineArgs(
35 + model=MODEL,
36 + tensor_parallel_size=tensor_parallel_size,
37 + pipeline_parallel_size=pipeline_parallel_size,
38 + max_model_len=max_model_len,
39 + gpu_memory_utilization=gpu_memory_utilization,
40 + distributed_executor_backend="ray",
41 + enforce_eager=True,
42 + )
43 + vllm_config = engine_args.create_engine_config()
44 +
45 + if placement_group is not None:
46 + vllm_config.parallel_config.placement_group = placement_group
47 +
48 + return vllm_config
49 +
50 +
51 +def ensure_ray_initialized():
52 + if not ray.is_initialized():
53 + ray.init(ignore_reinit_error=True)
54 +
55 +
56 +@pytest.fixture
57 +def create_placement_group(request):
58 + ensure_ray_initialized()
59 + num_gpus = request.param
60 + bundles = [{"GPU": 1, "CPU": 1} for _ in range(num_gpus)]
61 + pg = ray.util.placement_group(bundles, strategy="PACK")
62 + ray.get(pg.ready())
63 + yield pg
64 + ray.util.remove_placement_group(pg)
65 +
66 +
67 +@pytest.fixture
68 +def executor(request):
69 + """Create a RayExecutorV2 and shut it down after the test."""
70 + executor = RayExecutorV2(vllm_config=request.param)
71 + yield executor
72 + executor.shutdown()
73 +
74 +
75 +def assert_executor(executor, tp_size, pp_size):
76 + """Common assertions for executor initialization tests."""
77 + world_size = tp_size * pp_size
78 + expected_output_rank = (pp_size - 1) * tp_size
79 +
80 + assert executor.world_size == world_size
81 + assert len(executor.ray_worker_handles) == world_size
82 + assert len(executor.response_mqs) == world_size
83 + assert executor._get_output_rank() == expected_output_rank
84 +
85 + if pp_size > 1:
86 + assert executor.max_concurrent_batches == pp_size
87 +
88 + executor.check_health()
89 + assert not executor.is_failed
90 +
91 + ranks = sorted(h.rank for h in executor.ray_worker_handles)
92 + assert ranks == list(range(world_size))
93 +
94 + for handle in executor.ray_worker_handles:
95 + assert handle.node_id is not None
96 +
97 +
98 +@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
99 +def test_ray_v2_executor(tp_size, pp_size):
100 + """Validate RayExecutorV2 with various TP/PP configs."""
101 + vllm_config = create_vllm_config(
102 + tensor_parallel_size=tp_size,
103 + pipeline_parallel_size=pp_size,
104 + )
105 + executor = RayExecutorV2(vllm_config=vllm_config)
106 + try:
107 + assert_executor(executor, tp_size, pp_size)
108 + finally:
109 + executor.shutdown()
110 +
111 +
112 +@pytest.mark.parametrize(
113 + "tp_size, pp_size, create_placement_group",
114 + [(2, 1, 2), (4, 1, 4), (2, 2, 4)],
115 + indirect=["create_placement_group"],
116 +)
117 +def test_ray_v2_executor_pg(tp_size, pp_size, create_placement_group):
118 + """Validate RayExecutorV2 with various TP/PP configs using external PG."""
119 + vllm_config = create_vllm_config(
120 + tensor_parallel_size=tp_size,
121 + pipeline_parallel_size=pp_size,
122 + placement_group=create_placement_group,
123 + )
124 + executor = RayExecutorV2(vllm_config=vllm_config)
125 + try:
126 + assert_executor(executor, tp_size, pp_size)
127 + finally:
128 + executor.shutdown()
129 +
130 +
131 +@pytest.mark.parametrize(
132 + "executor",
133 + [create_vllm_config(tensor_parallel_size=2)],
134 + indirect=True,
135 +)
136 +def test_ray_v2_executor_failure_callback(executor):
137 + """Validate failure callback registration."""
138 + callback_invoked = False
139 +
140 + def test_callback():
141 + nonlocal callback_invoked
142 + callback_invoked = True
143 +
144 + executor.register_failure_callback(test_callback)
145 + assert not callback_invoked
146 +
147 + executor.is_failed = True
148 + executor.register_failure_callback(test_callback)
149 + assert callback_invoked
150 +
151 +
152 +@pytest.mark.parametrize(
153 + "executor",
154 + [create_vllm_config(tensor_parallel_size=2)],
155 + indirect=True,
156 +)
157 +def test_ray_v2_executor_collective_rpc(executor):
158 + """Validate collective RPC calls through MessageQueue."""
159 + executor.check_health()
160 + assert not executor.is_failed
161 + assert executor.rpc_broadcast_mq is not None
162 +
163 +
164 +@pytest.mark.parametrize(
165 + "executor",
166 + [create_vllm_config(tensor_parallel_size=2)],
167 + indirect=True,
168 +)
169 +def test_ray_v2_executor_driver_node_rank_0(executor):
170 + """Validate that driver node workers get the lowest ranks."""
171 + driver_node = ray.get_runtime_context().get_node_id()
172 +
173 + for handle in executor.ray_worker_handles:
174 + assert handle.node_id == driver_node
175 +
176 + rank0_handle = next(h for h in executor.ray_worker_handles if h.rank == 0)
177 + assert rank0_handle.node_id == driver_node
178 +
179 +
180 +@pytest.mark.parametrize(
181 + "executor",
182 + [create_vllm_config(tensor_parallel_size=2)],
183 + indirect=True,
184 +)
185 +def test_ray_v2_executor_worker_death(executor):
186 + """Validate executor detects worker death via ray.wait()."""
187 + callback_event = threading.Event()
188 +
189 + def on_failure():
190 + callback_event.set()
191 +
192 + executor.register_failure_callback(on_failure)
193 + assert not executor.is_failed
194 +
195 + # Kill one worker actor externally
196 + victim = executor.ray_worker_handles[1].actor
197 + ray.kill(victim, no_restart=True)
198 +
199 + # Monitor thread should detect the death and invoke callback
200 + assert callback_event.wait(timeout=30)
201 + assert executor.is_failed
202 + assert executor.shutting_down
203 +
204 +
205 +def test_ray_v2_executor_shutdown():
206 + """Validate graceful shutdown: ray.kill() terminates all worker actors."""
207 + executor = RayExecutorV2(vllm_config=create_vllm_config(tensor_parallel_size=2))
208 + assert executor.rpc_broadcast_mq is not None
209 + assert len(executor.response_mqs) == executor.world_size
210 +
211 + actors = [h.actor for h in executor.ray_worker_handles]
212 + executor.shutdown()
213 +
214 + for actor in actors:
215 + with pytest.raises(ray.exceptions.RayActorError):
216 + ray.get(actor.wait_for_init.remote(), timeout=5)
217 +
218 + assert executor.rpc_broadcast_mq is None
219 + assert len(executor.response_mqs) == 0
220 +
221 +
222 +@pytest.mark.parametrize(
223 + "executor",
224 + [create_vllm_config(tensor_parallel_size=2)],
225 + indirect=True,
226 +)
227 +def test_ray_v2_run_refs_stored_for_monitoring(executor):
228 + """Validate worker handles store run_ref for monitoring."""
229 + for handle in executor.ray_worker_handles:
230 + assert handle.run_ref is not None
231 + ready, _ = ray.wait([handle.run_ref], timeout=0)
232 + assert len(ready) == 0, "run_ref should be pending"
233 +
234 +
235 +@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
236 +def test_ray_v2_single_node_generation(tp_size, pp_size):
237 + """End-to-end LLM generation with RayExecutorV2."""
238 +
239 + llm = LLM(
240 + model=MODEL,
241 + tensor_parallel_size=tp_size,
242 + pipeline_parallel_size=pp_size,
243 + distributed_executor_backend="ray",
244 + enforce_eager=True,
245 + max_model_len=256,
246 + gpu_memory_utilization=0.3,
247 + )
248 + try:
249 + prompts = [
250 + "Hello, my name is",
251 + "The capital of France is",
252 + "The future of AI is",
253 + ]
254 + outputs = llm.generate(prompts)
255 +
256 + assert len(outputs) == len(prompts)
257 + for output in outputs:
258 + assert len(output.outputs) > 0
259 + assert len(output.outputs[0].text) > 0
260 + finally:
261 + llm.llm_engine.model_executor.shutdown()
262 + del llm
263 + gc.collect()
264 +
265 +
266 +@pytest.mark.parametrize(
267 + "bundle_indices, expected_bundle_ids, create_placement_group",
268 + [("2,3", [2, 3], 4), ("3,2", [3, 2], 4)],
269 + indirect=["create_placement_group"],
270 +)
271 +def test_ray_v2_bundle_indices_env(
272 + bundle_indices, expected_bundle_ids, create_placement_group, monkeypatch
273 +):
274 + """Validate explicit VLLM_RAY_BUNDLE_INDICES bundle placement."""
275 + monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", bundle_indices)
276 + vllm_config = create_vllm_config(
277 + tensor_parallel_size=2,
278 + placement_group=create_placement_group,
279 + )
280 + executor = RayExecutorV2(vllm_config=vllm_config)
281 + try:
282 + actual = [
283 + h.bundle_id_idx
284 + for h in sorted(executor.ray_worker_handles, key=lambda h: h.rank)
285 + ]
286 + assert actual == expected_bundle_ids
287 + assert_executor(executor, tp_size=2, pp_size=1)
288 + finally:
289 + executor.shutdown()
290 +
291 +
292 +@pytest.mark.parametrize(
293 + "bundle_indices, expected_error, create_placement_group",
294 + [
295 + ("1,1", "cannot have duplicate values,", 4),
296 + ("0,1,2", "must have the same size", 4),
297 + ],
298 + indirect=["create_placement_group"],
299 +)
300 +def test_ray_v2_invalid_bundle_indices(
301 + bundle_indices, expected_error, create_placement_group, monkeypatch
302 +):
303 + """Validate invalid bundle indices are rejected."""
304 + monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", bundle_indices)
305 + vllm_config = create_vllm_config(
306 + tensor_parallel_size=2, placement_group=create_placement_group
307 + )
308 + with pytest.raises(AssertionError, match=expected_error):
309 + RayExecutorV2(vllm_config=vllm_config)
310 +
311 +
312 +@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
313 +def test_ray_v2_single_node_generation_with_pg(tp_size, pp_size):
314 + """E2E LLM generation with a user-provided placement group."""
315 + ensure_ray_initialized()
316 + bundles = [{"GPU": 1, "CPU": 1} for _ in range(tp_size * pp_size)]
317 + pg = ray.util.placement_group(bundles, strategy="PACK")
318 + ray.get(pg.ready())
319 +
320 + try:
321 + with patch.object(ray.util, "get_current_placement_group", return_value=pg):
322 + llm = LLM(
323 + model=MODEL,
324 + tensor_parallel_size=tp_size,
325 + pipeline_parallel_size=pp_size,
326 + distributed_executor_backend="ray",
327 + enforce_eager=True,
328 + max_model_len=256,
329 + gpu_memory_utilization=0.3,
330 + )
331 + prompts = [
332 + "Hello, my name is",
333 + "The capital of France is",
334 + "The future of AI is",
335 + ]
336 + outputs = llm.generate(prompts)
337 +
338 + assert len(outputs) == len(prompts)
339 + for output in outputs:
340 + assert len(output.outputs) > 0
341 + assert len(output.outputs[0].text) > 0
342 + finally:
343 + llm.llm_engine.model_executor.shutdown()
344 + del llm
345 + gc.collect()
tests/distributed/test_ray_v2_executor_e2e.py new
+209
@@ -0,0 +1,209 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Orchestration-level integration tests for RayExecutorV2.
5 +"""
6 +
7 +import gc
8 +import os
9 +import pathlib
10 +
11 +import pytest
12 +import ray
13 +
14 +pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
15 +
16 +MODEL = "facebook/opt-125m"
17 +
18 +
19 +def _get_env_var(worker, name):
20 + return os.environ.get(name)
21 +
22 +
23 +def _ray_init():
24 + """Start Ray with the project root on workers' PYTHONPATH.
25 +
26 + Without this, workers cannot unpickle actor classes defined in the
27 + ``tests`` package, causing FunctionActorManager to fall back to
28 + TemporaryActor which drops async method signatures."""
29 + project_root = str(pathlib.Path(__file__).resolve().parents[2])
30 + ray.init(
31 + ignore_reinit_error=True,
32 + runtime_env={"env_vars": {"PYTHONPATH": project_root}},
33 + )
34 +
35 +
36 +@pytest.fixture
37 +def ray_init():
38 + _ray_init()
39 +
40 +
41 +class _AsyncLLMActor:
42 + def start(self, pg, bundle_indices=None, ray_runtime_env=None):
43 + os.environ["VLLM_USE_RAY_V2_EXECUTOR_BACKEND"] = "1"
44 + # Needed so collective_rpc can pickle _get_env_var over the
45 + # AsyncLLM -> EngineCore ZMQ boundary.
46 + os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
47 + if bundle_indices is not None:
48 + os.environ["VLLM_RAY_BUNDLE_INDICES"] = bundle_indices
49 + else:
50 + os.environ.pop("VLLM_RAY_BUNDLE_INDICES", None)
51 +
52 + from vllm.engine.arg_utils import AsyncEngineArgs
53 + from vllm.v1.engine.async_llm import AsyncLLM
54 + from vllm.v1.executor.abstract import Executor
55 +
56 + engine_args = AsyncEngineArgs(
57 + model=MODEL,
58 + tensor_parallel_size=2,
59 + distributed_executor_backend="ray",
60 + enforce_eager=True,
61 + max_model_len=256,
62 + gpu_memory_utilization=0.8,
63 + )
64 + vllm_config = engine_args.create_engine_config()
65 + vllm_config.parallel_config.placement_group = pg
66 + if ray_runtime_env is not None:
67 + vllm_config.parallel_config.ray_runtime_env = ray_runtime_env
68 +
69 + executor_class = Executor.get_class(vllm_config)
70 + self.engine = AsyncLLM(
71 + vllm_config=vllm_config,
72 + executor_class=executor_class,
73 + log_stats=False,
74 + log_requests=False,
75 + )
76 +
77 + async def generate(self, prompt):
78 + from vllm.sampling_params import SamplingParams
79 +
80 + params = SamplingParams(max_tokens=16)
81 + result = None
82 + async for output in self.engine.generate(
83 + prompt, params, request_id="test_request_id"
84 + ):
85 + result = output
86 + assert result is not None
87 + return result.outputs[0].text
88 +
89 + async def generate_and_get_worker_envs(self, prompt, env_names):
90 + from vllm.sampling_params import SamplingParams
91 +
92 + params = SamplingParams(max_tokens=16)
93 + result = None
94 + async for output in self.engine.generate(
95 + prompt, params, request_id="test_request_id"
96 + ):
97 + result = output
98 + assert result is not None
99 + text = result.outputs[0].text
100 +
101 + env_results = {}
102 + for name in env_names:
103 + vals = await self.engine.collective_rpc(
104 + _get_env_var, timeout=10, args=(name,)
105 + )
106 + env_results[name] = vals
107 + return text, env_results
108 +
109 + def shutdown(self):
110 + if engine := getattr(self, "engine", None):
111 + engine.shutdown()
112 + del self.engine
113 + gc.collect()
114 +
115 +
116 +AsyncLLMActor = ray.remote(num_cpus=0, max_concurrency=1)(_AsyncLLMActor)
117 +
118 +
119 +def test_multi_replicas(ray_init):
120 + pg1 = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
121 + pg2 = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
122 + ray.get([pg1.ready(), pg2.ready()])
123 +
124 + actor1 = AsyncLLMActor.remote()
125 + actor2 = AsyncLLMActor.remote()
126 +
127 + ray.get(actor1.start.remote(pg1))
128 + ray.get(actor2.start.remote(pg2))
129 +
130 + out1, out2 = ray.get(
131 + [
132 + actor1.generate.remote("Hello world"),
133 + actor2.generate.remote("Hello world"),
134 + ]
135 + )
136 + assert len(out1) > 0
137 + assert len(out2) > 0
138 +
139 +
140 +def test_multi_replicas_with_bundle_indices(ray_init):
141 + pg = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 4, strategy="PACK")
142 + ray.get(pg.ready())
143 +
144 + actor1 = AsyncLLMActor.remote()
145 + actor2 = AsyncLLMActor.remote()
146 +
147 + ray.get(actor1.start.remote(pg, bundle_indices="2,1"))
148 + ray.get(actor2.start.remote(pg, bundle_indices="0,3"))
149 +
150 + out1, out2 = ray.get(
151 + [
152 + actor1.generate.remote("Hello world"),
153 + actor2.generate.remote("Hello world"),
154 + ]
155 + )
156 + assert len(out1) > 0
157 + assert len(out2) > 0
158 +
159 +
160 +def test_env_var_and_runtime_env_propagation():
161 + """
162 + Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
163 + propagate to RayWorkerProc actors.
164 + """
165 + sentinel_vars = {
166 + "NCCL_DEBUG": "INFO",
167 + "HF_TOKEN": "test_sentinel_token",
168 + }
169 + for k, v in sentinel_vars.items():
170 + os.environ[k] = v
171 +
172 + try:
173 + # Called directly (not via the ray_init fixture) because sentinel
174 + # env vars must be in os.environ before ray.init() so that Ray
175 + # worker processes inherit them.
176 + _ray_init()
177 +
178 + pg = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
179 + ray.get(pg.ready())
180 +
181 + # Include the project root so that RayWorkerProc actors can
182 + # unpickle _get_env_var.
183 + project_root = str(pathlib.Path(__file__).resolve().parents[2])
184 + ray_runtime_env = {
185 + "env_vars": {
186 + "RAY_RUNTIME_ENV_TEST": "ray_runtime_env",
187 + "PYTHONPATH": project_root,
188 + },
189 + }
190 +
191 + actor = AsyncLLMActor.remote()
192 + ray.get(actor.start.remote(pg, ray_runtime_env=ray_runtime_env))
193 +
194 + all_env_names = list(sentinel_vars) + ["RAY_RUNTIME_ENV_TEST"]
195 + text, env_results = ray.get(
196 + actor.generate_and_get_worker_envs.remote("Hello world", all_env_names)
197 + )
198 + assert len(text) > 0
199 +
200 + for name, expected in sentinel_vars.items():
201 + for val in env_results[name]:
202 + assert val == expected
203 +
204 + for val in env_results["RAY_RUNTIME_ENV_TEST"]:
205 + assert val == "ray_runtime_env"
206 +
207 + finally:
208 + for k in sentinel_vars:
209 + os.environ.pop(k, None)
tests/test_ray_env_utils.py new
+51
@@ -0,0 +1,51 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for vllm.v1.executor.ray_env_utils."""
4 +
5 +import os
6 +from unittest.mock import patch
7 +
8 +from vllm.v1.executor.ray_env_utils import get_driver_env_vars
9 +
10 +WORKER_VARS: set[str] = {
11 + "CUDA_VISIBLE_DEVICES",
12 + "LOCAL_RANK",
13 +}
14 +
15 +
16 +class TestDefaultPropagation:
17 + """All env vars are propagated unless explicitly excluded."""
18 +
19 + @patch.dict(os.environ, {"NCCL_DEBUG": "INFO"}, clear=False)
20 + def test_nccl_prefix(self):
21 + assert get_driver_env_vars(WORKER_VARS)["NCCL_DEBUG"] == "INFO"
22 +
23 + @patch.dict(os.environ, {"HF_TOKEN": "secret"}, clear=False)
24 + def test_hf_token(self):
25 + assert "HF_TOKEN" in get_driver_env_vars(WORKER_VARS)
26 +
27 + @patch.dict(os.environ, {"LMCACHE_LOCAL_CPU": "True"}, clear=False)
28 + def test_lmcache_prefix(self):
29 + assert "LMCACHE_LOCAL_CPU" in get_driver_env_vars(WORKER_VARS)
30 +
31 + @patch.dict(os.environ, {"PYTHONHASHSEED": "42"}, clear=False)
32 + def test_pythonhashseed(self):
33 + assert get_driver_env_vars(WORKER_VARS)["PYTHONHASHSEED"] == "42"
34 +
35 + @patch.dict(os.environ, {"MYLIB_FOO": "bar"}, clear=False)
36 + def test_arbitrary_var_propagated(self):
37 + assert get_driver_env_vars(WORKER_VARS)["MYLIB_FOO"] == "bar"
38 +
39 +
40 +class TestExclusion:
41 + @patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0,1"}, clear=False)
42 + def test_worker_specific_excluded(self):
43 + assert "CUDA_VISIBLE_DEVICES" not in get_driver_env_vars(WORKER_VARS)
44 +
45 + @patch.dict(os.environ, {"LMCACHE_LOCAL_CPU": "True"}, clear=False)
46 + @patch(
47 + "vllm.v1.executor.ray_env_utils.RAY_NON_CARRY_OVER_ENV_VARS",
48 + {"LMCACHE_LOCAL_CPU"},
49 + )
50 + def test_non_carry_over_blacklist(self):
51 + assert "LMCACHE_LOCAL_CPU" not in get_driver_env_vars(WORKER_VARS)
tests/utils_/test_ray_utils.py new
+100
@@ -0,0 +1,100 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock, patch
5 +
6 +import pytest
7 +
8 +from vllm.v1.executor.ray_utils import get_bundles_sorted_by_node
9 +
10 +NODE_A = "node_a"
11 +NODE_B = "node_b"
12 +NODE_C = "node_c"
13 +
14 +IP_A = "10.0.0.1"
15 +IP_B = "10.0.0.2"
16 +IP_C = "10.0.0.3"
17 +
18 +NODE_ID_TO_IP = {NODE_A: IP_A, NODE_B: IP_B, NODE_C: IP_C}
19 +
20 +MOCK_RAY_NODES = [
21 + {"NodeID": NODE_A, "NodeManagerAddress": IP_A, "Alive": True},
22 + {"NodeID": NODE_B, "NodeManagerAddress": IP_B, "Alive": True},
23 + {"NodeID": NODE_C, "NodeManagerAddress": IP_C, "Alive": True},
24 +]
25 +
26 +
27 +@pytest.mark.parametrize(
28 + "bundles_to_node_id,bundle_specs,expected",
29 + [
30 + pytest.param(
31 + {0: NODE_C, 1: NODE_A, 2: NODE_B, 3: NODE_C, 4: NODE_A, 5: NODE_B},
32 + [{"GPU": 1}] * 6,
33 + [
34 + (1, NODE_A, IP_A),
35 + (4, NODE_A, IP_A),
36 + (2, NODE_B, IP_B),
37 + (5, NODE_B, IP_B),
38 + (0, NODE_C, IP_C),
39 + (3, NODE_C, IP_C),
40 + ],
41 + ),
42 + pytest.param(
43 + {0: NODE_B, 1: NODE_B, 2: NODE_A, 3: NODE_A},
44 + [{"GPU": 1}] * 4,
45 + [
46 + (2, NODE_A, IP_A),
47 + (3, NODE_A, IP_A),
48 + (0, NODE_B, IP_B),
49 + (1, NODE_B, IP_B),
50 + ],
51 + ),
52 + pytest.param(
53 + {0: NODE_C, 1: NODE_B, 2: NODE_C, 3: NODE_B},
54 + [{"GPU": 1}] * 4,
55 + [
56 + (1, NODE_B, IP_B),
57 + (3, NODE_B, IP_B),
58 + (0, NODE_C, IP_C),
59 + (2, NODE_C, IP_C),
60 + ],
61 + ),
62 + pytest.param(
63 + {0: NODE_A, 1: NODE_A, 2: NODE_A},
64 + [{"GPU": 1}] * 3,
65 + [(0, NODE_A, IP_A), (1, NODE_A, IP_A), (2, NODE_A, IP_A)],
66 + ),
67 + pytest.param(
68 + {},
69 + [],
70 + [],
71 + ),
72 + pytest.param(
73 + {0: NODE_A, 1: NODE_B, 2: NODE_A},
74 + [{"CPU": 1}, {"GPU": 1}, {"GPU": 1}],
75 + [(2, NODE_A, IP_A), (1, NODE_B, IP_B)],
76 + ),
77 + ],
78 +)
79 +def test_get_bundles_sorted_by_node(bundles_to_node_id, bundle_specs, expected):
80 + mock_pg = MagicMock()
81 + mock_pg.bundle_specs = bundle_specs
82 +
83 + mock_ctx = MagicMock()
84 + mock_ctx.get_node_id.return_value = NODE_A
85 +
86 + with (
87 + patch(
88 + "vllm.v1.executor.ray_utils.placement_group_table",
89 + return_value={"bundles_to_node_id": bundles_to_node_id},
90 + ),
91 + patch("vllm.v1.executor.ray_utils.ray") as mock_ray,
92 + patch("vllm.v1.executor.ray_utils.current_platform") as mock_platform,
93 + ):
94 + mock_ray.get_runtime_context.return_value = mock_ctx
95 + mock_ray.nodes.return_value = MOCK_RAY_NODES
96 + mock_platform.ray_device_key = "GPU"
97 +
98 + result = get_bundles_sorted_by_node(mock_pg)
99 +
100 + assert result == expected
vllm/envs.py
+7
@@ -59,6 +59,7 @@ if TYPE_CHECKING:
59 VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto"
60 VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False
61 VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True
62 + VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = False
63 VLLM_XLA_USE_SPMD: bool = False
64 VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn"] = "fork"
65 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
@@ -753,6 +754,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
754 "VLLM_USE_RAY_WRAPPED_PP_COMM": lambda: bool(
755 int(os.getenv("VLLM_USE_RAY_WRAPPED_PP_COMM", "1"))
756 ),
757 + # When True and distributed_executor_backend="ray", use RayExecutorV2
758 + # (MQ-based) instead of RayDistributedExecutor (compiled-graph backend).
759 + # TODO (jeffreywang): Enabled by default in vLLM 0.20.0.
760 + "VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
761 + int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "0"))
762 + ),
763 # Use dedicated multiprocess context for workers.
764 # Both spawn and fork work
765 "VLLM_WORKER_MULTIPROC_METHOD": env_with_choices(
vllm/v1/executor/abstract.py
+8 -2
@@ -7,6 +7,7 @@ from concurrent.futures import Future
7 from functools import cached_property
8 from typing import TYPE_CHECKING, Literal, TypeVar, overload
9
10 +import vllm.envs as envs
11 from vllm.config import VllmConfig
12 from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
13 from vllm.distributed.kv_transfer.kv_connector.v1.base import (
@@ -57,9 +58,14 @@ class Executor(ABC):
58 )
59 executor_class = distributed_executor_backend
60 elif distributed_executor_backend == "ray":
60 - from vllm.v1.executor.ray_executor import RayDistributedExecutor
61 + if envs.VLLM_USE_RAY_V2_EXECUTOR_BACKEND:
62 + from vllm.v1.executor.ray_executor_v2 import RayExecutorV2
63
62 - executor_class = RayDistributedExecutor
64 + executor_class = RayExecutorV2
65 + else:
66 + from vllm.v1.executor.ray_executor import RayDistributedExecutor
67 +
68 + executor_class = RayDistributedExecutor
69 elif distributed_executor_backend == "mp":
70 from vllm.v1.executor.multiproc_executor import MultiprocExecutor
71
vllm/v1/executor/ray_env_utils.py new
+18
@@ -0,0 +1,18 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import os
4 +
5 +from vllm.ray.ray_env import RAY_NON_CARRY_OVER_ENV_VARS
6 +
7 +
8 +def get_driver_env_vars(
9 + worker_specific_vars: set[str],
10 +) -> dict[str, str]:
11 + """Return driver env vars to propagate to Ray workers.
12 +
13 + Returns everything from ``os.environ`` except ``worker_specific_vars``
14 + and user-configured exclusions (``RAY_NON_CARRY_OVER_ENV_VARS``).
15 + """
16 + exclude_vars = worker_specific_vars | RAY_NON_CARRY_OVER_ENV_VARS
17 +
18 + return {key: value for key, value in os.environ.items() if key not in exclude_vars}
vllm/v1/executor/ray_executor.py
+2 -12
@@ -23,6 +23,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
23 from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
24 from vllm.v1.executor.abstract import Executor
25 from vllm.v1.executor.ray_utils import (
26 + WORKER_SPECIFIC_ENV_VARS,
27 FutureWrapper,
28 RayWorkerWrapper,
29 initialize_ray_cluster,
@@ -62,17 +63,6 @@ class RayWorkerMetaData:
63 class RayDistributedExecutor(Executor):
64 """Ray-based distributed executor"""
65
65 - # These env vars are worker-specific, therefore are NOT copied
66 - # from the driver to the workers
67 - WORKER_SPECIFIC_ENV_VARS = {
68 - "VLLM_HOST_IP",
69 - "VLLM_HOST_PORT",
70 - "LOCAL_RANK",
71 - "CUDA_VISIBLE_DEVICES",
72 - "HIP_VISIBLE_DEVICES",
73 - "ROCR_VISIBLE_DEVICES",
74 - }
75 -
66 uses_ray: bool = True
67 supports_pp: bool = True
68
@@ -335,7 +325,7 @@ class RayDistributedExecutor(Executor):
325
326 # Environment variables to copy from driver to workers
327 env_vars_to_copy = get_env_vars_to_copy(
338 - exclude_vars=self.WORKER_SPECIFIC_ENV_VARS,
328 + exclude_vars=WORKER_SPECIFIC_ENV_VARS,
329 additional_vars=set(current_platform.additional_env_vars),
330 destination="workers",
331 )
vllm/v1/executor/ray_executor_v2.py new
+524
@@ -0,0 +1,524 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import copy
4 +import os
5 +import threading
6 +import weakref
7 +from collections import defaultdict, deque
8 +from dataclasses import dataclass
9 +from typing import Any
10 +
11 +import vllm.envs as envs
12 +from vllm.config import VllmConfig
13 +from vllm.distributed.device_communicators.shm_broadcast import (
14 + Handle,
15 + MessageQueue,
16 +)
17 +from vllm.logger import init_logger
18 +from vllm.platforms import current_platform
19 +from vllm.utils.network_utils import (
20 + get_distributed_init_method,
21 + get_open_port,
22 +)
23 +from vllm.v1.executor.multiproc_executor import (
24 + FutureWrapper,
25 + MultiprocExecutor,
26 + WorkerProc,
27 +)
28 +from vllm.v1.executor.ray_env_utils import get_driver_env_vars
29 +from vllm.v1.executor.ray_utils import (
30 + WORKER_SPECIFIC_ENV_VARS,
31 + build_actor_name,
32 + get_bundles_for_indices,
33 + get_bundles_sorted_by_node,
34 + initialize_ray_cluster,
35 + ray,
36 +)
37 +
38 +if ray is not None:
39 + from ray.actor import ActorHandle
40 + from ray.types import ObjectRef
41 + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
42 +else:
43 + ActorHandle = None
44 +
45 +logger = init_logger(__name__)
46 +
47 +
48 +@dataclass
49 +class RayWorkerHandle:
50 + """Handle for a Ray worker actor, compatible with MultiprocExecutor."""
51 +
52 + actor: ActorHandle
53 + """Ray worker actor"""
54 +
55 + rank: int
56 + """Rank of the worker"""
57 +
58 + local_rank: int
59 + """Local rank of the worker"""
60 +
61 + node_id: str
62 + """Node ID of the worker"""
63 +
64 + bundle_id_idx: int = -1
65 + """Placement group bundle index for the worker"""
66 +
67 + run_ref: ObjectRef | None = None
68 + """run() ObjectRef used as a sentinel for health monitoring"""
69 +
70 + def run(self):
71 + """Start the worker's busy loop"""
72 + self.run_ref = self.actor.run.remote()
73 +
74 +
75 +class RayWorkerProc(WorkerProc):
76 + """Worker process that runs inside a Ray actor.
77 +
78 + Initialization is split into two phases:
79 + 1. __init__: lightweight setup, stores init args (no device/model init)
80 + 2. initialize_worker: called after GPU IDs are discovered, completes
81 + the full WorkerProc initialization with the correct local_rank and
82 + CUDA_VISIBLE_DEVICES.
83 +
84 + CUDA_VISIBLE_DEVICES setup flow:
85 +
86 + 1. RayExecutorV2 enables RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES so Ray does
87 + not set CUDA_VISIBLE_DEVICES on RayWorkerProc actors at creation time.
88 + 2. Each actor is scheduled with a placement group and bundle index; Ray resolves
89 + the physical GPU ID for that bundle at placement time.
90 + 3. After placement, the worker discovers that GPU ID and sets
91 + CUDA_VISIBLE_DEVICES before finishing WorkerProc initialization.
92 +
93 + There is no workaround for this unset-and-reset sequence when the placement group
94 + is externally managed: scheduling must complete before CUDA_VISIBLE_DEVICES can
95 + match the GPU tied to the worker's bundle.
96 +
97 + This sequence allows multiple vLLM instances to coexist on the same node:
98 + each instance is unaware which physical devices others hold, and the
99 + externally managed placement group avoids CUDA_VISIBLE_DEVICES conflicts
100 + by binding workers to specific placement group bundles.
101 + """
102 +
103 + def __init__(
104 + self,
105 + vllm_config: VllmConfig,
106 + rank: int,
107 + distributed_init_method: str,
108 + input_shm_handle: Handle,
109 + is_driver_worker: bool,
110 + is_driver_node: bool = False,
111 + ):
112 + # Defer WorkerProc.__init__ until GPU IDs are known.
113 + self._is_driver_node = is_driver_node
114 + self._init_kwargs = dict(
115 + vllm_config=vllm_config,
116 + rank=rank,
117 + distributed_init_method=distributed_init_method,
118 + input_shm_handle=input_shm_handle,
119 + shared_worker_lock=None,
120 + is_driver_worker=is_driver_worker,
121 + )
122 +
123 + def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
124 + """Return (node_id, gpu_ids) assigned to this actor by Ray."""
125 + node_id = ray.get_runtime_context().get_node_id()
126 + device_key = current_platform.ray_device_key
127 + if not device_key:
128 + raise RuntimeError(
129 + f"current platform {current_platform.device_name} does not support ray."
130 + )
131 + gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
132 + return node_id, [int(x) for x in gpu_ids]
133 +
134 + def initialize_worker(
135 + self,
136 + local_rank: int,
137 + env_vars: dict[str, str],
138 + driver_env_vars: dict[str, str] | None = None,
139 + ) -> None:
140 + """Complete initialization after GPU assignment is known.
141 +
142 + *driver_env_vars* are applied with ``setdefault`` — they fill
143 + in missing vars but never overwrite node-local values.
144 + *env_vars* (e.g. CUDA_VISIBLE_DEVICES) always overwrite.
145 + """
146 + if driver_env_vars:
147 + for key, value in driver_env_vars.items():
148 + os.environ.setdefault(key, value)
149 + for key, value in env_vars.items():
150 + os.environ[key] = value
151 +
152 + self.local_rank = local_rank
153 + super().__init__(
154 + local_rank=local_rank,
155 + **self._init_kwargs,
156 + )
157 +
158 + def _init_message_queues(
159 + self, input_shm_handle: Handle, vllm_config: VllmConfig
160 + ) -> None:
161 + """
162 + Workers on the same node as the executor use shared memory for
163 + both the broadcast (input) MQ and the response MQ. Workers on
164 + different nodes use TCP (n_local_reader=0).
165 + """
166 + self.rpc_broadcast_mq = MessageQueue.create_from_handle(
167 + input_shm_handle, self.worker.rank
168 + )
169 +
170 + n_local = 1 if self._is_driver_node else 0
171 + # Use ray.util.get_node_ip_address() to get Ray's internal IP.
172 + # get_ip() returns host's external IP which is typically not
173 + # routable between nodes within the cluster.
174 + self.worker_response_mq = MessageQueue(
175 + n_reader=1,
176 + n_local_reader=n_local,
177 + connect_ip=ray.util.get_node_ip_address(),
178 + )
179 + self.peer_response_handles: list[dict] = []
180 +
181 + def wait_for_init(self) -> dict:
182 + """Respond to the driver's wait_until_ready() barrier."""
183 + assert self.worker_response_mq is not None
184 + return {
185 + "status": self.READY_STR,
186 + "handle": self.worker_response_mq.export_handle(),
187 + }
188 +
189 + def run(self) -> None:
190 + """Main entry point called via actor.run.remote()."""
191 + try:
192 + assert self.rpc_broadcast_mq is not None
193 + self.rpc_broadcast_mq.wait_until_ready()
194 + assert self.worker_response_mq is not None
195 + self.worker_response_mq.wait_until_ready()
196 +
197 + self.worker_busy_loop()
198 + except Exception as e:
199 + logger.exception("RayWorkerProc failed: %s", e)
200 + raise
201 + finally:
202 + self.shutdown()
203 +
204 +
205 +class RayExecutorV2(MultiprocExecutor):
206 + """Ray-based distributed executor using MessageQueue communication.
207 +
208 + Inherits from MultiprocExecutor to reuse the MQ-based control plane
209 + and NCCL data plane. Workers are Ray actors.
210 +
211 + Async scheduling is enabled, inherited from MultiprocExecutor.
212 + This is cricitcal for RayExecutorV2 to be performant.
213 + """
214 +
215 + uses_ray: bool = True
216 + supports_pp: bool = True
217 +
218 + def __init__(self, vllm_config: VllmConfig):
219 + super().__init__(vllm_config)
220 +
221 + def _build_runtime_env(self) -> dict:
222 + """Build a runtime_env dict for RayWorkerProc actors.
223 +
224 + Driver env vars are applied separately via initialize_worker
225 + with setdefault semantics.
226 + """
227 + base = self.parallel_config.ray_runtime_env
228 + runtime_env: dict = copy.deepcopy(dict(base)) if base else {}
229 +
230 + env_vars = runtime_env.setdefault("env_vars", {})
231 + env_vars.update({v: "1" for v in current_platform.ray_noset_device_env_vars})
232 + if self.parallel_config.ray_workers_use_nsight:
233 + runtime_env["nsight"] = {
234 + "t": "cuda,cudnn,cublas",
235 + "o": "'worker_process_%p'",
236 + "cuda-graph-trace": "node",
237 + }
238 + return runtime_env
239 +
240 + @staticmethod
241 + def _get_actor_resource_kwargs() -> dict[str, Any]:
242 + """Return Ray actor resource kwargs for the current platform."""
243 + num_devices = envs.VLLM_RAY_PER_WORKER_GPUS
244 + device_key = current_platform.ray_device_key
245 + if device_key == "GPU":
246 + return {"num_gpus": num_devices}
247 + return {"num_gpus": 0, "resources": {device_key: num_devices}}
248 +
249 + def _init_executor(self) -> None:
250 + """Initialize the RayExecutorV2 executor."""
251 + self._finalizer = weakref.finalize(self, self.shutdown)
252 + self.is_failed = False
253 + self.failure_callback = None
254 + self.shutting_down = False
255 + self.shutdown_lock = threading.Lock()
256 +
257 + # Step 1: Initialize Ray cluster and retrieve placement group
258 + if ray is None:
259 + raise ImportError("Using Ray backend requires installation of ray.")
260 + initialize_ray_cluster(self.parallel_config, require_gpu_on_driver=False)
261 + placement_group = self.parallel_config.placement_group
262 +
263 + tp_size, pp_size, pcp_size = self._get_parallel_sizes()
264 + assert self.world_size == tp_size * pp_size * pcp_size, (
265 + f"world_size ({self.world_size}) must be equal to the "
266 + f"tensor_parallel_size ({tp_size}) x pipeline"
267 + f"_parallel_size ({pp_size}) x prefill_context"
268 + f"_parallel_size ({pcp_size}). "
269 + )
270 +
271 + # Step 2: Build bundle assignments for worker rank placement
272 + # while respecting VLLM_RAY_BUNDLE_INDICES.
273 + if envs.VLLM_RAY_BUNDLE_INDICES:
274 + bundle_to_node_id = get_bundles_for_indices(
275 + placement_group,
276 + list(map(int, envs.VLLM_RAY_BUNDLE_INDICES.split(","))),
277 + self.world_size,
278 + )
279 + else:
280 + bundle_to_node_id = get_bundles_sorted_by_node(placement_group)
281 + driver_node = ray.get_runtime_context().get_node_id()
282 +
283 + bundle_assignments: list[dict[str, Any]] = []
284 + for rank, (bundle_id_idx, node_id, node_ip) in enumerate(bundle_to_node_id):
285 + bundle_assignments.append(
286 + {
287 + "rank": rank,
288 + "bundle_id_idx": bundle_id_idx,
289 + "node_id": node_id,
290 + "node_ip": node_ip,
291 + }
292 + )
293 +
294 + # Step 3: Resolve the IP for torch.distributed TCPStore.
295 + # The TCPStore server runs on rank 0's node, so all workers
296 + # must be able to reach this address.
297 + dist_ip = bundle_assignments[0]["node_ip"]
298 + distributed_init_method = get_distributed_init_method(dist_ip, get_open_port())
299 +
300 + # Step 4: Create broadcast MessageQueue.
301 + # Workers on the driver node use shared memory; the rest use TCP.
302 + max_chunk_bytes = envs.VLLM_MQ_MAX_CHUNK_BYTES_MB * 1024 * 1024
303 + n_local = sum(1 for a in bundle_assignments if a["node_id"] == driver_node)
304 + self.rpc_broadcast_mq = MessageQueue(
305 + self.world_size,
306 + n_local,
307 + max_chunk_bytes=max_chunk_bytes,
308 + connect_ip=ray.util.get_node_ip_address(),
309 + )
310 + scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
311 +
312 + # Step 5: Spawn RayWorkerProc actors into PG bundles (deferred init).
313 + # Workers are created lightweight here; full initialization happens
314 + # in Step 7 after GPU IDs are discovered.
315 + self.ray_worker_handles: list[RayWorkerHandle] = []
316 + instance_id = self.vllm_config.instance_id
317 +
318 + # Collect driver env vars and apply but don't overwrite node-local values.
319 + self.driver_env_vars = get_driver_env_vars(
320 + worker_specific_vars=WORKER_SPECIFIC_ENV_VARS,
321 + )
322 +
323 + runtime_env = self._build_runtime_env()
324 + resource_kwargs = self._get_actor_resource_kwargs()
325 +
326 + for bundle_idx in range(self.world_size):
327 + bundle = bundle_assignments[bundle_idx]
328 + is_driver_worker = self._is_driver_worker(bundle["rank"])
329 + is_driver_node = bundle["node_id"] == driver_node
330 +
331 + scheduling_strategy = PlacementGroupSchedulingStrategy(
332 + placement_group=placement_group,
333 + placement_group_bundle_index=bundle["bundle_id_idx"],
334 + )
335 +
336 + actor_name = build_actor_name(
337 + instance_id, bundle["rank"], tp_size, pp_size, pcp_size
338 + )
339 +
340 + actor = (
341 + ray.remote(RayWorkerProc)
342 + .options(
343 + name=actor_name,
344 + num_cpus=0,
345 + **resource_kwargs,
346 + scheduling_strategy=scheduling_strategy,
347 + runtime_env=runtime_env,
348 + )
349 + .remote(
350 + vllm_config=self.vllm_config,
351 + rank=bundle["rank"],
352 + distributed_init_method=distributed_init_method,
353 + input_shm_handle=scheduler_output_handle,
354 + is_driver_worker=is_driver_worker,
355 + is_driver_node=is_driver_node,
356 + )
357 + )
358 +
359 + handle = RayWorkerHandle(
360 + actor=actor,
361 + rank=bundle["rank"],
362 + local_rank=-1, # Set in Step 7 after GPU ID discovery
363 + node_id=bundle["node_id"],
364 + bundle_id_idx=bundle["bundle_id_idx"],
365 + )
366 + self.ray_worker_handles.append(handle)
367 +
368 + # Step 6: Discover GPU IDs assigned to each worker via Ray runtime context.
369 + worker_node_and_gpu_ids = ray.get(
370 + [h.actor.get_node_and_gpu_ids.remote() for h in self.ray_worker_handles]
371 + )
372 +
373 + node_workers: dict[str, list[int]] = defaultdict(list)
374 + node_gpus: dict[str, list[int]] = defaultdict(list)
375 + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
376 + node_workers[node_id].append(i)
377 + node_gpus[node_id].extend(gpu_ids)
378 + for node_id, gpu_ids in node_gpus.items():
379 + node_gpus[node_id] = sorted(gpu_ids)
380 +
381 + # Step 7: Initialize workers with correct local_rank and
382 + # CUDA_VISIBLE_DEVICES. Each worker sees all GPUs assigned to
383 + # this executor on its node; local_rank indexes into that set.
384 + init_worker_refs = []
385 + for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
386 + local_rank = node_workers[node_id].index(i)
387 + worker_env_vars = {
388 + current_platform.device_control_env_var: ",".join(
389 + map(str, node_gpus[node_id])
390 + ),
391 + }
392 + self.ray_worker_handles[i].local_rank = local_rank
393 + init_worker_refs.append(
394 + self.ray_worker_handles[i].actor.initialize_worker.remote(
395 + local_rank, worker_env_vars, self.driver_env_vars
396 + )
397 + )
398 + ray.get(init_worker_refs)
399 +
400 + # Step 8: Collect response MQ handles
401 + init_results = ray.get(
402 + [h.actor.wait_for_init.remote() for h in self.ray_worker_handles]
403 + )
404 +
405 + self.response_mqs: list[MessageQueue] = []
406 + for i, result in enumerate(init_results):
407 + if result["status"] != RayWorkerProc.READY_STR:
408 + raise RuntimeError(f"Worker {i} failed to initialize: {result}")
409 + self.response_mqs.append(
410 + MessageQueue.create_from_handle(result["handle"], 0)
411 + )
412 +
413 + # Step 9: Start run() before wait_until_ready() to avoid
414 + # deadlock — workers send subscriptions inside run().
415 + for handle in self.ray_worker_handles:
416 + handle.run()
417 +
418 + # Step 10: wait_until_ready() barrier
419 + self.rpc_broadcast_mq.wait_until_ready()
420 + for response_mq in self.response_mqs:
421 + response_mq.wait_until_ready()
422 +
423 + self.futures_queue = deque[tuple[FutureWrapper, Any]]()
424 + self._post_init_executor()
425 +
426 + self.start_worker_monitor()
427 + self.output_rank = self._get_output_rank()
428 +
429 + def start_worker_monitor(self, inline=False) -> None:
430 + """Monitor worker liveness via ray.wait() on run() ObjectRefs."""
431 + run_refs = [h.run_ref for h in self.ray_worker_handles if h.run_ref is not None]
432 + if not run_refs:
433 + raise RuntimeError("Ray workers have not started successfully.")
434 +
435 + self_ref = weakref.ref(self)
436 + ref_to_rank = {
437 + h.run_ref: h.rank for h in self.ray_worker_handles if h.run_ref is not None
438 + }
439 +
440 + def _should_stop() -> bool:
441 + executor = self_ref()
442 + return not executor or executor.shutting_down
443 +
444 + def monitor_workers():
445 + # Poll with a timeout rather than blocking on ray.wait()
446 + # because a blocking call would segfault if Ray is torn down
447 + # while this thread is inside it.
448 + while not _should_stop() and ray.is_initialized():
449 + try:
450 + done, _ = ray.wait(run_refs, num_returns=1, timeout=5.0)
451 + except Exception:
452 + logger.exception(
453 + "RayWorkerMonitor: unexpected error, exiting monitor thread"
454 + )
455 + return
456 + if not done or _should_stop():
457 + continue
458 +
459 + dead_ranks = [ref_to_rank[r] for r in done]
460 + executor = self_ref()
461 + if not executor:
462 + return
463 + executor.is_failed = True
464 + logger.error(
465 + "RayWorkerProc rank=%s died unexpectedly, shutting down executor.",
466 + dead_ranks,
467 + )
468 + executor.shutdown()
469 + if executor.failure_callback is not None:
470 + callback = executor.failure_callback
471 + executor.failure_callback = None
472 + callback()
473 + return
474 +
475 + t = threading.Thread(
476 + target=monitor_workers, daemon=True, name="RayWorkerMonitor"
477 + )
478 + t.start()
479 + self._monitor_thread = t
480 +
481 + def _join_monitor_thread(self) -> None:
482 + """Wait for the monitor thread to exit.
483 +
484 + Must be called before tearing down Ray resources — the monitor
485 + may be inside ray.wait() which would segfault if Ray is shut
486 + down underneath it. When the monitor itself calls shutdown()
487 + on worker death, we skip the join because the thread is about
488 + to return anyway.
489 + """
490 + monitor = getattr(self, "_monitor_thread", None)
491 + if (
492 + monitor is not None
493 + and monitor.is_alive()
494 + and threading.current_thread() is not monitor
495 + ):
496 + monitor.join(timeout=10)
497 +
498 + def shutdown(self) -> None:
499 + """Properly shut down the executor and its workers."""
500 + lock = getattr(self, "shutdown_lock", None)
501 + if lock is None:
502 + return
503 +
504 + with lock:
505 + if getattr(self, "shutting_down", False):
506 + return
507 + self.shutting_down = True
508 +
509 + self._join_monitor_thread()
510 +
511 + for handle in getattr(self, "ray_worker_handles", []):
512 + try:
513 + ray.kill(handle.actor)
514 + logger.debug("Killed actor rank=%d", handle.rank)
515 + except Exception:
516 + logger.exception("Failed to kill actor rank=%d", handle.rank)
517 +
518 + if rpc_broadcast_mq := getattr(self, "rpc_broadcast_mq", None):
519 + rpc_broadcast_mq.shutdown()
520 + self.rpc_broadcast_mq = None
521 +
522 + for mq in getattr(self, "response_mqs", []):
523 + mq.shutdown()
524 + self.response_mqs = []
vllm/v1/executor/ray_utils.py
+155 -14
@@ -26,6 +26,17 @@ if TYPE_CHECKING:
26 logger = init_logger(__name__)
27 PG_WAIT_TIMEOUT = 1800
28
29 +# Env vars that are worker-specific and must NOT be copied from the
30 +# driver to Ray workers — they are set per-worker after GPU discovery.
31 +WORKER_SPECIFIC_ENV_VARS: set[str] = {
32 + "VLLM_HOST_IP",
33 + "VLLM_HOST_PORT",
34 + "LOCAL_RANK",
35 + "CUDA_VISIBLE_DEVICES",
36 + "HIP_VISIBLE_DEVICES",
37 + "ROCR_VISIBLE_DEVICES",
38 +}
39 +
40 try:
41 import ray
42 from ray.util import placement_group_table
@@ -51,6 +62,8 @@ try:
62 # that thread.
63 self.compiled_dag_cuda_device_set = False
64
65 + rpc_rank: int
66 +
67 def adjust_rank(self, rank_mapping: dict[int, int]) -> None:
68 """
69 Adjust the rpc_rank based on the given mapping.
@@ -214,13 +227,17 @@ def assert_ray_available():
227
228
229 def _verify_bundles(
217 - placement_group: "PlacementGroup", parallel_config: ParallelConfig, device_str: str
230 + placement_group: "PlacementGroup",
231 + parallel_config: ParallelConfig,
232 + device_str: str,
233 + require_gpu_on_driver: bool = True,
234 ):
235 """Verify a given placement group has bundles located in the right place.
236
237 There are 2 rules.
238 - Warn if all tensor parallel workers cannot fit in a single node.
223 - - Fail if driver node is not included in a placement group.
239 + - Fail if driver node is not included in a placement group
240 + (only when require_gpu_on_driver is True).
241 """
242 assert ray.is_initialized(), (
243 "Ray is not initialized although distributed-executor-backend is ray."
@@ -237,7 +254,7 @@ def _verify_bundles(
254 node_id_to_bundle[node_id].append(bundles[bundle_idx])
255 driver_node_id = ray.get_runtime_context().get_node_id()
256
240 - if driver_node_id not in node_id_to_bundle:
257 + if require_gpu_on_driver and driver_node_id not in node_id_to_bundle:
258 raise RuntimeError(
259 f"driver node id {driver_node_id} is not included in a placement "
260 f"group {placement_group.id}. Node id -> bundles "
@@ -266,6 +283,115 @@ def _verify_bundles(
283 )
284
285
286 +def build_actor_name(
287 + instance_id: str,
288 + rank: int,
289 + tp_size: int,
290 + pp_size: int,
291 + pcp_size: int,
292 +) -> str:
293 + """Build a descriptive Ray actor name for dashboard visibility."""
294 + name = f"vllm_Worker_{instance_id}"
295 + if tp_size > 1:
296 + name += f"_TP{rank % tp_size}"
297 + if pp_size > 1:
298 + name += f"_PP{(rank // tp_size) % pp_size}"
299 + if pcp_size > 1:
300 + name += f"_PCP{rank // (tp_size * pp_size)}"
301 + return name
302 +
303 +
304 +def get_bundles_for_indices(
305 + placement_group: "PlacementGroup",
306 + bundle_indices: list[int],
307 + world_size: int,
308 +) -> list[tuple[int, str, str]]:
309 + """
310 + Return GPU bundle indices paired with node IDs and node IPs for
311 + explicit bundle indices specified via VLLM_RAY_BUNDLE_INDICES.
312 + """
313 + assert len(bundle_indices) == world_size, (
314 + "VLLM_RAY_BUNDLE_INDICES must have the same size"
315 + f" as the world size, but got {bundle_indices=} "
316 + f"and {world_size=}"
317 + )
318 + assert len(set(bundle_indices)) == len(bundle_indices), (
319 + "VLLM_RAY_BUNDLE_INDICES cannot have duplicate values,"
320 + f" but got {bundle_indices=}"
321 + )
322 +
323 + pg_data = placement_group_table(placement_group)
324 + pg_bundle_to_node = pg_data["bundles_to_node_id"]
325 + node_id_to_ip = {
326 + n["NodeID"]: n["NodeManagerAddress"] for n in ray.nodes() if n["Alive"]
327 + }
328 + return [
329 + (bid, pg_bundle_to_node[bid], node_id_to_ip[pg_bundle_to_node[bid]])
330 + for bid in bundle_indices
331 + ]
332 +
333 +
334 +def get_bundles_sorted_by_node(
335 + placement_group: "PlacementGroup",
336 +) -> list[tuple[int, str, str]]:
337 + """
338 + Return GPU bundle indices paired with node IDs and node IPs,
339 + sorted driver-first.
340 +
341 + This utility has to be invoked from the driver node.
342 +
343 + Example: 3-node cluster, driver on node-A, PG bundles spread
344 + across nodes:
345 +
346 + Input: [
347 + (0, node-C),
348 + (1, node-A),
349 + (2, node-B),
350 + (3, node-C),
351 + (4, node-A),
352 + (5, node-B),
353 + ]
354 + Output: [
355 + (1, node-A),
356 + (4, node-A),
357 + (2, node-B),
358 + (5, node-B),
359 + (0, node-C),
360 + (3, node-C),
361 + ]
362 + """
363 + pg_data = placement_group_table(placement_group)
364 + bundle_to_node = pg_data["bundles_to_node_id"]
365 +
366 + ray_device_key = current_platform.ray_device_key
367 + if not ray_device_key:
368 + raise ValueError(
369 + f"current platform {current_platform.device_name} does not support ray."
370 + )
371 +
372 + node_id_to_ip = {
373 + n["NodeID"]: n["NodeManagerAddress"] for n in ray.nodes() if n["Alive"]
374 + }
375 +
376 + bundle_specs = placement_group.bundle_specs
377 + assert bundle_specs is not None
378 + bundle_to_node_id: list[tuple[int, str, str]] = []
379 + for bundle_idx, bundle in enumerate(bundle_specs):
380 + if bundle.get(ray_device_key):
381 + node_id = bundle_to_node.get(bundle_idx)
382 + bundle_to_node_id.append((bundle_idx, node_id, node_id_to_ip[node_id]))
383 +
384 + driver_node = ray.get_runtime_context().get_node_id()
385 +
386 + def _sort_key(item):
387 + _, node_id, _ = item
388 + return (0 if node_id == driver_node else 1, node_id)
389 +
390 + bundle_to_node_id.sort(key=_sort_key)
391 +
392 + return bundle_to_node_id
393 +
394 +
395 def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
396 """Wait until a placement group is ready.
397
@@ -352,6 +478,7 @@ def _wait_until_pg_removed(current_placement_group: "PlacementGroup"):
478 def initialize_ray_cluster(
479 parallel_config: ParallelConfig,
480 ray_address: str | None = None,
481 + require_gpu_on_driver: bool = True,
482 ):
483 """Initialize the distributed cluster with Ray.
484
@@ -363,10 +490,18 @@ def initialize_ray_cluster(
490 parallel_config: The configurations for parallel execution.
491 ray_address: The address of the Ray cluster. If None, uses
492 the default Ray cluster address.
493 + require_gpu_on_driver: If True (default), require at least one GPU
494 + on the current (driver) node and pin the first PG bundle to it.
495 + Set to False for executors like RayExecutorV2 where all GPU work
496 + is delegated to remote Ray actors.
497 """
498 assert_ray_available()
499 from vllm.platforms import current_platform
500
501 + # Disable Ray usage stats collection
502 + if os.environ.get("RAY_USAGE_STATS_ENABLED", "0") != "1":
503 + os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
504 +
505 # Prevalidate GPU requirements before Ray processing
506 if current_platform.is_cuda() and parallel_config.world_size > 1:
507 available_gpus = current_platform.device_count()
@@ -459,16 +594,20 @@ def initialize_ray_cluster(
594 current_ip = get_ip()
595 current_node_id = ray.get_runtime_context().get_node_id()
596 current_node_resource = available_resources_per_node()[current_node_id]
462 - if current_node_resource.get(device_str, 0) < 1:
463 - raise ValueError(
464 - f"Current node has no {device_str} available. "
465 - f"{current_node_resource=}. vLLM engine cannot start without "
466 - f"{device_str}. Make sure you have at least 1 {device_str} "
467 - f"available in a node {current_node_id=} {current_ip=}."
468 - )
469 - # This way, at least bundle is required to be created in a current
470 - # node.
471 - placement_group_specs[0][f"node:{current_ip}"] = 0.001
597 + # TODO (jeffreywang): require_gpu_on_driver should be always False
598 + # after deprecating RayDistributedExecutor.
599 + if require_gpu_on_driver:
600 + if current_node_resource.get(device_str, 0) < 1:
601 + raise ValueError(
602 + f"Current node has no {device_str} available. "
603 + f"{current_node_resource=}. vLLM engine cannot start "
604 + f"without {device_str}. Make sure you have at least 1 "
605 + f"{device_str} available in a node "
606 + f"{current_node_id=} {current_ip=}."
607 + )
608 + # This way, at least bundle is required to be created in a
609 + # current node.
610 + placement_group_specs[0][f"node:{current_ip}"] = 0.001
611
612 # By default, Ray packs resources as much as possible.
613 current_placement_group = ray.util.placement_group(
@@ -477,7 +616,9 @@ def initialize_ray_cluster(
616 _wait_until_pg_ready(current_placement_group)
617
618 assert current_placement_group is not None
480 - _verify_bundles(current_placement_group, parallel_config, device_str)
619 + _verify_bundles(
620 + current_placement_group, parallel_config, device_str, require_gpu_on_driver
621 + )
622 # Set the placement group in the parallel config
623 parallel_config.placement_group = current_placement_group
624
vllm/v1/worker/worker_base.py
+2 -2
@@ -195,8 +195,8 @@ class WorkerWrapperBase:
195 All workers have rpc_rank=0, but they have different ranks in the TP
196 group.
197 """
198 - self.rpc_rank = rpc_rank
199 - self.global_rank = self.rpc_rank if global_rank is None else global_rank
198 + self.rpc_rank: int = rpc_rank
199 + self.global_rank: int = self.rpc_rank if global_rank is None else global_rank
200
201 # Initialized after init_worker is called
202 self.worker: WorkerBase