A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Ray backend] Better error when pg topology is bad. (#7584)
Co-authored-by: youkaichao <youkaichao@126.com>
SangBin Cho committed
Aug 22, 2024 at 17:44 UTC
c01a6cb23144b67b473e569a25b6f9725bc3f85b
3 files changed
+197
-9
.buildkite/test-pipeline.yaml
+1
@@ -293,6 +293,7 @@ steps:
293
commands:
294
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
295
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
296
+ - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
297
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
298
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
299
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
tests/distributed/test_multi_node_assignment.py
new
+64
@@ -0,0 +1,64 @@
1
+"""Make sure ray assigns GPU workers to the correct node.
2
+
3
+Run:
4
+```sh
5
+cd $VLLM_PATH/tests
6
+
7
+pytest distributed/test_multi_node_assignment.py
8
+```
9
+"""
10
+
11
+import os
12
+
13
+import pytest
14
+import ray
15
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
16
+
17
+from vllm import initialize_ray_cluster
18
+from vllm.config import ParallelConfig
19
+from vllm.executor.ray_utils import _wait_until_pg_removed
20
+from vllm.utils import get_ip
21
+
22
+VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
23
+
24
+
25
+@pytest.mark.skipif(not VLLM_MULTI_NODE,
26
+ reason="Need at least 2 nodes to run the test.")
27
+def test_multi_node_assignment() -> None:
28
+
29
+ # NOTE: important to keep this class definition here
30
+ # to let ray use cloudpickle to serialize it.
31
+ class Actor:
32
+
33
+ def get_ip(self):
34
+ return get_ip()
35
+
36
+ for _ in range(10):
37
+ config = ParallelConfig(1, 2)
38
+ initialize_ray_cluster(config)
39
+
40
+ current_ip = get_ip()
41
+ workers = []
42
+ for bundle_id, bundle in enumerate(
43
+ config.placement_group.bundle_specs):
44
+ if not bundle.get("GPU", 0):
45
+ continue
46
+ scheduling_strategy = PlacementGroupSchedulingStrategy(
47
+ placement_group=config.placement_group,
48
+ placement_group_capture_child_tasks=True,
49
+ placement_group_bundle_index=bundle_id,
50
+ )
51
+
52
+ worker = ray.remote(
53
+ num_cpus=0,
54
+ num_gpus=1,
55
+ scheduling_strategy=scheduling_strategy,
56
+ )(Actor).remote()
57
+ worker_ip = ray.get(worker.get_ip.remote())
58
+ assert worker_ip == current_ip
59
+ workers.append(worker)
60
+
61
+ for worker in workers:
62
+ ray.kill(worker)
63
+
64
+ _wait_until_pg_removed(config.placement_group)
vllm/executor/ray_utils.py
+132
-9
@@ -1,4 +1,6 @@
1
-from typing import List, Optional, Tuple, Union
1
+import time
2
+from collections import defaultdict
3
+from typing import Dict, List, Optional, Tuple, Union
4
5
import msgspec
6
@@ -11,9 +13,13 @@ from vllm.utils import get_ip, is_hip, is_xpu
13
from vllm.worker.worker_base import WorkerWrapperBase
14
15
logger = init_logger(__name__)
16
+PG_WAIT_TIMEOUT = 1800
17
18
try:
19
import ray
20
+ from ray._private.state import available_resources_per_node
21
+ from ray.util import placement_group_table
22
+ from ray.util.placement_group import PlacementGroup
23
24
class RayWorkerWrapper(WorkerWrapperBase):
25
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
@@ -98,6 +104,106 @@ def assert_ray_available():
104
"`pip install ray`.") from ray_import_err
105
106
107
+def _verify_bundles(placement_group: "PlacementGroup",
108
+ parallel_config: ParallelConfig, device_str: str):
109
+ """Verify a given placement group has bundles located in the right place.
110
+
111
+ There are 2 rules.
112
+ - Warn if all tensor parallel workers cannot fit in a single node.
113
+ - Fail if driver node is not included in a placement group.
114
+ """
115
+ assert ray.is_initialized(), (
116
+ "Ray is not initialized although distributed-executor-backend is ray.")
117
+ pg_data = placement_group_table(placement_group)
118
+ # bundle_idx -> node_id
119
+ bundle_to_node_ids = pg_data["bundles_to_node_id"]
120
+ # bundle_idx -> bundle (e.g., {"GPU": 1})
121
+ bundles = pg_data["bundles"]
122
+ # node_id -> List of bundle (e.g., {"GPU": 1})
123
+ node_id_to_bundle: Dict[str, List[Dict[str, float]]] = defaultdict(list)
124
+
125
+ for bundle_idx, node_id in bundle_to_node_ids.items():
126
+ node_id_to_bundle[node_id].append(bundles[bundle_idx])
127
+ driver_node_id = ray.get_runtime_context().get_node_id()
128
+
129
+ if driver_node_id not in node_id_to_bundle:
130
+ raise RuntimeError(
131
+ f"driver node id {driver_node_id} is not included in a placement "
132
+ f"group {placement_group.id}. Node id -> bundles "
133
+ f"{node_id_to_bundle}. "
134
+ "You don't have enough GPUs available in a current node. Check "
135
+ "`ray status` to see if you have available GPUs in a node "
136
+ f"{driver_node_id} before starting an vLLM engine.")
137
+
138
+ for node_id, bundles in node_id_to_bundle.items():
139
+ if len(bundles) < parallel_config.tensor_parallel_size:
140
+ logger.warning(
141
+ "tensor_parallel_size=%d "
142
+ "is bigger than a reserved number of %ss (%d "
143
+ "%ss) in a node %s. Tensor parallel workers can be "
144
+ "spread out to 2+ nodes which can degrade the performance "
145
+ "unless you have fast interconnect across nodes, like "
146
+ "Infiniband. To resolve this issue, make sure you have more "
147
+ "than %d GPUs available at each node.",
148
+ parallel_config.tensor_parallel_size, device_str, len(bundles),
149
+ device_str, node_id, parallel_config.tensor_parallel_size)
150
+
151
+
152
+def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
153
+ """Wait until a placement group is ready.
154
+
155
+ It prints the informative log messages if the placement group is
156
+ not created within time.
157
+
158
+ """
159
+ # Wait until PG is ready - this will block until all
160
+ # requested resources are available, and will timeout
161
+ # if they cannot be provisioned.
162
+ placement_group_specs = current_placement_group.bundle_specs
163
+
164
+ s = time.time()
165
+ pg_ready_ref = current_placement_group.ready()
166
+ wait_interval = 10
167
+ while time.time() - s < PG_WAIT_TIMEOUT:
168
+ ready, _ = ray.wait([pg_ready_ref], timeout=wait_interval)
169
+ if len(ready) > 0:
170
+ break
171
+
172
+ # Exponential backoff for warning print.
173
+ wait_interval *= 2
174
+ logger.info(
175
+ "Waiting for creating a placement group of specs for "
176
+ "%d seconds. specs=%s. Check "
177
+ "`ray status` to see if you have enough resources.",
178
+ int(time.time() - s), placement_group_specs)
179
+
180
+ try:
181
+ ray.get(pg_ready_ref, timeout=0)
182
+ except ray.exceptions.GetTimeoutError:
183
+ raise ValueError(
184
+ "Cannot provide a placement group of "
185
+ f"{placement_group_specs=} within {PG_WAIT_TIMEOUT} seconds. See "
186
+ "`ray status` to make sure the cluster has enough resources."
187
+ ) from None
188
+
189
+
190
+def _wait_until_pg_removed(current_placement_group: "PlacementGroup"):
191
+ ray.util.remove_placement_group(current_placement_group)
192
+ s = time.time()
193
+ wait_interval = 10
194
+ while time.time() - s < PG_WAIT_TIMEOUT:
195
+ pg = ray.util.get_current_placement_group()
196
+ if pg is None:
197
+ break
198
+
199
+ # Exponential backoff for warning print.
200
+ wait_interval *= 2
201
+ logger.info(
202
+ "Waiting for removing a placement group of specs for "
203
+ "%d seconds.", int(time.time() - s))
204
+ time.sleep(wait_interval)
205
+
206
+
207
def initialize_ray_cluster(
208
parallel_config: ParallelConfig,
209
ray_address: Optional[str] = None,
@@ -156,15 +262,32 @@ def initialize_ray_cluster(
262
f"The number of required {device_str}s exceeds the total "
263
f"number of available {device_str}s in the placement group.")
264
# Create a new placement group
159
- placement_group_specs = ([{
160
- device_str: 1
161
- }] * parallel_config.world_size)
265
+ placement_group_specs: List[Dict[str, float]] = ([{
266
+ device_str: 1.0
267
+ } for _ in range(parallel_config.world_size)])
268
+
269
+ # vLLM engine is also a worker to execute model with an accelerator,
270
+ # so it requires to have the device in a current node. Check if
271
+ # the current node has at least one device.
272
+ current_ip = get_ip()
273
+ current_node_id = ray.get_runtime_context().get_node_id()
274
+ current_node_resource = available_resources_per_node()[current_node_id]
275
+ if current_node_resource.get(device_str, 0) < 1:
276
+ raise ValueError(
277
+ f"Current node has no {device_str} available. "
278
+ f"{current_node_resource=}. vLLM engine cannot start without "
279
+ f"{device_str}. Make sure you have at least 1 {device_str} "
280
+ f"available in a node {current_node_id=} {current_ip=}.")
281
+ # This way, at least bundle is required to be created in a current
282
+ # node.
283
+ placement_group_specs[0][f"node:{current_ip}"] = 0.001
284
+
285
+ # By default, Ray packs resources as much as possible.
286
current_placement_group = ray.util.placement_group(
163
- placement_group_specs)
164
- # Wait until PG is ready - this will block until all
165
- # requested resources are available, and will timeout
166
- # if they cannot be provisioned.
167
- ray.get(current_placement_group.ready(), timeout=1800)
287
+ placement_group_specs, strategy="PACK")
288
+ _wait_until_pg_ready(current_placement_group)
289
290
+ assert current_placement_group is not None
291
+ _verify_bundles(current_placement_group, parallel_config, device_str)
292
# Set the placement group in the parallel config
293
parallel_config.placement_group = current_placement_group