A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Ray placement group support (#397)
Antoni Baum committed
Jul 19, 2023 at 22:49 UTC
9925c179409f7b14aef55bcfde47b320bd1a263c
5 files changed
+185
-114
requirements.txt
+1
-1
@@ -1,6 +1,6 @@
1
ninja # For faster builds.
2
psutil
3
-ray
3
+ray >= 2.5.1
4
sentencepiece # Required for LLaMA tokenizer.
5
numpy
6
torch >= 2.0.0
vllm/engine/async_llm_engine.py
+2
-2
@@ -226,14 +226,14 @@ class AsyncLLMEngine:
226
engine_configs = engine_args.create_engine_configs()
227
parallel_config = engine_configs[2]
228
# Initialize the cluster.
229
- distributed_init_method, devices = initialize_cluster(
229
+ distributed_init_method, placement_group = initialize_cluster(
230
parallel_config, engine_args.engine_use_ray)
231
# Create the async LLM engine.
232
engine = cls(engine_args.worker_use_ray,
233
engine_args.engine_use_ray,
234
*engine_configs,
235
distributed_init_method,
236
- devices,
236
+ placement_group,
237
log_requests=not engine_args.disable_log_requests,
238
log_stats=not engine_args.disable_log_stats)
239
return engine
vllm/engine/llm_engine.py
+78
-27
@@ -1,11 +1,12 @@
1
import time
2
-from typing import Any, List, Optional
2
+from functools import partial
3
+from typing import Any, List, Optional, TYPE_CHECKING
4
5
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
6
SchedulerConfig)
7
from vllm.core.scheduler import Scheduler
8
from vllm.engine.arg_utils import EngineArgs
8
-from vllm.engine.ray_utils import DeviceID, initialize_cluster, ray
9
+from vllm.engine.ray_utils import initialize_cluster, ray, RayWorker
10
from vllm.logger import init_logger
11
from vllm.outputs import RequestOutput
12
from vllm.sampling_params import SamplingParams
@@ -13,7 +14,13 @@ from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
14
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
15
get_tokenizer)
16
from vllm.utils import Counter
16
-from vllm.worker.worker import Worker
17
+
18
+if ray:
19
+ from ray.air.util.torch_dist import init_torch_dist_process_group
20
+ from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
21
+
22
+if TYPE_CHECKING:
23
+ from ray.util.placement_group import PlacementGroup
24
25
logger = init_logger(__name__)
26
@@ -54,7 +61,7 @@ class LLMEngine:
61
parallel_config: ParallelConfig,
62
scheduler_config: SchedulerConfig,
63
distributed_init_method: str,
57
- stage_devices: List[List[DeviceID]],
64
+ placement_group: Optional["PlacementGroup"],
65
log_stats: bool,
66
) -> None:
67
logger.info(
@@ -85,31 +92,73 @@ class LLMEngine:
92
self.seq_counter = Counter()
93
94
# Create the parallel GPU workers.
88
- self.workers: List[Worker] = []
89
- assert len(stage_devices) == 1, "Only support one stage for now."
90
- for rank, node_resource, _ in stage_devices[0]:
91
- worker_cls = Worker
92
- if self.parallel_config.worker_use_ray:
93
- worker_cls = ray.remote(
94
- num_cpus=0,
95
- num_gpus=1,
96
- resources={node_resource: 1e-3},
97
- )(worker_cls).remote
98
-
99
- worker = worker_cls(
100
- model_config,
101
- parallel_config,
102
- scheduler_config,
103
- rank,
104
- distributed_init_method,
105
- )
106
- self.workers.append(worker)
95
+ if self.parallel_config.worker_use_ray:
96
+ self._init_workers_ray(placement_group)
97
+ else:
98
+ self._init_workers(distributed_init_method)
99
+
100
# Profile the memory usage and initialize the cache.
101
self._init_cache()
102
103
# Create the scheduler.
104
self.scheduler = Scheduler(scheduler_config, cache_config, log_stats)
105
106
+ def _init_workers(self, distributed_init_method: str):
107
+ # Lazy import the Worker to avoid importing torch.cuda/xformers
108
+ # before CUDA_VISIBLE_DEVICES is set in the Worker
109
+ from vllm.worker.worker import Worker # pylint: disable=import-outside-toplevel
110
+
111
+ assert self.parallel_config.world_size == 1, (
112
+ "Ray is required if parallel_config.world_size > 1.")
113
+
114
+ self.workers: List[Worker] = []
115
+ worker = Worker(
116
+ self.model_config,
117
+ self.parallel_config,
118
+ self.scheduler_config,
119
+ 0,
120
+ distributed_init_method,
121
+ )
122
+ self.workers.append(worker)
123
+ self._run_workers(
124
+ "init_model",
125
+ get_all_outputs=True,
126
+ )
127
+
128
+ def _init_workers_ray(self, placement_group: "PlacementGroup"):
129
+ # Lazy import the Worker to avoid importing torch.cuda/xformers
130
+ # before CUDA_VISIBLE_DEVICES is set in the Worker
131
+ from vllm.worker.worker import Worker # pylint: disable=import-outside-toplevel
132
+
133
+ self.workers: List[Worker] = []
134
+ for bundle in placement_group.bundle_specs:
135
+ if not bundle.get("GPU", 0):
136
+ continue
137
+ worker = ray.remote(
138
+ num_cpus=0,
139
+ num_gpus=1,
140
+ scheduling_strategy=PlacementGroupSchedulingStrategy(
141
+ placement_group=placement_group,
142
+ placement_group_capture_child_tasks=True),
143
+ )(RayWorker).remote()
144
+ self.workers.append(worker)
145
+
146
+ # Initialize torch distributed process group for the workers.
147
+ init_torch_dist_process_group(self.workers, backend="nccl")
148
+ self._run_workers("init_worker",
149
+ get_all_outputs=True,
150
+ worker_init_fn=lambda: Worker(
151
+ self.model_config,
152
+ self.parallel_config,
153
+ self.scheduler_config,
154
+ None,
155
+ None,
156
+ ))
157
+ self._run_workers(
158
+ "init_model",
159
+ get_all_outputs=True,
160
+ )
161
+
162
def _verify_args(self) -> None:
163
self.model_config.verify_with_parallel_config(self.parallel_config)
164
self.cache_config.verify_with_parallel_config(self.parallel_config)
@@ -152,11 +201,12 @@ class LLMEngine:
201
engine_configs = engine_args.create_engine_configs()
202
parallel_config = engine_configs[2]
203
# Initialize the cluster.
155
- distributed_init_method, devices = initialize_cluster(parallel_config)
204
+ distributed_init_method, placement_group = initialize_cluster(
205
+ parallel_config)
206
# Create the LLM engine.
207
engine = cls(*engine_configs,
208
distributed_init_method,
159
- devices,
209
+ placement_group,
210
log_stats=not engine_args.disable_log_stats)
211
return engine
212
@@ -326,9 +376,10 @@ class LLMEngine:
376
"""Runs the given method on all workers."""
377
all_outputs = []
378
for worker in self.workers:
329
- executor = getattr(worker, method)
379
if self.parallel_config.worker_use_ray:
331
- executor = executor.remote
380
+ executor = partial(worker.execute_method.remote, method)
381
+ else:
382
+ executor = getattr(worker, method)
383
384
output = executor(*args, **kwargs)
385
all_outputs.append(output)
vllm/engine/ray_utils.py
+56
-63
@@ -1,15 +1,35 @@
1
import socket
2
-from typing import List, Optional, Tuple
2
+from typing import Optional, Tuple, TYPE_CHECKING
3
+
4
+from vllm.config import ParallelConfig
5
6
try:
7
import ray
8
+ from ray.air.util.torch_dist import TorchDistributedWorker
9
+
10
+ class RayWorker(TorchDistributedWorker):
11
+ """Ray wrapper for vllm.worker.Worker, allowing Worker to be
12
+ lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
13
+
14
+ def __init__(self) -> None:
15
+ self.worker = None
16
+
17
+ def init_worker(self, worker_init_fn):
18
+ self.worker = worker_init_fn()
19
+
20
+ def __getattr__(self, name):
21
+ return getattr(self.worker, name)
22
+
23
+ def execute_method(self, method, *args, **kwargs):
24
+ executor = getattr(self, method)
25
+ return executor(*args, **kwargs)
26
+
27
except ImportError:
28
ray = None
29
+ TorchDistributedWorker = None
30
9
-from vllm.config import ParallelConfig
10
-
11
-# rank, node resource (node IP), device id
12
-DeviceID = Tuple[int, Optional[str], int]
31
+if TYPE_CHECKING:
32
+ from ray.util.placement_group import PlacementGroup
33
34
35
def get_open_port():
@@ -22,7 +42,7 @@ def initialize_cluster(
42
parallel_config: ParallelConfig,
43
engine_use_ray: bool = False,
44
ray_address: Optional[str] = None,
25
-) -> Tuple[str, List[List[DeviceID]]]:
45
+) -> Tuple[str, Optional["PlacementGroup"]]:
46
"""Initialize the distributed cluster probably with Ray.
47
48
Args:
@@ -52,63 +72,36 @@ def initialize_cluster(
72
# We need to setup the distributed init method to make sure
73
# the distributed megatron code (e.g., get world size) works correctly.
74
distributed_init_method = f"tcp://localhost:{port}"
55
- all_stage_devices = [[(0, None, 0)]]
56
- return distributed_init_method, all_stage_devices
57
-
58
- # Assume we have a uniform cluster that each node has the same number of
59
- # GPUs for now.
60
- valid_node_resources = []
61
- num_devices_per_node = None
62
- for node in ray.nodes():
63
- if (not node["Alive"]) or node["Resources"]["GPU"] <= 0:
64
- continue
65
- if num_devices_per_node is None:
66
- num_devices_per_node = node["Resources"]["GPU"]
67
- else:
68
- assert num_devices_per_node == node["Resources"]["GPU"], (
69
- "The number of GPUs per node is not uniform.")
70
- for key in node["Resources"]:
71
- if key.startswith("node:"):
72
- valid_node_resources.append(key)
73
-
74
- # Verify the parallel config.
75
- num_nodes = len(valid_node_resources)
76
- if parallel_config.world_size > num_nodes * num_devices_per_node:
77
- raise ValueError(
78
- "The number of required GPUs exceeds the total number of "
79
- "available GPUs.")
80
- if parallel_config.tensor_parallel_size >= num_devices_per_node:
81
- if parallel_config.tensor_parallel_size % num_devices_per_node != 0:
75
+ return distributed_init_method, None
76
+
77
+ current_placement_group = ray.util.get_current_placement_group()
78
+ if current_placement_group:
79
+ # We are in a placement group
80
+ bundles = current_placement_group.bundle_specs
81
+ # Verify that we can use the placement group.
82
+ gpu_bundles = 0
83
+ for bundle in bundles:
84
+ assert bundle.get("GPU", 0) > 1, (
85
+ "Placement group bundles cannot have more than 1 GPU")
86
+ if bundle.get("GPU", 0):
87
+ gpu_bundles += 1
88
+ if parallel_config.world_size > gpu_bundles:
89
raise ValueError(
83
- "The number of tensor parallelism is not divisible by the "
84
- "number of GPUs per node.")
90
+ "The number of required GPUs exceeds the total number of "
91
+ "available GPUs in the placement group.")
92
else:
86
- if num_devices_per_node % parallel_config.tensor_parallel_size != 0:
93
+ num_gpus_in_cluster = ray.cluster_resources().get("GPU", 0)
94
+ if parallel_config.world_size > num_gpus_in_cluster:
95
raise ValueError(
88
- "The number of GPUs per node is not divisible by the number "
89
- "of tensor parallelism.")
90
-
91
- # Assign GPUs to pipeline stages.
92
- rank = 0
93
- current_node_id = 0
94
- current_device_id = 0
95
- distributed_init_method = None
96
- all_stage_devices = []
97
-
98
- for _ in range(parallel_config.pipeline_parallel_size):
99
- stage_devices = []
100
- for _ in range(parallel_config.tensor_parallel_size):
101
- node_resource = valid_node_resources[current_node_id]
102
- stage_devices.append((rank, node_resource, current_device_id))
103
- if distributed_init_method is None:
104
- ip = node_resource.split("node:")[-1]
105
- port = get_open_port()
106
- distributed_init_method = f"tcp://{ip}:{port}"
107
- rank += 1
108
- current_device_id += 1
109
- if current_device_id >= num_devices_per_node:
110
- current_node_id += 1
111
- current_device_id = 0
112
- all_stage_devices.append(stage_devices)
113
-
114
- return distributed_init_method, all_stage_devices
96
+ "The number of required GPUs exceeds the total number of "
97
+ "available GPUs in the cluster.")
98
+ # Create a new placement group
99
+ current_placement_group = ray.util.placement_group([{
100
+ "GPU": 1
101
+ }] * parallel_config.world_size)
102
+ # Wait until PG is ready - this will block until all
103
+ # requested resources are available, and will timeout
104
+ # if they cannot be provisioned.
105
+ ray.get(current_placement_group.ready(), timeout=1800)
106
+
107
+ return None, current_placement_group
vllm/worker/worker.py
+48
-21
@@ -1,7 +1,9 @@
1
"""A GPU worker class."""
2
-from typing import Dict, List, Tuple
2
+import os
3
+from typing import Dict, List, Tuple, Optional
4
5
import torch
6
+import torch.distributed
7
8
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
@@ -27,8 +29,8 @@ class Worker:
29
model_config: ModelConfig,
30
parallel_config: ParallelConfig,
31
scheduler_config: SchedulerConfig,
30
- rank: int,
31
- distributed_init_method: str,
32
+ rank: Optional[int] = None,
33
+ distributed_init_method: Optional[str] = None,
34
) -> None:
35
self.model_config = model_config
36
self.parallel_config = parallel_config
@@ -36,27 +38,39 @@ class Worker:
38
self.rank = rank
39
self.distributed_init_method = distributed_init_method
40
41
+ # Uninitialized cache engine. Will be initialized by
42
+ # self.init_cache_engine().
43
+ self.cache_config = None
44
+ self.block_size = None
45
+ self.cache_engine = None
46
+ self.cache_events = None
47
+ self.gpu_cache = None
48
+
49
+ def init_model(self):
50
+ # This env var set by Ray causes exceptions with graph building.
51
+ os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
52
+ # Env vars will be set by Ray.
53
+ self.rank = self.rank if self.rank is not None else int(
54
+ os.getenv("RANK", "-1"))
55
+ local_rank = int(os.getenv("LOCAL_RANK", "0"))
56
+ self.device = torch.device(f"cuda:{local_rank}")
57
+ if self.rank < 0:
58
+ raise ValueError("Invalid or unspecified rank.")
59
+ torch.cuda.set_device(self.device)
60
+
61
# Initialize the distributed environment.
40
- _init_distributed_environment(parallel_config, rank,
41
- distributed_init_method)
62
+ _init_distributed_environment(self.parallel_config, self.rank,
63
+ self.distributed_init_method)
64
65
# Initialize the model.
66
set_random_seed(self.model_config.seed)
45
- self.model = get_model(model_config)
67
+ self.model = get_model(self.model_config)
68
initialize_all_reduce_launcher(
69
self.scheduler_config.max_num_batched_tokens,
70
self.model_config.get_hidden_size(),
71
self.model_config.dtype,
72
)
73
52
- # Uninitialized cache engine. Will be initialized by
53
- # self.init_cache_engine().
54
- self.cache_config = None
55
- self.block_size = None
56
- self.cache_engine = None
57
- self.cache_events = None
58
- self.gpu_cache = None
59
-
74
@torch.inference_mode()
75
def profile_num_available_blocks(
76
self,
@@ -294,15 +308,28 @@ class Worker:
308
def _init_distributed_environment(
309
parallel_config: ParallelConfig,
310
rank: int,
297
- distributed_init_method: str,
311
+ distributed_init_method: Optional[str] = None,
312
) -> None:
313
"""Initialize the distributed environment."""
300
- torch.distributed.init_process_group(
301
- backend="nccl",
302
- world_size=parallel_config.world_size,
303
- rank=rank,
304
- init_method=distributed_init_method,
305
- )
314
+ if torch.distributed.is_initialized():
315
+ torch_world_size = torch.distributed.get_world_size()
316
+ if torch_world_size != parallel_config.world_size:
317
+ raise RuntimeError(
318
+ "torch.distributed is already initialized but the torch world "
319
+ "size does not match parallel_config.world_size "
320
+ f"({torch_world_size} vs. {parallel_config.world_size}).")
321
+ elif not distributed_init_method:
322
+ raise ValueError(
323
+ "distributed_init_method must be set if torch.distributed "
324
+ "is not already initialized")
325
+ else:
326
+ torch.distributed.init_process_group(
327
+ backend="nccl",
328
+ world_size=parallel_config.world_size,
329
+ rank=rank,
330
+ init_method=distributed_init_method,
331
+ )
332
+
333
# A small all_reduce for warmup.
334
torch.distributed.all_reduce(torch.zeros(1).cuda())
335
initialize_model_parallel(parallel_config.tensor_parallel_size,