@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9925c17940
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Ray placement group support (#397)

Antoni Baum committed Jul 19, 2023 at 22:49 UTC 9925c179409f7b14aef55bcfde47b320bd1a263c
5 files changed +185 -114
requirements.txt
+1 -1
@@ -1,6 +1,6 @@
1 ninja # For faster builds.
2 psutil
3 -ray
3 +ray >= 2.5.1
4 sentencepiece # Required for LLaMA tokenizer.
5 numpy
6 torch >= 2.0.0
vllm/engine/async_llm_engine.py
+2 -2
@@ -226,14 +226,14 @@ class AsyncLLMEngine:
226 engine_configs = engine_args.create_engine_configs()
227 parallel_config = engine_configs[2]
228 # Initialize the cluster.
229 - distributed_init_method, devices = initialize_cluster(
229 + distributed_init_method, placement_group = initialize_cluster(
230 parallel_config, engine_args.engine_use_ray)
231 # Create the async LLM engine.
232 engine = cls(engine_args.worker_use_ray,
233 engine_args.engine_use_ray,
234 *engine_configs,
235 distributed_init_method,
236 - devices,
236 + placement_group,
237 log_requests=not engine_args.disable_log_requests,
238 log_stats=not engine_args.disable_log_stats)
239 return engine
vllm/engine/llm_engine.py
+78 -27
@@ -1,11 +1,12 @@
1 import time
2 -from typing import Any, List, Optional
2 +from functools import partial
3 +from typing import Any, List, Optional, TYPE_CHECKING
4
5 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
6 SchedulerConfig)
7 from vllm.core.scheduler import Scheduler
8 from vllm.engine.arg_utils import EngineArgs
8 -from vllm.engine.ray_utils import DeviceID, initialize_cluster, ray
9 +from vllm.engine.ray_utils import initialize_cluster, ray, RayWorker
10 from vllm.logger import init_logger
11 from vllm.outputs import RequestOutput
12 from vllm.sampling_params import SamplingParams
@@ -13,7 +14,13 @@ from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
14 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
15 get_tokenizer)
16 from vllm.utils import Counter
16 -from vllm.worker.worker import Worker
17 +
18 +if ray:
19 + from ray.air.util.torch_dist import init_torch_dist_process_group
20 + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
21 +
22 +if TYPE_CHECKING:
23 + from ray.util.placement_group import PlacementGroup
24
25 logger = init_logger(__name__)
26
@@ -54,7 +61,7 @@ class LLMEngine:
61 parallel_config: ParallelConfig,
62 scheduler_config: SchedulerConfig,
63 distributed_init_method: str,
57 - stage_devices: List[List[DeviceID]],
64 + placement_group: Optional["PlacementGroup"],
65 log_stats: bool,
66 ) -> None:
67 logger.info(
@@ -85,31 +92,73 @@ class LLMEngine:
92 self.seq_counter = Counter()
93
94 # Create the parallel GPU workers.
88 - self.workers: List[Worker] = []
89 - assert len(stage_devices) == 1, "Only support one stage for now."
90 - for rank, node_resource, _ in stage_devices[0]:
91 - worker_cls = Worker
92 - if self.parallel_config.worker_use_ray:
93 - worker_cls = ray.remote(
94 - num_cpus=0,
95 - num_gpus=1,
96 - resources={node_resource: 1e-3},
97 - )(worker_cls).remote
98 -
99 - worker = worker_cls(
100 - model_config,
101 - parallel_config,
102 - scheduler_config,
103 - rank,
104 - distributed_init_method,
105 - )
106 - self.workers.append(worker)
95 + if self.parallel_config.worker_use_ray:
96 + self._init_workers_ray(placement_group)
97 + else:
98 + self._init_workers(distributed_init_method)
99 +
100 # Profile the memory usage and initialize the cache.
101 self._init_cache()
102
103 # Create the scheduler.
104 self.scheduler = Scheduler(scheduler_config, cache_config, log_stats)
105
106 + def _init_workers(self, distributed_init_method: str):
107 + # Lazy import the Worker to avoid importing torch.cuda/xformers
108 + # before CUDA_VISIBLE_DEVICES is set in the Worker
109 + from vllm.worker.worker import Worker # pylint: disable=import-outside-toplevel
110 +
111 + assert self.parallel_config.world_size == 1, (
112 + "Ray is required if parallel_config.world_size > 1.")
113 +
114 + self.workers: List[Worker] = []
115 + worker = Worker(
116 + self.model_config,
117 + self.parallel_config,
118 + self.scheduler_config,
119 + 0,
120 + distributed_init_method,
121 + )
122 + self.workers.append(worker)
123 + self._run_workers(
124 + "init_model",
125 + get_all_outputs=True,
126 + )
127 +
128 + def _init_workers_ray(self, placement_group: "PlacementGroup"):
129 + # Lazy import the Worker to avoid importing torch.cuda/xformers
130 + # before CUDA_VISIBLE_DEVICES is set in the Worker
131 + from vllm.worker.worker import Worker # pylint: disable=import-outside-toplevel
132 +
133 + self.workers: List[Worker] = []
134 + for bundle in placement_group.bundle_specs:
135 + if not bundle.get("GPU", 0):
136 + continue
137 + worker = ray.remote(
138 + num_cpus=0,
139 + num_gpus=1,
140 + scheduling_strategy=PlacementGroupSchedulingStrategy(
141 + placement_group=placement_group,
142 + placement_group_capture_child_tasks=True),
143 + )(RayWorker).remote()
144 + self.workers.append(worker)
145 +
146 + # Initialize torch distributed process group for the workers.
147 + init_torch_dist_process_group(self.workers, backend="nccl")
148 + self._run_workers("init_worker",
149 + get_all_outputs=True,
150 + worker_init_fn=lambda: Worker(
151 + self.model_config,
152 + self.parallel_config,
153 + self.scheduler_config,
154 + None,
155 + None,
156 + ))
157 + self._run_workers(
158 + "init_model",
159 + get_all_outputs=True,
160 + )
161 +
162 def _verify_args(self) -> None:
163 self.model_config.verify_with_parallel_config(self.parallel_config)
164 self.cache_config.verify_with_parallel_config(self.parallel_config)
@@ -152,11 +201,12 @@ class LLMEngine:
201 engine_configs = engine_args.create_engine_configs()
202 parallel_config = engine_configs[2]
203 # Initialize the cluster.
155 - distributed_init_method, devices = initialize_cluster(parallel_config)
204 + distributed_init_method, placement_group = initialize_cluster(
205 + parallel_config)
206 # Create the LLM engine.
207 engine = cls(*engine_configs,
208 distributed_init_method,
159 - devices,
209 + placement_group,
210 log_stats=not engine_args.disable_log_stats)
211 return engine
212
@@ -326,9 +376,10 @@ class LLMEngine:
376 """Runs the given method on all workers."""
377 all_outputs = []
378 for worker in self.workers:
329 - executor = getattr(worker, method)
379 if self.parallel_config.worker_use_ray:
331 - executor = executor.remote
380 + executor = partial(worker.execute_method.remote, method)
381 + else:
382 + executor = getattr(worker, method)
383
384 output = executor(*args, **kwargs)
385 all_outputs.append(output)
vllm/engine/ray_utils.py
+56 -63
@@ -1,15 +1,35 @@
1 import socket
2 -from typing import List, Optional, Tuple
2 +from typing import Optional, Tuple, TYPE_CHECKING
3 +
4 +from vllm.config import ParallelConfig
5
6 try:
7 import ray
8 + from ray.air.util.torch_dist import TorchDistributedWorker
9 +
10 + class RayWorker(TorchDistributedWorker):
11 + """Ray wrapper for vllm.worker.Worker, allowing Worker to be
12 + lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
13 +
14 + def __init__(self) -> None:
15 + self.worker = None
16 +
17 + def init_worker(self, worker_init_fn):
18 + self.worker = worker_init_fn()
19 +
20 + def __getattr__(self, name):
21 + return getattr(self.worker, name)
22 +
23 + def execute_method(self, method, *args, **kwargs):
24 + executor = getattr(self, method)
25 + return executor(*args, **kwargs)
26 +
27 except ImportError:
28 ray = None
29 + TorchDistributedWorker = None
30
9 -from vllm.config import ParallelConfig
10 -
11 -# rank, node resource (node IP), device id
12 -DeviceID = Tuple[int, Optional[str], int]
31 +if TYPE_CHECKING:
32 + from ray.util.placement_group import PlacementGroup
33
34
35 def get_open_port():
@@ -22,7 +42,7 @@ def initialize_cluster(
42 parallel_config: ParallelConfig,
43 engine_use_ray: bool = False,
44 ray_address: Optional[str] = None,
25 -) -> Tuple[str, List[List[DeviceID]]]:
45 +) -> Tuple[str, Optional["PlacementGroup"]]:
46 """Initialize the distributed cluster probably with Ray.
47
48 Args:
@@ -52,63 +72,36 @@ def initialize_cluster(
72 # We need to setup the distributed init method to make sure
73 # the distributed megatron code (e.g., get world size) works correctly.
74 distributed_init_method = f"tcp://localhost:{port}"
55 - all_stage_devices = [[(0, None, 0)]]
56 - return distributed_init_method, all_stage_devices
57 -
58 - # Assume we have a uniform cluster that each node has the same number of
59 - # GPUs for now.
60 - valid_node_resources = []
61 - num_devices_per_node = None
62 - for node in ray.nodes():
63 - if (not node["Alive"]) or node["Resources"]["GPU"] <= 0:
64 - continue
65 - if num_devices_per_node is None:
66 - num_devices_per_node = node["Resources"]["GPU"]
67 - else:
68 - assert num_devices_per_node == node["Resources"]["GPU"], (
69 - "The number of GPUs per node is not uniform.")
70 - for key in node["Resources"]:
71 - if key.startswith("node:"):
72 - valid_node_resources.append(key)
73 -
74 - # Verify the parallel config.
75 - num_nodes = len(valid_node_resources)
76 - if parallel_config.world_size > num_nodes * num_devices_per_node:
77 - raise ValueError(
78 - "The number of required GPUs exceeds the total number of "
79 - "available GPUs.")
80 - if parallel_config.tensor_parallel_size >= num_devices_per_node:
81 - if parallel_config.tensor_parallel_size % num_devices_per_node != 0:
75 + return distributed_init_method, None
76 +
77 + current_placement_group = ray.util.get_current_placement_group()
78 + if current_placement_group:
79 + # We are in a placement group
80 + bundles = current_placement_group.bundle_specs
81 + # Verify that we can use the placement group.
82 + gpu_bundles = 0
83 + for bundle in bundles:
84 + assert bundle.get("GPU", 0) > 1, (
85 + "Placement group bundles cannot have more than 1 GPU")
86 + if bundle.get("GPU", 0):
87 + gpu_bundles += 1
88 + if parallel_config.world_size > gpu_bundles:
89 raise ValueError(
83 - "The number of tensor parallelism is not divisible by the "
84 - "number of GPUs per node.")
90 + "The number of required GPUs exceeds the total number of "
91 + "available GPUs in the placement group.")
92 else:
86 - if num_devices_per_node % parallel_config.tensor_parallel_size != 0:
93 + num_gpus_in_cluster = ray.cluster_resources().get("GPU", 0)
94 + if parallel_config.world_size > num_gpus_in_cluster:
95 raise ValueError(
88 - "The number of GPUs per node is not divisible by the number "
89 - "of tensor parallelism.")
90 -
91 - # Assign GPUs to pipeline stages.
92 - rank = 0
93 - current_node_id = 0
94 - current_device_id = 0
95 - distributed_init_method = None
96 - all_stage_devices = []
97 -
98 - for _ in range(parallel_config.pipeline_parallel_size):
99 - stage_devices = []
100 - for _ in range(parallel_config.tensor_parallel_size):
101 - node_resource = valid_node_resources[current_node_id]
102 - stage_devices.append((rank, node_resource, current_device_id))
103 - if distributed_init_method is None:
104 - ip = node_resource.split("node:")[-1]
105 - port = get_open_port()
106 - distributed_init_method = f"tcp://{ip}:{port}"
107 - rank += 1
108 - current_device_id += 1
109 - if current_device_id >= num_devices_per_node:
110 - current_node_id += 1
111 - current_device_id = 0
112 - all_stage_devices.append(stage_devices)
113 -
114 - return distributed_init_method, all_stage_devices
96 + "The number of required GPUs exceeds the total number of "
97 + "available GPUs in the cluster.")
98 + # Create a new placement group
99 + current_placement_group = ray.util.placement_group([{
100 + "GPU": 1
101 + }] * parallel_config.world_size)
102 + # Wait until PG is ready - this will block until all
103 + # requested resources are available, and will timeout
104 + # if they cannot be provisioned.
105 + ray.get(current_placement_group.ready(), timeout=1800)
106 +
107 + return None, current_placement_group
vllm/worker/worker.py
+48 -21
@@ -1,7 +1,9 @@
1 """A GPU worker class."""
2 -from typing import Dict, List, Tuple
2 +import os
3 +from typing import Dict, List, Tuple, Optional
4
5 import torch
6 +import torch.distributed
7
8 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
@@ -27,8 +29,8 @@ class Worker:
29 model_config: ModelConfig,
30 parallel_config: ParallelConfig,
31 scheduler_config: SchedulerConfig,
30 - rank: int,
31 - distributed_init_method: str,
32 + rank: Optional[int] = None,
33 + distributed_init_method: Optional[str] = None,
34 ) -> None:
35 self.model_config = model_config
36 self.parallel_config = parallel_config
@@ -36,27 +38,39 @@ class Worker:
38 self.rank = rank
39 self.distributed_init_method = distributed_init_method
40
41 + # Uninitialized cache engine. Will be initialized by
42 + # self.init_cache_engine().
43 + self.cache_config = None
44 + self.block_size = None
45 + self.cache_engine = None
46 + self.cache_events = None
47 + self.gpu_cache = None
48 +
49 + def init_model(self):
50 + # This env var set by Ray causes exceptions with graph building.
51 + os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
52 + # Env vars will be set by Ray.
53 + self.rank = self.rank if self.rank is not None else int(
54 + os.getenv("RANK", "-1"))
55 + local_rank = int(os.getenv("LOCAL_RANK", "0"))
56 + self.device = torch.device(f"cuda:{local_rank}")
57 + if self.rank < 0:
58 + raise ValueError("Invalid or unspecified rank.")
59 + torch.cuda.set_device(self.device)
60 +
61 # Initialize the distributed environment.
40 - _init_distributed_environment(parallel_config, rank,
41 - distributed_init_method)
62 + _init_distributed_environment(self.parallel_config, self.rank,
63 + self.distributed_init_method)
64
65 # Initialize the model.
66 set_random_seed(self.model_config.seed)
45 - self.model = get_model(model_config)
67 + self.model = get_model(self.model_config)
68 initialize_all_reduce_launcher(
69 self.scheduler_config.max_num_batched_tokens,
70 self.model_config.get_hidden_size(),
71 self.model_config.dtype,
72 )
73
52 - # Uninitialized cache engine. Will be initialized by
53 - # self.init_cache_engine().
54 - self.cache_config = None
55 - self.block_size = None
56 - self.cache_engine = None
57 - self.cache_events = None
58 - self.gpu_cache = None
59 -
74 @torch.inference_mode()
75 def profile_num_available_blocks(
76 self,
@@ -294,15 +308,28 @@ class Worker:
308 def _init_distributed_environment(
309 parallel_config: ParallelConfig,
310 rank: int,
297 - distributed_init_method: str,
311 + distributed_init_method: Optional[str] = None,
312 ) -> None:
313 """Initialize the distributed environment."""
300 - torch.distributed.init_process_group(
301 - backend="nccl",
302 - world_size=parallel_config.world_size,
303 - rank=rank,
304 - init_method=distributed_init_method,
305 - )
314 + if torch.distributed.is_initialized():
315 + torch_world_size = torch.distributed.get_world_size()
316 + if torch_world_size != parallel_config.world_size:
317 + raise RuntimeError(
318 + "torch.distributed is already initialized but the torch world "
319 + "size does not match parallel_config.world_size "
320 + f"({torch_world_size} vs. {parallel_config.world_size}).")
321 + elif not distributed_init_method:
322 + raise ValueError(
323 + "distributed_init_method must be set if torch.distributed "
324 + "is not already initialized")
325 + else:
326 + torch.distributed.init_process_group(
327 + backend="nccl",
328 + world_size=parallel_config.world_size,
329 + rank=rank,
330 + init_method=distributed_init_method,
331 + )
332 +
333 # A small all_reduce for warmup.
334 torch.distributed.all_reduce(torch.zeros(1).cuda())
335 initialize_model_parallel(parallel_config.tensor_parallel_size,