@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc1bdecebf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[core][distributed] exact ray placement control (#12732)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 6, 2025 at 02:03 UTC bc1bdecebf76cca0dfafe4924d529b30c8a24795
6 files changed +173 -13
.buildkite/test-pipeline.yaml
+2
@@ -128,6 +128,7 @@ steps:
128 - tests/spec_decode/e2e/test_integration_dist_tp4
129 - tests/compile
130 - examples/offline_inference/rlhf.py
131 + - examples/offline_inference/ray_placement.py
132 commands:
133 - pytest -v -s distributed/test_utils.py
134 - pytest -v -s compile/test_basic_correctness.py
@@ -136,6 +137,7 @@ steps:
137 # TODO: create a dedicated test section for multi-GPU example tests
138 # when we have multiple distributed example tests
139 - python3 ../examples/offline_inference/rlhf.py
140 + - RAY_DEDUP_LOGS=0 python3 ../examples/offline_inference/ray_placement.py
141
142 - label: Metrics, Tracing Test # 10min
143 num_gpus: 2
examples/offline_inference/ray_placement.py new
+121
@@ -0,0 +1,121 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""
3 +a simple demonstration to show how to control
4 +the placement of the vLLM workers with Ray.
5 +The key is to set VLLM_RAY_PER_WORKER_GPUS and
6 +VLLM_RAY_BUNDLE_INDICES properly.
7 +"""
8 +import os
9 +
10 +import ray
11 +from ray.util.placement_group import placement_group
12 +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
13 +
14 +from vllm import LLM
15 +from vllm.worker.worker import Worker
16 +
17 +
18 +class MyWorker(Worker):
19 +
20 + def report_device_id(self) -> str:
21 + from vllm.platforms import current_platform
22 + return current_platform.get_device_uuid(self.device.index)
23 +
24 +
25 +class MyLLM(LLM):
26 +
27 + def __init__(self, *args, bundle_indices: list, **kwargs):
28 + # a hack to make the script work.
29 + # stop ray from manipulating CUDA_VISIBLE_DEVICES
30 + # at the top-level
31 + del os.environ["CUDA_VISIBLE_DEVICES"]
32 + # every worker will use 0.4 GPU, so that we can schedule
33 + # 2 instances on the same GPUs.
34 + os.environ["VLLM_RAY_PER_WORKER_GPUS"] = "0.4"
35 + os.environ["VLLM_RAY_BUNDLE_INDICES"] = ",".join(
36 + map(str, bundle_indices))
37 + print(f"creating LLM with bundle_indices={bundle_indices}")
38 + super().__init__(*args, **kwargs)
39 +
40 +
41 +class RayTrainingActor:
42 +
43 + def report_device_id(self) -> str:
44 + # the argument for get_device_uuid is the index
45 + # of the GPU in the visible devices.
46 + # ray will set CUDA_VISIBLE_DEVICES to the assigned GPUs
47 + from vllm.platforms import current_platform
48 + return current_platform.get_device_uuid(0)
49 +
50 +
51 +# ray manages 4 GPUs
52 +os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
53 +ray.init()
54 +
55 +# we want to co-locate vLLM instance and the training actor
56 +# on the same set of GPUs.
57 +# the placement plan is as follows:
58 +# GPU 0 and 1: training actor 0, 1, and vLLM instance 0 (with TP=2)
59 +# GPU 2 and 3: training actor 2, 3, and vLLM instance 1 (with TP=2)
60 +
61 +pg = placement_group([{"GPU": 1, "CPU": 0}] * 4)
62 +ray.get(pg.ready())
63 +print(f"placement group has bundles {pg.bundle_specs=}")
64 +
65 +training_actors = []
66 +training_actor_device_ids = []
67 +inference_engines = []
68 +inference_engine_device_ids = []
69 +
70 +for bundle_index in [0, 1, 2, 3]:
71 + training_actor = ray.remote(
72 + num_cpus=0,
73 + num_gpus=0.4,
74 + scheduling_strategy=PlacementGroupSchedulingStrategy(
75 + placement_group=pg,
76 + placement_group_capture_child_tasks=True,
77 + placement_group_bundle_index=bundle_index,
78 + ),
79 + )(RayTrainingActor).remote()
80 + training_actors.append(training_actor)
81 + device_id = ray.get(training_actor.report_device_id.remote())
82 + print(f"training actor {bundle_index} is on {device_id}")
83 + training_actor_device_ids.append(device_id)
84 +
85 +for (i, bundle_indices) in enumerate([[0, 1], [2, 3]]):
86 + # IMPORTANT: when creating vLLM instances, we need to
87 + # make sure there are no GPU activities on the target GPUs,
88 + # otherwise, they will interfere with the vLLM memory profiling,
89 + # and cause unexpected behaviors.
90 + llm = ray.remote(
91 + num_cpus=0,
92 + num_gpus=0,
93 + scheduling_strategy=PlacementGroupSchedulingStrategy(
94 + placement_group=pg,
95 + placement_group_capture_child_tasks=True,
96 + ),
97 + )(MyLLM).remote(
98 + model="facebook/opt-125m",
99 + enforce_eager=True,
100 + worker_cls=MyWorker,
101 + tensor_parallel_size=2,
102 + distributed_executor_backend="ray",
103 + gpu_memory_utilization=0.4,
104 + bundle_indices=bundle_indices,
105 + )
106 + inference_engines.append(llm)
107 + # don't call any method on the inference engine here,
108 + # otherwise it will block until the vLLM instance is created.
109 +
110 +for i, llm in enumerate(inference_engines):
111 + inference_engine_device_ids.append(
112 + ray.get(llm.collective_rpc.remote("report_device_id", args=tuple())))
113 + print(f"inference engine {i} is on {inference_engine_device_ids[-1]}")
114 +
115 +# check the placement
116 +# the first two training actors should be
117 +# on the same GPUs as the first inference engine
118 +assert training_actor_device_ids[:2] == inference_engine_device_ids[0]
119 +# the last two training actors should be
120 +# on the same GPUs as the second inference engine
121 +assert training_actor_device_ids[2:] == inference_engine_device_ids[1]
vllm/envs.py
+14
@@ -85,6 +85,8 @@ if TYPE_CHECKING:
85 VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
86 VLLM_MLA_CUDA_MEM_ALIGN_KV_CACHE: bool = True
87 VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
88 + VLLM_RAY_PER_WORKER_GPUS: float = 1.0
89 + VLLM_RAY_BUNDLE_INDICES: str = ""
90
91
92 def get_default_cache_root():
@@ -550,6 +552,18 @@ environment_variables: Dict[str, Callable[[], Any]] = {
552 lambda: bool(int(os.getenv("VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON", "0"))
553 ),
554
555 + # Number of GPUs per worker in Ray, if it is set to be a fraction,
556 + # it allows ray to schedule multiple actors on a single GPU,
557 + # so that users can colocate other actors on the same GPUs as vLLM.
558 + "VLLM_RAY_PER_WORKER_GPUS":
559 + lambda: float(os.getenv("VLLM_RAY_PER_WORKER_GPUS", "1.0")),
560 +
561 + # Bundle indices for Ray, if it is set, it can control precisely
562 + # which indices are used for the Ray bundle, for every worker.
563 + # Format: comma-separated list of integers, e.g. "0,1,2,3"
564 + "VLLM_RAY_BUNDLE_INDICES":
565 + lambda: os.getenv("VLLM_RAY_BUNDLE_INDICES", ""),
566 +
567 # When on a Nvidia GPU aligns single entries (within a page) so they are 256
568 # byte aligned for better performance, this increases the memory usage of
569 # the cache. Currently this only affects MLA that results in non-256
vllm/executor/ray_distributed_executor.py
+23 -13
@@ -129,13 +129,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
129
130 def _init_workers_ray(self, placement_group: "PlacementGroup",
131 **ray_remote_kwargs):
132 - if (self.parallel_config.tensor_parallel_size == 1
133 - and self.parallel_config.pipeline_parallel_size == 1):
134 - # For single GPU case, we use a ray worker with constrained memory.
135 - num_gpus = self.cache_config.gpu_memory_utilization
136 - else:
137 - # Otherwise, the ray workers are allocated with a full GPU.
138 - num_gpus = 1
132 + num_gpus = envs.VLLM_RAY_PER_WORKER_GPUS
133
134 # The driver dummy worker does not actually use any resources.
135 # It holds the resource for the driver worker.
@@ -155,12 +149,29 @@ class RayDistributedExecutor(DistributedExecutorBase):
149 logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
150
151 # Create the workers.
158 - driver_ip = get_ip()
159 - rank = 0
152 + bundle_indices: List[int]
153 + if envs.VLLM_RAY_BUNDLE_INDICES:
154 + # Use the bundle indices specified by the user.
155 + bundle_indices = list(
156 + map(int, envs.VLLM_RAY_BUNDLE_INDICES.split(",")))
157 + assert len(bundle_indices) == self.parallel_config.world_size, \
158 + ("VLLM_RAY_BUNDLE_INDICES must have the same size"
159 + f" as the world size, but got {bundle_indices=} "
160 + f"and {self.parallel_config.world_size=}")
161 + assert len(set(bundle_indices)) == len(bundle_indices), \
162 + ("VLLM_RAY_BUNDLE_INDICES cannot have duplicate values,"
163 + f" but got {bundle_indices=}")
164 + else:
165 + # use the first N bundles that have GPU resources.
166 + bundle_indices = []
167 + for bundle_id, bundle in enumerate(placement_group.bundle_specs):
168 + if bundle.get(current_platform.ray_device_key, 0):
169 + bundle_indices.append(bundle_id)
170 + bundle_indices = bundle_indices[:self.parallel_config.world_size]
171 +
172 worker_metadata: List[RayWorkerMetaData] = []
161 - for bundle_id, bundle in enumerate(placement_group.bundle_specs):
162 - if not bundle.get(current_platform.ray_device_key, 0):
163 - continue
173 + driver_ip = get_ip()
174 + for rank, bundle_id in enumerate(bundle_indices):
175 scheduling_strategy = PlacementGroupSchedulingStrategy(
176 placement_group=placement_group,
177 placement_group_capture_child_tasks=True,
@@ -187,7 +198,6 @@ class RayDistributedExecutor(DistributedExecutorBase):
198 rpc_rank=rank)
199 worker_metadata.append(
200 RayWorkerMetaData(worker=worker, created_rank=rank))
190 - rank += 1
201
202 worker_ips = ray.get([
203 each.worker.get_node_ip.remote() # type: ignore[attr-defined]
vllm/platforms/cuda.py
+8
@@ -275,6 +275,14 @@ class NvmlCudaPlatform(CudaPlatformBase):
275 physical_device_id = device_id_to_physical_device_id(device_id)
276 return cls._get_physical_device_name(physical_device_id)
277
278 + @classmethod
279 + @lru_cache(maxsize=8)
280 + @with_nvml_context
281 + def get_device_uuid(cls, device_id: int = 0) -> str:
282 + physical_device_id = device_id_to_physical_device_id(device_id)
283 + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
284 + return pynvml.nvmlDeviceGetUUID(handle)
285 +
286 @classmethod
287 @lru_cache(maxsize=8)
288 @with_nvml_context
vllm/platforms/interface.py
+5
@@ -183,6 +183,11 @@ class Platform:
183 """Get the name of a device."""
184 raise NotImplementedError
185
186 + @classmethod
187 + def get_device_uuid(cls, device_id: int = 0) -> str:
188 + """Get the uuid of a device, e.g. the PCI bus ID."""
189 + raise NotImplementedError
190 +
191 @classmethod
192 def get_device_total_memory(cls, device_id: int = 0) -> int:
193 """Get the total memory of a device in bytes."""