A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Introduce `DistributedGPUExecutor` abstract class (#4348)
Nick Hill committed
Apr 26, 2024 at 21:14 UTC
258a2c58d08fc7a242556120877a89404861fbce
2 files changed
+122
-86
vllm/executor/distributed_gpu_executor.py
new
+114
@@ -0,0 +1,114 @@
1
+from abc import abstractmethod
2
+from typing import Any, Dict, Optional, Set, Tuple
3
+
4
+from vllm.executor.executor_base import ExecutorAsyncBase
5
+from vllm.executor.gpu_executor import GPUExecutor
6
+from vllm.logger import init_logger
7
+from vllm.lora.request import LoRARequest
8
+from vllm.sequence import SamplerOutput
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+class DistributedGPUExecutor(GPUExecutor):
14
+ """Abstract superclass of multi-GPU executor implementations."""
15
+
16
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
17
+ """Determine the number of available KV blocks.
18
+
19
+ This invokes `determine_num_available_blocks` on each worker and takes
20
+ the min of the results, guaranteeing that the selected cache sizes are
21
+ compatible with all workers.
22
+
23
+ Returns:
24
+ - tuple[num_gpu_blocks, num_cpu_blocks]
25
+ """
26
+ # Get the maximum number of blocks that can be allocated on GPU and CPU.
27
+ num_blocks = self._run_workers("determine_num_available_blocks", )
28
+
29
+ # Since we use a shared centralized controller, we take the minimum
30
+ # number of blocks across all workers to make sure all the memory
31
+ # operators can be applied to all workers.
32
+ num_gpu_blocks = min(b[0] for b in num_blocks)
33
+ num_cpu_blocks = min(b[1] for b in num_blocks)
34
+
35
+ return num_gpu_blocks, num_cpu_blocks
36
+
37
+ def initialize_cache(self, num_gpu_blocks: int,
38
+ num_cpu_blocks: int) -> None:
39
+ """Initialize the KV cache in all workers.
40
+ """
41
+
42
+ # NOTE: We log here to avoid multiple logs when number of workers is
43
+ # greater than one. We could log in the engine, but not all executors
44
+ # have GPUs.
45
+ logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
46
+ num_cpu_blocks)
47
+
48
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
49
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
50
+
51
+ self._run_workers("initialize_cache",
52
+ num_gpu_blocks=num_gpu_blocks,
53
+ num_cpu_blocks=num_cpu_blocks)
54
+
55
+ def execute_model(self, *args, **kwargs) -> SamplerOutput:
56
+ all_outputs = self._run_workers("execute_model",
57
+ driver_args=args,
58
+ driver_kwargs=kwargs)
59
+
60
+ # Only the driver worker returns the sampling results.
61
+ return all_outputs[0]
62
+
63
+ def add_lora(self, lora_request: LoRARequest) -> bool:
64
+ assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
65
+ return self._run_workers(
66
+ "add_lora",
67
+ lora_request=lora_request,
68
+ )
69
+
70
+ def remove_lora(self, lora_id: int) -> bool:
71
+ assert lora_id > 0, "lora_id must be greater than 0."
72
+ return self._run_workers(
73
+ "remove_lora",
74
+ lora_id=lora_id,
75
+ )
76
+
77
+ def list_loras(self) -> Set[int]:
78
+ return self._run_workers("list_loras")
79
+
80
+ @abstractmethod
81
+ def _run_workers(
82
+ self,
83
+ method: str,
84
+ *args,
85
+ driver_args: Optional[Tuple[Any, ...]] = None,
86
+ driver_kwargs: Optional[Dict[str, Any]] = None,
87
+ max_concurrent_workers: Optional[int] = None,
88
+ **kwargs,
89
+ ) -> Any:
90
+ """Runs the given method on all workers."""
91
+ raise NotImplementedError
92
+
93
+
94
+class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
95
+
96
+ @abstractmethod
97
+ async def _run_workers_async(
98
+ self,
99
+ method: str,
100
+ *args,
101
+ driver_args: Optional[Tuple[Any, ...]] = None,
102
+ driver_kwargs: Optional[Dict[str, Any]] = None,
103
+ **kwargs,
104
+ ) -> Any:
105
+ """Runs the given method on all workers."""
106
+ raise NotImplementedError
107
+
108
+ async def execute_model_async(self, *args, **kwargs) -> SamplerOutput:
109
+ all_outputs = await self._run_workers_async("execute_model",
110
+ driver_args=args,
111
+ driver_kwargs=kwargs)
112
+
113
+ # Only the driver worker returns the sampling results.
114
+ return all_outputs[0]
vllm/executor/ray_gpu_executor.py
+8
-86
@@ -3,12 +3,12 @@ import os
3
import pickle
4
from collections import defaultdict
5
from itertools import islice, repeat
6
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
6
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8
+from vllm.executor.distributed_gpu_executor import ( # yapf: disable
9
+ DistributedGPUExecutor, DistributedGPUExecutorAsync)
10
from vllm.executor.ray_utils import RayWorkerWrapper, ray
11
from vllm.logger import init_logger
11
-from vllm.lora.request import LoRARequest
12
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
13
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
14
get_vllm_instance_id, make_async)
@@ -27,7 +27,7 @@ logger = init_logger(__name__)
27
USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
28
29
30
-class RayGPUExecutor(ExecutorBase):
30
+class RayGPUExecutor(DistributedGPUExecutor):
31
32
def _init_executor(self) -> None:
33
assert (not self.speculative_config
@@ -179,50 +179,9 @@ class RayGPUExecutor(ExecutorBase):
179
self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
180
181
self._run_workers("init_device")
182
- self._run_workers(
183
- "load_model",
184
- max_concurrent_workers=self.parallel_config.
185
- max_parallel_loading_workers,
186
- )
187
-
188
- def determine_num_available_blocks(self) -> Tuple[int, int]:
189
- """Determine the number of available KV blocks.
190
-
191
- This invokes `determine_num_available_blocks` on each worker and takes
192
- the min of the results, guaranteeing that the selected cache sizes are
193
- compatible with all workers.
194
-
195
- Returns:
196
- - Tuple[num_gpu_blocks, num_cpu_blocks]
197
- """
198
- # Get the maximum number of blocks that can be allocated on GPU and CPU.
199
- num_blocks = self._run_workers("determine_num_available_blocks", )
200
-
201
- # Since we use a shared centralized controller, we take the minimum
202
- # number of blocks across all workers to make sure all the memory
203
- # operators can be applied to all workers.
204
- num_gpu_blocks = min(b[0] for b in num_blocks)
205
- num_cpu_blocks = min(b[1] for b in num_blocks)
206
-
207
- return num_gpu_blocks, num_cpu_blocks
208
-
209
- def initialize_cache(self, num_gpu_blocks: int,
210
- num_cpu_blocks: int) -> None:
211
- """Initialize the KV cache in all workers.
212
- """
213
-
214
- # NOTE: We log here to avoid multiple logs when number of workers is
215
- # greater than one. We could log in the engine, but not all executors
216
- # have GPUs.
217
- logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
218
- num_cpu_blocks)
219
-
220
- self.cache_config.num_gpu_blocks = num_gpu_blocks
221
- self.cache_config.num_cpu_blocks = num_cpu_blocks
222
-
223
- self._run_workers("initialize_cache",
224
- num_gpu_blocks=num_gpu_blocks,
225
- num_cpu_blocks=num_cpu_blocks)
182
+ self._run_workers("load_model",
183
+ max_concurrent_workers=self.parallel_config.
184
+ max_parallel_loading_workers)
185
186
def execute_model(self,
187
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -244,23 +203,6 @@ class RayGPUExecutor(ExecutorBase):
203
output = all_outputs[0]
204
return output
205
247
- def add_lora(self, lora_request: LoRARequest) -> bool:
248
- assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
249
- return self._run_workers(
250
- "add_lora",
251
- lora_request=lora_request,
252
- )
253
-
254
- def remove_lora(self, lora_id: int) -> bool:
255
- assert lora_id > 0, "lora_id must be greater than 0."
256
- return self._run_workers(
257
- "remove_lora",
258
- lora_id=lora_id,
259
- )
260
-
261
- def list_loras(self) -> Set[int]:
262
- return self._run_workers("list_loras")
263
-
206
def _run_workers(
207
self,
208
method: str,
@@ -378,7 +320,7 @@ class RayGPUExecutor(ExecutorBase):
320
f"Dead Workers: {dead_actors}. ")
321
322
381
-class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
323
+class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
324
325
def __init__(self, *args, **kwargs):
326
super().__init__(*args, **kwargs)
@@ -409,23 +351,3 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
351
352
all_outputs = await asyncio.gather(*coros)
353
return all_outputs
412
-
413
- async def execute_model_async(
414
- self,
415
- seq_group_metadata_list: List[SequenceGroupMetadata],
416
- blocks_to_swap_in: Dict[int, int],
417
- blocks_to_swap_out: Dict[int, int],
418
- blocks_to_copy: Dict[int, List[int]],
419
- ) -> SamplerOutput:
420
- all_outputs = await self._run_workers_async(
421
- "execute_model",
422
- driver_kwargs={
423
- "seq_group_metadata_list": seq_group_metadata_list,
424
- "blocks_to_swap_in": blocks_to_swap_in,
425
- "blocks_to_swap_out": blocks_to_swap_out,
426
- "blocks_to_copy": blocks_to_copy,
427
- })
428
-
429
- # Only the driver worker returns the sampling results.
430
- output = all_outputs[0]
431
- return output