A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add distributed model executor abstraction (#3191)
Zhuohan Li committed
Mar 11, 2024 at 11:03 UTC
4c922709b65ff5c0652ae36b93047016bdeaace8
13 files changed
+818
-509
docs/source/dev/engine/llm_engine.rst
+1
-1
@@ -2,5 +2,5 @@ LLMEngine
2
=================================
3
4
.. autoclass:: vllm.engine.llm_engine.LLMEngine
5
- :members: add_request, abort_request, step, _init_cache
5
+ :members: add_request, abort_request, step
6
:show-inheritance:
\ No newline at end of file
format.sh
+6
-2
@@ -95,13 +95,17 @@ echo 'vLLM yapf: Done'
95
# echo 'vLLM mypy:'
96
# mypy
97
98
+CODESPELL_EXCLUDES=(
99
+ '--skip' '*docs/source/_build/**'
100
+)
101
+
102
# check spelling of specified files
103
spell_check() {
104
codespell "$@"
105
}
106
107
spell_check_all(){
104
- codespell --toml pyproject.toml
108
+ codespell --toml pyproject.toml "${CODESPELL_EXCLUDES[@]}"
109
}
110
111
# Spelling check of files that differ from main branch.
@@ -116,7 +120,7 @@ spell_check_changed() {
120
121
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
122
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
119
- codespell
123
+ codespell "${CODESPELL_EXCLUDES[@]}"
124
fi
125
}
126
tests/lora/conftest.py
+2
-1
@@ -152,4 +152,5 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
152
@pytest.fixture
153
def llama_2_7b_model_extra_embeddings(
154
llama_2_7b_engine_extra_embeddings) -> nn.Module:
155
- yield llama_2_7b_engine_extra_embeddings.driver_worker.model_runner.model
155
+ yield (llama_2_7b_engine_extra_embeddings.model_executor.driver_worker.
156
+ model_runner.model)
vllm/__init__.py
+2
-2
@@ -3,7 +3,7 @@
3
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4
from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
-from vllm.engine.ray_utils import initialize_cluster
6
+from vllm.engine.ray_utils import initialize_ray_cluster
7
from vllm.entrypoints.llm import LLM
8
from vllm.outputs import CompletionOutput, RequestOutput
9
from vllm.sampling_params import SamplingParams
@@ -19,5 +19,5 @@ __all__ = [
19
"EngineArgs",
20
"AsyncLLMEngine",
21
"AsyncEngineArgs",
22
- "initialize_cluster",
22
+ "initialize_ray_cluster",
23
]
vllm/config.py
+6
-1
@@ -1,4 +1,4 @@
1
-from typing import Optional, Union, ClassVar
1
+from typing import TYPE_CHECKING, Optional, Union, ClassVar
2
from dataclasses import dataclass
3
import os
4
from packaging.version import Version
@@ -10,6 +10,9 @@ from vllm.logger import init_logger
10
from vllm.transformers_utils.config import get_config
11
from vllm.utils import get_cpu_memory, is_hip, is_neuron, get_nvcc_cuda_version
12
13
+if TYPE_CHECKING:
14
+ from ray.util.placement_group import PlacementGroup
15
+
16
logger = init_logger(__name__)
17
18
_GB = 1 << 30
@@ -397,6 +400,7 @@ class ParallelConfig:
400
max_parallel_loading_workers: Optional[int] = None,
401
disable_custom_all_reduce: bool = False,
402
ray_workers_use_nsight: bool = False,
403
+ placement_group: Optional["PlacementGroup"] = None,
404
) -> None:
405
self.pipeline_parallel_size = pipeline_parallel_size
406
if is_neuron():
@@ -412,6 +416,7 @@ class ParallelConfig:
416
self.max_parallel_loading_workers = max_parallel_loading_workers
417
self.disable_custom_all_reduce = disable_custom_all_reduce
418
self.ray_workers_use_nsight = ray_workers_use_nsight
419
+ self.placement_group = placement_group
420
421
self.world_size = pipeline_parallel_size * self.tensor_parallel_size
422
# Ray worker is not supported for Neuron backend.
vllm/engine/async_llm_engine.py
+38
-68
@@ -2,8 +2,8 @@ import asyncio
2
import os
3
import time
4
from functools import partial
5
-from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
6
- Union, AsyncIterator, Callable)
5
+from typing import (Callable, Dict, Iterable, List, Optional, Set, Tuple, Type,
6
+ Union, AsyncIterator)
7
8
from transformers import PreTrainedTokenizer
9
@@ -11,7 +11,7 @@ from vllm.lora.request import LoRARequest
11
from vllm.config import ModelConfig
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
from vllm.engine.llm_engine import LLMEngine
14
-from vllm.engine.ray_utils import initialize_cluster, ray
14
+from vllm.engine.ray_utils import initialize_ray_cluster, ray
15
from vllm.logger import init_logger
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
@@ -208,17 +208,10 @@ class _AsyncLLMEngine(LLMEngine):
208
209
if not scheduler_outputs.is_empty():
210
# Execute the model.
211
- all_outputs = await self._run_workers_async(
212
- "execute_model",
213
- driver_kwargs={
214
- "seq_group_metadata_list": seq_group_metadata_list,
215
- "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
216
- "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
217
- "blocks_to_copy": scheduler_outputs.blocks_to_copy,
218
- })
219
-
220
- # Only the driver worker returns the sampling results.
221
- output = all_outputs[0]
211
+ output = await self.model_executor.execute_model_async(
212
+ seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
213
+ scheduler_outputs.blocks_to_swap_out,
214
+ scheduler_outputs.blocks_to_copy)
215
else:
216
output = []
217
@@ -268,37 +261,8 @@ class _AsyncLLMEngine(LLMEngine):
261
lora_request=lora_request,
262
)
263
271
- async def _run_workers_async(
272
- self,
273
- method: str,
274
- *args,
275
- driver_args: Optional[List[Any]] = None,
276
- driver_kwargs: Optional[Dict[str, Any]] = None,
277
- **kwargs,
278
- ) -> Any:
279
- """Runs the given method on all workers."""
280
- coros = []
281
-
282
- if driver_args is None:
283
- driver_args = args
284
- if driver_kwargs is None:
285
- driver_kwargs = kwargs
286
-
287
- # Run the driver worker asynchronously.
288
- driver_executor = getattr(self.driver_worker, method)
289
- coros.append(asyncio.get_event_loop().run_in_executor(
290
- None, partial(driver_executor, *driver_args, **driver_kwargs)))
291
-
292
- # Run the ray workers asynchronously.
293
- for worker in self.workers:
294
- coros.append(worker.execute_method.remote(method, *args, **kwargs))
295
-
296
- all_outputs = await asyncio.gather(*coros)
297
- return all_outputs
298
-
299
- async def check_health_async(self):
300
- """Raises an error if engine is unhealthy."""
301
- self._check_if_any_actor_is_dead()
264
+ async def check_health_async(self) -> None:
265
+ self.model_executor.check_health()
266
267
268
class AsyncLLMEngine:
@@ -353,6 +317,34 @@ class AsyncLLMEngine:
317
self._request_tracker: Optional[RequestTracker] = None
318
self._errored_with: Optional[BaseException] = None
319
320
+ @classmethod
321
+ def from_engine_args(cls,
322
+ engine_args: AsyncEngineArgs,
323
+ start_engine_loop: bool = True) -> "AsyncLLMEngine":
324
+ """Creates an async LLM engine from the engine arguments."""
325
+ # Create the engine configs.
326
+ engine_configs = engine_args.create_engine_configs()
327
+ parallel_config = engine_configs[2]
328
+ if parallel_config.worker_use_ray or engine_args.engine_use_ray:
329
+ initialize_ray_cluster(parallel_config)
330
+ from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
331
+ executor_class = RayGPUExecutorAsync
332
+ else:
333
+ assert parallel_config.world_size == 1, (
334
+ "Ray is required if parallel_config.world_size > 1.")
335
+ from vllm.executor.gpu_executor import GPUExecutorAsync
336
+ executor_class = GPUExecutorAsync
337
+ # Create the async LLM engine.
338
+ engine = cls(parallel_config.worker_use_ray,
339
+ engine_args.engine_use_ray,
340
+ *engine_configs,
341
+ executor_class,
342
+ log_requests=not engine_args.disable_log_requests,
343
+ log_stats=not engine_args.disable_log_stats,
344
+ max_log_len=engine_args.max_log_len,
345
+ start_engine_loop=start_engine_loop)
346
+ return engine
347
+
348
@property
349
def is_running(self) -> bool:
350
return (self.background_loop is not None
@@ -670,35 +662,13 @@ class AsyncLLMEngine:
662
else:
663
return self.engine.get_model_config()
664
673
- @classmethod
674
- def from_engine_args(cls,
675
- engine_args: AsyncEngineArgs,
676
- start_engine_loop: bool = True) -> "AsyncLLMEngine":
677
- """Creates an async LLM engine from the engine arguments."""
678
- # Create the engine configs.
679
- engine_configs = engine_args.create_engine_configs()
680
- parallel_config = engine_configs[2]
681
- # Initialize the cluster.
682
- placement_group = initialize_cluster(parallel_config,
683
- engine_args.engine_use_ray)
684
- # Create the async LLM engine.
685
- engine = cls(parallel_config.worker_use_ray,
686
- engine_args.engine_use_ray,
687
- *engine_configs,
688
- placement_group,
689
- log_requests=not engine_args.disable_log_requests,
690
- log_stats=not engine_args.disable_log_stats,
691
- max_log_len=engine_args.max_log_len,
692
- start_engine_loop=start_engine_loop)
693
- return engine
694
-
665
async def do_log_stats(self) -> None:
666
if self.engine_use_ray:
667
await self.engine.do_log_stats.remote()
668
else:
669
self.engine.do_log_stats()
670
701
- async def check_health(self):
671
+ async def check_health(self) -> None:
672
"""Raises an error if engine is unhealthy."""
673
t = time.perf_counter()
674
logger.debug("Starting health check...")
vllm/engine/llm_engine.py
+44
-402
@@ -1,11 +1,5 @@
1
-import copy
2
-from collections import defaultdict
3
-import os
1
import time
5
-import pickle
6
-import importlib
7
-from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8
- Union)
2
+from typing import Dict, Iterable, List, Optional, Tuple, Type, Union
3
4
from transformers import PreTrainedTokenizer
5
@@ -15,8 +9,9 @@ from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
9
ParallelConfig, SchedulerConfig, LoRAConfig)
10
from vllm.core.scheduler import Scheduler, SchedulerOutputs
11
from vllm.engine.arg_utils import EngineArgs
12
+from vllm.executor.executor_base import ExecutorBase
13
from vllm.engine.metrics import StatLogger, Stats
19
-from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
14
+from vllm.engine.ray_utils import initialize_ray_cluster
15
from vllm.logger import init_logger
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
@@ -24,29 +19,11 @@ from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
19
SequenceGroupOutput, SequenceOutput, SequenceStatus)
20
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
21
TokenizerGroup)
27
-from vllm.utils import (Counter, set_cuda_visible_devices, get_ip,
28
- get_open_port, get_distributed_init_method)
29
-
30
-if ray:
31
- from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
32
-
33
-if TYPE_CHECKING:
34
- from ray.util.placement_group import PlacementGroup
22
+from vllm.utils import Counter
23
24
logger = init_logger(__name__)
25
_LOCAL_LOGGING_INTERVAL_SEC = 5
26
39
-# A map between the device type (in device config) to its worker module.
40
-DEVICE_TO_WORKER_MODULE_MAP = {
41
- "cuda": "vllm.worker.worker",
42
- "neuron": "vllm.worker.neuron_worker",
43
-}
44
-
45
-# If the env var is set, it uses the Ray's compiled DAG API
46
-# which optimizes the control plane overhead.
47
-# Run VLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
48
-USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
49
-
27
28
class LLMEngine:
29
"""An LLM engine that receives requests and generates texts.
@@ -71,8 +48,8 @@ class LLMEngine:
48
parallel_config: The configuration related to distributed execution.
49
scheduler_config: The configuration related to the request scheduler.
50
device_config: The configuration related to the device.
74
- placement_group: Ray placement group for distributed execution.
75
- Required for distributed execution.
51
+ executor_class: The model executor class for managing distributed
52
+ execution.
53
log_stats: Whether to log statistics.
54
"""
55
@@ -84,7 +61,7 @@ class LLMEngine:
61
scheduler_config: SchedulerConfig,
62
device_config: DeviceConfig,
63
lora_config: Optional[LoRAConfig],
87
- placement_group: Optional["PlacementGroup"],
64
+ executor_class: Type[ExecutorBase],
65
log_stats: bool,
66
) -> None:
67
logger.info(
@@ -121,33 +98,13 @@ class LLMEngine:
98
self._init_tokenizer()
99
self.seq_counter = Counter()
100
124
- # Create the parallel GPU workers.
125
- if self.parallel_config.worker_use_ray:
126
- # Disable Ray usage stats collection.
127
- ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
128
- if ray_usage != "1":
129
- os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
130
- # Pass additional arguments to initialize the worker
131
- additional_ray_args = {}
132
- if self.parallel_config.ray_workers_use_nsight:
133
- logger.info("Configuring Ray workers to use nsight.")
134
- additional_ray_args = {
135
- "runtime_env": {
136
- "nsight": {
137
- "t": "cuda,cudnn,cublas",
138
- "o": "'worker_process_%p'",
139
- "cuda-graph-trace": "node",
140
- }
141
- }
142
- }
143
- self._init_workers_ray(placement_group, **additional_ray_args)
144
- else:
145
- self._init_workers()
146
-
147
- # Profile the memory usage and initialize the cache.
148
- self._init_cache()
101
+ self.model_executor = executor_class(model_config, cache_config,
102
+ parallel_config, scheduler_config,
103
+ device_config, lora_config)
104
105
# Create the scheduler.
106
+ # NOTE: the cache_config here have been updated with the numbers of
107
+ # GPU and CPU blocks, which are profiled in the distributed executor.
108
self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
109
110
# Metric Logging.
@@ -157,9 +114,29 @@ class LLMEngine:
114
labels=dict(model_name=model_config.model))
115
self.stat_logger.info("cache_config", self.cache_config)
116
160
- self.forward_dag = None
161
- if USE_RAY_COMPILED_DAG:
162
- self.forward_dag = self._compiled_ray_dag()
117
+ @classmethod
118
+ def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
119
+ """Creates an LLM engine from the engine arguments."""
120
+ # Create the engine configs.
121
+ engine_configs = engine_args.create_engine_configs()
122
+ parallel_config = engine_configs[2]
123
+
124
+ # Initialize the cluster and specify the executor class.
125
+ if parallel_config.worker_use_ray:
126
+ initialize_ray_cluster(parallel_config)
127
+ from vllm.executor.ray_gpu_executor import RayGPUExecutor
128
+ executor_class = RayGPUExecutor
129
+ else:
130
+ assert parallel_config.world_size == 1, (
131
+ "Ray is required if parallel_config.world_size > 1.")
132
+ from vllm.executor.gpu_executor import GPUExecutor
133
+ executor_class = GPUExecutor
134
+
135
+ # Create the LLM engine.
136
+ engine = cls(*engine_configs,
137
+ executor_class=executor_class,
138
+ log_stats=not engine_args.disable_log_stats)
139
+ return engine
140
141
def __reduce__(self):
142
# This is to ensure that the LLMEngine is not referenced in
@@ -173,39 +150,6 @@ class LLMEngine:
150
sequence: Sequence) -> "PreTrainedTokenizer":
151
return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
152
176
- def _dispatch_worker(self):
177
- worker_module = DEVICE_TO_WORKER_MODULE_MAP[
178
- self.device_config.device_type]
179
- imported_worker = importlib.import_module(worker_module)
180
- Worker = imported_worker.Worker
181
- return Worker
182
-
183
- def _init_workers(self):
184
- # Lazy import the Worker to avoid importing torch.cuda/xformers
185
- # before CUDA_VISIBLE_DEVICES is set in the Worker
186
- Worker = self._dispatch_worker()
187
-
188
- assert self.parallel_config.world_size == 1, (
189
- "Ray is required if parallel_config.world_size > 1.")
190
-
191
- self.workers: List[Worker] = []
192
- distributed_init_method = get_distributed_init_method(
193
- get_ip(), get_open_port())
194
- self.driver_worker = Worker(
195
- self.model_config,
196
- self.parallel_config,
197
- self.scheduler_config,
198
- self.device_config,
199
- local_rank=0,
200
- rank=0,
201
- distributed_init_method=distributed_init_method,
202
- lora_config=self.lora_config,
203
- kv_cache_dtype=self.cache_config.cache_dtype,
204
- is_driver_worker=True,
205
- )
206
- self._run_workers("init_model")
207
- self._run_workers("load_model")
208
-
153
def _init_tokenizer(self, **tokenizer_init_kwargs):
154
init_kwargs = dict(
155
enable_lora=bool(self.lora_config),
@@ -218,126 +162,6 @@ class LLMEngine:
162
self.tokenizer: TokenizerGroup = TokenizerGroup(
163
self.model_config.tokenizer, **init_kwargs)
164
221
- def _init_workers_ray(self, placement_group: "PlacementGroup",
222
- **ray_remote_kwargs):
223
- if self.parallel_config.tensor_parallel_size == 1:
224
- num_gpus = self.cache_config.gpu_memory_utilization
225
- else:
226
- num_gpus = 1
227
-
228
- self.driver_dummy_worker: RayWorkerVllm = None
229
- self.workers: List[RayWorkerVllm] = []
230
-
231
- driver_ip = get_ip()
232
- for bundle_id, bundle in enumerate(placement_group.bundle_specs):
233
- if not bundle.get("GPU", 0):
234
- continue
235
- scheduling_strategy = PlacementGroupSchedulingStrategy(
236
- placement_group=placement_group,
237
- placement_group_capture_child_tasks=True,
238
- placement_group_bundle_index=bundle_id,
239
- )
240
- worker = ray.remote(
241
- num_cpus=0,
242
- num_gpus=num_gpus,
243
- scheduling_strategy=scheduling_strategy,
244
- **ray_remote_kwargs,
245
- )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
246
-
247
- worker_ip = ray.get(worker.get_node_ip.remote())
248
- if worker_ip == driver_ip and self.driver_dummy_worker is None:
249
- # If the worker is on the same node as the driver, we use it
250
- # as the resource holder for the driver process.
251
- self.driver_dummy_worker = worker
252
- else:
253
- self.workers.append(worker)
254
-
255
- if self.driver_dummy_worker is None:
256
- raise ValueError(
257
- "Ray does not allocate any GPUs on the driver node. Consider "
258
- "adjusting the Ray placement group or running the driver on a "
259
- "GPU node.")
260
-
261
- driver_node_id, driver_gpu_ids = ray.get(
262
- self.driver_dummy_worker.get_node_and_gpu_ids.remote())
263
- worker_node_and_gpu_ids = ray.get(
264
- [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
265
-
266
- node_workers = defaultdict(list)
267
- node_gpus = defaultdict(list)
268
-
269
- node_workers[driver_node_id].append(0)
270
- node_gpus[driver_node_id].extend(driver_gpu_ids)
271
- for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
272
- start=1):
273
- node_workers[node_id].append(i)
274
- node_gpus[node_id].extend(gpu_ids)
275
- for node_id, gpu_ids in node_gpus.items():
276
- node_gpus[node_id] = sorted(gpu_ids)
277
-
278
- # Set CUDA_VISIBLE_DEVICES for the driver.
279
- set_cuda_visible_devices(node_gpus[driver_node_id])
280
- for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
281
- worker.set_cuda_visible_devices.remote(node_gpus[node_id])
282
-
283
- distributed_init_method = get_distributed_init_method(
284
- driver_ip, get_open_port())
285
-
286
- # Lazy import the Worker to avoid importing torch.cuda/xformers
287
- # before CUDA_VISIBLE_DEVICES is set in the Worker
288
- Worker = self._dispatch_worker()
289
-
290
- # Initialize torch distributed process group for the workers.
291
- model_config = copy.deepcopy(self.model_config)
292
- parallel_config = copy.deepcopy(self.parallel_config)
293
- scheduler_config = copy.deepcopy(self.scheduler_config)
294
- device_config = copy.deepcopy(self.device_config)
295
- lora_config = copy.deepcopy(self.lora_config)
296
- kv_cache_dtype = self.cache_config.cache_dtype
297
-
298
- for rank, (worker, (node_id,
299
- _)) in enumerate(zip(self.workers,
300
- worker_node_and_gpu_ids),
301
- start=1):
302
- local_rank = node_workers[node_id].index(rank)
303
- worker.init_worker.remote(
304
- lambda rank=rank, local_rank=local_rank: Worker(
305
- model_config,
306
- parallel_config,
307
- scheduler_config,
308
- device_config,
309
- local_rank,
310
- rank,
311
- distributed_init_method,
312
- lora_config=lora_config,
313
- kv_cache_dtype=kv_cache_dtype,
314
- ))
315
-
316
- driver_rank = 0
317
- driver_local_rank = node_workers[driver_node_id].index(driver_rank)
318
- self.driver_worker = Worker(
319
- self.model_config,
320
- self.parallel_config,
321
- self.scheduler_config,
322
- self.device_config,
323
- driver_local_rank,
324
- driver_rank,
325
- distributed_init_method,
326
- lora_config=self.lora_config,
327
- kv_cache_dtype=kv_cache_dtype,
328
- is_driver_worker=True,
329
- )
330
-
331
- # don't use cupy for eager mode
332
- self._run_workers("init_model",
333
- cupy_port=get_open_port()
334
- if not model_config.enforce_eager else None)
335
- self._run_workers(
336
- "load_model",
337
- max_concurrent_workers=self.parallel_config.
338
- max_parallel_loading_workers,
339
- )
340
-
165
def _verify_args(self) -> None:
166
self.model_config.verify_with_parallel_config(self.parallel_config)
167
self.cache_config.verify_with_parallel_config(self.parallel_config)
@@ -346,81 +170,6 @@ class LLMEngine:
170
self.lora_config.verify_with_scheduler_config(
171
self.scheduler_config)
172
349
- def _init_cache(self) -> None:
350
- """Profiles the memory usage and initializes the KV cache.
351
-
352
- The engine will first conduct a profiling of the existing memory usage.
353
- Then, it calculate the maximum possible number of GPU and CPU blocks
354
- that can be allocated with the remaining free memory.
355
- More details can be found in the
356
- :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
357
- from class :class:`~vllm.worker.Worker`.
358
-
359
- Afterwards, as there may be multiple workers,
360
- we take the minimum number of blocks across all workers
361
- to ensure this can be applied to all of them.
362
-
363
- Finally, the engine will initialize the KV cache
364
- with the calculated number of blocks.
365
-
366
- .. tip::
367
- You may limit the usage of GPU memory
368
- by adjusting the `gpu_memory_utilization` parameters.
369
- """
370
- # Get the maximum number of blocks that can be allocated on GPU and CPU.
371
- num_blocks = self._run_workers(
372
- "profile_num_available_blocks",
373
- block_size=self.cache_config.block_size,
374
- gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
375
- cpu_swap_space=self.cache_config.swap_space_bytes,
376
- cache_dtype=self.cache_config.cache_dtype,
377
- )
378
-
379
- # Since we use a shared centralized controller, we take the minimum
380
- # number of blocks across all workers to make sure all the memory
381
- # operators can be applied to all workers.
382
- num_gpu_blocks = min(b[0] for b in num_blocks)
383
- num_cpu_blocks = min(b[1] for b in num_blocks)
384
- # FIXME(woosuk): Change to debug log.
385
- logger.info(f"# GPU blocks: {num_gpu_blocks}, "
386
- f"# CPU blocks: {num_cpu_blocks}")
387
-
388
- if num_gpu_blocks <= 0:
389
- raise ValueError("No available memory for the cache blocks. "
390
- "Try increasing `gpu_memory_utilization` when "
391
- "initializing the engine.")
392
- max_seq_len = self.cache_config.block_size * num_gpu_blocks
393
- if self.model_config.max_model_len > max_seq_len:
394
- raise ValueError(
395
- f"The model's max seq len ({self.model_config.max_model_len}) "
396
- "is larger than the maximum number of tokens that can be "
397
- f"stored in KV cache ({max_seq_len}). Try increasing "
398
- "`gpu_memory_utilization` or decreasing `max_model_len` when "
399
- "initializing the engine.")
400
-
401
- self.cache_config.num_gpu_blocks = num_gpu_blocks
402
- self.cache_config.num_cpu_blocks = num_cpu_blocks
403
-
404
- # Initialize the cache.
405
- self._run_workers("init_cache_engine", cache_config=self.cache_config)
406
- # Warm up the model. This includes capturing the model into CUDA graph
407
- # if enforce_eager is False.
408
- self._run_workers("warm_up_model")
409
-
410
- @classmethod
411
- def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
412
- """Creates an LLM engine from the engine arguments."""
413
- # Create the engine configs.
414
- engine_configs = engine_args.create_engine_configs()
415
- parallel_config = engine_configs[2]
416
- # Initialize the cluster.
417
- placement_group = initialize_cluster(parallel_config)
418
- # Create the LLM engine.
419
- engine = cls(*engine_configs,
420
- placement_group,
421
- log_stats=not engine_args.disable_log_stats)
422
- return engine
423
-
173
def encode_request(
174
self,
175
request_id: str, # pylint: disable=unused-argument
@@ -826,7 +575,7 @@ class LLMEngine:
575
- A Sequence Group (SG) refer to a group of sequences
576
that are generated from the same prompt.
577
829
- - Step 2: Calls the workers to execute the model.
578
+ - Step 2: Calls the distributed executor to execute the model.
579
- Step 3: Processes the model output. This mainly includes:
580
581
- Decodes the relevant outputs.
@@ -862,19 +611,10 @@ class LLMEngine:
611
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
612
613
if not scheduler_outputs.is_empty():
865
- # Execute the model.
866
- all_outputs = self._run_workers(
867
- "execute_model",
868
- driver_kwargs={
869
- "seq_group_metadata_list": seq_group_metadata_list,
870
- "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
871
- "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
872
- "blocks_to_copy": scheduler_outputs.blocks_to_copy,
873
- },
874
- use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
875
-
876
- # Only the driver worker returns the sampling results.
877
- output = all_outputs[0]
614
+ output = self.model_executor.execute_model(
615
+ seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
616
+ scheduler_outputs.blocks_to_swap_out,
617
+ scheduler_outputs.blocks_to_copy)
618
else:
619
output = []
620
@@ -1043,111 +783,13 @@ class LLMEngine:
783
seq.output_text = seq.output_text[:-len(stop_string)]
784
785
def add_lora(self, lora_request: LoRARequest) -> bool:
1046
- assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
1047
- return self._run_workers(
1048
- "add_lora",
1049
- lora_request=lora_request,
1050
- )
786
+ return self.model_executor.add_lora(lora_request)
787
788
def remove_lora(self, lora_id: int) -> bool:
1053
- assert lora_id > 0, "lora_id must be greater than 0."
1054
- return self._run_workers(
1055
- "remove_lora",
1056
- lora_id=lora_id,
1057
- )
789
+ return self.model_executor.remove_lora(lora_id)
790
791
def list_loras(self) -> List[int]:
1060
- return self._run_workers("list_loras")
1061
-
1062
- def _run_workers(
1063
- self,
1064
- method: str,
1065
- *args,
1066
- driver_args: Optional[List[Any]] = None,
1067
- driver_kwargs: Optional[Dict[str, Any]] = None,
1068
- max_concurrent_workers: Optional[int] = None,
1069
- use_ray_compiled_dag: bool = False,
1070
- **kwargs,
1071
- ) -> Any:
1072
- """Runs the given method on all workers."""
1073
-
1074
- if max_concurrent_workers:
1075
- raise NotImplementedError(
1076
- "max_concurrent_workers is not supported yet.")
1077
-
1078
- if use_ray_compiled_dag:
1079
- # Right now, compiled DAG can only accept a single
1080
- # input. TODO(sang): Fix it.
1081
- output_channels = self.forward_dag.execute(1)
1082
- else:
1083
- # Start the ray workers first.
1084
- ray_worker_outputs = [
1085
- worker.execute_method.remote(method, *args, **kwargs)
1086
- for worker in self.workers
1087
- ]
1088
-
1089
- if driver_args is None:
1090
- driver_args = args
1091
- if driver_kwargs is None:
1092
- driver_kwargs = kwargs
1093
-
1094
- # Start the driver worker after all the ray workers.
1095
- driver_worker_output = getattr(self.driver_worker,
1096
- method)(*driver_args, **driver_kwargs)
1097
-
1098
- # Get the results of the ray workers.
1099
- if self.workers:
1100
- if use_ray_compiled_dag:
1101
- try:
1102
- ray_worker_outputs = [
1103
- pickle.loads(chan.begin_read())
1104
- for chan in output_channels
1105
- ]
1106
- finally:
1107
- # Has to call end_read in order to reuse the DAG.
1108
- for chan in output_channels:
1109
- chan.end_read()
1110
- else:
1111
- ray_worker_outputs = ray.get(ray_worker_outputs)
1112
-
1113
- return [driver_worker_output] + ray_worker_outputs
1114
-
1115
- def _compiled_ray_dag(self):
1116
- import pkg_resources
1117
- required_version = "2.9"
1118
- current_version = pkg_resources.get_distribution("ray").version
1119
- if current_version < required_version:
1120
- raise ValueError(f"Ray version {required_version} or greater is "
1121
- f"required, but found {current_version}")
1122
-
1123
- from ray.dag import MultiOutputNode, InputNode
1124
- assert self.parallel_config.worker_use_ray
1125
-
1126
- # Right now, compiled DAG requires at least 1 arg. We send
1127
- # a dummy value for now. It will be fixed soon.
1128
- with InputNode() as input_data:
1129
- forward_dag = MultiOutputNode([
1130
- worker.execute_model_compiled_dag_remote.bind(input_data)
1131
- for worker in self.workers
1132
- ])
1133
- return forward_dag.experimental_compile()
792
+ return self.model_executor.list_loras()
793
794
def check_health(self) -> None:
1136
- """Raises an error if engine is unhealthy."""
1137
- self._check_if_any_actor_is_dead()
1138
-
1139
- def _check_if_any_actor_is_dead(self):
1140
- if not self.parallel_config.worker_use_ray:
1141
- return
1142
-
1143
- if not self.workers:
1144
- return
1145
-
1146
- dead_actors = []
1147
- for actor in self.workers:
1148
- actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
1149
- if actor_state["State"] == "DEAD":
1150
- dead_actors.append(actor)
1151
- if dead_actors:
1152
- raise RuntimeError("At least one Worker is dead. "
1153
- f"Dead Workers: {dead_actors}. ")
795
+ self.model_executor.check_health()
vllm/engine/ray_utils.py
+26
-32
@@ -1,6 +1,6 @@
1
import pickle
2
3
-from typing import Optional, List, Tuple, TYPE_CHECKING
3
+from typing import Optional, List, Tuple
4
5
from vllm.config import ParallelConfig
6
from vllm.logger import init_logger
@@ -65,45 +65,38 @@ except ImportError as e:
65
ray = None
66
RayWorkerVllm = None
67
68
-if TYPE_CHECKING:
69
- from ray.util.placement_group import PlacementGroup
68
71
-
72
-def initialize_cluster(
69
+def initialize_ray_cluster(
70
parallel_config: ParallelConfig,
74
- engine_use_ray: bool = False,
71
ray_address: Optional[str] = None,
76
-) -> Optional["PlacementGroup"]:
77
- """Initialize the distributed cluster probably with Ray.
72
+):
73
+ """Initialize the distributed cluster with Ray.
74
+
75
+ it will connect to the Ray cluster and create a placement group
76
+ for the workers, which includes the specification of the resources
77
+ for each distributed worker.
78
79
Args:
80
parallel_config: The configurations for parallel execution.
81
- engine_use_ray: Whether to use Ray for async engine.
81
ray_address: The address of the Ray cluster. If None, uses
82
the default Ray cluster address.
84
-
85
- Returns:
86
- An optional `PlacementGroup`. It includes the specification
87
- of the resources for each distributed worker. None if Ray is
88
- not used.
83
"""
90
- if parallel_config.worker_use_ray or engine_use_ray:
91
- if ray is None:
92
- raise ImportError(
93
- "Ray is not installed. Please install Ray to use distributed "
94
- "serving.")
95
- # Connect to a ray cluster.
96
- if is_hip():
97
- ray.init(address=ray_address,
98
- ignore_reinit_error=True,
99
- num_gpus=parallel_config.world_size)
100
- else:
101
- ray.init(address=ray_address, ignore_reinit_error=True)
102
-
103
- if not parallel_config.worker_use_ray:
104
- assert parallel_config.world_size == 1, (
105
- "Ray is required if parallel_config.world_size > 1.")
106
- return None
84
+ if ray is None:
85
+ raise ImportError(
86
+ "Ray is not installed. Please install Ray to use distributed "
87
+ "serving.")
88
+
89
+ # Connect to a ray cluster.
90
+ if is_hip():
91
+ ray.init(address=ray_address,
92
+ ignore_reinit_error=True,
93
+ num_gpus=parallel_config.world_size)
94
+ else:
95
+ ray.init(address=ray_address, ignore_reinit_error=True)
96
+
97
+ if parallel_config.placement_group:
98
+ # Placement group is already set.
99
+ return
100
101
# Create placement group for worker processes
102
current_placement_group = ray.util.get_current_placement_group()
@@ -138,4 +131,5 @@ def initialize_cluster(
131
# if they cannot be provisioned.
132
ray.get(current_placement_group.ready(), timeout=1800)
133
141
- return current_placement_group
134
+ # Set the placement group in the parallel config
135
+ parallel_config.placement_group = current_placement_group
vllm/executor/__init__.py
vllm/executor/executor_base.py
new
+75
@@ -0,0 +1,75 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Dict, List, Optional
3
+
4
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5
+ ParallelConfig, SchedulerConfig, LoRAConfig)
6
+from vllm.lora.request import LoRARequest
7
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
+
9
+
10
+class ExecutorBase(ABC):
11
+ """Base class for all executors.
12
+
13
+ An executor is responsible for executing the model on a specific device
14
+ type (e.g., CPU, GPU, Neuron, etc.). Or it can be a distributed executor
15
+ that can execute the model on multiple devices.
16
+ """
17
+
18
+ @abstractmethod
19
+ def __init__(
20
+ self,
21
+ model_config: ModelConfig,
22
+ cache_config: CacheConfig,
23
+ parallel_config: ParallelConfig,
24
+ scheduler_config: SchedulerConfig,
25
+ device_config: DeviceConfig,
26
+ lora_config: Optional[LoRAConfig],
27
+ ) -> None:
28
+ raise NotImplementedError
29
+
30
+ @abstractmethod
31
+ def execute_model(self,
32
+ seq_group_metadata_list: List[SequenceGroupMetadata],
33
+ blocks_to_swap_in: Dict[int, int],
34
+ blocks_to_swap_out: Dict[int, int],
35
+ blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
36
+ """Executes one model step on the given sequences."""
37
+ raise NotImplementedError
38
+
39
+ @abstractmethod
40
+ def add_lora(self, lora_request: LoRARequest) -> bool:
41
+ raise NotImplementedError
42
+
43
+ @abstractmethod
44
+ def remove_lora(self, lora_id: int) -> bool:
45
+ raise NotImplementedError
46
+
47
+ @abstractmethod
48
+ def list_loras(self) -> List[int]:
49
+ raise NotImplementedError
50
+
51
+ @abstractmethod
52
+ def check_health(self) -> None:
53
+ """Checks if the executor is healthy. If not, it should raise an
54
+ exception."""
55
+ raise NotImplementedError
56
+
57
+
58
+class ExecutorAsyncBase(ExecutorBase):
59
+
60
+ @abstractmethod
61
+ async def execute_model_async(
62
+ self,
63
+ seq_group_metadata_list: List[SequenceGroupMetadata],
64
+ blocks_to_swap_in: Dict[int, int],
65
+ blocks_to_swap_out: Dict[int, int],
66
+ blocks_to_copy: Dict[int, List[int]],
67
+ ) -> SamplerOutput:
68
+ """Executes one model step on the given sequences."""
69
+ raise NotImplementedError
70
+
71
+ @abstractmethod
72
+ async def check_health_async(self) -> None:
73
+ """Checks if the executor is healthy. If not, it should raise an
74
+ exception."""
75
+ raise NotImplementedError
vllm/executor/gpu_executor.py
new
+163
@@ -0,0 +1,163 @@
1
+import importlib
2
+from typing import Dict, List, Optional
3
+
4
+from vllm.lora.request import LoRARequest
5
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
6
+ ParallelConfig, SchedulerConfig, LoRAConfig)
7
+from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8
+from vllm.executor.utils import check_block_size_valid
9
+from vllm.logger import init_logger
10
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
11
+from vllm.utils import (get_ip, get_open_port, get_distributed_init_method,
12
+ make_async)
13
+
14
+logger = init_logger(__name__)
15
+
16
+# A map between the device type (in device config) to its worker module.
17
+DEVICE_TO_WORKER_MODULE_MAP = {
18
+ "cuda": "vllm.worker.worker",
19
+ "neuron": "vllm.worker.neuron_worker",
20
+}
21
+
22
+
23
+class GPUExecutor(ExecutorBase):
24
+
25
+ def __init__(
26
+ self,
27
+ model_config: ModelConfig,
28
+ cache_config: CacheConfig,
29
+ parallel_config: ParallelConfig,
30
+ scheduler_config: SchedulerConfig,
31
+ device_config: DeviceConfig,
32
+ lora_config: Optional[LoRAConfig],
33
+ ) -> None:
34
+ self.model_config = model_config
35
+ self.cache_config = cache_config
36
+ self.lora_config = lora_config
37
+ self.parallel_config = parallel_config
38
+ self.scheduler_config = scheduler_config
39
+ self.device_config = device_config
40
+
41
+ # Instantiate the worker and load the model to GPU.
42
+ self._init_worker()
43
+
44
+ # Profile the memory usage and initialize the cache.
45
+ self._init_cache()
46
+
47
+ def _dispatch_worker(self):
48
+ worker_module = DEVICE_TO_WORKER_MODULE_MAP[
49
+ self.device_config.device_type]
50
+ imported_worker = importlib.import_module(worker_module)
51
+ Worker = imported_worker.Worker
52
+ return Worker
53
+
54
+ def _init_worker(self):
55
+ # Lazy import the Worker to avoid importing torch.cuda/xformers
56
+ # before CUDA_VISIBLE_DEVICES is set in the Worker
57
+ Worker = self._dispatch_worker()
58
+
59
+ assert self.parallel_config.world_size == 1, (
60
+ "GPUExecutor only supports single GPU.")
61
+
62
+ distributed_init_method = get_distributed_init_method(
63
+ get_ip(), get_open_port())
64
+ self.driver_worker = Worker(
65
+ self.model_config,
66
+ self.parallel_config,
67
+ self.scheduler_config,
68
+ self.device_config,
69
+ local_rank=0,
70
+ rank=0,
71
+ distributed_init_method=distributed_init_method,
72
+ lora_config=self.lora_config,
73
+ kv_cache_dtype=self.cache_config.cache_dtype,
74
+ is_driver_worker=True,
75
+ )
76
+ self.driver_worker.init_model()
77
+ self.driver_worker.load_model()
78
+
79
+ def _init_cache(self) -> None:
80
+ """Profiles the memory usage and initializes the KV cache.
81
+
82
+ The engine first profiles the existing memory usage.
83
+ Then, it allocates the remaining memory for KV blocks.
84
+
85
+ .. tip::
86
+ You may limit the usage of GPU memory
87
+ by adjusting the `gpu_memory_utilization` parameter.
88
+ """
89
+ # Get the maximum number of blocks that can be allocated on GPU and CPU.
90
+ num_gpu_blocks, num_cpu_blocks = (
91
+ self.driver_worker.profile_num_available_blocks(
92
+ block_size=self.cache_config.block_size,
93
+ gpu_memory_utilization=self.cache_config.
94
+ gpu_memory_utilization,
95
+ cpu_swap_space=self.cache_config.swap_space_bytes,
96
+ cache_dtype=self.cache_config.cache_dtype,
97
+ ))
98
+
99
+ logger.info(f"# GPU blocks: {num_gpu_blocks}, "
100
+ f"# CPU blocks: {num_cpu_blocks}")
101
+
102
+ check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
103
+ self.model_config.max_model_len)
104
+
105
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
106
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
107
+
108
+ # Initialize the cache.
109
+ self.driver_worker.init_cache_engine(cache_config=self.cache_config)
110
+ # Warm up the model. This includes capturing the model into CUDA graph
111
+ # if enforce_eager is False.
112
+ self.driver_worker.warm_up_model()
113
+
114
+ def execute_model(self,
115
+ seq_group_metadata_list: List[SequenceGroupMetadata],
116
+ blocks_to_swap_in: Dict[int, int],
117
+ blocks_to_swap_out: Dict[int, int],
118
+ blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
119
+ output = self.driver_worker.execute_model(
120
+ seq_group_metadata_list=seq_group_metadata_list,
121
+ blocks_to_swap_in=blocks_to_swap_in,
122
+ blocks_to_swap_out=blocks_to_swap_out,
123
+ blocks_to_copy=blocks_to_copy,
124
+ )
125
+ return output
126
+
127
+ def add_lora(self, lora_request: LoRARequest) -> bool:
128
+ assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
129
+ return self.driver_worker.add_lora(lora_request)
130
+
131
+ def remove_lora(self, lora_id: int) -> bool:
132
+ assert lora_id > 0, "lora_id must be greater than 0."
133
+ return self.driver_worker.remove_lora(lora_id)
134
+
135
+ def list_loras(self) -> List[int]:
136
+ return self.driver_worker.list_loras()
137
+
138
+ def check_health(self) -> None:
139
+ # GPUExecutor will always be healthy as long as
140
+ # it's running.
141
+ return
142
+
143
+
144
+class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
145
+
146
+ async def execute_model_async(
147
+ self,
148
+ seq_group_metadata_list: List[SequenceGroupMetadata],
149
+ blocks_to_swap_in: Dict[int, int],
150
+ blocks_to_swap_out: Dict[int, int],
151
+ blocks_to_copy: Dict[int, List[int]],
152
+ ) -> SamplerOutput:
153
+ output = await make_async(self.driver_worker.execute_model)(
154
+ seq_group_metadata_list=seq_group_metadata_list,
155
+ blocks_to_swap_in=blocks_to_swap_in,
156
+ blocks_to_swap_out=blocks_to_swap_out,
157
+ blocks_to_copy=blocks_to_copy)
158
+ return output
159
+
160
+ async def check_health_async(self) -> None:
161
+ # GPUExecutor will always be healthy as long as
162
+ # it's running.
163
+ return
vllm/executor/ray_gpu_executor.py
new
+442
@@ -0,0 +1,442 @@
1
+import asyncio
2
+import copy
3
+from collections import defaultdict
4
+import os
5
+import pickle
6
+import importlib
7
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
8
+
9
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10
+ ParallelConfig, SchedulerConfig, LoRAConfig)
11
+from vllm.engine.ray_utils import RayWorkerVllm, ray
12
+from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13
+from vllm.executor.utils import check_block_size_valid
14
+from vllm.logger import init_logger
15
+from vllm.lora.request import LoRARequest
16
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17
+from vllm.utils import (set_cuda_visible_devices, get_ip, get_open_port,
18
+ get_distributed_init_method, make_async)
19
+
20
+if ray is not None:
21
+ from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
22
+
23
+if TYPE_CHECKING:
24
+ from ray.util.placement_group import PlacementGroup
25
+
26
+logger = init_logger(__name__)
27
+
28
+# A map between the device type (in device config) to its worker module.
29
+DEVICE_TO_WORKER_MODULE_MAP = {
30
+ "cuda": "vllm.worker.worker",
31
+ "neuron": "vllm.worker.neuron_worker",
32
+}
33
+
34
+# If the env var is set, it uses the Ray's compiled DAG API
35
+# which optimizes the control plane overhead.
36
+# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
37
+USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
38
+
39
+
40
+class RayGPUExecutor(ExecutorBase):
41
+
42
+ def __init__(
43
+ self,
44
+ model_config: ModelConfig,
45
+ cache_config: CacheConfig,
46
+ parallel_config: ParallelConfig,
47
+ scheduler_config: SchedulerConfig,
48
+ device_config: DeviceConfig,
49
+ lora_config: Optional[LoRAConfig],
50
+ ) -> None:
51
+ self.model_config = model_config
52
+ self.cache_config = cache_config
53
+ self.lora_config = lora_config
54
+ self.parallel_config = parallel_config
55
+ self.scheduler_config = scheduler_config
56
+ self.device_config = device_config
57
+
58
+ assert self.parallel_config.worker_use_ray
59
+ placement_group = self.parallel_config.placement_group
60
+
61
+ # Disable Ray usage stats collection.
62
+ ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
63
+ if ray_usage != "1":
64
+ os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
65
+
66
+ # Create the parallel GPU workers.
67
+ self._init_workers_ray(placement_group)
68
+
69
+ # Profile the memory usage and initialize the cache.
70
+ self._init_cache()
71
+
72
+ self.forward_dag = None
73
+ if USE_RAY_COMPILED_DAG:
74
+ self.forward_dag = self._compiled_ray_dag()
75
+
76
+ def _dispatch_worker(self):
77
+ worker_module = DEVICE_TO_WORKER_MODULE_MAP[
78
+ self.device_config.device_type]
79
+ imported_worker = importlib.import_module(worker_module)
80
+ Worker = imported_worker.Worker
81
+ return Worker
82
+
83
+ def _init_workers_ray(self, placement_group: "PlacementGroup",
84
+ **ray_remote_kwargs):
85
+ if self.parallel_config.tensor_parallel_size == 1:
86
+ # For single GPU case, we use a ray worker with constrained memory.
87
+ num_gpus = self.cache_config.gpu_memory_utilization
88
+ else:
89
+ # Otherwise, the ray workers are allocated with a full GPU.
90
+ num_gpus = 1
91
+
92
+ # The driver dummy worker does not actually use any resources.
93
+ # It holds the resource for the driver worker.
94
+ self.driver_dummy_worker: RayWorkerVllm = None
95
+ # The remaining workers are the actual ray actors.
96
+ self.workers: List[RayWorkerVllm] = []
97
+
98
+ # Create the workers.
99
+ driver_ip = get_ip()
100
+ for bundle_id, bundle in enumerate(placement_group.bundle_specs):
101
+ if not bundle.get("GPU", 0):
102
+ continue
103
+ scheduling_strategy = PlacementGroupSchedulingStrategy(
104
+ placement_group=placement_group,
105
+ placement_group_capture_child_tasks=True,
106
+ placement_group_bundle_index=bundle_id,
107
+ )
108
+ worker = ray.remote(
109
+ num_cpus=0,
110
+ num_gpus=num_gpus,
111
+ scheduling_strategy=scheduling_strategy,
112
+ **ray_remote_kwargs,
113
+ )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
114
+
115
+ worker_ip = ray.get(worker.get_node_ip.remote())
116
+ if worker_ip == driver_ip and self.driver_dummy_worker is None:
117
+ # If the worker is on the same node as the driver, we use it
118
+ # as the resource holder for the driver process.
119
+ self.driver_dummy_worker = worker
120
+ else:
121
+ # Else, added to the list of workers.
122
+ self.workers.append(worker)
123
+
124
+ if self.driver_dummy_worker is None:
125
+ raise ValueError(
126
+ "Ray does not allocate any GPUs on the driver node. Consider "
127
+ "adjusting the Ray placement group or running the driver on a "
128
+ "GPU node.")
129
+
130
+ # Get the set of GPU IDs used on each node.
131
+ driver_node_id, driver_gpu_ids = ray.get(
132
+ self.driver_dummy_worker.get_node_and_gpu_ids.remote())
133
+ worker_node_and_gpu_ids = ray.get(
134
+ [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
135
+
136
+ node_workers = defaultdict(list)
137
+ node_gpus = defaultdict(list)
138
+
139
+ node_workers[driver_node_id].append(0)
140
+ node_gpus[driver_node_id].extend(driver_gpu_ids)
141
+ for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
142
+ start=1):
143
+ node_workers[node_id].append(i)
144
+ node_gpus[node_id].extend(gpu_ids)
145
+ for node_id, gpu_ids in node_gpus.items():
146
+ node_gpus[node_id] = sorted(gpu_ids)
147
+
148
+ # Set CUDA_VISIBLE_DEVICES for the driver and workers.
149
+ set_cuda_visible_devices(node_gpus[driver_node_id])
150
+ for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
151
+ worker.set_cuda_visible_devices.remote(node_gpus[node_id])
152
+
153
+ distributed_init_method = get_distributed_init_method(
154
+ driver_ip, get_open_port())
155
+
156
+ # Lazy import the Worker to avoid importing torch.cuda/xformers
157
+ # before CUDA_VISIBLE_DEVICES is set in the Worker
158
+ Worker = self._dispatch_worker()
159
+
160
+ model_config = copy.deepcopy(self.model_config)
161
+ parallel_config = copy.deepcopy(self.parallel_config)
162
+ scheduler_config = copy.deepcopy(self.scheduler_config)
163
+ device_config = copy.deepcopy(self.device_config)
164
+ lora_config = copy.deepcopy(self.lora_config)
165
+ kv_cache_dtype = self.cache_config.cache_dtype
166
+
167
+ # Initialize the actual workers with the Worker class.
168
+ for rank, (worker, (node_id, _)) in enumerate(
169
+ zip(self.workers, worker_node_and_gpu_ids),
170
+ start=1,
171
+ ):
172
+ local_rank = node_workers[node_id].index(rank)
173
+ worker.init_worker.remote(
174
+ lambda rank=rank, local_rank=local_rank: Worker(
175
+ model_config,
176
+ parallel_config,
177
+ scheduler_config,
178
+ device_config,
179
+ local_rank,
180
+ rank,
181
+ distributed_init_method,
182
+ lora_config=lora_config,
183
+ kv_cache_dtype=kv_cache_dtype,
184
+ ))
185
+
186
+ # Initialize the driver worker with the Worker class.
187
+ driver_rank = 0
188
+ driver_local_rank = node_workers[driver_node_id].index(driver_rank)
189
+ self.driver_worker = Worker(
190
+ self.model_config,
191
+ self.parallel_config,
192
+ self.scheduler_config,
193
+ self.device_config,
194
+ driver_local_rank,
195
+ driver_rank,
196
+ distributed_init_method,
197
+ lora_config=self.lora_config,
198
+ kv_cache_dtype=kv_cache_dtype,
199
+ is_driver_worker=True,
200
+ )
201
+
202
+ # FIXME(woosuk): We are not properly initializing cupy NCCL when
203
+ # we have multiple nodes.
204
+ self._run_workers("init_model",
205
+ cupy_port=get_open_port()
206
+ if not model_config.enforce_eager else None)
207
+ self._run_workers(
208
+ "load_model",
209
+ max_concurrent_workers=self.parallel_config.
210
+ max_parallel_loading_workers,
211
+ )
212
+
213
+ def _init_cache(self) -> None:
214
+ """Profiles the memory usage and initializes the KV cache.
215
+
216
+ The engine will first conduct a profiling of the existing memory usage.
217
+ Then, it calculate the maximum possible number of GPU and CPU blocks
218
+ that can be allocated with the remaining free memory.
219
+ More details can be found in the
220
+ :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
221
+ from class :class:`~vllm.worker.Worker`.
222
+
223
+ Afterwards, as there may be multiple workers,
224
+ we take the minimum number of blocks across all workers
225
+ to ensure this can be applied to all of them.
226
+
227
+ Finally, the engine will initialize the KV cache
228
+ with the calculated number of blocks.
229
+
230
+ .. tip::
231
+ You may limit the usage of GPU memory
232
+ by adjusting the `gpu_memory_utilization` parameter.
233
+ """
234
+ # Get the maximum number of blocks that can be allocated on GPU and CPU.
235
+ num_blocks = self._run_workers(
236
+ "profile_num_available_blocks",
237
+ block_size=self.cache_config.block_size,
238
+ gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
239
+ cpu_swap_space=self.cache_config.swap_space_bytes,
240
+ cache_dtype=self.cache_config.cache_dtype,
241
+ )
242
+
243
+ # Since we use a shared centralized controller, we take the minimum
244
+ # number of blocks across all workers to make sure all the memory
245
+ # operators can be applied to all workers.
246
+ num_gpu_blocks = min(b[0] for b in num_blocks)
247
+ num_cpu_blocks = min(b[1] for b in num_blocks)
248
+ logger.info(f"# GPU blocks: {num_gpu_blocks}, "
249
+ f"# CPU blocks: {num_cpu_blocks}")
250
+
251
+ check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
252
+ self.model_config.max_model_len)
253
+
254
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
255
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
256
+
257
+ # Initialize the cache.
258
+ self._run_workers("init_cache_engine", cache_config=self.cache_config)
259
+ # Warm up the model. This includes capturing the model into CUDA graph
260
+ # if enforce_eager is False.
261
+ self._run_workers("warm_up_model")
262
+
263
+ def execute_model(self,
264
+ seq_group_metadata_list: List[SequenceGroupMetadata],
265
+ blocks_to_swap_in: Dict[int, int],
266
+ blocks_to_swap_out: Dict[int, int],
267
+ blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
268
+ all_outputs = self._run_workers(
269
+ "execute_model",
270
+ driver_kwargs={
271
+ "seq_group_metadata_list": seq_group_metadata_list,
272
+ "blocks_to_swap_in": blocks_to_swap_in,
273
+ "blocks_to_swap_out": blocks_to_swap_out,
274
+ "blocks_to_copy": blocks_to_copy,
275
+ },
276
+ use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
277
+
278
+ # Only the driver worker returns the sampling results.
279
+ output = all_outputs[0]
280
+ return output
281
+
282
+ def add_lora(self, lora_request: LoRARequest) -> bool:
283
+ assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
284
+ return self._run_workers(
285
+ "add_lora",
286
+ lora_request=lora_request,
287
+ )
288
+
289
+ def remove_lora(self, lora_id: int) -> bool:
290
+ assert lora_id > 0, "lora_id must be greater than 0."
291
+ return self._run_workers(
292
+ "remove_lora",
293
+ lora_id=lora_id,
294
+ )
295
+
296
+ def list_loras(self) -> List[int]:
297
+ return self._run_workers("list_loras")
298
+
299
+ def _run_workers(
300
+ self,
301
+ method: str,
302
+ *args,
303
+ driver_args: Optional[List[Any]] = None,
304
+ driver_kwargs: Optional[Dict[str, Any]] = None,
305
+ max_concurrent_workers: Optional[int] = None,
306
+ use_ray_compiled_dag: bool = False,
307
+ **kwargs,
308
+ ) -> Any:
309
+ """Runs the given method on all workers."""
310
+
311
+ if max_concurrent_workers:
312
+ raise NotImplementedError(
313
+ "max_concurrent_workers is not supported yet.")
314
+
315
+ if use_ray_compiled_dag:
316
+ # Right now, compiled DAG can only accept a single
317
+ # input. TODO(sang): Fix it.
318
+ output_channels = self.forward_dag.execute(1)
319
+ else:
320
+ # Start the ray workers first.
321
+ ray_worker_outputs = [
322
+ worker.execute_method.remote(method, *args, **kwargs)
323
+ for worker in self.workers
324
+ ]
325
+
326
+ if driver_args is None:
327
+ driver_args = args
328
+ if driver_kwargs is None:
329
+ driver_kwargs = kwargs
330
+
331
+ # Start the driver worker after all the ray workers.
332
+ driver_worker_output = getattr(self.driver_worker,
333
+ method)(*driver_args, **driver_kwargs)
334
+
335
+ # Get the results of the ray workers.
336
+ if self.workers:
337
+ if use_ray_compiled_dag:
338
+ try:
339
+ ray_worker_outputs = [
340
+ pickle.loads(chan.begin_read())
341
+ for chan in output_channels
342
+ ]
343
+ finally:
344
+ # Has to call end_read in order to reuse the DAG.
345
+ for chan in output_channels:
346
+ chan.end_read()
347
+ else:
348
+ ray_worker_outputs = ray.get(ray_worker_outputs)
349
+
350
+ return [driver_worker_output] + ray_worker_outputs
351
+
352
+ def _compiled_ray_dag(self):
353
+ import pkg_resources
354
+ required_version = "2.9"
355
+ current_version = pkg_resources.get_distribution("ray").version
356
+ if current_version < required_version:
357
+ raise ValueError(f"Ray version {required_version} or greater is "
358
+ f"required, but found {current_version}")
359
+
360
+ from ray.dag import MultiOutputNode, InputNode
361
+ assert self.parallel_config.worker_use_ray
362
+
363
+ # Right now, compiled DAG requires at least 1 arg. We send
364
+ # a dummy value for now. It will be fixed soon.
365
+ with InputNode() as input_data:
366
+ forward_dag = MultiOutputNode([
367
+ worker.execute_model_compiled_dag_remote.bind(input_data)
368
+ for worker in self.workers
369
+ ])
370
+ return forward_dag.experimental_compile()
371
+
372
+ def check_health(self) -> None:
373
+ """Raises an error if engine is unhealthy."""
374
+ self._check_if_any_actor_is_dead()
375
+
376
+ def _check_if_any_actor_is_dead(self):
377
+ if not self.workers:
378
+ return
379
+
380
+ dead_actors = []
381
+ for actor in self.workers:
382
+ actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
383
+ if actor_state["State"] == "DEAD":
384
+ dead_actors.append(actor)
385
+ if dead_actors:
386
+ raise RuntimeError("At least one Worker is dead. "
387
+ f"Dead Workers: {dead_actors}. ")
388
+
389
+
390
+class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
391
+
392
+ async def _run_workers_async(
393
+ self,
394
+ method: str,
395
+ *args,
396
+ driver_args: Optional[List[Any]] = None,
397
+ driver_kwargs: Optional[Dict[str, Any]] = None,
398
+ **kwargs,
399
+ ) -> Any:
400
+ """Runs the given method on all workers."""
401
+ coros = []
402
+
403
+ if driver_args is None:
404
+ driver_args = args
405
+ if driver_kwargs is None:
406
+ driver_kwargs = kwargs
407
+
408
+ # Run the driver worker asynchronously.
409
+ driver_executor = make_async(getattr(self.driver_worker, method))
410
+ coros.append(driver_executor(*driver_args, **driver_kwargs))
411
+
412
+ # Run the ray workers asynchronously.
413
+ for worker in self.workers:
414
+ coros.append(worker.execute_method.remote(method, *args, **kwargs))
415
+
416
+ all_outputs = await asyncio.gather(*coros)
417
+ return all_outputs
418
+
419
+ async def execute_model_async(
420
+ self,
421
+ seq_group_metadata_list: List[SequenceGroupMetadata],
422
+ blocks_to_swap_in: Dict[int, int],
423
+ blocks_to_swap_out: Dict[int, int],
424
+ blocks_to_copy: Dict[int, List[int]],
425
+ ) -> SamplerOutput:
426
+ all_outputs = await self._run_workers_async(
427
+ "execute_model",
428
+ driver_kwargs={
429
+ "seq_group_metadata_list": seq_group_metadata_list,
430
+ "blocks_to_swap_in": blocks_to_swap_in,
431
+ "blocks_to_swap_out": blocks_to_swap_out,
432
+ "blocks_to_copy": blocks_to_copy,
433
+ },
434
+ use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
435
+
436
+ # Only the driver worker returns the sampling results.
437
+ output = all_outputs[0]
438
+ return output
439
+
440
+ async def check_health_async(self) -> None:
441
+ """Raises an error if engine is unhealthy."""
442
+ self._check_if_any_actor_is_dead()
vllm/executor/utils.py
new
+13
@@ -0,0 +1,13 @@
1
+def check_block_size_valid(num_gpu_blocks, block_size, max_model_len) -> None:
2
+ if num_gpu_blocks <= 0:
3
+ raise ValueError("No available memory for the cache blocks. "
4
+ "Try increasing `gpu_memory_utilization` when "
5
+ "initializing the engine.")
6
+ max_seq_len = block_size * num_gpu_blocks
7
+ if max_model_len > max_seq_len:
8
+ raise ValueError(
9
+ f"The model's max seq len ({max_model_len}) "
10
+ "is larger than the maximum number of tokens that can be "
11
+ f"stored in KV cache ({max_seq_len}). Try increasing "
12
+ "`gpu_memory_utilization` or decreasing `max_model_len` when "
13
+ "initializing the engine.")