@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c922709b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add distributed model executor abstraction (#3191)

Zhuohan Li committed Mar 11, 2024 at 11:03 UTC 4c922709b65ff5c0652ae36b93047016bdeaace8
13 files changed +818 -509
docs/source/dev/engine/llm_engine.rst
+1 -1
@@ -2,5 +2,5 @@ LLMEngine
2 =================================
3
4 .. autoclass:: vllm.engine.llm_engine.LLMEngine
5 - :members: add_request, abort_request, step, _init_cache
5 + :members: add_request, abort_request, step
6 :show-inheritance:
\ No newline at end of file
format.sh
+6 -2
@@ -95,13 +95,17 @@ echo 'vLLM yapf: Done'
95 # echo 'vLLM mypy:'
96 # mypy
97
98 +CODESPELL_EXCLUDES=(
99 + '--skip' '*docs/source/_build/**'
100 +)
101 +
102 # check spelling of specified files
103 spell_check() {
104 codespell "$@"
105 }
106
107 spell_check_all(){
104 - codespell --toml pyproject.toml
108 + codespell --toml pyproject.toml "${CODESPELL_EXCLUDES[@]}"
109 }
110
111 # Spelling check of files that differ from main branch.
@@ -116,7 +120,7 @@ spell_check_changed() {
120
121 if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
122 git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
119 - codespell
123 + codespell "${CODESPELL_EXCLUDES[@]}"
124 fi
125 }
126
tests/lora/conftest.py
+2 -1
@@ -152,4 +152,5 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
152 @pytest.fixture
153 def llama_2_7b_model_extra_embeddings(
154 llama_2_7b_engine_extra_embeddings) -> nn.Module:
155 - yield llama_2_7b_engine_extra_embeddings.driver_worker.model_runner.model
155 + yield (llama_2_7b_engine_extra_embeddings.model_executor.driver_worker.
156 + model_runner.model)
vllm/__init__.py
+2 -2
@@ -3,7 +3,7 @@
3 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4 from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 -from vllm.engine.ray_utils import initialize_cluster
6 +from vllm.engine.ray_utils import initialize_ray_cluster
7 from vllm.entrypoints.llm import LLM
8 from vllm.outputs import CompletionOutput, RequestOutput
9 from vllm.sampling_params import SamplingParams
@@ -19,5 +19,5 @@ __all__ = [
19 "EngineArgs",
20 "AsyncLLMEngine",
21 "AsyncEngineArgs",
22 - "initialize_cluster",
22 + "initialize_ray_cluster",
23 ]
vllm/config.py
+6 -1
@@ -1,4 +1,4 @@
1 -from typing import Optional, Union, ClassVar
1 +from typing import TYPE_CHECKING, Optional, Union, ClassVar
2 from dataclasses import dataclass
3 import os
4 from packaging.version import Version
@@ -10,6 +10,9 @@ from vllm.logger import init_logger
10 from vllm.transformers_utils.config import get_config
11 from vllm.utils import get_cpu_memory, is_hip, is_neuron, get_nvcc_cuda_version
12
13 +if TYPE_CHECKING:
14 + from ray.util.placement_group import PlacementGroup
15 +
16 logger = init_logger(__name__)
17
18 _GB = 1 << 30
@@ -397,6 +400,7 @@ class ParallelConfig:
400 max_parallel_loading_workers: Optional[int] = None,
401 disable_custom_all_reduce: bool = False,
402 ray_workers_use_nsight: bool = False,
403 + placement_group: Optional["PlacementGroup"] = None,
404 ) -> None:
405 self.pipeline_parallel_size = pipeline_parallel_size
406 if is_neuron():
@@ -412,6 +416,7 @@ class ParallelConfig:
416 self.max_parallel_loading_workers = max_parallel_loading_workers
417 self.disable_custom_all_reduce = disable_custom_all_reduce
418 self.ray_workers_use_nsight = ray_workers_use_nsight
419 + self.placement_group = placement_group
420
421 self.world_size = pipeline_parallel_size * self.tensor_parallel_size
422 # Ray worker is not supported for Neuron backend.
vllm/engine/async_llm_engine.py
+38 -68
@@ -2,8 +2,8 @@ import asyncio
2 import os
3 import time
4 from functools import partial
5 -from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
6 - Union, AsyncIterator, Callable)
5 +from typing import (Callable, Dict, Iterable, List, Optional, Set, Tuple, Type,
6 + Union, AsyncIterator)
7
8 from transformers import PreTrainedTokenizer
9
@@ -11,7 +11,7 @@ from vllm.lora.request import LoRARequest
11 from vllm.config import ModelConfig
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 from vllm.engine.llm_engine import LLMEngine
14 -from vllm.engine.ray_utils import initialize_cluster, ray
14 +from vllm.engine.ray_utils import initialize_ray_cluster, ray
15 from vllm.logger import init_logger
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
@@ -208,17 +208,10 @@ class _AsyncLLMEngine(LLMEngine):
208
209 if not scheduler_outputs.is_empty():
210 # Execute the model.
211 - all_outputs = await self._run_workers_async(
212 - "execute_model",
213 - driver_kwargs={
214 - "seq_group_metadata_list": seq_group_metadata_list,
215 - "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
216 - "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
217 - "blocks_to_copy": scheduler_outputs.blocks_to_copy,
218 - })
219 -
220 - # Only the driver worker returns the sampling results.
221 - output = all_outputs[0]
211 + output = await self.model_executor.execute_model_async(
212 + seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
213 + scheduler_outputs.blocks_to_swap_out,
214 + scheduler_outputs.blocks_to_copy)
215 else:
216 output = []
217
@@ -268,37 +261,8 @@ class _AsyncLLMEngine(LLMEngine):
261 lora_request=lora_request,
262 )
263
271 - async def _run_workers_async(
272 - self,
273 - method: str,
274 - *args,
275 - driver_args: Optional[List[Any]] = None,
276 - driver_kwargs: Optional[Dict[str, Any]] = None,
277 - **kwargs,
278 - ) -> Any:
279 - """Runs the given method on all workers."""
280 - coros = []
281 -
282 - if driver_args is None:
283 - driver_args = args
284 - if driver_kwargs is None:
285 - driver_kwargs = kwargs
286 -
287 - # Run the driver worker asynchronously.
288 - driver_executor = getattr(self.driver_worker, method)
289 - coros.append(asyncio.get_event_loop().run_in_executor(
290 - None, partial(driver_executor, *driver_args, **driver_kwargs)))
291 -
292 - # Run the ray workers asynchronously.
293 - for worker in self.workers:
294 - coros.append(worker.execute_method.remote(method, *args, **kwargs))
295 -
296 - all_outputs = await asyncio.gather(*coros)
297 - return all_outputs
298 -
299 - async def check_health_async(self):
300 - """Raises an error if engine is unhealthy."""
301 - self._check_if_any_actor_is_dead()
264 + async def check_health_async(self) -> None:
265 + self.model_executor.check_health()
266
267
268 class AsyncLLMEngine:
@@ -353,6 +317,34 @@ class AsyncLLMEngine:
317 self._request_tracker: Optional[RequestTracker] = None
318 self._errored_with: Optional[BaseException] = None
319
320 + @classmethod
321 + def from_engine_args(cls,
322 + engine_args: AsyncEngineArgs,
323 + start_engine_loop: bool = True) -> "AsyncLLMEngine":
324 + """Creates an async LLM engine from the engine arguments."""
325 + # Create the engine configs.
326 + engine_configs = engine_args.create_engine_configs()
327 + parallel_config = engine_configs[2]
328 + if parallel_config.worker_use_ray or engine_args.engine_use_ray:
329 + initialize_ray_cluster(parallel_config)
330 + from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
331 + executor_class = RayGPUExecutorAsync
332 + else:
333 + assert parallel_config.world_size == 1, (
334 + "Ray is required if parallel_config.world_size > 1.")
335 + from vllm.executor.gpu_executor import GPUExecutorAsync
336 + executor_class = GPUExecutorAsync
337 + # Create the async LLM engine.
338 + engine = cls(parallel_config.worker_use_ray,
339 + engine_args.engine_use_ray,
340 + *engine_configs,
341 + executor_class,
342 + log_requests=not engine_args.disable_log_requests,
343 + log_stats=not engine_args.disable_log_stats,
344 + max_log_len=engine_args.max_log_len,
345 + start_engine_loop=start_engine_loop)
346 + return engine
347 +
348 @property
349 def is_running(self) -> bool:
350 return (self.background_loop is not None
@@ -670,35 +662,13 @@ class AsyncLLMEngine:
662 else:
663 return self.engine.get_model_config()
664
673 - @classmethod
674 - def from_engine_args(cls,
675 - engine_args: AsyncEngineArgs,
676 - start_engine_loop: bool = True) -> "AsyncLLMEngine":
677 - """Creates an async LLM engine from the engine arguments."""
678 - # Create the engine configs.
679 - engine_configs = engine_args.create_engine_configs()
680 - parallel_config = engine_configs[2]
681 - # Initialize the cluster.
682 - placement_group = initialize_cluster(parallel_config,
683 - engine_args.engine_use_ray)
684 - # Create the async LLM engine.
685 - engine = cls(parallel_config.worker_use_ray,
686 - engine_args.engine_use_ray,
687 - *engine_configs,
688 - placement_group,
689 - log_requests=not engine_args.disable_log_requests,
690 - log_stats=not engine_args.disable_log_stats,
691 - max_log_len=engine_args.max_log_len,
692 - start_engine_loop=start_engine_loop)
693 - return engine
694 -
665 async def do_log_stats(self) -> None:
666 if self.engine_use_ray:
667 await self.engine.do_log_stats.remote()
668 else:
669 self.engine.do_log_stats()
670
701 - async def check_health(self):
671 + async def check_health(self) -> None:
672 """Raises an error if engine is unhealthy."""
673 t = time.perf_counter()
674 logger.debug("Starting health check...")
vllm/engine/llm_engine.py
+44 -402
@@ -1,11 +1,5 @@
1 -import copy
2 -from collections import defaultdict
3 -import os
1 import time
5 -import pickle
6 -import importlib
7 -from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8 - Union)
2 +from typing import Dict, Iterable, List, Optional, Tuple, Type, Union
3
4 from transformers import PreTrainedTokenizer
5
@@ -15,8 +9,9 @@ from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
9 ParallelConfig, SchedulerConfig, LoRAConfig)
10 from vllm.core.scheduler import Scheduler, SchedulerOutputs
11 from vllm.engine.arg_utils import EngineArgs
12 +from vllm.executor.executor_base import ExecutorBase
13 from vllm.engine.metrics import StatLogger, Stats
19 -from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
14 +from vllm.engine.ray_utils import initialize_ray_cluster
15 from vllm.logger import init_logger
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
@@ -24,29 +19,11 @@ from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
19 SequenceGroupOutput, SequenceOutput, SequenceStatus)
20 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
21 TokenizerGroup)
27 -from vllm.utils import (Counter, set_cuda_visible_devices, get_ip,
28 - get_open_port, get_distributed_init_method)
29 -
30 -if ray:
31 - from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
32 -
33 -if TYPE_CHECKING:
34 - from ray.util.placement_group import PlacementGroup
22 +from vllm.utils import Counter
23
24 logger = init_logger(__name__)
25 _LOCAL_LOGGING_INTERVAL_SEC = 5
26
39 -# A map between the device type (in device config) to its worker module.
40 -DEVICE_TO_WORKER_MODULE_MAP = {
41 - "cuda": "vllm.worker.worker",
42 - "neuron": "vllm.worker.neuron_worker",
43 -}
44 -
45 -# If the env var is set, it uses the Ray's compiled DAG API
46 -# which optimizes the control plane overhead.
47 -# Run VLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
48 -USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
49 -
27
28 class LLMEngine:
29 """An LLM engine that receives requests and generates texts.
@@ -71,8 +48,8 @@ class LLMEngine:
48 parallel_config: The configuration related to distributed execution.
49 scheduler_config: The configuration related to the request scheduler.
50 device_config: The configuration related to the device.
74 - placement_group: Ray placement group for distributed execution.
75 - Required for distributed execution.
51 + executor_class: The model executor class for managing distributed
52 + execution.
53 log_stats: Whether to log statistics.
54 """
55
@@ -84,7 +61,7 @@ class LLMEngine:
61 scheduler_config: SchedulerConfig,
62 device_config: DeviceConfig,
63 lora_config: Optional[LoRAConfig],
87 - placement_group: Optional["PlacementGroup"],
64 + executor_class: Type[ExecutorBase],
65 log_stats: bool,
66 ) -> None:
67 logger.info(
@@ -121,33 +98,13 @@ class LLMEngine:
98 self._init_tokenizer()
99 self.seq_counter = Counter()
100
124 - # Create the parallel GPU workers.
125 - if self.parallel_config.worker_use_ray:
126 - # Disable Ray usage stats collection.
127 - ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
128 - if ray_usage != "1":
129 - os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
130 - # Pass additional arguments to initialize the worker
131 - additional_ray_args = {}
132 - if self.parallel_config.ray_workers_use_nsight:
133 - logger.info("Configuring Ray workers to use nsight.")
134 - additional_ray_args = {
135 - "runtime_env": {
136 - "nsight": {
137 - "t": "cuda,cudnn,cublas",
138 - "o": "'worker_process_%p'",
139 - "cuda-graph-trace": "node",
140 - }
141 - }
142 - }
143 - self._init_workers_ray(placement_group, **additional_ray_args)
144 - else:
145 - self._init_workers()
146 -
147 - # Profile the memory usage and initialize the cache.
148 - self._init_cache()
101 + self.model_executor = executor_class(model_config, cache_config,
102 + parallel_config, scheduler_config,
103 + device_config, lora_config)
104
105 # Create the scheduler.
106 + # NOTE: the cache_config here have been updated with the numbers of
107 + # GPU and CPU blocks, which are profiled in the distributed executor.
108 self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
109
110 # Metric Logging.
@@ -157,9 +114,29 @@ class LLMEngine:
114 labels=dict(model_name=model_config.model))
115 self.stat_logger.info("cache_config", self.cache_config)
116
160 - self.forward_dag = None
161 - if USE_RAY_COMPILED_DAG:
162 - self.forward_dag = self._compiled_ray_dag()
117 + @classmethod
118 + def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
119 + """Creates an LLM engine from the engine arguments."""
120 + # Create the engine configs.
121 + engine_configs = engine_args.create_engine_configs()
122 + parallel_config = engine_configs[2]
123 +
124 + # Initialize the cluster and specify the executor class.
125 + if parallel_config.worker_use_ray:
126 + initialize_ray_cluster(parallel_config)
127 + from vllm.executor.ray_gpu_executor import RayGPUExecutor
128 + executor_class = RayGPUExecutor
129 + else:
130 + assert parallel_config.world_size == 1, (
131 + "Ray is required if parallel_config.world_size > 1.")
132 + from vllm.executor.gpu_executor import GPUExecutor
133 + executor_class = GPUExecutor
134 +
135 + # Create the LLM engine.
136 + engine = cls(*engine_configs,
137 + executor_class=executor_class,
138 + log_stats=not engine_args.disable_log_stats)
139 + return engine
140
141 def __reduce__(self):
142 # This is to ensure that the LLMEngine is not referenced in
@@ -173,39 +150,6 @@ class LLMEngine:
150 sequence: Sequence) -> "PreTrainedTokenizer":
151 return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
152
176 - def _dispatch_worker(self):
177 - worker_module = DEVICE_TO_WORKER_MODULE_MAP[
178 - self.device_config.device_type]
179 - imported_worker = importlib.import_module(worker_module)
180 - Worker = imported_worker.Worker
181 - return Worker
182 -
183 - def _init_workers(self):
184 - # Lazy import the Worker to avoid importing torch.cuda/xformers
185 - # before CUDA_VISIBLE_DEVICES is set in the Worker
186 - Worker = self._dispatch_worker()
187 -
188 - assert self.parallel_config.world_size == 1, (
189 - "Ray is required if parallel_config.world_size > 1.")
190 -
191 - self.workers: List[Worker] = []
192 - distributed_init_method = get_distributed_init_method(
193 - get_ip(), get_open_port())
194 - self.driver_worker = Worker(
195 - self.model_config,
196 - self.parallel_config,
197 - self.scheduler_config,
198 - self.device_config,
199 - local_rank=0,
200 - rank=0,
201 - distributed_init_method=distributed_init_method,
202 - lora_config=self.lora_config,
203 - kv_cache_dtype=self.cache_config.cache_dtype,
204 - is_driver_worker=True,
205 - )
206 - self._run_workers("init_model")
207 - self._run_workers("load_model")
208 -
153 def _init_tokenizer(self, **tokenizer_init_kwargs):
154 init_kwargs = dict(
155 enable_lora=bool(self.lora_config),
@@ -218,126 +162,6 @@ class LLMEngine:
162 self.tokenizer: TokenizerGroup = TokenizerGroup(
163 self.model_config.tokenizer, **init_kwargs)
164
221 - def _init_workers_ray(self, placement_group: "PlacementGroup",
222 - **ray_remote_kwargs):
223 - if self.parallel_config.tensor_parallel_size == 1:
224 - num_gpus = self.cache_config.gpu_memory_utilization
225 - else:
226 - num_gpus = 1
227 -
228 - self.driver_dummy_worker: RayWorkerVllm = None
229 - self.workers: List[RayWorkerVllm] = []
230 -
231 - driver_ip = get_ip()
232 - for bundle_id, bundle in enumerate(placement_group.bundle_specs):
233 - if not bundle.get("GPU", 0):
234 - continue
235 - scheduling_strategy = PlacementGroupSchedulingStrategy(
236 - placement_group=placement_group,
237 - placement_group_capture_child_tasks=True,
238 - placement_group_bundle_index=bundle_id,
239 - )
240 - worker = ray.remote(
241 - num_cpus=0,
242 - num_gpus=num_gpus,
243 - scheduling_strategy=scheduling_strategy,
244 - **ray_remote_kwargs,
245 - )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
246 -
247 - worker_ip = ray.get(worker.get_node_ip.remote())
248 - if worker_ip == driver_ip and self.driver_dummy_worker is None:
249 - # If the worker is on the same node as the driver, we use it
250 - # as the resource holder for the driver process.
251 - self.driver_dummy_worker = worker
252 - else:
253 - self.workers.append(worker)
254 -
255 - if self.driver_dummy_worker is None:
256 - raise ValueError(
257 - "Ray does not allocate any GPUs on the driver node. Consider "
258 - "adjusting the Ray placement group or running the driver on a "
259 - "GPU node.")
260 -
261 - driver_node_id, driver_gpu_ids = ray.get(
262 - self.driver_dummy_worker.get_node_and_gpu_ids.remote())
263 - worker_node_and_gpu_ids = ray.get(
264 - [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
265 -
266 - node_workers = defaultdict(list)
267 - node_gpus = defaultdict(list)
268 -
269 - node_workers[driver_node_id].append(0)
270 - node_gpus[driver_node_id].extend(driver_gpu_ids)
271 - for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
272 - start=1):
273 - node_workers[node_id].append(i)
274 - node_gpus[node_id].extend(gpu_ids)
275 - for node_id, gpu_ids in node_gpus.items():
276 - node_gpus[node_id] = sorted(gpu_ids)
277 -
278 - # Set CUDA_VISIBLE_DEVICES for the driver.
279 - set_cuda_visible_devices(node_gpus[driver_node_id])
280 - for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
281 - worker.set_cuda_visible_devices.remote(node_gpus[node_id])
282 -
283 - distributed_init_method = get_distributed_init_method(
284 - driver_ip, get_open_port())
285 -
286 - # Lazy import the Worker to avoid importing torch.cuda/xformers
287 - # before CUDA_VISIBLE_DEVICES is set in the Worker
288 - Worker = self._dispatch_worker()
289 -
290 - # Initialize torch distributed process group for the workers.
291 - model_config = copy.deepcopy(self.model_config)
292 - parallel_config = copy.deepcopy(self.parallel_config)
293 - scheduler_config = copy.deepcopy(self.scheduler_config)
294 - device_config = copy.deepcopy(self.device_config)
295 - lora_config = copy.deepcopy(self.lora_config)
296 - kv_cache_dtype = self.cache_config.cache_dtype
297 -
298 - for rank, (worker, (node_id,
299 - _)) in enumerate(zip(self.workers,
300 - worker_node_and_gpu_ids),
301 - start=1):
302 - local_rank = node_workers[node_id].index(rank)
303 - worker.init_worker.remote(
304 - lambda rank=rank, local_rank=local_rank: Worker(
305 - model_config,
306 - parallel_config,
307 - scheduler_config,
308 - device_config,
309 - local_rank,
310 - rank,
311 - distributed_init_method,
312 - lora_config=lora_config,
313 - kv_cache_dtype=kv_cache_dtype,
314 - ))
315 -
316 - driver_rank = 0
317 - driver_local_rank = node_workers[driver_node_id].index(driver_rank)
318 - self.driver_worker = Worker(
319 - self.model_config,
320 - self.parallel_config,
321 - self.scheduler_config,
322 - self.device_config,
323 - driver_local_rank,
324 - driver_rank,
325 - distributed_init_method,
326 - lora_config=self.lora_config,
327 - kv_cache_dtype=kv_cache_dtype,
328 - is_driver_worker=True,
329 - )
330 -
331 - # don't use cupy for eager mode
332 - self._run_workers("init_model",
333 - cupy_port=get_open_port()
334 - if not model_config.enforce_eager else None)
335 - self._run_workers(
336 - "load_model",
337 - max_concurrent_workers=self.parallel_config.
338 - max_parallel_loading_workers,
339 - )
340 -
165 def _verify_args(self) -> None:
166 self.model_config.verify_with_parallel_config(self.parallel_config)
167 self.cache_config.verify_with_parallel_config(self.parallel_config)
@@ -346,81 +170,6 @@ class LLMEngine:
170 self.lora_config.verify_with_scheduler_config(
171 self.scheduler_config)
172
349 - def _init_cache(self) -> None:
350 - """Profiles the memory usage and initializes the KV cache.
351 -
352 - The engine will first conduct a profiling of the existing memory usage.
353 - Then, it calculate the maximum possible number of GPU and CPU blocks
354 - that can be allocated with the remaining free memory.
355 - More details can be found in the
356 - :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
357 - from class :class:`~vllm.worker.Worker`.
358 -
359 - Afterwards, as there may be multiple workers,
360 - we take the minimum number of blocks across all workers
361 - to ensure this can be applied to all of them.
362 -
363 - Finally, the engine will initialize the KV cache
364 - with the calculated number of blocks.
365 -
366 - .. tip::
367 - You may limit the usage of GPU memory
368 - by adjusting the `gpu_memory_utilization` parameters.
369 - """
370 - # Get the maximum number of blocks that can be allocated on GPU and CPU.
371 - num_blocks = self._run_workers(
372 - "profile_num_available_blocks",
373 - block_size=self.cache_config.block_size,
374 - gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
375 - cpu_swap_space=self.cache_config.swap_space_bytes,
376 - cache_dtype=self.cache_config.cache_dtype,
377 - )
378 -
379 - # Since we use a shared centralized controller, we take the minimum
380 - # number of blocks across all workers to make sure all the memory
381 - # operators can be applied to all workers.
382 - num_gpu_blocks = min(b[0] for b in num_blocks)
383 - num_cpu_blocks = min(b[1] for b in num_blocks)
384 - # FIXME(woosuk): Change to debug log.
385 - logger.info(f"# GPU blocks: {num_gpu_blocks}, "
386 - f"# CPU blocks: {num_cpu_blocks}")
387 -
388 - if num_gpu_blocks <= 0:
389 - raise ValueError("No available memory for the cache blocks. "
390 - "Try increasing `gpu_memory_utilization` when "
391 - "initializing the engine.")
392 - max_seq_len = self.cache_config.block_size * num_gpu_blocks
393 - if self.model_config.max_model_len > max_seq_len:
394 - raise ValueError(
395 - f"The model's max seq len ({self.model_config.max_model_len}) "
396 - "is larger than the maximum number of tokens that can be "
397 - f"stored in KV cache ({max_seq_len}). Try increasing "
398 - "`gpu_memory_utilization` or decreasing `max_model_len` when "
399 - "initializing the engine.")
400 -
401 - self.cache_config.num_gpu_blocks = num_gpu_blocks
402 - self.cache_config.num_cpu_blocks = num_cpu_blocks
403 -
404 - # Initialize the cache.
405 - self._run_workers("init_cache_engine", cache_config=self.cache_config)
406 - # Warm up the model. This includes capturing the model into CUDA graph
407 - # if enforce_eager is False.
408 - self._run_workers("warm_up_model")
409 -
410 - @classmethod
411 - def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
412 - """Creates an LLM engine from the engine arguments."""
413 - # Create the engine configs.
414 - engine_configs = engine_args.create_engine_configs()
415 - parallel_config = engine_configs[2]
416 - # Initialize the cluster.
417 - placement_group = initialize_cluster(parallel_config)
418 - # Create the LLM engine.
419 - engine = cls(*engine_configs,
420 - placement_group,
421 - log_stats=not engine_args.disable_log_stats)
422 - return engine
423 -
173 def encode_request(
174 self,
175 request_id: str, # pylint: disable=unused-argument
@@ -826,7 +575,7 @@ class LLMEngine:
575 - A Sequence Group (SG) refer to a group of sequences
576 that are generated from the same prompt.
577
829 - - Step 2: Calls the workers to execute the model.
578 + - Step 2: Calls the distributed executor to execute the model.
579 - Step 3: Processes the model output. This mainly includes:
580
581 - Decodes the relevant outputs.
@@ -862,19 +611,10 @@ class LLMEngine:
611 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
612
613 if not scheduler_outputs.is_empty():
865 - # Execute the model.
866 - all_outputs = self._run_workers(
867 - "execute_model",
868 - driver_kwargs={
869 - "seq_group_metadata_list": seq_group_metadata_list,
870 - "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
871 - "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
872 - "blocks_to_copy": scheduler_outputs.blocks_to_copy,
873 - },
874 - use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
875 -
876 - # Only the driver worker returns the sampling results.
877 - output = all_outputs[0]
614 + output = self.model_executor.execute_model(
615 + seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
616 + scheduler_outputs.blocks_to_swap_out,
617 + scheduler_outputs.blocks_to_copy)
618 else:
619 output = []
620
@@ -1043,111 +783,13 @@ class LLMEngine:
783 seq.output_text = seq.output_text[:-len(stop_string)]
784
785 def add_lora(self, lora_request: LoRARequest) -> bool:
1046 - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
1047 - return self._run_workers(
1048 - "add_lora",
1049 - lora_request=lora_request,
1050 - )
786 + return self.model_executor.add_lora(lora_request)
787
788 def remove_lora(self, lora_id: int) -> bool:
1053 - assert lora_id > 0, "lora_id must be greater than 0."
1054 - return self._run_workers(
1055 - "remove_lora",
1056 - lora_id=lora_id,
1057 - )
789 + return self.model_executor.remove_lora(lora_id)
790
791 def list_loras(self) -> List[int]:
1060 - return self._run_workers("list_loras")
1061 -
1062 - def _run_workers(
1063 - self,
1064 - method: str,
1065 - *args,
1066 - driver_args: Optional[List[Any]] = None,
1067 - driver_kwargs: Optional[Dict[str, Any]] = None,
1068 - max_concurrent_workers: Optional[int] = None,
1069 - use_ray_compiled_dag: bool = False,
1070 - **kwargs,
1071 - ) -> Any:
1072 - """Runs the given method on all workers."""
1073 -
1074 - if max_concurrent_workers:
1075 - raise NotImplementedError(
1076 - "max_concurrent_workers is not supported yet.")
1077 -
1078 - if use_ray_compiled_dag:
1079 - # Right now, compiled DAG can only accept a single
1080 - # input. TODO(sang): Fix it.
1081 - output_channels = self.forward_dag.execute(1)
1082 - else:
1083 - # Start the ray workers first.
1084 - ray_worker_outputs = [
1085 - worker.execute_method.remote(method, *args, **kwargs)
1086 - for worker in self.workers
1087 - ]
1088 -
1089 - if driver_args is None:
1090 - driver_args = args
1091 - if driver_kwargs is None:
1092 - driver_kwargs = kwargs
1093 -
1094 - # Start the driver worker after all the ray workers.
1095 - driver_worker_output = getattr(self.driver_worker,
1096 - method)(*driver_args, **driver_kwargs)
1097 -
1098 - # Get the results of the ray workers.
1099 - if self.workers:
1100 - if use_ray_compiled_dag:
1101 - try:
1102 - ray_worker_outputs = [
1103 - pickle.loads(chan.begin_read())
1104 - for chan in output_channels
1105 - ]
1106 - finally:
1107 - # Has to call end_read in order to reuse the DAG.
1108 - for chan in output_channels:
1109 - chan.end_read()
1110 - else:
1111 - ray_worker_outputs = ray.get(ray_worker_outputs)
1112 -
1113 - return [driver_worker_output] + ray_worker_outputs
1114 -
1115 - def _compiled_ray_dag(self):
1116 - import pkg_resources
1117 - required_version = "2.9"
1118 - current_version = pkg_resources.get_distribution("ray").version
1119 - if current_version < required_version:
1120 - raise ValueError(f"Ray version {required_version} or greater is "
1121 - f"required, but found {current_version}")
1122 -
1123 - from ray.dag import MultiOutputNode, InputNode
1124 - assert self.parallel_config.worker_use_ray
1125 -
1126 - # Right now, compiled DAG requires at least 1 arg. We send
1127 - # a dummy value for now. It will be fixed soon.
1128 - with InputNode() as input_data:
1129 - forward_dag = MultiOutputNode([
1130 - worker.execute_model_compiled_dag_remote.bind(input_data)
1131 - for worker in self.workers
1132 - ])
1133 - return forward_dag.experimental_compile()
792 + return self.model_executor.list_loras()
793
794 def check_health(self) -> None:
1136 - """Raises an error if engine is unhealthy."""
1137 - self._check_if_any_actor_is_dead()
1138 -
1139 - def _check_if_any_actor_is_dead(self):
1140 - if not self.parallel_config.worker_use_ray:
1141 - return
1142 -
1143 - if not self.workers:
1144 - return
1145 -
1146 - dead_actors = []
1147 - for actor in self.workers:
1148 - actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
1149 - if actor_state["State"] == "DEAD":
1150 - dead_actors.append(actor)
1151 - if dead_actors:
1152 - raise RuntimeError("At least one Worker is dead. "
1153 - f"Dead Workers: {dead_actors}. ")
795 + self.model_executor.check_health()
vllm/engine/ray_utils.py
+26 -32
@@ -1,6 +1,6 @@
1 import pickle
2
3 -from typing import Optional, List, Tuple, TYPE_CHECKING
3 +from typing import Optional, List, Tuple
4
5 from vllm.config import ParallelConfig
6 from vllm.logger import init_logger
@@ -65,45 +65,38 @@ except ImportError as e:
65 ray = None
66 RayWorkerVllm = None
67
68 -if TYPE_CHECKING:
69 - from ray.util.placement_group import PlacementGroup
68
71 -
72 -def initialize_cluster(
69 +def initialize_ray_cluster(
70 parallel_config: ParallelConfig,
74 - engine_use_ray: bool = False,
71 ray_address: Optional[str] = None,
76 -) -> Optional["PlacementGroup"]:
77 - """Initialize the distributed cluster probably with Ray.
72 +):
73 + """Initialize the distributed cluster with Ray.
74 +
75 + it will connect to the Ray cluster and create a placement group
76 + for the workers, which includes the specification of the resources
77 + for each distributed worker.
78
79 Args:
80 parallel_config: The configurations for parallel execution.
81 - engine_use_ray: Whether to use Ray for async engine.
81 ray_address: The address of the Ray cluster. If None, uses
82 the default Ray cluster address.
84 -
85 - Returns:
86 - An optional `PlacementGroup`. It includes the specification
87 - of the resources for each distributed worker. None if Ray is
88 - not used.
83 """
90 - if parallel_config.worker_use_ray or engine_use_ray:
91 - if ray is None:
92 - raise ImportError(
93 - "Ray is not installed. Please install Ray to use distributed "
94 - "serving.")
95 - # Connect to a ray cluster.
96 - if is_hip():
97 - ray.init(address=ray_address,
98 - ignore_reinit_error=True,
99 - num_gpus=parallel_config.world_size)
100 - else:
101 - ray.init(address=ray_address, ignore_reinit_error=True)
102 -
103 - if not parallel_config.worker_use_ray:
104 - assert parallel_config.world_size == 1, (
105 - "Ray is required if parallel_config.world_size > 1.")
106 - return None
84 + if ray is None:
85 + raise ImportError(
86 + "Ray is not installed. Please install Ray to use distributed "
87 + "serving.")
88 +
89 + # Connect to a ray cluster.
90 + if is_hip():
91 + ray.init(address=ray_address,
92 + ignore_reinit_error=True,
93 + num_gpus=parallel_config.world_size)
94 + else:
95 + ray.init(address=ray_address, ignore_reinit_error=True)
96 +
97 + if parallel_config.placement_group:
98 + # Placement group is already set.
99 + return
100
101 # Create placement group for worker processes
102 current_placement_group = ray.util.get_current_placement_group()
@@ -138,4 +131,5 @@ def initialize_cluster(
131 # if they cannot be provisioned.
132 ray.get(current_placement_group.ready(), timeout=1800)
133
141 - return current_placement_group
134 + # Set the placement group in the parallel config
135 + parallel_config.placement_group = current_placement_group
vllm/executor/__init__.py
vllm/executor/executor_base.py new
+75
@@ -0,0 +1,75 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Dict, List, Optional
3 +
4 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5 + ParallelConfig, SchedulerConfig, LoRAConfig)
6 +from vllm.lora.request import LoRARequest
7 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8 +
9 +
10 +class ExecutorBase(ABC):
11 + """Base class for all executors.
12 +
13 + An executor is responsible for executing the model on a specific device
14 + type (e.g., CPU, GPU, Neuron, etc.). Or it can be a distributed executor
15 + that can execute the model on multiple devices.
16 + """
17 +
18 + @abstractmethod
19 + def __init__(
20 + self,
21 + model_config: ModelConfig,
22 + cache_config: CacheConfig,
23 + parallel_config: ParallelConfig,
24 + scheduler_config: SchedulerConfig,
25 + device_config: DeviceConfig,
26 + lora_config: Optional[LoRAConfig],
27 + ) -> None:
28 + raise NotImplementedError
29 +
30 + @abstractmethod
31 + def execute_model(self,
32 + seq_group_metadata_list: List[SequenceGroupMetadata],
33 + blocks_to_swap_in: Dict[int, int],
34 + blocks_to_swap_out: Dict[int, int],
35 + blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
36 + """Executes one model step on the given sequences."""
37 + raise NotImplementedError
38 +
39 + @abstractmethod
40 + def add_lora(self, lora_request: LoRARequest) -> bool:
41 + raise NotImplementedError
42 +
43 + @abstractmethod
44 + def remove_lora(self, lora_id: int) -> bool:
45 + raise NotImplementedError
46 +
47 + @abstractmethod
48 + def list_loras(self) -> List[int]:
49 + raise NotImplementedError
50 +
51 + @abstractmethod
52 + def check_health(self) -> None:
53 + """Checks if the executor is healthy. If not, it should raise an
54 + exception."""
55 + raise NotImplementedError
56 +
57 +
58 +class ExecutorAsyncBase(ExecutorBase):
59 +
60 + @abstractmethod
61 + async def execute_model_async(
62 + self,
63 + seq_group_metadata_list: List[SequenceGroupMetadata],
64 + blocks_to_swap_in: Dict[int, int],
65 + blocks_to_swap_out: Dict[int, int],
66 + blocks_to_copy: Dict[int, List[int]],
67 + ) -> SamplerOutput:
68 + """Executes one model step on the given sequences."""
69 + raise NotImplementedError
70 +
71 + @abstractmethod
72 + async def check_health_async(self) -> None:
73 + """Checks if the executor is healthy. If not, it should raise an
74 + exception."""
75 + raise NotImplementedError
vllm/executor/gpu_executor.py new
+163
@@ -0,0 +1,163 @@
1 +import importlib
2 +from typing import Dict, List, Optional
3 +
4 +from vllm.lora.request import LoRARequest
5 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
6 + ParallelConfig, SchedulerConfig, LoRAConfig)
7 +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8 +from vllm.executor.utils import check_block_size_valid
9 +from vllm.logger import init_logger
10 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
11 +from vllm.utils import (get_ip, get_open_port, get_distributed_init_method,
12 + make_async)
13 +
14 +logger = init_logger(__name__)
15 +
16 +# A map between the device type (in device config) to its worker module.
17 +DEVICE_TO_WORKER_MODULE_MAP = {
18 + "cuda": "vllm.worker.worker",
19 + "neuron": "vllm.worker.neuron_worker",
20 +}
21 +
22 +
23 +class GPUExecutor(ExecutorBase):
24 +
25 + def __init__(
26 + self,
27 + model_config: ModelConfig,
28 + cache_config: CacheConfig,
29 + parallel_config: ParallelConfig,
30 + scheduler_config: SchedulerConfig,
31 + device_config: DeviceConfig,
32 + lora_config: Optional[LoRAConfig],
33 + ) -> None:
34 + self.model_config = model_config
35 + self.cache_config = cache_config
36 + self.lora_config = lora_config
37 + self.parallel_config = parallel_config
38 + self.scheduler_config = scheduler_config
39 + self.device_config = device_config
40 +
41 + # Instantiate the worker and load the model to GPU.
42 + self._init_worker()
43 +
44 + # Profile the memory usage and initialize the cache.
45 + self._init_cache()
46 +
47 + def _dispatch_worker(self):
48 + worker_module = DEVICE_TO_WORKER_MODULE_MAP[
49 + self.device_config.device_type]
50 + imported_worker = importlib.import_module(worker_module)
51 + Worker = imported_worker.Worker
52 + return Worker
53 +
54 + def _init_worker(self):
55 + # Lazy import the Worker to avoid importing torch.cuda/xformers
56 + # before CUDA_VISIBLE_DEVICES is set in the Worker
57 + Worker = self._dispatch_worker()
58 +
59 + assert self.parallel_config.world_size == 1, (
60 + "GPUExecutor only supports single GPU.")
61 +
62 + distributed_init_method = get_distributed_init_method(
63 + get_ip(), get_open_port())
64 + self.driver_worker = Worker(
65 + self.model_config,
66 + self.parallel_config,
67 + self.scheduler_config,
68 + self.device_config,
69 + local_rank=0,
70 + rank=0,
71 + distributed_init_method=distributed_init_method,
72 + lora_config=self.lora_config,
73 + kv_cache_dtype=self.cache_config.cache_dtype,
74 + is_driver_worker=True,
75 + )
76 + self.driver_worker.init_model()
77 + self.driver_worker.load_model()
78 +
79 + def _init_cache(self) -> None:
80 + """Profiles the memory usage and initializes the KV cache.
81 +
82 + The engine first profiles the existing memory usage.
83 + Then, it allocates the remaining memory for KV blocks.
84 +
85 + .. tip::
86 + You may limit the usage of GPU memory
87 + by adjusting the `gpu_memory_utilization` parameter.
88 + """
89 + # Get the maximum number of blocks that can be allocated on GPU and CPU.
90 + num_gpu_blocks, num_cpu_blocks = (
91 + self.driver_worker.profile_num_available_blocks(
92 + block_size=self.cache_config.block_size,
93 + gpu_memory_utilization=self.cache_config.
94 + gpu_memory_utilization,
95 + cpu_swap_space=self.cache_config.swap_space_bytes,
96 + cache_dtype=self.cache_config.cache_dtype,
97 + ))
98 +
99 + logger.info(f"# GPU blocks: {num_gpu_blocks}, "
100 + f"# CPU blocks: {num_cpu_blocks}")
101 +
102 + check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
103 + self.model_config.max_model_len)
104 +
105 + self.cache_config.num_gpu_blocks = num_gpu_blocks
106 + self.cache_config.num_cpu_blocks = num_cpu_blocks
107 +
108 + # Initialize the cache.
109 + self.driver_worker.init_cache_engine(cache_config=self.cache_config)
110 + # Warm up the model. This includes capturing the model into CUDA graph
111 + # if enforce_eager is False.
112 + self.driver_worker.warm_up_model()
113 +
114 + def execute_model(self,
115 + seq_group_metadata_list: List[SequenceGroupMetadata],
116 + blocks_to_swap_in: Dict[int, int],
117 + blocks_to_swap_out: Dict[int, int],
118 + blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
119 + output = self.driver_worker.execute_model(
120 + seq_group_metadata_list=seq_group_metadata_list,
121 + blocks_to_swap_in=blocks_to_swap_in,
122 + blocks_to_swap_out=blocks_to_swap_out,
123 + blocks_to_copy=blocks_to_copy,
124 + )
125 + return output
126 +
127 + def add_lora(self, lora_request: LoRARequest) -> bool:
128 + assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
129 + return self.driver_worker.add_lora(lora_request)
130 +
131 + def remove_lora(self, lora_id: int) -> bool:
132 + assert lora_id > 0, "lora_id must be greater than 0."
133 + return self.driver_worker.remove_lora(lora_id)
134 +
135 + def list_loras(self) -> List[int]:
136 + return self.driver_worker.list_loras()
137 +
138 + def check_health(self) -> None:
139 + # GPUExecutor will always be healthy as long as
140 + # it's running.
141 + return
142 +
143 +
144 +class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
145 +
146 + async def execute_model_async(
147 + self,
148 + seq_group_metadata_list: List[SequenceGroupMetadata],
149 + blocks_to_swap_in: Dict[int, int],
150 + blocks_to_swap_out: Dict[int, int],
151 + blocks_to_copy: Dict[int, List[int]],
152 + ) -> SamplerOutput:
153 + output = await make_async(self.driver_worker.execute_model)(
154 + seq_group_metadata_list=seq_group_metadata_list,
155 + blocks_to_swap_in=blocks_to_swap_in,
156 + blocks_to_swap_out=blocks_to_swap_out,
157 + blocks_to_copy=blocks_to_copy)
158 + return output
159 +
160 + async def check_health_async(self) -> None:
161 + # GPUExecutor will always be healthy as long as
162 + # it's running.
163 + return
vllm/executor/ray_gpu_executor.py new
+442
@@ -0,0 +1,442 @@
1 +import asyncio
2 +import copy
3 +from collections import defaultdict
4 +import os
5 +import pickle
6 +import importlib
7 +from typing import TYPE_CHECKING, Any, Dict, List, Optional
8 +
9 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10 + ParallelConfig, SchedulerConfig, LoRAConfig)
11 +from vllm.engine.ray_utils import RayWorkerVllm, ray
12 +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13 +from vllm.executor.utils import check_block_size_valid
14 +from vllm.logger import init_logger
15 +from vllm.lora.request import LoRARequest
16 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17 +from vllm.utils import (set_cuda_visible_devices, get_ip, get_open_port,
18 + get_distributed_init_method, make_async)
19 +
20 +if ray is not None:
21 + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
22 +
23 +if TYPE_CHECKING:
24 + from ray.util.placement_group import PlacementGroup
25 +
26 +logger = init_logger(__name__)
27 +
28 +# A map between the device type (in device config) to its worker module.
29 +DEVICE_TO_WORKER_MODULE_MAP = {
30 + "cuda": "vllm.worker.worker",
31 + "neuron": "vllm.worker.neuron_worker",
32 +}
33 +
34 +# If the env var is set, it uses the Ray's compiled DAG API
35 +# which optimizes the control plane overhead.
36 +# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
37 +USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
38 +
39 +
40 +class RayGPUExecutor(ExecutorBase):
41 +
42 + def __init__(
43 + self,
44 + model_config: ModelConfig,
45 + cache_config: CacheConfig,
46 + parallel_config: ParallelConfig,
47 + scheduler_config: SchedulerConfig,
48 + device_config: DeviceConfig,
49 + lora_config: Optional[LoRAConfig],
50 + ) -> None:
51 + self.model_config = model_config
52 + self.cache_config = cache_config
53 + self.lora_config = lora_config
54 + self.parallel_config = parallel_config
55 + self.scheduler_config = scheduler_config
56 + self.device_config = device_config
57 +
58 + assert self.parallel_config.worker_use_ray
59 + placement_group = self.parallel_config.placement_group
60 +
61 + # Disable Ray usage stats collection.
62 + ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
63 + if ray_usage != "1":
64 + os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
65 +
66 + # Create the parallel GPU workers.
67 + self._init_workers_ray(placement_group)
68 +
69 + # Profile the memory usage and initialize the cache.
70 + self._init_cache()
71 +
72 + self.forward_dag = None
73 + if USE_RAY_COMPILED_DAG:
74 + self.forward_dag = self._compiled_ray_dag()
75 +
76 + def _dispatch_worker(self):
77 + worker_module = DEVICE_TO_WORKER_MODULE_MAP[
78 + self.device_config.device_type]
79 + imported_worker = importlib.import_module(worker_module)
80 + Worker = imported_worker.Worker
81 + return Worker
82 +
83 + def _init_workers_ray(self, placement_group: "PlacementGroup",
84 + **ray_remote_kwargs):
85 + if self.parallel_config.tensor_parallel_size == 1:
86 + # For single GPU case, we use a ray worker with constrained memory.
87 + num_gpus = self.cache_config.gpu_memory_utilization
88 + else:
89 + # Otherwise, the ray workers are allocated with a full GPU.
90 + num_gpus = 1
91 +
92 + # The driver dummy worker does not actually use any resources.
93 + # It holds the resource for the driver worker.
94 + self.driver_dummy_worker: RayWorkerVllm = None
95 + # The remaining workers are the actual ray actors.
96 + self.workers: List[RayWorkerVllm] = []
97 +
98 + # Create the workers.
99 + driver_ip = get_ip()
100 + for bundle_id, bundle in enumerate(placement_group.bundle_specs):
101 + if not bundle.get("GPU", 0):
102 + continue
103 + scheduling_strategy = PlacementGroupSchedulingStrategy(
104 + placement_group=placement_group,
105 + placement_group_capture_child_tasks=True,
106 + placement_group_bundle_index=bundle_id,
107 + )
108 + worker = ray.remote(
109 + num_cpus=0,
110 + num_gpus=num_gpus,
111 + scheduling_strategy=scheduling_strategy,
112 + **ray_remote_kwargs,
113 + )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
114 +
115 + worker_ip = ray.get(worker.get_node_ip.remote())
116 + if worker_ip == driver_ip and self.driver_dummy_worker is None:
117 + # If the worker is on the same node as the driver, we use it
118 + # as the resource holder for the driver process.
119 + self.driver_dummy_worker = worker
120 + else:
121 + # Else, added to the list of workers.
122 + self.workers.append(worker)
123 +
124 + if self.driver_dummy_worker is None:
125 + raise ValueError(
126 + "Ray does not allocate any GPUs on the driver node. Consider "
127 + "adjusting the Ray placement group or running the driver on a "
128 + "GPU node.")
129 +
130 + # Get the set of GPU IDs used on each node.
131 + driver_node_id, driver_gpu_ids = ray.get(
132 + self.driver_dummy_worker.get_node_and_gpu_ids.remote())
133 + worker_node_and_gpu_ids = ray.get(
134 + [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
135 +
136 + node_workers = defaultdict(list)
137 + node_gpus = defaultdict(list)
138 +
139 + node_workers[driver_node_id].append(0)
140 + node_gpus[driver_node_id].extend(driver_gpu_ids)
141 + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
142 + start=1):
143 + node_workers[node_id].append(i)
144 + node_gpus[node_id].extend(gpu_ids)
145 + for node_id, gpu_ids in node_gpus.items():
146 + node_gpus[node_id] = sorted(gpu_ids)
147 +
148 + # Set CUDA_VISIBLE_DEVICES for the driver and workers.
149 + set_cuda_visible_devices(node_gpus[driver_node_id])
150 + for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
151 + worker.set_cuda_visible_devices.remote(node_gpus[node_id])
152 +
153 + distributed_init_method = get_distributed_init_method(
154 + driver_ip, get_open_port())
155 +
156 + # Lazy import the Worker to avoid importing torch.cuda/xformers
157 + # before CUDA_VISIBLE_DEVICES is set in the Worker
158 + Worker = self._dispatch_worker()
159 +
160 + model_config = copy.deepcopy(self.model_config)
161 + parallel_config = copy.deepcopy(self.parallel_config)
162 + scheduler_config = copy.deepcopy(self.scheduler_config)
163 + device_config = copy.deepcopy(self.device_config)
164 + lora_config = copy.deepcopy(self.lora_config)
165 + kv_cache_dtype = self.cache_config.cache_dtype
166 +
167 + # Initialize the actual workers with the Worker class.
168 + for rank, (worker, (node_id, _)) in enumerate(
169 + zip(self.workers, worker_node_and_gpu_ids),
170 + start=1,
171 + ):
172 + local_rank = node_workers[node_id].index(rank)
173 + worker.init_worker.remote(
174 + lambda rank=rank, local_rank=local_rank: Worker(
175 + model_config,
176 + parallel_config,
177 + scheduler_config,
178 + device_config,
179 + local_rank,
180 + rank,
181 + distributed_init_method,
182 + lora_config=lora_config,
183 + kv_cache_dtype=kv_cache_dtype,
184 + ))
185 +
186 + # Initialize the driver worker with the Worker class.
187 + driver_rank = 0
188 + driver_local_rank = node_workers[driver_node_id].index(driver_rank)
189 + self.driver_worker = Worker(
190 + self.model_config,
191 + self.parallel_config,
192 + self.scheduler_config,
193 + self.device_config,
194 + driver_local_rank,
195 + driver_rank,
196 + distributed_init_method,
197 + lora_config=self.lora_config,
198 + kv_cache_dtype=kv_cache_dtype,
199 + is_driver_worker=True,
200 + )
201 +
202 + # FIXME(woosuk): We are not properly initializing cupy NCCL when
203 + # we have multiple nodes.
204 + self._run_workers("init_model",
205 + cupy_port=get_open_port()
206 + if not model_config.enforce_eager else None)
207 + self._run_workers(
208 + "load_model",
209 + max_concurrent_workers=self.parallel_config.
210 + max_parallel_loading_workers,
211 + )
212 +
213 + def _init_cache(self) -> None:
214 + """Profiles the memory usage and initializes the KV cache.
215 +
216 + The engine will first conduct a profiling of the existing memory usage.
217 + Then, it calculate the maximum possible number of GPU and CPU blocks
218 + that can be allocated with the remaining free memory.
219 + More details can be found in the
220 + :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
221 + from class :class:`~vllm.worker.Worker`.
222 +
223 + Afterwards, as there may be multiple workers,
224 + we take the minimum number of blocks across all workers
225 + to ensure this can be applied to all of them.
226 +
227 + Finally, the engine will initialize the KV cache
228 + with the calculated number of blocks.
229 +
230 + .. tip::
231 + You may limit the usage of GPU memory
232 + by adjusting the `gpu_memory_utilization` parameter.
233 + """
234 + # Get the maximum number of blocks that can be allocated on GPU and CPU.
235 + num_blocks = self._run_workers(
236 + "profile_num_available_blocks",
237 + block_size=self.cache_config.block_size,
238 + gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
239 + cpu_swap_space=self.cache_config.swap_space_bytes,
240 + cache_dtype=self.cache_config.cache_dtype,
241 + )
242 +
243 + # Since we use a shared centralized controller, we take the minimum
244 + # number of blocks across all workers to make sure all the memory
245 + # operators can be applied to all workers.
246 + num_gpu_blocks = min(b[0] for b in num_blocks)
247 + num_cpu_blocks = min(b[1] for b in num_blocks)
248 + logger.info(f"# GPU blocks: {num_gpu_blocks}, "
249 + f"# CPU blocks: {num_cpu_blocks}")
250 +
251 + check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
252 + self.model_config.max_model_len)
253 +
254 + self.cache_config.num_gpu_blocks = num_gpu_blocks
255 + self.cache_config.num_cpu_blocks = num_cpu_blocks
256 +
257 + # Initialize the cache.
258 + self._run_workers("init_cache_engine", cache_config=self.cache_config)
259 + # Warm up the model. This includes capturing the model into CUDA graph
260 + # if enforce_eager is False.
261 + self._run_workers("warm_up_model")
262 +
263 + def execute_model(self,
264 + seq_group_metadata_list: List[SequenceGroupMetadata],
265 + blocks_to_swap_in: Dict[int, int],
266 + blocks_to_swap_out: Dict[int, int],
267 + blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
268 + all_outputs = self._run_workers(
269 + "execute_model",
270 + driver_kwargs={
271 + "seq_group_metadata_list": seq_group_metadata_list,
272 + "blocks_to_swap_in": blocks_to_swap_in,
273 + "blocks_to_swap_out": blocks_to_swap_out,
274 + "blocks_to_copy": blocks_to_copy,
275 + },
276 + use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
277 +
278 + # Only the driver worker returns the sampling results.
279 + output = all_outputs[0]
280 + return output
281 +
282 + def add_lora(self, lora_request: LoRARequest) -> bool:
283 + assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
284 + return self._run_workers(
285 + "add_lora",
286 + lora_request=lora_request,
287 + )
288 +
289 + def remove_lora(self, lora_id: int) -> bool:
290 + assert lora_id > 0, "lora_id must be greater than 0."
291 + return self._run_workers(
292 + "remove_lora",
293 + lora_id=lora_id,
294 + )
295 +
296 + def list_loras(self) -> List[int]:
297 + return self._run_workers("list_loras")
298 +
299 + def _run_workers(
300 + self,
301 + method: str,
302 + *args,
303 + driver_args: Optional[List[Any]] = None,
304 + driver_kwargs: Optional[Dict[str, Any]] = None,
305 + max_concurrent_workers: Optional[int] = None,
306 + use_ray_compiled_dag: bool = False,
307 + **kwargs,
308 + ) -> Any:
309 + """Runs the given method on all workers."""
310 +
311 + if max_concurrent_workers:
312 + raise NotImplementedError(
313 + "max_concurrent_workers is not supported yet.")
314 +
315 + if use_ray_compiled_dag:
316 + # Right now, compiled DAG can only accept a single
317 + # input. TODO(sang): Fix it.
318 + output_channels = self.forward_dag.execute(1)
319 + else:
320 + # Start the ray workers first.
321 + ray_worker_outputs = [
322 + worker.execute_method.remote(method, *args, **kwargs)
323 + for worker in self.workers
324 + ]
325 +
326 + if driver_args is None:
327 + driver_args = args
328 + if driver_kwargs is None:
329 + driver_kwargs = kwargs
330 +
331 + # Start the driver worker after all the ray workers.
332 + driver_worker_output = getattr(self.driver_worker,
333 + method)(*driver_args, **driver_kwargs)
334 +
335 + # Get the results of the ray workers.
336 + if self.workers:
337 + if use_ray_compiled_dag:
338 + try:
339 + ray_worker_outputs = [
340 + pickle.loads(chan.begin_read())
341 + for chan in output_channels
342 + ]
343 + finally:
344 + # Has to call end_read in order to reuse the DAG.
345 + for chan in output_channels:
346 + chan.end_read()
347 + else:
348 + ray_worker_outputs = ray.get(ray_worker_outputs)
349 +
350 + return [driver_worker_output] + ray_worker_outputs
351 +
352 + def _compiled_ray_dag(self):
353 + import pkg_resources
354 + required_version = "2.9"
355 + current_version = pkg_resources.get_distribution("ray").version
356 + if current_version < required_version:
357 + raise ValueError(f"Ray version {required_version} or greater is "
358 + f"required, but found {current_version}")
359 +
360 + from ray.dag import MultiOutputNode, InputNode
361 + assert self.parallel_config.worker_use_ray
362 +
363 + # Right now, compiled DAG requires at least 1 arg. We send
364 + # a dummy value for now. It will be fixed soon.
365 + with InputNode() as input_data:
366 + forward_dag = MultiOutputNode([
367 + worker.execute_model_compiled_dag_remote.bind(input_data)
368 + for worker in self.workers
369 + ])
370 + return forward_dag.experimental_compile()
371 +
372 + def check_health(self) -> None:
373 + """Raises an error if engine is unhealthy."""
374 + self._check_if_any_actor_is_dead()
375 +
376 + def _check_if_any_actor_is_dead(self):
377 + if not self.workers:
378 + return
379 +
380 + dead_actors = []
381 + for actor in self.workers:
382 + actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
383 + if actor_state["State"] == "DEAD":
384 + dead_actors.append(actor)
385 + if dead_actors:
386 + raise RuntimeError("At least one Worker is dead. "
387 + f"Dead Workers: {dead_actors}. ")
388 +
389 +
390 +class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
391 +
392 + async def _run_workers_async(
393 + self,
394 + method: str,
395 + *args,
396 + driver_args: Optional[List[Any]] = None,
397 + driver_kwargs: Optional[Dict[str, Any]] = None,
398 + **kwargs,
399 + ) -> Any:
400 + """Runs the given method on all workers."""
401 + coros = []
402 +
403 + if driver_args is None:
404 + driver_args = args
405 + if driver_kwargs is None:
406 + driver_kwargs = kwargs
407 +
408 + # Run the driver worker asynchronously.
409 + driver_executor = make_async(getattr(self.driver_worker, method))
410 + coros.append(driver_executor(*driver_args, **driver_kwargs))
411 +
412 + # Run the ray workers asynchronously.
413 + for worker in self.workers:
414 + coros.append(worker.execute_method.remote(method, *args, **kwargs))
415 +
416 + all_outputs = await asyncio.gather(*coros)
417 + return all_outputs
418 +
419 + async def execute_model_async(
420 + self,
421 + seq_group_metadata_list: List[SequenceGroupMetadata],
422 + blocks_to_swap_in: Dict[int, int],
423 + blocks_to_swap_out: Dict[int, int],
424 + blocks_to_copy: Dict[int, List[int]],
425 + ) -> SamplerOutput:
426 + all_outputs = await self._run_workers_async(
427 + "execute_model",
428 + driver_kwargs={
429 + "seq_group_metadata_list": seq_group_metadata_list,
430 + "blocks_to_swap_in": blocks_to_swap_in,
431 + "blocks_to_swap_out": blocks_to_swap_out,
432 + "blocks_to_copy": blocks_to_copy,
433 + },
434 + use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
435 +
436 + # Only the driver worker returns the sampling results.
437 + output = all_outputs[0]
438 + return output
439 +
440 + async def check_health_async(self) -> None:
441 + """Raises an error if engine is unhealthy."""
442 + self._check_if_any_actor_is_dead()
vllm/executor/utils.py new
+13
@@ -0,0 +1,13 @@
1 +def check_block_size_valid(num_gpu_blocks, block_size, max_model_len) -> None:
2 + if num_gpu_blocks <= 0:
3 + raise ValueError("No available memory for the cache blocks. "
4 + "Try increasing `gpu_memory_utilization` when "
5 + "initializing the engine.")
6 + max_seq_len = block_size * num_gpu_blocks
7 + if max_model_len > max_seq_len:
8 + raise ValueError(
9 + f"The model's max seq len ({max_model_len}) "
10 + "is larger than the maximum number of tokens that can be "
11 + f"stored in KV cache ({max_seq_len}). Try increasing "
12 + "`gpu_memory_utilization` or decreasing `max_model_len` when "
13 + "initializing the engine.")