@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad34c0df0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] platform agnostic executor via collective_rpc (#11256)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 15, 2025 at 13:45 UTC ad34c0df0f1b26b303a590133685b29e3daad20e
43 files changed +852 -2642
tests/engine/test_custom_executor.py
+12 -16
@@ -1,12 +1,13 @@
1 import asyncio
2 import os
3 +from typing import Any, Dict, List, Optional, Tuple
4
5 import pytest
6
7 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
8 from vllm.engine.async_llm_engine import AsyncLLMEngine
9 from vllm.engine.llm_engine import LLMEngine
9 -from vllm.executor.gpu_executor import GPUExecutor, GPUExecutorAsync
10 +from vllm.executor.uniproc_executor import UniProcExecutor
11 from vllm.sampling_params import SamplingParams
12
13
@@ -14,21 +15,20 @@ class Mock:
15 ...
16
17
17 -class CustomGPUExecutor(GPUExecutor):
18 +class CustomUniExecutor(UniProcExecutor):
19
19 - def execute_model(self, *args, **kwargs):
20 + def collective_rpc(self,
21 + method: str,
22 + timeout: Optional[float] = None,
23 + args: Tuple = (),
24 + kwargs: Optional[Dict] = None) -> List[Any]:
25 # Drop marker to show that this was ran
26 with open(".marker", "w"):
27 ...
23 - return super().execute_model(*args, **kwargs)
28 + return super().collective_rpc(method, timeout, args, kwargs)
29
30
26 -class CustomGPUExecutorAsync(GPUExecutorAsync):
27 -
28 - async def execute_model_async(self, *args, **kwargs):
29 - with open(".marker", "w"):
30 - ...
31 - return await super().execute_model_async(*args, **kwargs)
31 +CustomUniExecutorAsync = CustomUniExecutor
32
33
34 @pytest.mark.parametrize("model", ["facebook/opt-125m"])
@@ -41,10 +41,6 @@ def test_custom_executor_type_checking(model):
41 engine_args = AsyncEngineArgs(model=model,
42 distributed_executor_backend=Mock)
43 AsyncLLMEngine.from_engine_args(engine_args)
44 - with pytest.raises(TypeError):
45 - engine_args = AsyncEngineArgs(
46 - model=model, distributed_executor_backend=CustomGPUExecutor)
47 - AsyncLLMEngine.from_engine_args(engine_args)
44
45
46 @pytest.mark.parametrize("model", ["facebook/opt-125m"])
@@ -55,7 +51,7 @@ def test_custom_executor(model, tmp_path):
51 assert not os.path.exists(".marker")
52
53 engine_args = EngineArgs(
58 - model=model, distributed_executor_backend=CustomGPUExecutor)
54 + model=model, distributed_executor_backend=CustomUniExecutor)
55 engine = LLMEngine.from_engine_args(engine_args)
56 sampling_params = SamplingParams(max_tokens=1)
57
@@ -75,7 +71,7 @@ def test_custom_executor_async(model, tmp_path):
71 assert not os.path.exists(".marker")
72
73 engine_args = AsyncEngineArgs(
78 - model=model, distributed_executor_backend=CustomGPUExecutorAsync)
74 + model=model, distributed_executor_backend=CustomUniExecutorAsync)
75 engine = AsyncLLMEngine.from_engine_args(engine_args)
76 sampling_params = SamplingParams(max_tokens=1)
77
tests/engine/test_multiproc_workers.py
+6 -6
@@ -6,16 +6,15 @@ from typing import Any, List, Tuple
6
7 import pytest
8
9 +from vllm.config import VllmConfig
10 from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
11 ResultHandler, WorkerMonitor)
12 +from vllm.worker.worker_base import WorkerWrapperBase
13
14
13 -class DummyWorker:
15 +class DummyWorkerWrapper(WorkerWrapperBase):
16 """Dummy version of vllm.worker.worker.Worker"""
17
16 - def __init__(self, rank: int):
17 - self.rank = rank
18 -
18 def worker_method(self, worker_input: Any) -> Tuple[int, Any]:
19 sleep(0.05)
20
@@ -28,9 +27,10 @@ class DummyWorker:
27
28 def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:
29 result_handler = ResultHandler()
30 + vllm_config = VllmConfig()
31 workers = [
32 - ProcessWorkerWrapper(result_handler, partial(DummyWorker, rank=rank))
33 - for rank in range(8)
32 + ProcessWorkerWrapper(result_handler, DummyWorkerWrapper, vllm_config,
33 + rank) for rank in range(8)
34 ]
35
36 worker_monitor = WorkerMonitor(workers, result_handler)
tests/test_utils.py
+5 -1
@@ -2,6 +2,7 @@ import asyncio
2 import os
3 import socket
4 from typing import AsyncIterator, Tuple
5 +from unittest.mock import patch
6
7 import pytest
8 import torch
@@ -390,7 +391,10 @@ def test_bind_kv_cache_encoder_decoder():
391
392
393 def test_bind_kv_cache_pp():
393 - cfg = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=2))
394 + with patch("vllm.utils.cuda_device_count_stateless", lambda: 2):
395 + # this test runs with 1 GPU, but we simulate 2 GPUs
396 + cfg = VllmConfig(
397 + parallel_config=ParallelConfig(pipeline_parallel_size=2))
398 with set_current_vllm_config(cfg):
399 from vllm.attention import Attention
400
vllm/config.py
+8 -4
@@ -1294,8 +1294,11 @@ class ParallelConfig:
1294 from vllm.executor import ray_utils
1295 backend = "mp"
1296 ray_found = ray_utils.ray_is_available()
1297 - if (current_platform.is_cuda()
1298 - and cuda_device_count_stateless() < self.world_size):
1297 + if current_platform.is_neuron():
1298 + # neuron uses single process to control multiple devices
1299 + backend = "uni"
1300 + elif (current_platform.is_cuda()
1301 + and cuda_device_count_stateless() < self.world_size):
1302 if not ray_found:
1303 raise ValueError("Unable to load Ray which is "
1304 "required for multi-node inference, "
@@ -1328,13 +1331,14 @@ class ParallelConfig:
1331 from vllm.executor.executor_base import ExecutorBase
1332 from vllm.platforms import current_platform
1333 if self.distributed_executor_backend not in (
1331 - "ray", "mp", None) and not (isinstance(
1334 + "ray", "mp", "uni", None) and not (isinstance(
1335 self.distributed_executor_backend, type) and issubclass(
1336 self.distributed_executor_backend, ExecutorBase)):
1337 raise ValueError(
1338 "Unrecognized distributed executor backend "
1339 f"{self.distributed_executor_backend}. Supported "
1337 - "values are 'ray', 'mp' or custom ExecutorBase subclass.")
1340 + "values are 'ray', 'mp' 'uni', or custom ExecutorBase"
1341 + " subclass.")
1342 if self.use_ray:
1343 from vllm.executor import ray_utils
1344 ray_utils.assert_ray_available()
vllm/distributed/parallel_state.py
+4 -2
@@ -862,12 +862,14 @@ def init_model_parallel_group(
862 ) -> GroupCoordinator:
863 if use_custom_allreduce is None:
864 use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
865 + from vllm.platforms import current_platform
866 return GroupCoordinator(
867 group_ranks=group_ranks,
868 local_rank=local_rank,
869 torch_distributed_backend=backend,
869 - use_pynccl=True,
870 - use_custom_allreduce=use_custom_allreduce,
870 + use_pynccl=current_platform.is_cuda_alike(),
871 + use_custom_allreduce=current_platform.is_cuda_alike()
872 + and use_custom_allreduce,
873 use_tpu_communicator=True,
874 use_hpu_communicator=True,
875 use_xpu_communicator=True,
vllm/engine/async_llm_engine.py
+6 -82
@@ -18,9 +18,7 @@ from vllm.engine.async_timeout import asyncio_timeout
18 from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
19 from vllm.engine.metrics_types import StatLoggerBase
20 from vllm.engine.protocol import EngineClient
21 -from vllm.executor.executor_base import ExecutorAsyncBase
22 -from vllm.executor.gpu_executor import GPUExecutorAsync
23 -from vllm.executor.ray_utils import initialize_ray_cluster
21 +from vllm.executor.executor_base import ExecutorBase
22 from vllm.inputs import PromptType
23 from vllm.inputs.preprocess import InputPreprocessor
24 from vllm.logger import init_logger
@@ -620,69 +618,9 @@ class AsyncLLMEngine(EngineClient):
618 rt.new_requests_event.set()
619
620 @classmethod
623 - def _get_executor_cls(
624 - cls, engine_config: VllmConfig) -> Type[ExecutorAsyncBase]:
625 - distributed_executor_backend = (
626 - engine_config.parallel_config.distributed_executor_backend)
627 - if isinstance(distributed_executor_backend, type):
628 - if not issubclass(distributed_executor_backend, ExecutorAsyncBase):
629 - raise TypeError(
630 - "distributed_executor_backend must be a subclass of "
631 - f"ExecutorAsyncBase. Got {distributed_executor_backend}.")
632 - executor_class = distributed_executor_backend
633 - elif engine_config.device_config.device_type == "neuron":
634 - from vllm.executor.neuron_executor import NeuronExecutorAsync
635 - executor_class = NeuronExecutorAsync
636 - elif engine_config.device_config.device_type == "tpu":
637 - if distributed_executor_backend == "ray":
638 - from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync
639 - executor_class = RayTPUExecutorAsync
640 - else:
641 - assert distributed_executor_backend is None
642 - from vllm.executor.tpu_executor import TPUExecutorAsync
643 - executor_class = TPUExecutorAsync
644 - elif engine_config.device_config.device_type == "cpu":
645 - from vllm.executor.cpu_executor import CPUExecutorAsync
646 - executor_class = CPUExecutorAsync
647 - elif engine_config.device_config.device_type == "hpu":
648 - if distributed_executor_backend == "ray":
649 - initialize_ray_cluster(engine_config.parallel_config)
650 - from vllm.executor.ray_hpu_executor import RayHPUExecutorAsync
651 - executor_class = RayHPUExecutorAsync
652 - else:
653 - from vllm.executor.hpu_executor import HPUExecutorAsync
654 - executor_class = HPUExecutorAsync
655 - elif engine_config.device_config.device_type == "openvino":
656 - assert distributed_executor_backend is None, (
657 - "Distributed execution is not supported with "
658 - "the OpenVINO backend.")
659 - from vllm.executor.openvino_executor import OpenVINOExecutorAsync
660 - executor_class = OpenVINOExecutorAsync
661 - elif engine_config.device_config.device_type == "xpu":
662 - if distributed_executor_backend is None:
663 - from vllm.executor.xpu_executor import XPUExecutorAsync
664 - executor_class = XPUExecutorAsync
665 - elif distributed_executor_backend == "ray":
666 - from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
667 - executor_class = RayXPUExecutorAsync
668 - elif distributed_executor_backend == "mp":
669 - from vllm.executor.multiproc_xpu_executor import (
670 - MultiprocessingXPUExecutorAsync)
671 - executor_class = MultiprocessingXPUExecutorAsync
672 - else:
673 - raise RuntimeError(
674 - "Not supported distributed execution model on XPU device.")
675 - elif distributed_executor_backend == "ray":
676 - from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
677 - executor_class = RayGPUExecutorAsync
678 - elif distributed_executor_backend == "mp":
679 - from vllm.executor.multiproc_gpu_executor import (
680 - MultiprocessingGPUExecutorAsync)
681 - executor_class = MultiprocessingGPUExecutorAsync
682 - else:
683 - from vllm.executor.gpu_executor import GPUExecutorAsync
684 - executor_class = GPUExecutorAsync
685 - return executor_class
621 + def _get_executor_cls(cls,
622 + engine_config: VllmConfig) -> Type[ExecutorBase]:
623 + return LLMEngine._get_executor_cls(engine_config)
624
625 @classmethod
626 def from_engine_args(
@@ -700,9 +638,6 @@ class AsyncLLMEngine(EngineClient):
638
639 executor_class = cls._get_executor_cls(engine_config)
640
703 - if executor_class.uses_ray:
704 - initialize_ray_cluster(engine_config.parallel_config)
705 -
641 # Create the async LLM engine.
642 engine = cls(
643 vllm_config=engine_config,
@@ -1242,23 +1177,12 @@ class AsyncLLMEngine(EngineClient):
1177 self.engine.remove_logger(logger_name=logger_name)
1178
1179 async def start_profile(self) -> None:
1245 - # using type instead of isinstance to check to avoid capturing
1246 - # inherited classes
1247 - if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1248 - self.engine.model_executor.start_profile()
1249 - else:
1250 - self.engine.model_executor._run_workers("start_profile")
1180 + self.engine.start_profile()
1181
1182 async def stop_profile(self) -> None:
1253 - # using type instead of isinstance to check to avoid capturing
1254 - # inherited classes
1255 - if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1256 - self.engine.model_executor.stop_profile()
1257 - else:
1258 - self.engine.model_executor._run_workers("stop_profile")
1183 + self.engine.stop_profile()
1184
1185 async def add_lora(self, lora_request: LoRARequest) -> None:
1261 - """Load a new LoRA adapter into the engine for future requests."""
1186 self.engine.add_lora(lora_request)
1187
1188
vllm/engine/llm_engine.py
+22 -72
@@ -28,8 +28,6 @@ from vllm.engine.output_processor.util import create_output_by_sequence_group
28 from vllm.entrypoints.openai.logits_processors import (
29 get_logits_processors as get_openai_logits_processors)
30 from vllm.executor.executor_base import ExecutorBase
31 -from vllm.executor.gpu_executor import GPUExecutor
32 -from vllm.executor.ray_utils import initialize_ray_cluster
31 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
32 PromptType, SingletonInputsAdapter)
33 from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
@@ -442,64 +440,26 @@ class LLMEngine:
440 raise TypeError(
441 "distributed_executor_backend must be a subclass of "
442 f"ExecutorBase. Got {distributed_executor_backend}.")
445 - if distributed_executor_backend.uses_ray: # type: ignore
446 - initialize_ray_cluster(engine_config.parallel_config)
443 executor_class = distributed_executor_backend
448 - elif engine_config.device_config.device_type == "neuron":
449 - from vllm.executor.neuron_executor import NeuronExecutor
450 - executor_class = NeuronExecutor
451 - elif engine_config.device_config.device_type == "tpu":
444 + elif engine_config.parallel_config.world_size > 1:
445 if distributed_executor_backend == "ray":
453 - initialize_ray_cluster(engine_config.parallel_config)
454 - from vllm.executor.ray_tpu_executor import RayTPUExecutor
455 - executor_class = RayTPUExecutor
456 - else:
457 - assert distributed_executor_backend is None
458 - from vllm.executor.tpu_executor import TPUExecutor
459 - executor_class = TPUExecutor
460 - elif engine_config.device_config.device_type == "cpu":
461 - from vllm.executor.cpu_executor import CPUExecutor
462 - executor_class = CPUExecutor
463 - elif engine_config.device_config.device_type == "hpu":
464 - if distributed_executor_backend == "ray":
465 - initialize_ray_cluster(engine_config.parallel_config)
466 - from vllm.executor.ray_hpu_executor import RayHPUExecutor
467 - executor_class = RayHPUExecutor
468 - else:
469 - from vllm.executor.hpu_executor import HPUExecutor
470 - executor_class = HPUExecutor
471 - elif engine_config.device_config.device_type == "openvino":
472 - from vllm.executor.openvino_executor import OpenVINOExecutor
473 - executor_class = OpenVINOExecutor
474 - elif engine_config.device_config.device_type == "xpu":
475 - if distributed_executor_backend == "ray":
476 - initialize_ray_cluster(engine_config.parallel_config)
477 - from vllm.executor.ray_xpu_executor import RayXPUExecutor
478 - executor_class = RayXPUExecutor
446 + from vllm.executor.ray_distributed_executor import (
447 + RayDistributedExecutor)
448 + executor_class = RayDistributedExecutor
449 elif distributed_executor_backend == "mp":
480 - # FIXME(kunshang):
481 - # spawn needs calling `if __name__ == '__main__':``
482 - # fork is not supported for xpu start new process.
483 - logger.error(
484 - "Both start methods (spawn and fork) have issue "
485 - "on XPU if you use mp backend, Please try ray instead.")
486 - else:
487 - from vllm.executor.xpu_executor import XPUExecutor
488 - executor_class = XPUExecutor
489 - elif distributed_executor_backend == "ray":
490 - initialize_ray_cluster(engine_config.parallel_config)
491 - from vllm.executor.ray_gpu_executor import RayGPUExecutor
492 - executor_class = RayGPUExecutor
493 - elif distributed_executor_backend == "mp":
494 - from vllm.executor.multiproc_gpu_executor import (
495 - MultiprocessingGPUExecutor)
496 - assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
497 - "multiprocessing distributed executor backend does not "
498 - "support VLLM_USE_RAY_SPMD_WORKER=1")
499 - executor_class = MultiprocessingGPUExecutor
450 + from vllm.executor.mp_distributed_executor import (
451 + MultiprocessingDistributedExecutor)
452 + assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
453 + "multiprocessing distributed executor backend does not "
454 + "support VLLM_USE_RAY_SPMD_WORKER=1")
455 + executor_class = MultiprocessingDistributedExecutor
456 + elif distributed_executor_backend == "uni":
457 + # JAX-style, single-process, multi-device executor.
458 + from vllm.executor.uniproc_executor import UniProcExecutor
459 + executor_class = UniProcExecutor
460 else:
501 - from vllm.executor.gpu_executor import GPUExecutor
502 - executor_class = GPUExecutor
461 + from vllm.executor.uniproc_executor import UniProcExecutor
462 + executor_class = UniProcExecutor
463 return executor_class
464
465 @classmethod
@@ -1845,27 +1805,17 @@ class LLMEngine:
1805 def list_prompt_adapters(self) -> List[int]:
1806 return self.model_executor.list_prompt_adapters()
1807
1808 + def start_profile(self) -> None:
1809 + self.model_executor.start_profile()
1810 +
1811 + def stop_profile(self) -> None:
1812 + self.model_executor.stop_profile()
1813 +
1814 def check_health(self) -> None:
1815 if self.tokenizer:
1816 self.tokenizer.check_health()
1817 self.model_executor.check_health()
1818
1853 - def start_profile(self) -> None:
1854 - # using type instead of isinstance to check to avoid capturing
1855 - # inherited classes (MultiprocessingGPUExecutor)
1856 - if type(self.model_executor) == GPUExecutor: # noqa: E721
1857 - self.model_executor.start_profile()
1858 - else:
1859 - self.model_executor._run_workers("start_profile")
1860 -
1861 - def stop_profile(self) -> None:
1862 - # using type instead of isinstance to check to avoid capturing
1863 - # inherited classes (MultiprocessingGPUExecutor)
1864 - if type(self.model_executor) == GPUExecutor: # noqa: E721
1865 - self.model_executor.stop_profile()
1866 - else:
1867 - self.model_executor._run_workers("stop_profile")
1868 -
1819 def is_tracing_enabled(self) -> bool:
1820 return self.tracer is not None
1821
vllm/engine/multiprocessing/engine.py
+2 -9
@@ -20,7 +20,6 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
20 RPCStartupResponse,
21 RPCUProfileRequest)
22 # yapf: enable
23 -from vllm.executor.gpu_executor import GPUExecutor
23 from vllm.logger import init_logger
24 from vllm.outputs import RequestOutput
25 from vllm.usage.usage_lib import UsageContext
@@ -356,16 +355,10 @@ class MQLLMEngine:
355 self._errored_with = e
356
357 def start_profile(self) -> None:
359 - if type(self.engine.model_executor) is GPUExecutor:
360 - self.engine.model_executor.start_profile()
361 - else:
362 - self.engine.model_executor._run_workers("start_profile")
358 + self.engine.start_profile()
359
360 def stop_profile(self) -> None:
365 - if type(self.engine.model_executor) is GPUExecutor:
366 - self.engine.model_executor.stop_profile()
367 - else:
368 - self.engine.model_executor._run_workers("stop_profile")
361 + self.engine.stop_profile()
362
363
364 def signal_handler(*_) -> None:
vllm/executor/cpu_executor.py deleted
-299
@@ -1,299 +0,0 @@
1 -import os
2 -from functools import partial
3 -from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
4 -
5 -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 -from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
7 - ResultHandler, WorkerMonitor)
8 -from vllm.logger import init_logger
9 -from vllm.lora.request import LoRARequest
10 -from vllm.model_executor.layers.sampler import SamplerOutput
11 -from vllm.prompt_adapter.request import PromptAdapterRequest
12 -from vllm.sequence import ExecuteModelRequest
13 -from vllm.utils import get_distributed_init_method, get_open_port, make_async
14 -from vllm.worker.worker_base import WorkerWrapperBase
15 -
16 -logger = init_logger(__name__)
17 -
18 -
19 -class CPUExecutor(ExecutorBase):
20 -
21 - uses_ray: bool = False
22 -
23 - def _init_executor(self) -> None:
24 - assert self.device_config.device_type == "cpu"
25 -
26 - #
27 - # Environment variables for CPU executor
28 - #
29 -
30 - # Disable torch async compiling which won't work with daemonic processes
31 - os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
32 -
33 - # Intel OpenMP setting
34 - ld_prealod_str = os.getenv("LD_PRELOAD", "")
35 - if "libiomp5.so" in ld_prealod_str:
36 - # The time(milliseconds) that a thread should wait after
37 - # completing the execution of a parallel region, before sleeping.
38 - os.environ['KMP_BLOCKTIME'] = "1"
39 - # Prevents the CPU to run into low performance state
40 - os.environ['KMP_TPAUSE'] = "0"
41 - # Provides fine granularity parallelism
42 - os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
43 - os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
44 - os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
45 -
46 - # To hint IPEX uses shared memory based AllReduce
47 - os.environ["LOCAL_WORLD_SIZE"] = str(
48 - self.parallel_config.tensor_parallel_size)
49 -
50 - # Multiprocessing-based executor does not support multi-node setting.
51 - # Since it only works for single node, we can use the loopback address
52 - # 127.0.0.1 for communication.
53 - ip = "127.0.0.1"
54 - port = get_open_port()
55 - self.distributed_init_method = get_distributed_init_method(ip, port)
56 -
57 - is_async = isinstance(self, CPUExecutorAsync)
58 -
59 - world_size = self.parallel_config.tensor_parallel_size
60 - result_handler = ResultHandler()
61 - self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
62 - self.workers = []
63 -
64 - if is_async:
65 - self.workers = [
66 - ProcessWorkerWrapper(
67 - result_handler,
68 - partial(
69 - self._create_worker,
70 - rank=rank,
71 - local_rank=rank,
72 - )) for rank in range(0, world_size)
73 - ]
74 - self.driver_worker = self.workers[0]
75 - self.workers = self.workers[1:]
76 - self.driver_method_invoker = _async_driver_method_invoker
77 - else:
78 - self.driver_worker = self._create_worker()
79 - self.driver_method_invoker = _driver_method_invoker
80 -
81 - if world_size != 1:
82 - self.workers = [
83 - ProcessWorkerWrapper(
84 - result_handler,
85 - partial(
86 - self._create_worker,
87 - rank=rank,
88 - local_rank=rank,
89 - )) for rank in range(1, world_size)
90 - ]
91 -
92 - self.worker_monitor = None
93 - if world_size != 1 or is_async:
94 - if is_async:
95 - async_worker_list = self.workers + [self.driver_worker]
96 - else:
97 - async_worker_list = self.workers
98 - self.worker_monitor = WorkerMonitor(async_worker_list,
99 - result_handler)
100 - result_handler.start()
101 - self.worker_monitor.start()
102 -
103 - self._run_workers("init_device")
104 - self._run_workers("load_model")
105 -
106 - def _create_worker(
107 - self,
108 - local_rank: int = 0,
109 - rank: int = 0,
110 - ):
111 -
112 - wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
113 -
114 - assert self.distributed_init_method is not None
115 -
116 - kwargs = dict(
117 - vllm_config=self.vllm_config,
118 - local_rank=local_rank,
119 - rank=rank,
120 - distributed_init_method=self.distributed_init_method,
121 - kv_cache_dtype=self.cache_config.cache_dtype,
122 - is_driver_worker=rank == 0,
123 - )
124 - wrapper.init_worker(**kwargs)
125 -
126 - return wrapper.worker
127 -
128 - def _run_workers(
129 - self,
130 - method: str,
131 - *args,
132 - async_run_remote_workers_only: bool = False,
133 - max_concurrent_workers: Optional[int] = None,
134 - **kwargs,
135 - ) -> Any:
136 - """Runs the given method on all workers.
137 -
138 - Args:
139 - async_run_remote_workers_only: If True the method will be run only
140 - in the remote workers, not the driver worker. It will also be
141 - run asynchronously and return a list of futures rather than
142 - blocking on the results.
143 - """
144 -
145 - if max_concurrent_workers:
146 - raise NotImplementedError(
147 - "max_concurrent_workers is not supported yet.")
148 -
149 - # Start the workers first.
150 - worker_outputs = [
151 - worker.execute_method(method, *args, **kwargs)
152 - for worker in self.workers
153 - ]
154 -
155 - if async_run_remote_workers_only:
156 - # Just return futures
157 - return worker_outputs
158 -
159 - driver_worker_output = self.driver_method_invoker(
160 - self.driver_worker, method, *args, **kwargs)
161 -
162 - # Get the results of the workers.
163 - return [driver_worker_output
164 - ] + [output.get() for output in worker_outputs]
165 -
166 - def determine_num_available_blocks(self) -> Tuple[int, int]:
167 - """Determine the number of available KV blocks by invoking the
168 - underlying worker.
169 - """
170 - return self.driver_method_invoker(self.driver_worker,
171 - "determine_num_available_blocks")
172 -
173 - def initialize_cache(self, num_gpu_blocks: int,
174 - num_cpu_blocks: int) -> None:
175 - """Initialize the KV cache by invoking the underlying worker.
176 - """
177 - # NOTE: We log here to avoid multiple logs when number of workers is
178 - # greater than one. We could log in the engine, but not all executors
179 - # have GPUs.
180 - # NOTE: `cpu block` for CPU backend is located on CPU memory but is
181 - # referred as `gpu block`. Because we want to reuse the existing block
182 - # management procedure.
183 - logger.info("# CPU blocks: %d", num_gpu_blocks)
184 -
185 - self._run_workers("initialize_cache",
186 - num_gpu_blocks=num_gpu_blocks,
187 - num_cpu_blocks=num_cpu_blocks)
188 -
189 - def execute_model(
190 - self,
191 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
192 - if (self.parallel_config.tensor_parallel_size > 1
193 - and self.parallel_worker_tasks is None):
194 - self.parallel_worker_tasks = self._run_workers(
195 - "start_worker_execution_loop",
196 - async_run_remote_workers_only=True,
197 - )
198 - output = self.driver_method_invoker(self.driver_worker,
199 - "execute_model", execute_model_req)
200 - return output
201 -
202 - def stop_remote_worker_execution_loop(self) -> None:
203 - if self.parallel_worker_tasks is None:
204 - return
205 - """
206 - Passing None will cause the driver to stop the model execution
207 - loop running in each of the remote workers.
208 - """
209 - self.driver_method_invoker(self.driver_worker, "execute_model", None)
210 - parallel_worker_tasks = self.parallel_worker_tasks
211 - self.parallel_worker_tasks = None
212 - # Ensure that workers exit model loop cleanly
213 - # (this will raise otherwise)
214 - self._wait_for_tasks_completion(parallel_worker_tasks)
215 -
216 - def add_lora(self, lora_request: LoRARequest) -> bool:
217 - return all(self._run_workers("add_lora", lora_request))
218 -
219 - def remove_lora(self, lora_id: int) -> bool:
220 - return all(self._run_workers("remove_lora", lora_id))
221 -
222 - def pin_lora(self, lora_id: int) -> bool:
223 - assert lora_id > 0, "lora_id must be greater than 0."
224 - return all(self._run_workers(
225 - "pin_lora",
226 - lora_id=lora_id,
227 - ))
228 -
229 - def list_loras(self) -> Set[int]:
230 - return self.driver_method_invoker(self.driver_worker, "list_loras")
231 -
232 - def add_prompt_adapter(
233 - self, prompt_adapter_request: PromptAdapterRequest) -> bool:
234 - return all(
235 - self._run_workers(
236 - "add_prompt_adapter",
237 - prompt_adapter_request,
238 - ))
239 -
240 - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
241 - return all(
242 - self._run_workers(
243 - "remove_prompt_adapter",
244 - prompt_adapter_id,
245 - ))
246 -
247 - def list_prompt_adapters(self) -> Set[int]:
248 - return self.driver_method_invoker(self.driver_worker,
249 - "list_prompt_adapters")
250 -
251 - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
252 - return all(self._run_workers(
253 - "pin_prompt_adapter",
254 - prompt_adapter_id,
255 - ))
256 -
257 - def check_health(self) -> None:
258 - """Raises an error if engine is unhealthy."""
259 - if self.worker_monitor is not None and not self.worker_monitor.is_alive(
260 - ):
261 - raise RuntimeError("Worker processes are not running")
262 -
263 - def shutdown(self):
264 - if (worker_monitor := getattr(self, "worker_monitor",
265 - None)) is not None:
266 - worker_monitor.close()
267 -
268 - def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
269 - """Wait for futures returned from _run_workers() with
270 - async_run_remote_workers_only to complete."""
271 - for result in parallel_worker_tasks:
272 - result.get()
273 -
274 - def start_profile(self) -> None:
275 - self.driver_method_invoker(self.driver_worker, "start_profile")
276 -
277 - def stop_profile(self) -> None:
278 - self.driver_method_invoker(self.driver_worker, "stop_profile")
279 -
280 -
281 -class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
282 -
283 - async def execute_model_async(
284 - self,
285 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
286 - output = await make_async(self.execute_model
287 - )(execute_model_req=execute_model_req, )
288 - return output
289 -
290 - async def check_health_async(self) -> None:
291 - self.check_health()
292 -
293 -
294 -def _driver_method_invoker(driver, method: str, *args, **kwargs):
295 - return getattr(driver, method)(*args, **kwargs)
296 -
297 -
298 -def _async_driver_method_invoker(driver, method: str, *args, **kwargs):
299 - return driver.execute_method(method, *args, **kwargs).get()
vllm/executor/distributed_gpu_executor.py deleted
-212
@@ -1,212 +0,0 @@
1 -import asyncio
2 -from abc import abstractmethod
3 -from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
4 -
5 -from vllm.executor.executor_base import ExecutorAsyncBase
6 -from vllm.executor.gpu_executor import GPUExecutor
7 -from vllm.logger import init_logger
8 -from vllm.lora.request import LoRARequest
9 -from vllm.model_executor.layers.sampler import SamplerOutput
10 -from vllm.sequence import ExecuteModelRequest
11 -
12 -logger = init_logger(__name__)
13 -
14 -
15 -class DistributedGPUExecutor(GPUExecutor):
16 - """Abstract superclass of multi-GPU executor implementations."""
17 -
18 - def __init__(self, *args, **kwargs):
19 - # This is non-None when the execute model loop is running
20 - # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
21 - self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
22 - # Updated by implementations that require additional args to be passed
23 - # to the _run_workers execute_model call
24 - self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
25 -
26 - super().__init__(*args, **kwargs)
27 -
28 - def determine_num_available_blocks(self) -> Tuple[int, int]:
29 - """Determine the number of available KV blocks.
30 -
31 - This invokes `determine_num_available_blocks` on each worker and takes
32 - the min of the results, guaranteeing that the selected cache sizes are
33 - compatible with all workers.
34 -
35 - Returns:
36 - - tuple[num_gpu_blocks, num_cpu_blocks]
37 - """
38 - # Get the maximum number of blocks that can be allocated on GPU and CPU.
39 - num_blocks = self._run_workers("determine_num_available_blocks", )
40 -
41 - # Since we use a shared centralized controller, we take the minimum
42 - # number of blocks across all workers to make sure all the memory
43 - # operators can be applied to all workers.
44 - num_gpu_blocks = min(b[0] for b in num_blocks)
45 - num_cpu_blocks = min(b[1] for b in num_blocks)
46 -
47 - return num_gpu_blocks, num_cpu_blocks
48 -
49 - def initialize_cache(self, num_gpu_blocks: int,
50 - num_cpu_blocks: int) -> None:
51 - """Initialize the KV cache in all workers.
52 - """
53 -
54 - # NOTE: We log here to avoid multiple logs when number of workers is
55 - # greater than one. We could log in the engine, but not all executors
56 - # have GPUs.
57 - logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
58 - num_cpu_blocks)
59 - max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
60 - self.model_config.max_model_len)
61 - logger.info("Maximum concurrency for %s tokens per request: %.2fx",
62 - self.model_config.max_model_len, max_concurrency)
63 -
64 - self.cache_config.num_gpu_blocks = num_gpu_blocks
65 - self.cache_config.num_cpu_blocks = num_cpu_blocks
66 -
67 - self._run_workers("initialize_cache",
68 - num_gpu_blocks=num_gpu_blocks,
69 - num_cpu_blocks=num_cpu_blocks)
70 -
71 - def execute_model(
72 - self,
73 - execute_model_req: ExecuteModelRequest,
74 - ) -> List[SamplerOutput]:
75 - if self.parallel_worker_tasks is None:
76 - self.parallel_worker_tasks = self._run_workers(
77 - "start_worker_execution_loop",
78 - async_run_tensor_parallel_workers_only=True,
79 - **self.extra_execute_model_run_workers_kwargs)
80 -
81 - # Only the driver worker returns the sampling results.
82 - driver_outputs = self._driver_execute_model(execute_model_req)
83 - assert driver_outputs is not None
84 - return driver_outputs
85 -
86 - def stop_remote_worker_execution_loop(self) -> None:
87 - if self.parallel_worker_tasks is None:
88 - return
89 -
90 - self._driver_execute_model(execute_model_req=None)
91 - parallel_worker_tasks = self.parallel_worker_tasks
92 - self.parallel_worker_tasks = None
93 - # Ensure that workers exit model loop cleanly
94 - # (this will raise otherwise)
95 - self._wait_for_tasks_completion(parallel_worker_tasks)
96 -
97 - def add_lora(self, lora_request: LoRARequest) -> bool:
98 - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
99 - return self._run_workers(
100 - "add_lora",
101 - lora_request=lora_request,
102 - )
103 -
104 - def remove_lora(self, lora_id: int) -> bool:
105 - assert lora_id > 0, "lora_id must be greater than 0."
106 - return self._run_workers(
107 - "remove_lora",
108 - lora_id=lora_id,
109 - )
110 -
111 - def pin_lora(self, lora_id: int) -> bool:
112 - assert lora_id > 0, "lora_id must be greater than 0."
113 - return self._run_workers(
114 - "pin_lora",
115 - lora_id=lora_id,
116 - )
117 -
118 - def list_loras(self) -> Set[int]:
119 - return self._run_workers("list_loras")
120 -
121 - def save_sharded_state(
122 - self,
123 - path: str,
124 - pattern: Optional[str] = None,
125 - max_size: Optional[int] = None,
126 - ) -> None:
127 - self._run_workers("save_sharded_state",
128 - path=path,
129 - pattern=pattern,
130 - max_size=max_size)
131 -
132 - @abstractmethod
133 - def _driver_execute_model(
134 - self, execute_model_req: Optional[ExecuteModelRequest]
135 - ) -> Optional[List[SamplerOutput]]:
136 - """Run execute_model in the driver worker.
137 -
138 - Passing None will cause the driver to stop the model execution loop
139 - running in each of the remote workers. In this case, this method
140 - returns None. Otherwise, this method returns the model output.
141 - """
142 - raise NotImplementedError
143 -
144 - @abstractmethod
145 - def _run_workers(
146 - self,
147 - method: str,
148 - *args,
149 - async_run_tensor_parallel_workers_only: bool = False,
150 - max_concurrent_workers: Optional[int] = None,
151 - **kwargs,
152 - ) -> Any:
153 - """Runs the given method on all workers.
154 -
155 - Args:
156 - async_run_tensor_parallel_workers_only: If True the method will be
157 - run only in the remote TP workers, not the driver worker.
158 - It will also be run asynchronously and return a list of futures
159 - rather than blocking on the results.
160 - """
161 - raise NotImplementedError
162 -
163 - @abstractmethod
164 - def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
165 - """Wait for futures returned from _run_workers() with
166 - async_run_remote_workers_only to complete."""
167 - raise NotImplementedError
168 -
169 -
170 -class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
171 -
172 - async def execute_model_async(
173 - self,
174 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
175 - if self.parallel_worker_tasks is None:
176 - # Start model execution loop running in the parallel workers
177 - self.parallel_worker_tasks = asyncio.create_task(
178 - self._start_worker_execution_loop())
179 -
180 - # Only the driver worker returns the sampling results.
181 - return await self._driver_execute_model_async(execute_model_req)
182 -
183 - async def stop_remote_worker_execution_loop_async(self) -> None:
184 - if self.parallel_worker_tasks is None:
185 - return
186 -
187 - await self._driver_execute_model_async()
188 - parallel_worker_tasks = self.parallel_worker_tasks
189 - self.parallel_worker_tasks = None
190 - # Ensure that workers exit model loop cleanly
191 - # (this will raise otherwise)
192 - await parallel_worker_tasks
193 -
194 - @abstractmethod
195 - async def _driver_execute_model_async(
196 - self,
197 - execute_model_req: Optional[ExecuteModelRequest] = None,
198 - ) -> List[SamplerOutput]:
199 - """Execute the model asynchronously in the driver worker.
200 -
201 - Passing None will cause the driver to stop the model execution
202 - loop running in each of the remote workers.
203 - """
204 - raise NotImplementedError
205 -
206 - @abstractmethod
207 - async def _start_worker_execution_loop(self):
208 - """Run execution loop on all workers. It guarantees all workers run
209 - the loop or None of them is running the loop. Loop can be stopped by
210 - `stop_remote_worker_execution_loop`.
211 - The API is idempotent (guarantee only 1 loop run at any moment)."""
212 - raise NotImplementedError
vllm/executor/executor_base.py
+232 -35
@@ -1,18 +1,24 @@
1 +import asyncio
2 from abc import ABC, abstractmethod
2 -from typing import List, Optional, Set, Tuple
3 +from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
4
5 from vllm.config import VllmConfig
6 +from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 +from vllm.platforms import current_platform
10 from vllm.prompt_adapter.request import PromptAdapterRequest
8 -from vllm.sequence import ExecuteModelRequest
11 +from vllm.sequence import ExecuteModelRequest, PoolerOutput
12 +from vllm.utils import make_async
13 +
14 +logger = init_logger(__name__)
15
16
17 class ExecutorBase(ABC):
18 """Base class for all executors.
19
14 - An executor is responsible for executing the model on a specific device
15 - type (e.g., CPU, GPU, Neuron, etc.). Or it can be a distributed executor
20 + An executor is responsible for executing the model on one device,
21 + or it can be a distributed executor
22 that can execute the model on multiple devices.
23 """
24
@@ -40,6 +46,20 @@ class ExecutorBase(ABC):
46 pass
47
48 @abstractmethod
49 + def collective_rpc(self,
50 + method: str,
51 + timeout: Optional[float] = None,
52 + args: Tuple = (),
53 + kwargs: Optional[Dict] = None) -> List[Any]:
54 + """
55 + The main interface of the executor to run a method on all workers,
56 + with homogeneous arguments.
57 + If the args are heterogeneous, then we can pack them into a list,
58 + and unpack them in the method of every worker, because every worker
59 + knows their own rank.
60 + """
61 + pass
62 +
63 def determine_num_available_blocks(self) -> Tuple[int, int]:
64 """Determine the number of available blocks for the GPU KV cache and
65 swappable CPU KV cache.
@@ -53,58 +73,113 @@ class ExecutorBase(ABC):
73 num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
74 appended to.
75 """
56 - raise NotImplementedError
76 + results = self.collective_rpc("determine_num_available_blocks")
77 + a = min([r[0] for r in results])
78 + b = min([r[1] for r in results])
79 + return a, b
80
58 - @abstractmethod
59 - def initialize_cache(self, num_gpu_blocks: int,
60 - num_cpu_blocks: int) -> None:
61 - """Initialize the KV cache with the given size in blocks.
81 + def initialize(self, num_gpu_blocks: int) -> None:
82 """
63 - raise NotImplementedError
83 + Initialize the KV caches and begin the model execution loop of the
84 + underlying workers.
85 + For V1 compatibility.
86 + """
87 + logger.info("# GPU blocks: %d", num_gpu_blocks)
88 + self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
89 + self.collective_rpc("compile_or_warm_up_model")
90 +
91 + def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
92 + """Initialize the KV cache by invoking the underlying worker.
93 + """
94 + # NOTE: This is logged in the executor because there can be >1 workers.
95 + logger.info("# %s blocks: %d, # CPU blocks: %d",
96 + current_platform.dispatch_key, num_gpu_blocks,
97 + num_cpu_blocks)
98 + max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
99 + self.model_config.max_model_len)
100 + logger.info("Maximum concurrency for %s tokens per request: %.2fx",
101 + self.model_config.max_model_len, max_concurrency)
102 +
103 + self.cache_config.num_gpu_blocks = num_gpu_blocks
104 + self.cache_config.num_cpu_blocks = num_cpu_blocks
105 +
106 + self.collective_rpc("initialize_cache",
107 + args=(num_gpu_blocks, num_cpu_blocks))
108
65 - @abstractmethod
109 def execute_model(
110 self, execute_model_req: ExecuteModelRequest
68 - ) -> Optional[List[SamplerOutput]]:
69 - """Executes at least one model step on the given sequences."""
70 - raise NotImplementedError
111 + ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
112 + output = self.collective_rpc("execute_model",
113 + args=(execute_model_req, ))
114 + return output[0]
115
116 def stop_remote_worker_execution_loop(self) -> None:
117 """Releases parallel workers from model loop."""
118 return
119
76 - @abstractmethod
120 def add_lora(self, lora_request: LoRARequest) -> bool:
78 - raise NotImplementedError
121 + assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
122 + return all(self.collective_rpc("add_lora", args=(lora_request, )))
123
80 - @abstractmethod
124 def remove_lora(self, lora_id: int) -> bool:
82 - raise NotImplementedError
125 + assert lora_id > 0, "lora_id must be greater than 0."
126 + return all(self.collective_rpc("remove_lora", args=(lora_id, )))
127
84 - @abstractmethod
128 def pin_lora(self, lora_id: int) -> bool:
86 - raise NotImplementedError # type: ignore
129 + assert lora_id > 0, "lora_id must be greater than 0."
130 + return all(self.collective_rpc("pin_lora", args=(lora_id, )))
131
88 - @abstractmethod
132 def list_loras(self) -> Set[int]:
90 - raise NotImplementedError
133 + sets = self.collective_rpc("list_loras")
134 + for s in sets:
135 + assert s == sets[0], "All workers should have the same LORAs."
136 + return sets[0]
137
92 - @abstractmethod
138 def add_prompt_adapter(
139 self, prompt_adapter_request: PromptAdapterRequest) -> bool:
95 - raise NotImplementedError
140 + assert prompt_adapter_request.prompt_adapter_id > 0, \
141 + "prompt_adapter_id must be greater than 0."
142 + return all(
143 + self.collective_rpc("add_prompt_adapter",
144 + args=(prompt_adapter_request, )))
145
97 - @abstractmethod
146 def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
99 - raise NotImplementedError
147 + assert prompt_adapter_id > 0, \
148 + "prompt_adapter_id must be greater than 0."
149 + return all(
150 + self.collective_rpc("remove_prompt_adapter",
151 + args=(prompt_adapter_id, )))
152
101 - @abstractmethod
153 def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
103 - raise NotImplementedError # type: ignore
154 + assert prompt_adapter_id > 0, \
155 + "prompt_adapter_id must be greater than 0."
156 + return all(
157 + self.collective_rpc("pin_prompt_adapter",
158 + args=(prompt_adapter_id, )))
159
105 - @abstractmethod
160 def list_prompt_adapters(self) -> Set[int]:
107 - raise NotImplementedError
161 + sets = self.collective_rpc("list_prompt_adapters")
162 + for s in sets:
163 + assert (s == sets[0]
164 + ), "All workers should have the same prompt adapters."
165 + return sets[0]
166 +
167 + def start_profile(self) -> None:
168 + self.collective_rpc("start_profile")
169 +
170 + def stop_profile(self) -> None:
171 + self.collective_rpc("stop_profile")
172 +
173 + def save_sharded_state(
174 + self,
175 + path: str,
176 + pattern: Optional[str] = None,
177 + max_size: Optional[int] = None,
178 + ) -> None:
179 + self.collective_rpc("save_sharded_state",
180 + kwargs=dict(path=path,
181 + pattern=pattern,
182 + max_size=max_size))
183
184 @abstractmethod
185 def check_health(self) -> None:
@@ -119,15 +194,12 @@ class ExecutorBase(ABC):
194 def __del__(self):
195 self.shutdown()
196
122 -
123 -class ExecutorAsyncBase(ExecutorBase):
124 -
125 - @abstractmethod
197 async def execute_model_async(
198 self,
199 execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
200 """Executes one model step on the given sequences."""
130 - raise NotImplementedError
201 + output = await make_async(self.execute_model)(execute_model_req)
202 + return output
203
204 async def stop_remote_worker_execution_loop_async(self) -> None:
205 """Releases parallel workers from model loop."""
@@ -137,3 +209,128 @@ class ExecutorAsyncBase(ExecutorBase):
209 """Checks if the executor is healthy. If not, it should raise an
210 exception."""
211 self.check_health()
212 +
213 +
214 +class DistributedExecutorBase(ExecutorBase):
215 + """Abstract superclass of distributed executor implementations."""
216 +
217 + def __init__(self, *args, **kwargs):
218 + # This is non-None when the execute model loop is running
219 + # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
220 + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
221 +
222 + super().__init__(*args, **kwargs)
223 +
224 + def execute_model(
225 + self,
226 + execute_model_req: ExecuteModelRequest,
227 + ) -> List[SamplerOutput]:
228 + # TODO: unify into collective_rpc
229 + if self.parallel_worker_tasks is None:
230 + self.parallel_worker_tasks = self._run_workers(
231 + "start_worker_execution_loop",
232 + async_run_tensor_parallel_workers_only=True)
233 +
234 + # Only the driver worker returns the sampling results.
235 + driver_outputs = self._driver_execute_model(execute_model_req)
236 + assert driver_outputs is not None
237 + return driver_outputs
238 +
239 + def stop_remote_worker_execution_loop(self) -> None:
240 + if self.parallel_worker_tasks is None:
241 + return
242 +
243 + self._driver_execute_model(execute_model_req=None)
244 + parallel_worker_tasks = self.parallel_worker_tasks
245 + self.parallel_worker_tasks = None
246 + # Ensure that workers exit model loop cleanly
247 + # (this will raise otherwise)
248 + self._wait_for_tasks_completion(parallel_worker_tasks)
249 +
250 + @abstractmethod
251 + def _driver_execute_model(
252 + self, execute_model_req: Optional[ExecuteModelRequest]
253 + ) -> Optional[List[SamplerOutput]]:
254 + """Run execute_model in the driver worker.
255 +
256 + Passing None will cause the driver to stop the model execution loop
257 + running in each of the remote workers. In this case, this method
258 + returns None. Otherwise, this method returns the model output.
259 + """
260 + raise NotImplementedError
261 +
262 + def collective_rpc(self,
263 + method: str,
264 + timeout: Optional[float] = None,
265 + args: Tuple = (),
266 + kwargs: Optional[Dict] = None) -> List[Any]:
267 + return self._run_workers(method, *args, **(kwargs or {}))
268 +
269 + @abstractmethod
270 + def _run_workers(
271 + self,
272 + method: str,
273 + *args,
274 + async_run_tensor_parallel_workers_only: bool = False,
275 + max_concurrent_workers: Optional[int] = None,
276 + **kwargs,
277 + ) -> Any:
278 + """Runs the given method on all workers.
279 +
280 + Args:
281 + async_run_tensor_parallel_workers_only: If True the method will be
282 + run only in the remote TP workers, not the driver worker.
283 + It will also be run asynchronously and return a list of futures
284 + rather than blocking on the results.
285 +
286 + # TODO: simplify and merge with collective_rpc
287 + """
288 + raise NotImplementedError
289 +
290 + @abstractmethod
291 + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
292 + """Wait for futures returned from _run_workers() with
293 + async_run_remote_workers_only to complete."""
294 + raise NotImplementedError
295 +
296 + async def execute_model_async(
297 + self,
298 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
299 + if self.parallel_worker_tasks is None:
300 + # Start model execution loop running in the parallel workers
301 + self.parallel_worker_tasks = asyncio.create_task(
302 + self._start_worker_execution_loop())
303 +
304 + # Only the driver worker returns the sampling results.
305 + return await self._driver_execute_model_async(execute_model_req)
306 +
307 + async def stop_remote_worker_execution_loop_async(self) -> None:
308 + if self.parallel_worker_tasks is None:
309 + return
310 +
311 + await self._driver_execute_model_async()
312 + parallel_worker_tasks = self.parallel_worker_tasks
313 + self.parallel_worker_tasks = None
314 + # Ensure that workers exit model loop cleanly
315 + # (this will raise otherwise)
316 + await parallel_worker_tasks
317 +
318 + @abstractmethod
319 + async def _driver_execute_model_async(
320 + self,
321 + execute_model_req: Optional[ExecuteModelRequest] = None,
322 + ) -> List[SamplerOutput]:
323 + """Execute the model asynchronously in the driver worker.
324 +
325 + Passing None will cause the driver to stop the model execution
326 + loop running in each of the remote workers.
327 + """
328 + raise NotImplementedError
329 +
330 + @abstractmethod
331 + async def _start_worker_execution_loop(self):
332 + """Run execution loop on all workers. It guarantees all workers run
333 + the loop or None of them is running the loop. Loop can be stopped by
334 + `stop_remote_worker_execution_loop`.
335 + The API is idempotent (guarantee only 1 loop run at any moment)."""
336 + raise NotImplementedError
vllm/executor/gpu_executor.py deleted
-145
@@ -1,145 +0,0 @@
1 -from typing import Any, Dict, List, Optional, Set, Tuple, Union
2 -
3 -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 -from vllm.logger import init_logger
5 -from vllm.lora.request import LoRARequest
6 -from vllm.model_executor.layers.sampler import SamplerOutput
7 -from vllm.prompt_adapter.request import PromptAdapterRequest
8 -from vllm.sequence import ExecuteModelRequest, PoolerOutput
9 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10 - make_async)
11 -from vllm.worker.worker_base import WorkerWrapperBase
12 -
13 -logger = init_logger(__name__)
14 -
15 -
16 -def create_worker(**kwargs):
17 - vllm_config = kwargs.get("vllm_config")
18 - wrapper = WorkerWrapperBase(vllm_config=vllm_config)
19 - wrapper.init_worker(**kwargs)
20 - return wrapper.worker
21 -
22 -
23 -class GPUExecutor(ExecutorBase):
24 -
25 - uses_ray: bool = False
26 -
27 - def _init_executor(self) -> None:
28 - """Initialize the worker and load the model.
29 - """
30 - assert self.parallel_config.world_size == 1, (
31 - "GPUExecutor only supports single GPU.")
32 -
33 - self.driver_worker = self._create_worker()
34 - self.driver_worker.init_device()
35 - self.driver_worker.load_model()
36 -
37 - def _get_worker_kwargs(
38 - self,
39 - local_rank: int = 0,
40 - rank: int = 0,
41 - distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
42 - """Return worker init args for a given rank."""
43 - if distributed_init_method is None:
44 - distributed_init_method = get_distributed_init_method(
45 - get_ip(), get_open_port())
46 - return dict(
47 - vllm_config=self.vllm_config,
48 - local_rank=local_rank,
49 - rank=rank,
50 - distributed_init_method=distributed_init_method,
51 - is_driver_worker=(not self.parallel_config)
52 - or (rank % self.parallel_config.tensor_parallel_size == 0),
53 - )
54 -
55 - def _create_worker(self,
56 - local_rank: int = 0,
57 - rank: int = 0,
58 - distributed_init_method: Optional[str] = None):
59 - return create_worker(**self._get_worker_kwargs(
60 - local_rank=local_rank,
61 - rank=rank,
62 - distributed_init_method=distributed_init_method))
63 -
64 - def determine_num_available_blocks(self) -> Tuple[int, int]:
65 - """Determine the number of available KV blocks by invoking the
66 - underlying worker.
67 - """
68 - return self.driver_worker.determine_num_available_blocks()
69 -
70 - def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
71 - """Initialize the KV cache by invoking the underlying worker.
72 - """
73 - # NOTE: This is logged in the executor because there can be >1 worker
74 - # with other executors. We could log in the engine level, but work
75 - # remains to abstract away the device for non-GPU configurations.
76 - logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
77 - num_cpu_blocks)
78 - max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
79 - self.model_config.max_model_len)
80 - logger.info("Maximum concurrency for %s tokens per request: %.2fx",
81 - self.model_config.max_model_len, max_concurrency)
82 -
83 - self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
84 -
85 - def execute_model(
86 - self, execute_model_req: ExecuteModelRequest
87 - ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
88 - output = self.driver_worker.execute_model(execute_model_req)
89 - return output
90 -
91 - def add_lora(self, lora_request: LoRARequest) -> bool:
92 - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
93 - return self.driver_worker.add_lora(lora_request)
94 -
95 - def remove_lora(self, lora_id: int) -> bool:
96 - assert lora_id > 0, "lora_id must be greater than 0."
97 - return self.driver_worker.remove_lora(lora_id)
98 -
99 - def pin_lora(self, lora_id: int) -> bool:
100 - assert lora_id > 0, "lora_id must be greater than 0."
101 - return self.driver_worker.pin_lora(lora_id)
102 -
103 - def list_loras(self) -> Set[int]:
104 - return self.driver_worker.list_loras()
105 -
106 - def add_prompt_adapter(
107 - self, prompt_adapter_request: PromptAdapterRequest) -> bool:
108 - assert prompt_adapter_request.prompt_adapter_id > 0, \
109 - "prompt_adapter_id must be greater than 0."
110 - return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
111 -
112 - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
113 - assert prompt_adapter_id > 0, \
114 - "prompt_adapter_id must be greater than 0."
115 - return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
116 -
117 - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
118 - assert prompt_adapter_id > 0, \
119 - "prompt_adapter_id must be greater than 0."
120 - return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
121 -
122 - def list_prompt_adapters(self) -> Set[int]:
123 - return self.driver_worker.list_prompt_adapters()
124 -
125 - def check_health(self) -> None:
126 - # GPUExecutor will always be healthy as long as
127 - # it's running.
128 - return
129 -
130 - def start_profile(self) -> None:
131 - self.driver_worker.start_profile()
132 -
133 - def stop_profile(self) -> None:
134 - self.driver_worker.stop_profile()
135 -
136 -
137 -class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
138 -
139 - async def execute_model_async(
140 - self,
141 - execute_model_req: ExecuteModelRequest,
142 - ) -> List[Union[SamplerOutput, PoolerOutput]]:
143 - output = await make_async(self.driver_worker.execute_model
144 - )(execute_model_req=execute_model_req)
145 - return output
vllm/executor/hpu_executor.py deleted
-202
@@ -1,202 +0,0 @@
1 -###############################################################################
2 -# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company
3 -###############################################################################
4 -
5 -import contextlib
6 -import os
7 -from typing import Any, Dict, List, Optional, Set, Tuple
8 -
9 -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10 -from vllm.logger import init_logger
11 -from vllm.lora.request import LoRARequest
12 -from vllm.model_executor.layers.sampler import SamplerOutput
13 -from vllm.prompt_adapter.request import PromptAdapterRequest
14 -from vllm.sequence import ExecuteModelRequest
15 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16 - make_async)
17 -from vllm.worker.worker_base import WorkerWrapperBase
18 -
19 -logger = init_logger(__name__)
20 -
21 -
22 -class HPUExecutor(ExecutorBase):
23 -
24 - uses_ray: bool = False
25 -
26 - def _init_executor(self) -> None:
27 - """Initialize the worker and load the model."""
28 - self._init_worker()
29 -
30 - def _get_worker_kwargs(
31 - self,
32 - local_rank: int = 0,
33 - rank: int = 0,
34 - distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
35 - """Return worker init args for a given rank."""
36 - if distributed_init_method is None:
37 - distributed_init_method = get_distributed_init_method(
38 - get_ip(), get_open_port())
39 - return dict(
40 - vllm_config=self.vllm_config,
41 - local_rank=local_rank,
42 - rank=rank,
43 - distributed_init_method=distributed_init_method,
44 - is_driver_worker=rank == 0,
45 - )
46 -
47 - def _create_worker(self,
48 - local_rank: int = 0,
49 - rank: int = 0,
50 - distributed_init_method: Optional[str] = None):
51 - wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
52 - wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
53 - distributed_init_method))
54 - return wrapper.worker
55 -
56 - def _init_worker(self):
57 - assert self.parallel_config.world_size == 1, (
58 - "GPUExecutor only supports single GPU.")
59 -
60 - self.driver_worker = self._create_worker()
61 - self.driver_worker.init_device()
62 - self.driver_worker.load_model()
63 -
64 - def determine_num_available_blocks(self) -> Tuple[int, int]:
65 - """Determine the number of available KV blocks by invoking the
66 - underlying worker.
67 - """
68 - return self.driver_worker.determine_num_available_blocks()
69 -
70 - def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
71 - """Initialize the KV cache by invoking the underlying worker.
72 - """
73 - # NOTE: This is logged in the executor because there can be >1 worker
74 - # with other executors. We could log in the engine level, but work
75 - # remains to abstract away the device for non-GPU configurations.
76 - logger.info("# HPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
77 - num_cpu_blocks)
78 - from vllm_hpu_extension.profiler import HabanaMemoryProfiler
79 - with HabanaMemoryProfiler() as cache_init_m:
80 - self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
81 - msg = f"init_cache_engine took {cache_init_m.get_summary_string()}"
82 - logger.info(msg)
83 -
84 - def finish_measurements(self):
85 - self.driver_worker.finish_measurements()
86 -
87 - def execute_model(
88 - self,
89 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
90 - # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION - will log graph compilations per engine step, only when there was any - highly recommended to use alongside PT_HPU_METRICS_GC_DETAILS! # noqa:E501
91 - # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL - will log graph compilations per engine step, always, even if there were none # noqa:E501
92 - # VLLM_HPU_LOG_STEP_CPU_FALLBACKS - will log cpu fallbacks per engine step, only when there was any # noqa:E501
93 - # VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL - will log cpu fallbacks per engine step, always, even if there were none # noqa:E501
94 - log_graph_compilation_all = os.environ.get(
95 - 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL', '0') != '0'
96 - log_graph_compilation = os.environ.get(
97 - 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION',
98 - '0') != '0' or log_graph_compilation_all
99 - log_cpu_fallbacks_all = os.environ.get(
100 - 'VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL', '0') != '0'
101 - log_cpu_fallbacks = os.environ.get('VLLM_HPU_LOG_STEP_CPU_FALLBACKS',
102 - '0') != '0' or log_cpu_fallbacks_all
103 - if log_graph_compilation or log_cpu_fallbacks:
104 - from habana_frameworks.torch.hpu.metrics import metric_localcontext
105 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list
106 - is_prompt = any([
107 - seq_group_metadata.is_prompt
108 - for seq_group_metadata in seq_group_metadata_list
109 - ])
110 - max_context_len = max([
111 - max([
112 - len(v.prompt_token_ids) + len(v.output_token_ids)
113 - for v in seq_group_metadata.seq_data.values()
114 - ]) for seq_group_metadata in seq_group_metadata_list
115 - ]) # whoa, that's some spicy stuff right here
116 - max_num_blocks = (
117 - (max_context_len - 1) // self.cache_config.block_size) + 1
118 - input_stats = (f'is_prompt: {is_prompt}, '
119 - f'num_seqs: {len(seq_group_metadata_list)}, '
120 - f'max_context_len: {max_context_len}, '
121 - f'max_num_blocks {max_num_blocks}')
122 - gc_ctx = metric_localcontext(
123 - "graph_compilation"
124 - ) if log_graph_compilation else contextlib.nullcontext()
125 - cpu_fallback_ctx = metric_localcontext(
126 - "cpu_fallback"
127 - ) if log_cpu_fallbacks else contextlib.nullcontext()
128 - with gc_ctx as gc_local_metric, \
129 - cpu_fallback_ctx as cpu_fallback_local_metric:
130 - output = self.driver_worker.execute_model(execute_model_req)
131 - if (log_graph_compilation and gc_local_metric.stats()[0][1] > 0
132 - ) or log_graph_compilation_all:
133 - msg = ("VLLM_HPU_STEP_GRAPH_COMPILATION: "
134 - f"{gc_local_metric.stats()}, {input_stats}")
135 - logger.warning(msg)
136 - if (log_cpu_fallbacks and cpu_fallback_local_metric.stats()[0][1] >
137 - 0) or log_cpu_fallbacks_all:
138 - msg = ("VLLM_HPU_STEP_CPU_FALLBACK: "
139 - f"{cpu_fallback_local_metric.stats()}, {input_stats}")
140 - logger.warning(msg)
141 -
142 - return output
143 -
144 - output = self.driver_worker.execute_model(execute_model_req)
145 - return output
146 -
147 - def add_lora(self, lora_request: LoRARequest) -> bool:
148 - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
149 - return self.driver_worker.add_lora(lora_request)
150 -
151 - def remove_lora(self, lora_id: int) -> bool:
152 - assert lora_id > 0, "lora_id must be greater than 0."
153 - return self.driver_worker.remove_lora(lora_id)
154 -
155 - def pin_lora(self, lora_id: int) -> bool:
156 - assert lora_id > 0, "lora_id must be greater than 0."
157 - return self.driver_worker.pin_lora(lora_id)
158 -
159 - def list_loras(self) -> Set[int]:
160 - return self.driver_worker.list_loras()
161 -
162 - def add_prompt_adapter(
163 - self, prompt_adapter_request: PromptAdapterRequest) -> bool:
164 - raise NotImplementedError(
165 - "Prompt Adapter is not implemented for HPU backend.")
166 -
167 - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
168 - raise NotImplementedError(
169 - "Prompt Adapter is not implemented for HPU backend.")
170 -
171 - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
172 - raise NotImplementedError(
173 - "Prompt Adapter is not implemented for HPU backend.")
174 -
175 - def list_prompt_adapters(self) -> Set[int]:
176 - raise NotImplementedError(
177 - "Prompt Adapter is not implemented for HPU backend.")
178 -
179 - def check_health(self) -> None:
180 - # GPUExecutor will always be healthy as long as
181 - # it's running.
182 - return
183 -
184 - def start_profile(self) -> None:
185 - self.driver_worker.start_profile()
186 -
187 - def stop_profile(self) -> None:
188 - self.driver_worker.stop_profile()
189 -
190 - def shutdown(self) -> None:
191 - self.driver_worker.shutdown_inc()
192 -
193 -
194 -class HPUExecutorAsync(HPUExecutor, ExecutorAsyncBase):
195 -
196 - async def execute_model_async(
197 - self,
198 - execute_model_req: ExecuteModelRequest,
199 - ) -> List[SamplerOutput]:
200 - output = await make_async(self.driver_worker.execute_model
201 - )(execute_model_req=execute_model_req, )
202 - return output
vllm/executor/mp_distributed_executor.py renamed
+29 -52
@@ -1,32 +1,26 @@
1 import asyncio
2 -import os
3 -from functools import partial
2 from typing import Any, List, Optional
3
6 -from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7 - DistributedGPUExecutor, DistributedGPUExecutorAsync)
8 -from vllm.executor.gpu_executor import create_worker
4 +from vllm.executor.executor_base import DistributedExecutorBase
5 from vllm.executor.multiproc_worker_utils import (
6 ProcessWorkerWrapper, ResultHandler, WorkerMonitor,
7 set_multiprocessing_worker_envs)
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.sampler import SamplerOutput
10 from vllm.sequence import ExecuteModelRequest
15 -from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
16 - get_distributed_init_method, get_open_port, make_async,
17 - update_environment_variables)
11 +from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
12 + get_ip, get_open_port, make_async)
13 +from vllm.worker.worker_base import WorkerWrapperBase
14
15 logger = init_logger(__name__)
16
17
22 -class MultiprocessingGPUExecutor(DistributedGPUExecutor):
23 - """Python multiprocessing-based multi-GPU executor"""
18 +class MultiprocessingDistributedExecutor(DistributedExecutorBase):
19 + """Python multiprocessing-based distributed executor"""
20
21 uses_ray: bool = False
22
23 def _init_executor(self) -> None:
28 - self._check_executor_parameters()
29 -
24 # Create the parallel GPU workers.
25 world_size = self.parallel_config.world_size
26 tensor_parallel_size = self.parallel_config.tensor_parallel_size
@@ -55,15 +49,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
49 else:
50 result_handler = ResultHandler()
51 for rank in range(1, world_size):
58 - worker = ProcessWorkerWrapper(
59 - result_handler,
60 - partial(
61 - create_worker,
62 - **self._get_worker_kwargs(
63 - rank=rank,
64 - local_rank=rank,
65 - distributed_init_method=distributed_init_method,
66 - )))
52 + worker = ProcessWorkerWrapper(result_handler,
53 + WorkerWrapperBase,
54 + self.vllm_config, rank)
55 self.workers.append(worker)
56 if rank % tensor_parallel_size == 0:
57 self.tp_driver_workers.append(worker)
@@ -77,32 +65,30 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
65 # Set up signal handlers to shutdown the executor cleanly
66 # sometimes gc does not work well
67
80 - self.driver_worker = self._create_worker(
81 - distributed_init_method=distributed_init_method)
68 + self.driver_worker = WorkerWrapperBase(self.vllm_config, 0)
69 +
70 + all_kwargs = []
71 + distributed_init_method = get_distributed_init_method(
72 + get_ip(), get_open_port())
73 + for i in range(world_size):
74 + local_rank = i
75 + rank = i
76 + kwargs = dict(
77 + vllm_config=self.vllm_config,
78 + local_rank=local_rank,
79 + rank=rank,
80 + distributed_init_method=distributed_init_method,
81 + is_driver_worker=(not self.parallel_config)
82 + or (rank % self.parallel_config.tensor_parallel_size == 0),
83 + )
84 + all_kwargs.append(kwargs)
85 + self._run_workers("init_worker", all_kwargs)
86 self._run_workers("init_device")
87 self._run_workers("load_model",
88 max_concurrent_workers=self.parallel_config.
89 max_parallel_loading_workers)
86 -
87 - def _check_executor_parameters(self):
88 - world_size = self.parallel_config.world_size
89 - tensor_parallel_size = self.parallel_config.tensor_parallel_size
90 -
91 - # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
92 - if "CUDA_VISIBLE_DEVICES" not in os.environ:
93 - update_environment_variables({
94 - "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
95 - })
96 -
97 - cuda_device_count = cuda_device_count_stateless()
98 - # Use confusing message for more common TP-only case.
99 - assert tensor_parallel_size <= cuda_device_count, (
100 - f"please set tensor_parallel_size ({tensor_parallel_size}) "
101 - f"to less than max local gpu count ({cuda_device_count})")
102 -
103 - assert world_size <= cuda_device_count, (
104 - f"please ensure that world_size ({world_size}) "
105 - f"is less than than max local gpu count ({cuda_device_count})")
90 + self.driver_exec_model = make_async(self.driver_worker.execute_model)
91 + self.pp_locks: Optional[List[asyncio.Lock]] = None
92
93 def shutdown(self):
94 if (worker_monitor := getattr(self, "worker_monitor",
@@ -172,15 +158,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
158 for result in parallel_worker_tasks:
159 result.get()
160
175 -
176 -class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
177 - DistributedGPUExecutorAsync):
178 -
179 - def __init__(self, *args, **kwargs):
180 - super().__init__(*args, **kwargs)
181 - self.driver_exec_model = make_async(self.driver_worker.execute_model)
182 - self.pp_locks: Optional[List[asyncio.Lock]] = None
183 -
161 async def _driver_execute_model_async(
162 self,
163 execute_model_req: Optional[ExecuteModelRequest] = None
vllm/executor/multiproc_worker_utils.py
+9 -3
@@ -12,6 +12,7 @@ from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
12
13 import torch
14
15 +from vllm.config import VllmConfig
16 from vllm.logger import init_logger
17 from vllm.triton_utils.importing import HAS_TRITON
18 from vllm.utils import _check_multiproc_method, get_mp_context
@@ -147,7 +148,8 @@ class ProcessWorkerWrapper:
148 for handling single-node multi-GPU tensor parallel."""
149
150 def __init__(self, result_handler: ResultHandler,
150 - worker_factory: Callable[[], Any]) -> None:
151 + worker_factory: Callable[[VllmConfig, int], Any],
152 + vllm_config: VllmConfig, rank: int) -> None:
153 self.mp = get_mp_context()
154 self._task_queue = self.mp.Queue()
155 self.result_queue = result_handler.result_queue
@@ -159,6 +161,8 @@ class ProcessWorkerWrapper:
161 worker_factory=worker_factory,
162 task_queue=self._task_queue,
163 result_queue=self.result_queue,
164 + vllm_config=vllm_config,
165 + rank=rank,
166 ),
167 daemon=True)
168
@@ -199,9 +203,11 @@ class ProcessWorkerWrapper:
203
204
205 def _run_worker_process(
202 - worker_factory: Callable[[], Any],
206 + worker_factory: Callable[[VllmConfig, int], Any],
207 task_queue: Queue,
208 result_queue: Queue,
209 + vllm_config: VllmConfig,
210 + rank: int,
211 ) -> None:
212 """Worker process event loop"""
213
@@ -212,7 +218,7 @@ def _run_worker_process(
218 _add_prefix(sys.stderr, process_name, pid)
219
220 # Initialize worker
215 - worker = worker_factory()
221 + worker = worker_factory(vllm_config, rank)
222 del worker_factory
223
224 # Accept tasks from the engine in task_queue
vllm/executor/multiproc_xpu_executor.py deleted
-26
@@ -1,26 +0,0 @@
1 -import vllm.envs as envs
2 -from vllm.executor.multiproc_gpu_executor import (
3 - MultiprocessingGPUExecutor, MultiprocessingGPUExecutorAsync)
4 -from vllm.executor.xpu_executor import XPUExecutor
5 -from vllm.logger import init_logger
6 -from vllm.utils import make_async
7 -
8 -logger = init_logger(__name__)
9 -
10 -
11 -class MultiprocessingXPUExecutor(MultiprocessingGPUExecutor, XPUExecutor):
12 - """Python multiprocessing-based multi-XPU executor"""
13 -
14 - def _check_executor_parameters(self):
15 - mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
16 - if mp_method != "spawn":
17 - raise RuntimeError(
18 - "XPU multiprocess executor only support spawn as mp method")
19 -
20 -
21 -class MultiprocessingXPUExecutorAsync(MultiprocessingXPUExecutor,
22 - MultiprocessingGPUExecutorAsync):
23 -
24 - def __init__(self, *args, **kwargs):
25 - super().__init__(*args, **kwargs)
26 - self.driver_exec_model = make_async(self.driver_worker.execute_model)
vllm/executor/neuron_executor.py deleted
-114
@@ -1,114 +0,0 @@
1 -from typing import List, Set, Tuple
2 -
3 -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 -from vllm.logger import init_logger
5 -from vllm.lora.request import LoRARequest
6 -from vllm.model_executor.layers.sampler import SamplerOutput
7 -from vllm.sequence import ExecuteModelRequest
8 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9 - make_async)
10 -from vllm.worker.worker_base import WorkerWrapperBase
11 -
12 -logger = init_logger(__name__)
13 -
14 -
15 -class NeuronExecutor(ExecutorBase):
16 -
17 - uses_ray: bool = False
18 -
19 - def _init_executor(self) -> None:
20 - assert (self.lora_config is
21 - None), "LoRA is not supported for Neuron backend."
22 - assert (not self.speculative_config
23 - ), "Speculative decoding not yet supported for Neuron backend."
24 -
25 - # Instantiate the worker and load the model to the device.
26 - self._init_worker()
27 -
28 - def _init_worker(self):
29 - wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
30 - distributed_init_method = get_distributed_init_method(
31 - get_ip(), get_open_port())
32 - wrapper.init_worker(
33 - vllm_config=self.vllm_config,
34 - local_rank=0,
35 - rank=0,
36 - distributed_init_method=distributed_init_method,
37 - )
38 - self.driver_worker = wrapper.worker
39 - self.driver_worker.init_device()
40 - self.driver_worker.load_model()
41 -
42 - def determine_num_available_blocks(self) -> Tuple[int, int]:
43 - """Determine the number of available KV blocks by invoking the
44 - underlying worker.
45 - """
46 - return self.driver_worker.determine_num_available_blocks()
47 -
48 - def initialize_cache(self, num_gpu_blocks: int,
49 - num_cpu_blocks: int) -> None:
50 - """Initialize the KV cache by invoking the underlying worker.
51 - """
52 - self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
53 -
54 - def execute_model(
55 - self,
56 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
57 - assert (not execute_model_req.blocks_to_swap_in
58 - and not execute_model_req.blocks_to_swap_out
59 - and not execute_model_req.blocks_to_copy), (
60 - "Cache operations are not supported for Neuron backend.")
61 - assert execute_model_req.num_lookahead_slots == 0, (
62 - "lookahead not supported for Neuron backend.")
63 -
64 - output = self.driver_worker.execute_model(execute_model_req)
65 - return output
66 -
67 - def add_lora(self, lora_request: LoRARequest) -> bool:
68 - return self.driver_worker.add_lora(lora_request)
69 -
70 - def remove_lora(self, lora_id: int) -> bool:
71 - return self.driver_worker.remove_lora(lora_id)
72 -
73 - def pin_lora(self, lora_id: int) -> bool:
74 - return self.driver_worker.pin_lora(lora_id)
75 -
76 - def list_loras(self) -> Set[int]:
77 - return self.driver_worker.list_loras()
78 -
79 - def add_prompt_adapter(self, prompt_adapter_request) -> bool:
80 - raise NotImplementedError(
81 - "Soft prompt is currently not supported by the Neuron backend.")
82 -
83 - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
84 - raise NotImplementedError(
85 - "Soft prompt is currently not supported by the Neuron backend.")
86 -
87 - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
88 - raise NotImplementedError(
89 - "Soft prompt is currently not supported by the Neuron backend.")
90 -
91 - def list_prompt_adapters(self) -> Set[int]:
92 - raise NotImplementedError(
93 - "Soft prompt is currently not supported by the Neuron backend.")
94 -
95 - def check_health(self) -> None:
96 - # NeuronExecutor will always be healthy as long as
97 - # it's running.
98 - return
99 -
100 -
101 -class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
102 -
103 - async def execute_model_async(
104 - self,
105 - execute_model_req: ExecuteModelRequest,
106 - ) -> List[SamplerOutput]:
107 - output = await make_async(self.driver_worker.execute_model
108 - )(execute_model_req=execute_model_req, )
109 - return output
110 -
111 - async def check_health_async(self) -> None:
112 - # NeuronExecutor will always be healthy as long as
113 - # it's running.
114 - return
vllm/executor/openvino_executor.py deleted
-125
@@ -1,125 +0,0 @@
1 -from typing import List, Set, Tuple
2 -
3 -import openvino as ov
4 -
5 -import vllm.envs as envs
6 -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7 -from vllm.logger import init_logger
8 -from vllm.lora.request import LoRARequest
9 -from vllm.model_executor.layers.sampler import SamplerOutput
10 -from vllm.platforms import current_platform
11 -from vllm.sequence import ExecuteModelRequest
12 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13 - make_async)
14 -from vllm.worker.worker_base import WorkerWrapperBase
15 -
16 -logger = init_logger(__name__)
17 -
18 -
19 -class OpenVINOExecutor(ExecutorBase):
20 -
21 - uses_ray: bool = False
22 -
23 - def _init_executor(self) -> None:
24 - assert self.device_config.device_type == "openvino"
25 - assert self.lora_config is None, "OpenVINO backend doesn't support LoRA"
26 - assert current_platform.is_openvino_cpu() or \
27 - current_platform.is_openvino_gpu(), \
28 - "OpenVINO backend supports only CPU and GPU devices"
29 -
30 - # Instantiate the worker and load the model to CPU.
31 - self._init_worker()
32 -
33 - def _init_worker(self):
34 -
35 - wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
36 -
37 - distributed_init_method = get_distributed_init_method(
38 - get_ip(), get_open_port())
39 - wrapper.init_worker(
40 - ov_core=ov.Core(),
41 - vllm_config=self.vllm_config,
42 - local_rank=0,
43 - rank=0,
44 - distributed_init_method=distributed_init_method,
45 - kv_cache_dtype=self.cache_config.cache_dtype,
46 - is_driver_worker=True,
47 - )
48 - self.driver_worker = wrapper.worker
49 - self.driver_worker.init_device()
50 - self.driver_worker.load_model()
51 -
52 - def determine_num_available_blocks(self) -> Tuple[int, int]:
53 - """Determine the number of available KV blocks by invoking the
54 - underlying worker.
55 - """
56 - return self.driver_worker.determine_num_available_blocks()
57 -
58 - def initialize_cache(self, num_gpu_blocks: int,
59 - num_cpu_blocks: int) -> None:
60 - """Initialize the KV cache by invoking the underlying worker."""
61 - # NOTE: We log here to avoid multiple logs when number of workers is
62 - # greater than one. We could log in the engine, but not all executors
63 - # have GPUs.
64 - # NOTE: In case of a CPU device, `cpu block` for OpenVINO backend
65 - # is located on CPU memory but is referred as `gpu block`.
66 - # Because we want to reuse the existing block management procedure.
67 - device_blocks = num_gpu_blocks
68 - swap_blocks = num_cpu_blocks
69 - logger.info("OpenVINO %s: # device blocks: %d; # swap blocks: %d",
70 - envs.VLLM_OPENVINO_DEVICE, device_blocks, swap_blocks)
71 - self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
72 -
73 - def execute_model(
74 - self,
75 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
76 - output = self.driver_worker.execute_model(execute_model_req)
77 - return output
78 -
79 - def add_lora(self, lora_request: LoRARequest) -> bool:
80 - return self.driver_worker.add_lora(lora_request)
81 -
82 - def remove_lora(self, lora_id: int) -> bool:
83 - return self.driver_worker.remove_lora(lora_id)
84 -
85 - def pin_lora(self, lora_id: int) -> bool:
86 - return self.driver_worker.pin_lora(lora_id)
87 -
88 - def list_loras(self) -> Set[int]:
89 - return self.driver_worker.list_loras()
90 -
91 - def add_prompt_adapter(self, prompt_adapter_request) -> bool:
92 - raise NotImplementedError(
93 - "Soft prompt is currently not supported by the OPENVINO backend.")
94 -
95 - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
96 - raise NotImplementedError(
97 - "Soft prompt is currently not supported by the OPENVINO backend.")
98 -
99 - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
100 - raise NotImplementedError(
101 - "Soft prompt is currently not supported by the OPENVINO backend.")
102 -
103 - def list_prompt_adapters(self) -> Set[int]:
104 - raise NotImplementedError(
105 - "Soft prompt is currently not supported by the OPENVINO backend.")
106 -
107 - def check_health(self) -> None:
108 - # OpenVINOExecutor will always be healthy as long as
109 - # it's running.
110 - return
111 -
112 -
113 -class OpenVINOExecutorAsync(OpenVINOExecutor, ExecutorAsyncBase):
114 -
115 - async def execute_model_async(
116 - self,
117 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
118 - output = await make_async(self.driver_worker.execute_model
119 - )(execute_model_req=execute_model_req, )
120 - return output
121 -
122 - async def check_health_async(self) -> None:
123 - # OpenVINOExecutor will always be healthy as long as
124 - # it's running.
125 - return
vllm/executor/ray_distributed_executor.py renamed
+144 -96
@@ -1,24 +1,29 @@
1 import asyncio
2 import os
3 from collections import defaultdict
4 -from itertools import islice, repeat
5 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
4 +from dataclasses import dataclass
5 +from typing import TYPE_CHECKING, Any, Dict, List, Optional
6
7 import msgspec
8
9 import vllm.envs as envs
10 -from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11 - DistributedGPUExecutor, DistributedGPUExecutorAsync)
10 +from vllm.executor.executor_base import (
11 + DistributedExecutorBase) # yapf: disable
12 from vllm.executor.msgspec_utils import encode_hook
13 -from vllm.executor.ray_utils import RayWorkerWrapper, ray
13 +from vllm.executor.ray_utils import (RayWorkerWrapper, initialize_ray_cluster,
14 + ray)
15 from vllm.logger import init_logger
16 from vllm.model_executor.layers.sampler import SamplerOutput
17 +from vllm.platforms import current_platform
18 from vllm.sequence import ExecuteModelRequest
19 from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
20 get_ip, get_open_port, make_async)
21
22 if ray is not None:
23 + from ray.actor import ActorHandle
24 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
25 +else:
26 + ActorHandle = None
27
28 if TYPE_CHECKING:
29 from ray.util.placement_group import PlacementGroup
@@ -26,12 +31,29 @@ if TYPE_CHECKING:
31 logger = init_logger(__name__)
32
33
29 -class RayGPUExecutor(DistributedGPUExecutor):
34 +@dataclass
35 +class RayWorkerMetaData:
36 + """
37 + Metadata for a Ray worker.
38 + The order of ray worker creation can be random,
39 + and we need to reset the rank after creating all workers.
40 + """
41 + worker: ActorHandle
42 + created_rank: int
43 + adjusted_rank: int = -1
44 + ip: str = ""
45 +
46 +
47 +class RayDistributedExecutor(DistributedExecutorBase):
48
49 uses_ray: bool = True
50
51 def _init_executor(self) -> None:
52 self.forward_dag: Optional[ray.dag.CompiledDAG] = None
53 + if envs.VLLM_USE_V1:
54 + # v1 always uses the compiled DAG and SPMD worker.
55 + os.environ["VLLM_USE_RAY_SPMD_WORKER"] = "1"
56 + os.environ["VLLM_USE_RAY_COMPILED_DAG"] = "1"
57 # If the env var is set, it uses the Ray's compiled DAG API
58 # which optimizes the control plane overhead.
59 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -53,6 +75,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
75 "VLLM_USE_RAY_COMPILED_DAG=1")
76
77 assert self.uses_ray
78 + initialize_ray_cluster(self.parallel_config)
79 placement_group = self.parallel_config.placement_group
80
81 # Disable Ray usage stats collection.
@@ -66,6 +89,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
89 self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
90 self.output_decoder = msgspec.msgpack.Decoder(
91 Optional[List[SamplerOutput]])
92 + self.use_v1 = envs.VLLM_USE_V1
93 +
94 + self.pp_locks: Optional[List[asyncio.Lock]] = None
95 + self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
96 + if not self.use_ray_compiled_dag:
97 + self.driver_exec_method = make_async(
98 + self.driver_worker.execute_method)
99
100 def shutdown(self) -> None:
101 if hasattr(self, "forward_dag") and self.forward_dag is not None:
@@ -123,9 +153,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
153
154 # Create the workers.
155 driver_ip = get_ip()
126 - workers = []
156 + rank = 0
157 + worker_metadata: List[RayWorkerMetaData] = []
158 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
128 - if not bundle.get("GPU", 0):
159 + if not bundle.get(current_platform.ray_device_key, 0):
160 continue
161 scheduling_strategy = PlacementGroupSchedulingStrategy(
162 placement_group=placement_group,
@@ -133,38 +164,51 @@ class RayGPUExecutor(DistributedGPUExecutor):
164 placement_group_bundle_index=bundle_id,
165 )
166
136 - worker = ray.remote(
137 - num_cpus=0,
138 - num_gpus=num_gpus,
139 - scheduling_strategy=scheduling_strategy,
140 - **ray_remote_kwargs,
141 - )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
142 - workers.append(worker)
143 -
144 - worker_ip_refs = [
145 - worker.get_node_ip.remote() # type: ignore[attr-defined]
146 - for worker in workers
147 - ]
148 - worker_ips = ray.get(worker_ip_refs)
167 + if current_platform.ray_device_key == "GPU":
168 + # NV+AMD GPUs, and Intel XPUs
169 + worker = ray.remote(
170 + num_cpus=0,
171 + num_gpus=num_gpus,
172 + scheduling_strategy=scheduling_strategy,
173 + **ray_remote_kwargs,
174 + )(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
175 + rank=rank)
176 + else:
177 + worker = ray.remote(
178 + num_cpus=0,
179 + num_gpus=0,
180 + resources={current_platform.ray_device_key: num_gpus},
181 + scheduling_strategy=scheduling_strategy,
182 + **ray_remote_kwargs,
183 + )(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
184 + rank=rank)
185 + worker_metadata.append(
186 + RayWorkerMetaData(worker=worker, created_rank=rank))
187 + rank += 1
188 +
189 + worker_ips = ray.get([
190 + each.worker.get_node_ip.remote() # type: ignore[attr-defined]
191 + for each in worker_metadata
192 + ])
193 +
194 + for each, ip in zip(worker_metadata, worker_ips):
195 + each.ip = ip
196
197 if not self.use_ray_spmd_worker:
151 - for i in range(len(workers)):
152 - worker = workers[i]
153 - worker_ip = worker_ips[i]
198 + for i, each in enumerate(worker_metadata):
199 + # find and remove the dummy worker from the list
200 + worker = each.worker
201 + worker_ip = each.ip
202 if self.driver_dummy_worker is None and worker_ip == driver_ip:
203 # If the worker is on the same node as the driver, we use it
204 # as the resource holder for the driver process.
205 self.driver_dummy_worker = worker
206 self.driver_worker = RayWorkerWrapper(
159 - vllm_config=self.vllm_config)
160 - workers.pop(i)
161 - worker_ips.pop(i)
162 - self.workers = workers
207 + vllm_config=self.vllm_config, rank=0)
208 + worker_metadata.pop(i)
209 break
164 - else:
165 - self.workers = workers
210
167 - logger.debug("workers: %s", self.workers)
211 + logger.debug("workers: %s", worker_metadata)
212 logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
213 if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
214 raise ValueError(
@@ -176,9 +220,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
220 for ip in worker_ips:
221 ip_counts[ip] = ip_counts.get(ip, 0) + 1
222
179 - worker_to_ip = dict(zip(self.workers, worker_ips))
180 -
181 - def sort_by_driver_then_worker_ip(worker):
223 + def sort_by_driver_then_worker_ip(item: RayWorkerMetaData):
224 """
225 Sort the workers based on 3 properties:
226 1. If the worker is on the same node as the driver (vllm engine),
@@ -188,13 +230,23 @@ class RayGPUExecutor(DistributedGPUExecutor):
230 3. Finally, if the work is on a node with smaller IP address, it
231 should be placed first.
232 """
191 - ip = worker_to_ip[worker]
192 - return (ip != driver_ip, ip_counts[ip], ip)
233 + ip = item.ip
234 + return (0 if ip == driver_ip else 1, ip_counts[ip], ip)
235
236 # After sorting, the workers on the same node will be
237 # close to each other, and the workers on the driver
238 # node will be placed first.
197 - self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
239 + sorted_worker_metadata = sorted(worker_metadata,
240 + key=sort_by_driver_then_worker_ip)
241 + start_rank = 0 if self.use_ray_spmd_worker else 1
242 + for i, item in enumerate(sorted_worker_metadata):
243 + item.adjusted_rank = i + start_rank
244 + self.workers = [item.worker for item in sorted_worker_metadata]
245 + rerank_mapping = {
246 + item.created_rank: item.adjusted_rank
247 + for item in sorted_worker_metadata
248 + }
249 + self._run_workers("adjust_rank", rerank_mapping)
250
251 # Get the set of GPU IDs used on each node.
252 worker_node_and_gpu_ids = []
@@ -235,21 +287,29 @@ class RayGPUExecutor(DistributedGPUExecutor):
287 " each node.")
288
289 # Set environment variables for the driver and workers.
238 - all_args_to_update_environment_variables = [({
239 - "CUDA_VISIBLE_DEVICES":
290 + all_args_to_update_environment_variables = [{
291 + current_platform.device_control_env_var:
292 ",".join(map(str, node_gpus[node_id])),
241 - "VLLM_TRACE_FUNCTION":
242 - str(envs.VLLM_TRACE_FUNCTION),
243 - **({
244 - "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
245 - } if envs.VLLM_ATTENTION_BACKEND is not None else {})
246 - }, ) for (node_id, _) in worker_node_and_gpu_ids]
293 + } for (node_id, _) in worker_node_and_gpu_ids]
294 +
295 + for args in all_args_to_update_environment_variables:
296 + # some carry-over env vars from the driver
297 + # TODO: refactor platform-specific env vars
298 + for name in [
299 + "VLLM_ATTENTION_BACKEND",
300 + "TPU_CHIPS_PER_HOST_BOUNDS",
301 + "TPU_HOST_BOUNDS",
302 + "VLLM_USE_V1",
303 + "VLLM_TRACE_FUNCTION",
304 + ]:
305 + if name in os.environ:
306 + args[name] = os.environ[name]
307
308 self._env_vars_for_all_workers = (
309 all_args_to_update_environment_variables)
310
311 self._run_workers("update_environment_variables",
252 - all_args=self._get_env_vars_to_be_updated())
312 + self._get_env_vars_to_be_updated())
313
314 if len(node_gpus) == 1:
315 # in single node case, we don't need to get the IP address.
@@ -265,14 +325,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
325 driver_ip, get_open_port())
326
327 # Initialize the actual workers inside worker wrapper.
268 - init_worker_all_kwargs = [
269 - self._get_worker_kwargs(
270 - local_rank=node_workers[node_id].index(rank),
328 + all_kwargs = []
329 + for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
330 + local_rank = node_workers[node_id].index(rank)
331 + kwargs = dict(
332 + vllm_config=self.vllm_config,
333 + local_rank=local_rank,
334 rank=rank,
335 distributed_init_method=distributed_init_method,
273 - ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
274 - ]
275 - self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
336 + is_driver_worker=(not self.parallel_config)
337 + or (rank % self.parallel_config.tensor_parallel_size == 0),
338 + )
339 + all_kwargs.append(kwargs)
340 + self._run_workers("init_worker", all_kwargs)
341
342 self._run_workers("init_device")
343 self._run_workers("load_model",
@@ -332,9 +397,15 @@ class RayGPUExecutor(DistributedGPUExecutor):
397 if self.forward_dag is None:
398 self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
399
335 - serialized_data = self.input_encoder.encode(execute_model_req)
400 + if self.use_v1:
401 + serialized_data = execute_model_req
402 + else:
403 + serialized_data = self.input_encoder.encode(execute_model_req)
404 outputs = ray.get(self.forward_dag.execute(serialized_data))
337 - output = self.output_decoder.decode(outputs[0])
405 + if self.use_v1:
406 + output = outputs[0]
407 + else:
408 + output = self.output_decoder.decode(outputs[0])
409 return output
410
411 def _run_workers(
@@ -342,8 +413,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
413 method: str,
414 *args,
415 async_run_tensor_parallel_workers_only: bool = False,
345 - all_args: Optional[List[Tuple[Any, ...]]] = None,
346 - all_kwargs: Optional[List[Dict[str, Any]]] = None,
416 max_concurrent_workers: Optional[int] = None,
417 **kwargs,
418 ) -> Any:
@@ -356,8 +425,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
425 It will also be run asynchronously and return a list of futures
426 rather than blocking on the results.
427 - args/kwargs: All workers share the same args/kwargs
359 - - all_args/all_kwargs: args/kwargs for each worker are specified
360 - individually
428 """
429 if self.use_ray_spmd_worker:
430 assert not async_run_tensor_parallel_workers_only, (
@@ -368,26 +435,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
435 raise NotImplementedError(
436 "max_concurrent_workers is not supported yet.")
437
371 - count = len(self.workers) if not \
372 - async_run_tensor_parallel_workers_only \
373 - else len(self.non_driver_workers)
374 - # If using SPMD worker, all workers are the same, so we should execute
375 - # the args on all workers. Otherwise, we skip the first worker's args
376 - # because those args will go to the driver worker.
377 - first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1
378 - all_worker_args = repeat(args, count) if all_args is None \
379 - else islice(all_args, first_worker_args_index, None)
380 - all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
381 - else islice(all_kwargs, first_worker_args_index, None)
382 -
438 # Start the ray workers first.
439 ray_workers = self.workers
440 if async_run_tensor_parallel_workers_only:
441 ray_workers = self.non_driver_workers
442 ray_worker_outputs = [
388 - worker.execute_method.remote(method, *worker_args, **worker_kwargs)
389 - for (worker, worker_args, worker_kwargs
390 - ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
443 + worker.execute_method.remote(method, *args, **kwargs)
444 + for worker in ray_workers
445 ]
446
447 if async_run_tensor_parallel_workers_only:
@@ -399,13 +453,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
453 # so we only explicitly execute on the driver worker if using a
454 # non-SPMD worker class.
455 if not self.use_ray_spmd_worker:
402 - driver_args = args if all_args is None else all_args[0]
403 - driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
404 -
456 # Start the driver worker after all the ray workers.
457 driver_worker_output = [
407 - self.driver_worker.execute_method(method, *driver_args,
408 - **driver_kwargs)
458 + self.driver_worker.execute_method(method, *args, **kwargs)
459 ]
460
461 # Get the results of the ray workers.
@@ -467,11 +517,18 @@ class RayGPUExecutor(DistributedGPUExecutor):
517 for pp_rank, tp_group in enumerate(self.pp_tp_workers):
518 # Each PP worker takes in the output of the previous PP worker,
519 # and the TP group executes in SPMD fashion.
470 - outputs = [
471 - worker.execute_model_spmd.
472 - bind( # type: ignore[attr-defined]
473 - outputs[i]) for i, worker in enumerate(tp_group)
474 - ]
520 + if self.use_v1:
521 + outputs = [
522 + worker.execute_model.
523 + bind( # type: ignore[attr-defined]
524 + outputs[i]) for i, worker in enumerate(tp_group)
525 + ]
526 + else:
527 + outputs = [
528 + worker.execute_model_spmd.
529 + bind( # type: ignore[attr-defined]
530 + outputs[i]) for i, worker in enumerate(tp_group)
531 + ]
532
533 last_pp_rank = len(self.pp_tp_workers) - 1
534 if pp_rank < last_pp_rank:
@@ -497,17 +554,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
554 def __del__(self):
555 self.shutdown()
556
500 -
501 -class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
502 -
503 - def __init__(self, *args, **kwargs):
504 - super().__init__(*args, **kwargs)
505 - self.pp_locks: Optional[List[asyncio.Lock]] = None
506 - self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
507 - if not self.use_ray_compiled_dag:
508 - self.driver_exec_method = make_async(
509 - self.driver_worker.execute_method)
510 -
557 async def execute_model_async(
558 self,
559 execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
@@ -568,5 +614,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
614 ]
615 return await asyncio.gather(*coros)
616
571 - def __del__(self):
572 - self.shutdown()
617 + def check_health(self) -> None:
618 + # Assume that the Ray workers are healthy.
619 + # TODO: check the health of the Ray workers
620 + return
vllm/executor/ray_hpu_executor.py deleted
-515
@@ -1,515 +0,0 @@
1 -import asyncio
2 -import os
3 -from collections import defaultdict
4 -from itertools import islice, repeat
5 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6 -
7 -import msgspec
8 -
9 -import vllm.envs as envs
10 -from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11 - DistributedGPUExecutor, DistributedGPUExecutorAsync)
12 -from vllm.executor.msgspec_utils import encode_hook
13 -from vllm.executor.ray_utils import RayWorkerWrapper, ray
14 -from vllm.logger import init_logger
15 -from vllm.model_executor.layers.sampler import SamplerOutput
16 -from vllm.sequence import ExecuteModelRequest
17 -from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18 - get_ip, get_open_port, make_async)
19 -
20 -if ray is not None:
21 - from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
22 -
23 -if TYPE_CHECKING:
24 - from ray.util.placement_group import PlacementGroup
25 -
26 -logger = init_logger(__name__)
27 -
28 -
29 -class RayHPUExecutor(DistributedGPUExecutor):
30 -
31 - uses_ray: bool = True
32 -
33 - def _init_executor(self) -> None:
34 - self.forward_dag: Optional[ray.dag.CompiledDAG] = None
35 - # If the env var is set, it uses the Ray's compiled DAG API
36 - # which optimizes the control plane overhead.
37 - # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
38 - # Currently, this requires USE_RAY_SPMD_WORKER=True.
39 - self.use_ray_compiled_dag = envs.VLLM_USE_RAY_COMPILED_DAG
40 - # If the env var is set, then we do not distinguish between the
41 - # "driver worker" vs other workers. Also, the rank 0 worker will
42 - # be executed in a remote Ray worker. Currently this requires
43 - # USE_RAY_COMPILED_DAG=True.
44 - self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
45 - if self.use_ray_compiled_dag:
46 - assert self.use_ray_spmd_worker, (
47 - "VLLM_USE_RAY_COMPILED_DAG=1 requires "
48 - "VLLM_USE_RAY_SPMD_WORKER=1")
49 - if self.use_ray_spmd_worker:
50 - # TODO: Support SPMD worker for non-DAG Ray executor.
51 - assert self.use_ray_compiled_dag, (
52 - "VLLM_USE_RAY_SPMD_WORKER=1 requires "
53 - "VLLM_USE_RAY_COMPILED_DAG=1")
54 -
55 - assert self.uses_ray
56 - placement_group = self.parallel_config.placement_group
57 -
58 - # Disable Ray usage stats collection.
59 - ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
60 - if ray_usage != "1":
61 - os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
62 -
63 - # Create the parallel GPU workers.
64 - self._init_workers_ray(placement_group)
65 -
66 - self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
67 - self.output_decoder = msgspec.msgpack.Decoder(
68 - Optional[List[SamplerOutput]])
69 -
70 - def shutdown(self) -> None:
71 - if hasattr(self, "forward_dag") and self.forward_dag is not None:
72 - self.forward_dag.teardown()
73 - import ray
74 - for worker in self.workers:
75 - ray.kill(worker)
76 - self.forward_dag = None
77 -
78 - def finish_measurements(self):
79 - self._run_workers("finish_measurements")
80 -
81 - def _init_workers_ray(self, placement_group: "PlacementGroup",
82 - **ray_remote_kwargs):
83 - # Otherwise, the ray workers are allocated with a full GPU.
84 - num_gpus = 1
85 -
86 - # The driver dummy worker does not actually use any resources.
87 - # It holds the resource for the driver worker.
88 - self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
89 - # The remaining workers are the actual ray actors.
90 - self.workers: List[RayWorkerWrapper] = []
91 -
92 - # Used in ray compiled DAG: indexed first by PP rank,
93 - # and then TP rank. In other words, the inner list is
94 - # the TP group of workers for a PP rank.
95 - self.pp_tp_workers: List[List[RayWorkerWrapper]] = []
96 -
97 - logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
98 -
99 - # Create the workers.
100 - driver_ip = get_ip()
101 - for bundle_id, bundle in enumerate(placement_group.bundle_specs):
102 - if not bundle.get("HPU", 0):
103 - continue
104 - scheduling_strategy = PlacementGroupSchedulingStrategy(
105 - placement_group=placement_group,
106 - placement_group_capture_child_tasks=True,
107 - placement_group_bundle_index=bundle_id,
108 - )
109 -
110 - worker = ray.remote(
111 - num_cpus=0,
112 - num_gpus=0,
113 - resources={'HPU': num_gpus},
114 - scheduling_strategy=scheduling_strategy,
115 - **ray_remote_kwargs,
116 - )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
117 -
118 - if self.use_ray_spmd_worker:
119 - self.workers.append(worker)
120 - else:
121 - worker_ip = ray.get(worker.get_node_ip.remote())
122 - if worker_ip == driver_ip and self.driver_dummy_worker is None:
123 - # If the worker is on the same node as the driver, we use it
124 - # as the resource holder for the driver process.
125 - self.driver_dummy_worker = worker
126 - self.driver_worker = RayWorkerWrapper(
127 - vllm_config=self.vllm_config)
128 - else:
129 - # Else, added to the list of workers.
130 - self.workers.append(worker)
131 -
132 - logger.debug("workers: %s", self.workers)
133 - logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
134 - if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
135 - raise ValueError(
136 - "Ray does not allocate any GPUs on the driver node. Consider "
137 - "adjusting the Ray placement group or running the driver on a "
138 - "GPU node.")
139 -
140 - worker_ips = [
141 - ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
142 - for worker in self.workers
143 - ]
144 - ip_counts: Dict[str, int] = {}
145 - for ip in worker_ips:
146 - ip_counts[ip] = ip_counts.get(ip, 0) + 1
147 -
148 - def sort_by_driver_then_worker_ip(worker):
149 - """
150 - Sort the workers based on 3 properties:
151 - 1. If the worker is on the same node as the driver (vllm engine),
152 - it should be placed first.
153 - 2. Then, if the worker is on a node with fewer workers, it should
154 - be placed first.
155 - 3. Finally, if the work is on a node with smaller IP address, it
156 - should be placed first.
157 - """
158 - ip = ray.get(worker.get_node_ip.remote())
159 - return (ip != driver_ip, ip_counts[ip], ip)
160 -
161 - # After sorting, the workers on the same node will be
162 - # close to each other, and the workers on the driver
163 - # node will be placed first.
164 - self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
165 -
166 - worker_node_and_gpu_ids = []
167 - for worker in [self.driver_dummy_worker] + self.workers:
168 - if worker is None:
169 - # driver_dummy_worker can be None when using ray spmd worker.
170 - continue
171 - worker_node_and_gpu_ids.append(
172 - ray.get(worker.get_node_and_gpu_ids.remote()) \
173 - ) # type: ignore
174 -
175 - node_workers = defaultdict(list) # node id -> list of worker ranks
176 - node_gpus = defaultdict(list) # node id -> list of gpu ids
177 -
178 - for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
179 - node_workers[node_id].append(i)
180 - # `gpu_ids` can be a list of strings or integers.
181 - # convert them to integers for consistency.
182 - # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
183 - # string sorting is not sufficient.
184 - # see https://github.com/vllm-project/vllm/issues/5590
185 - gpu_ids = [int(x) for x in gpu_ids]
186 - node_gpus[node_id].extend(gpu_ids)
187 - for node_id, gpu_ids in node_gpus.items():
188 - node_gpus[node_id] = sorted(gpu_ids)
189 -
190 - all_ips = set(worker_ips + [driver_ip])
191 - n_ips = len(all_ips)
192 - n_nodes = len(node_workers)
193 -
194 - if n_nodes != n_ips:
195 - raise RuntimeError(
196 - f"Every node should have a unique IP address. Got {n_nodes}"
197 - f" nodes with node ids {list(node_workers.keys())} and "
198 - f"{n_ips} unique IP addresses {all_ips}. Please check your"
199 - " network configuration. If you set `VLLM_HOST_IP` "
200 - "environment variable, make sure it is unique for"
201 - " each node.")
202 -
203 - # Set environment variables for the driver and workers.
204 - all_args_to_update_environment_variables = [({
205 - "VLLM_TRACE_FUNCTION":
206 - str(envs.VLLM_TRACE_FUNCTION),
207 - }, ) for (node_id, _) in worker_node_and_gpu_ids]
208 - self._run_workers("update_environment_variables",
209 - all_args=all_args_to_update_environment_variables)
210 -
211 - if len(node_gpus) == 1:
212 - # in single node case, we don't need to get the IP address.
213 - # the loopback address is sufficient
214 - # NOTE: a node may have several IP addresses, one for each
215 - # network interface. `get_ip()` might return any of them,
216 - # while they might not work for communication inside the node
217 - # if the network setup is complicated. Using the loopback address
218 - # solves this issue, as it always works for communication inside
219 - # the node.
220 - driver_ip = "127.0.0.1"
221 - distributed_init_method = get_distributed_init_method(
222 - driver_ip, get_open_port())
223 -
224 - # Initialize the actual workers inside worker wrapper.
225 - init_worker_all_kwargs = [
226 - self._get_worker_kwargs(
227 - local_rank=node_workers[node_id].index(rank),
228 - rank=rank,
229 - distributed_init_method=distributed_init_method,
230 - ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
231 - ]
232 - self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
233 -
234 - self._run_workers("init_device")
235 - self._run_workers("load_model",
236 - max_concurrent_workers=self.parallel_config.
237 - max_parallel_loading_workers)
238 -
239 - if self.use_ray_spmd_worker:
240 - for pp_rank in range(self.parallel_config.pipeline_parallel_size):
241 - self.pp_tp_workers.append([])
242 - for tp_rank in range(
243 - self.parallel_config.tensor_parallel_size):
244 - # PP=2, TP=4
245 - # pp_tp_workers = [[0, 1, 2, 3], [4, 5, 6, 7]]
246 - rank = (pp_rank * self.parallel_config.tensor_parallel_size
247 - ) + tp_rank
248 - assert len(self.pp_tp_workers[pp_rank]) == tp_rank
249 - assert pp_rank < len(self.pp_tp_workers)
250 - self.pp_tp_workers[pp_rank].append(self.workers[rank])
251 -
252 - # This is the list of workers that are rank 0 of each TP group EXCEPT
253 - # global rank 0. These are the workers that will broadcast to the
254 - # rest of the workers.
255 - self.tp_driver_workers: List[RayWorkerWrapper] = []
256 - # This is the list of workers that are not drivers and not the first
257 - # worker in a TP group. These are the workers that will be
258 - # broadcasted to.
259 - self.non_driver_workers: List[RayWorkerWrapper] = []
260 -
261 - # Enforce rank order for correct rank to return final output.
262 - for index, worker in enumerate(self.workers):
263 - # The driver worker is rank 0 and not in self.workers.
264 - rank = index + 1
265 - if rank % self.parallel_config.tensor_parallel_size == 0:
266 - self.tp_driver_workers.append(worker)
267 - else:
268 - self.non_driver_workers.append(worker)
269 -
270 - def _driver_execute_model(
271 - self, execute_model_req: Optional[ExecuteModelRequest]
272 - ) -> Optional[List[SamplerOutput]]:
273 - """Run execute_model in the driver worker.
274 -
275 - Passing None will cause the driver to stop the model execution
276 - loop running in each of the remote workers.
277 - """
278 - assert not self.use_ray_spmd_worker, (
279 - "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
280 - return self.driver_worker.execute_method("execute_model",
281 - execute_model_req)
282 -
283 - def execute_model(
284 - self,
285 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
286 - if not self.use_ray_spmd_worker:
287 - return super().execute_model(execute_model_req)
288 -
289 - if self.forward_dag is None:
290 - self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
291 -
292 - serialized_data = self.input_encoder.encode(execute_model_req)
293 - outputs = ray.get(self.forward_dag.execute(serialized_data))
294 - output = self.output_decoder.decode(outputs[0])
295 - return output
296 -
297 - def _run_workers(
298 - self,
299 - method: str,
300 - *args,
301 - async_run_tensor_parallel_workers_only: bool = False,
302 - all_args: Optional[List[Tuple[Any, ...]]] = None,
303 - all_kwargs: Optional[List[Dict[str, Any]]] = None,
304 - max_concurrent_workers: Optional[int] = None,
305 - **kwargs,
306 - ) -> Any:
307 - """Runs the given method on all workers. Can be used in the following
308 - ways:
309 -
310 - Args:
311 - - async_run_tensor_parallel_workers_only: If True the method will be
312 - run only in the remote TP workers, not the driver worker.
313 - It will also be run asynchronously and return a list of futures
314 - rather than blocking on the results.
315 - - args/kwargs: All workers share the same args/kwargs
316 - - all_args/all_kwargs: args/kwargs for each worker are specified
317 - individually
318 - """
319 - if self.use_ray_spmd_worker:
320 - assert not async_run_tensor_parallel_workers_only, (
321 - "async_run_tensor_parallel_workers_only is not supported for "
322 - "spmd mode.")
323 -
324 - if max_concurrent_workers:
325 - raise NotImplementedError(
326 - "max_concurrent_workers is not supported yet.")
327 -
328 - count = len(self.workers) if not \
329 - async_run_tensor_parallel_workers_only \
330 - else len(self.non_driver_workers)
331 - # If using SPMD worker, all workers are the same, so we should execute
332 - # the args on all workers. Otherwise, we skip the first worker's args
333 - # because those args will go to the driver worker.
334 - first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1
335 - all_worker_args = repeat(args, count) if all_args is None \
336 - else islice(all_args, first_worker_args_index, None)
337 - all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
338 - else islice(all_kwargs, first_worker_args_index, None)
339 -
340 - # Start the ray workers first.
341 - ray_workers = self.workers
342 - if async_run_tensor_parallel_workers_only:
343 - ray_workers = self.non_driver_workers
344 - ray_worker_outputs = [
345 - worker.execute_method.remote(method, *worker_args, **worker_kwargs)
346 - for (worker, worker_args, worker_kwargs
347 - ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
348 - ]
349 -
350 - if async_run_tensor_parallel_workers_only:
351 - # Just return futures
352 - return ray_worker_outputs
353 -
354 - driver_worker_output = []
355 - # In SPMD mode, the driver worker is the same as any other worker,
356 - # so we only explicitly execute on the driver worker if using a
357 - # non-SPMD worker class.
358 - if not self.use_ray_spmd_worker:
359 - driver_args = args if all_args is None else all_args[0]
360 - driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
361 -
362 - # Start the driver worker after all the ray workers.
363 - driver_worker_output = [
364 - self.driver_worker.execute_method(method, *driver_args,
365 - **driver_kwargs)
366 - ]
367 -
368 - # Get the results of the ray workers.
369 - if self.workers:
370 - ray_worker_outputs = ray.get(ray_worker_outputs)
371 -
372 - return driver_worker_output + ray_worker_outputs
373 -
374 - def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
375 - """Wait for futures returned from _run_workers() with
376 - async_run_remote_workers_only to complete."""
377 - ray.get(parallel_worker_tasks)
378 -
379 - def _check_ray_adag_installation(self):
380 - import pkg_resources
381 - from packaging import version
382 -
383 - required_version = version.parse("2.35")
384 - current_version = version.parse(
385 - pkg_resources.get_distribution("ray").version)
386 - # TODO: update the constraint once we adapt to the backward
387 - # incompatible API change from ray 2.36
388 - if current_version != required_version:
389 - raise ValueError(f"Ray version {required_version} is "
390 - f"required, but found {current_version}")
391 -
392 - import importlib.util
393 - adag_spec = importlib.util.find_spec(
394 - "ray.experimental.compiled_dag_ref")
395 - if adag_spec is None:
396 - raise ValueError("Ray accelerated DAG is not installed. "
397 - "Run `pip install ray[adag]` to install it.")
398 -
399 - def _compiled_ray_dag(self, enable_asyncio: bool):
400 - assert self.parallel_config.use_ray
401 - self._check_ray_adag_installation()
402 - from ray.dag import InputNode, MultiOutputNode
403 - from ray.experimental.channel.torch_tensor_type import TorchTensorType
404 -
405 - with InputNode() as input_data:
406 - # Example DAG: PP=2, TP=4
407 - # (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501
408 - # -> 1 -> (ExecuteModelReq, IntermediateOutput) -> 5 -> SamplerOutput # noqa: E501
409 - # -> 2 -> (ExecuteModelReq, IntermediateOutput) -> 6 -> SamplerOutput # noqa: E501
410 - # -> 3 -> (ExecuteModelReq, IntermediateOutput) -> 7 -> SamplerOutput # noqa: E501
411 -
412 - # All workers in the first TP group will take in the
413 - # ExecuteModelRequest as input.
414 - outputs = [input_data for _ in self.pp_tp_workers[0]]
415 - for pp_rank, tp_group in enumerate(self.pp_tp_workers):
416 - # Each PP worker takes in the output of the previous PP worker,
417 - # and the TP group executes in SPMD fashion.
418 - outputs = [
419 - worker.execute_model_spmd.
420 - bind( # type: ignore[attr-defined]
421 - outputs[i]) for i, worker in enumerate(tp_group)
422 - ]
423 -
424 - last_pp_rank = len(self.pp_tp_workers) - 1
425 - if pp_rank < last_pp_rank:
426 - # Specify how intermediate tensors should be passed
427 - # between pp stages, no need to specify for the last
428 - # pp stage.
429 - transport = "auto"
430 - outputs = [
431 - output.with_type_hint(
432 - TorchTensorType(transport=transport))
433 - for output in outputs
434 - ]
435 -
436 - forward_dag = MultiOutputNode(outputs)
437 -
438 - return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
439 -
440 - def __del__(self):
441 - self.shutdown()
442 -
443 -
444 -class RayHPUExecutorAsync(RayHPUExecutor, DistributedGPUExecutorAsync):
445 -
446 - def __init__(self, *args, **kwargs):
447 - super().__init__(*args, **kwargs)
448 - self.pp_locks: Optional[List[asyncio.Lock]] = None
449 - self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
450 - if not self.use_ray_compiled_dag:
451 - self.driver_exec_method = make_async(
452 - self.driver_worker.execute_method)
453 -
454 - async def execute_model_async(
455 - self,
456 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
457 - if not self.use_ray_spmd_worker:
458 - return await super().execute_model_async(execute_model_req)
459 -
460 - if self.forward_dag is None:
461 - self.forward_dag = self._compiled_ray_dag(enable_asyncio=True)
462 -
463 - serialized_data = self.input_encoder.encode(execute_model_req)
464 - dag_future = await self.forward_dag.execute_async(serialized_data)
465 - outputs = await dag_future
466 - return self.output_decoder.decode(outputs[0])
467 -
468 - async def _driver_execute_model_async(
469 - self,
470 - execute_model_req: Optional[ExecuteModelRequest] = None
471 - ) -> List[SamplerOutput]:
472 - assert not self.use_ray_spmd_worker, (
473 - "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
474 - if not self.tp_driver_workers:
475 - return await self.driver_exec_method("execute_model",
476 - execute_model_req)
477 - if self.pp_locks is None:
478 - # This locks each pipeline parallel stage so multiple virtual
479 - # engines can't execute on the same stage at the same time
480 - # We create the locks here to avoid creating them in the constructor
481 - # which uses a different asyncio loop.
482 - self.pp_locks = [
483 - asyncio.Lock()
484 - for _ in range(self.parallel_config.pipeline_parallel_size)
485 - ]
486 -
487 - tasks = [
488 - asyncio.create_task(
489 - _run_task_with_lock(self.driver_exec_method, self.pp_locks[0],
490 - "execute_model", execute_model_req))
491 - ]
492 - for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
493 - start=1):
494 - tasks.append(
495 - asyncio.create_task(
496 - _run_task_with_lock(driver_worker.execute_method.remote,
497 - self.pp_locks[pp_rank],
498 - "execute_model", execute_model_req)))
499 -
500 - results = await asyncio.gather(*tasks)
501 -
502 - # Only the last PP stage has the final results.
503 - return results[-1]
504 -
505 - async def _start_worker_execution_loop(self):
506 - assert not self.use_ray_spmd_worker, (
507 - "worker loop is disabled for VLLM_USE_RAY_SPMD_WORKER=1")
508 - coros = [
509 - worker.execute_method.remote("start_worker_execution_loop")
510 - for worker in self.non_driver_workers
511 - ]
512 - return await asyncio.gather(*coros)
513 -
514 - def __del__(self):
515 - self.shutdown()
vllm/executor/ray_tpu_executor.py deleted
-343
@@ -1,343 +0,0 @@
1 -import asyncio
2 -import os
3 -from collections import defaultdict
4 -from itertools import islice, repeat
5 -from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Tuple,
6 - Union)
7 -
8 -import vllm.envs as envs
9 -from vllm.executor.executor_base import ExecutorAsyncBase
10 -from vllm.executor.ray_utils import RayWorkerWrapper, ray
11 -from vllm.executor.tpu_executor import TPUExecutor
12 -from vllm.logger import init_logger
13 -from vllm.model_executor.layers.sampler import SamplerOutput
14 -from vllm.sequence import ExecuteModelRequest
15 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16 - make_async)
17 -
18 -if ray is not None:
19 - from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
20 -
21 -if TYPE_CHECKING:
22 - from ray.util.placement_group import PlacementGroup
23 -
24 -logger = init_logger(__name__)
25 -
26 -
27 -class RayTPUExecutor(TPUExecutor):
28 -
29 - uses_ray: bool = True
30 -
31 - def __init__(self, *args, **kwargs):
32 - # This is non-None when the execute model loop is running
33 - # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
34 - self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
35 - # Updated by implementations that require additional args to be passed
36 - # to the _run_workers execute_model call
37 - self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
38 -
39 - super().__init__(*args, **kwargs)
40 -
41 - def _init_executor(self) -> None:
42 - assert self.parallel_config.distributed_executor_backend == "ray"
43 - placement_group = self.parallel_config.placement_group
44 -
45 - # Disable Ray usage stats collection.
46 - ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
47 - if ray_usage != "1":
48 - os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
49 -
50 - # Create the parallel TPU workers.
51 - self._init_workers_ray(placement_group)
52 -
53 - def _init_workers_ray(self, placement_group: "PlacementGroup",
54 - **ray_remote_kwargs):
55 - # The driver dummy worker does not actually use any resources.
56 - # It holds the resource for the driver worker.
57 - self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
58 - # The remaining workers are the actual ray actors.
59 - self.workers: List[RayWorkerWrapper] = []
60 -
61 - # Create the workers.
62 - driver_ip = get_ip()
63 - for bundle_id, bundle in enumerate(placement_group.bundle_specs):
64 - if not bundle.get("TPU", 0):
65 - continue
66 - scheduling_strategy = PlacementGroupSchedulingStrategy(
67 - placement_group=placement_group,
68 - placement_group_capture_child_tasks=True,
69 - placement_group_bundle_index=bundle_id,
70 - )
71 -
72 - # GKE does not fetch environment information from metadata server
73 - # and instead sets these from within the Ray process. Therefore we
74 - # need to override the Ray environment variables manually.
75 - override_env = {}
76 - if "TPU_CHIPS_PER_HOST_BOUNDS" in os.environ:
77 - override_env.update({
78 - "TPU_CHIPS_PER_HOST_BOUNDS":
79 - os.environ["TPU_CHIPS_PER_HOST_BOUNDS"]
80 - })
81 - if "TPU_HOST_BOUNDS" in os.environ:
82 - override_env.update(
83 - {"TPU_HOST_BOUNDS": os.environ["TPU_HOST_BOUNDS"]})
84 -
85 - worker = ray.remote(
86 - num_cpus=0,
87 - resources={"TPU": 1},
88 - scheduling_strategy=scheduling_strategy,
89 - **ray_remote_kwargs,
90 - )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
91 - if override_env:
92 - worker.override_env_vars.remote(override_env)
93 -
94 - worker_ip = ray.get(worker.get_node_ip.remote())
95 - if worker_ip == driver_ip and self.driver_dummy_worker is None:
96 - # If the worker is on the same node as the driver, we use it
97 - # as the resource holder for the driver process.
98 - self.driver_dummy_worker = worker
99 - self.driver_worker = RayWorkerWrapper(
100 - vllm_config=self.vllm_config)
101 - else:
102 - # Else, added to the list of workers.
103 - self.workers.append(worker)
104 -
105 - logger.debug("workers: %s", self.workers)
106 - logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
107 - if self.driver_dummy_worker is None:
108 - raise ValueError(
109 - "Ray does not allocate any TPUs on the driver node. Consider "
110 - "adjusting the Ray placement group or running the driver on a "
111 - "TPU node.")
112 -
113 - worker_ips = [
114 - ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
115 - for worker in self.workers
116 - ]
117 - ip_counts: Dict[str, int] = {}
118 - for ip in worker_ips:
119 - ip_counts[ip] = ip_counts.get(ip, 0) + 1
120 -
121 - def sort_by_driver_then_worker_ip(worker):
122 - """
123 - Sort the workers based on 3 properties:
124 - 1. If the worker is on the same node as the driver (vllm engine),
125 - it should be placed first.
126 - 2. Then, if the worker is on a node with fewer workers, it should
127 - be placed first.
128 - 3. Finally, if the work is on a node with smaller IP address, it
129 - should be placed first.
130 - """
131 - ip = ray.get(worker.get_node_ip.remote())
132 - return (ip != driver_ip, ip_counts[ip], ip)
133 -
134 - # After sorting, the workers on the same node will be
135 - # close to each other, and the workers on the driver
136 - # node will be placed first.
137 - self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
138 -
139 - # Get the set of TPU IDs used on each node.
140 - worker_node_and_gpu_ids = []
141 - for worker in [self.driver_dummy_worker] + self.workers:
142 - if worker is None:
143 - # driver_dummy_worker can be None when using ray spmd worker.
144 - continue
145 - worker_node_and_gpu_ids.append(
146 - ray.get(worker.get_node_and_gpu_ids.remote()) \
147 - ) # type: ignore
148 -
149 - node_workers = defaultdict(list)
150 - for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
151 - node_workers[node_id].append(i)
152 -
153 - # Set environment variables for the driver and workers.
154 - all_args_to_update_environment_variables = [({
155 - "VLLM_TRACE_FUNCTION":
156 - str(envs.VLLM_TRACE_FUNCTION),
157 - }, ) for _ in worker_node_and_gpu_ids]
158 - self._run_workers("update_environment_variables",
159 - all_args=all_args_to_update_environment_variables)
160 -
161 - if len(node_workers) == 1:
162 - # in single node case, we don't need to get the IP address.
163 - # the loopback address is sufficient
164 - # NOTE: a node may have several IP addresses, one for each
165 - # network interface. `get_ip()` might return any of them,
166 - # while they might not work for communication inside the node
167 - # if the network setup is complicated. Using the loopback address
168 - # solves this issue, as it always works for communication inside
169 - # the node.
170 - driver_ip = "127.0.0.1"
171 - distributed_init_method = get_distributed_init_method(
172 - driver_ip, get_open_port())
173 -
174 - # Initialize the actual workers inside worker wrapper.
175 - init_worker_all_kwargs = [
176 - self._get_worker_kwargs(
177 - local_rank=node_workers[node_id].index(rank),
178 - rank=rank,
179 - distributed_init_method=distributed_init_method,
180 - ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
181 - ]
182 - self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
183 -
184 - self._run_workers("init_device")
185 - self._run_workers("load_model",
186 - max_concurrent_workers=self.parallel_config.
187 - max_parallel_loading_workers)
188 -
189 - def _driver_execute_model(
190 - self,
191 - execute_model_req: Optional[ExecuteModelRequest] = None
192 - ) -> List[SamplerOutput]:
193 - """Run execute_model in the driver worker.
194 -
195 - Passing None will cause the driver to stop the model execution
196 - loop running in each of the remote workers.
197 - """
198 - return self.driver_worker.execute_method("execute_model",
199 - execute_model_req)
200 -
201 - def _run_workers(
202 - self,
203 - method: str,
204 - *args,
205 - async_run_remote_workers_only: bool = False,
206 - all_args: Optional[List[Tuple[Any, ...]]] = None,
207 - all_kwargs: Optional[List[Dict[str, Any]]] = None,
208 - max_concurrent_workers: Optional[int] = None,
209 - use_ray_compiled_dag: bool = False,
210 - **kwargs,
211 - ) -> Any:
212 - """Runs the given method on all workers. Can be used in the following
213 - ways:
214 -
215 - - async_run_remote_workers_only: If True the method will be run only
216 - in the remote workers, not the driver worker. It will also be
217 - run asynchronously and return a list of futures rather than blocking
218 - on the results.
219 - - args/kwargs: All workers share the same args/kwargs
220 - - all_args/all_kwargs: args/kwargs for each worker are specified
221 - individually
222 - """
223 -
224 - if max_concurrent_workers:
225 - raise NotImplementedError(
226 - "max_concurrent_workers is not supported yet.")
227 -
228 - count = len(self.workers)
229 - all_worker_args = repeat(args, count) if all_args is None \
230 - else islice(all_args, 1, None)
231 - all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
232 - else islice(all_kwargs, 1, None)
233 -
234 - # Start the ray workers first.
235 - ray_worker_outputs = [
236 - worker.execute_method.remote(method, *worker_args, **worker_kwargs)
237 - for (worker, worker_args, worker_kwargs
238 - ) in zip(self.workers, all_worker_args, all_worker_kwargs)
239 - ]
240 -
241 - if async_run_remote_workers_only:
242 - # Just return futures
243 - return ray_worker_outputs
244 -
245 - driver_args = args if all_args is None else all_args[0]
246 - driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
247 -
248 - # Start the driver worker after all the ray workers.
249 - driver_worker_output = self.driver_worker.execute_method(
250 - method, *driver_args, **driver_kwargs)
251 - # Get the results of the ray workers.
252 - if self.workers:
253 - ray_worker_outputs = ray.get(ray_worker_outputs)
254 -
255 - return [driver_worker_output] + ray_worker_outputs
256 -
257 - def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
258 - """Wait for futures returned from _run_workers() with
259 - async_run_remote_workers_only to complete."""
260 - ray.get(parallel_worker_tasks)
261 -
262 - def determine_num_available_blocks(self) -> Tuple[int, int]:
263 - num_blocks = self._run_workers("determine_num_available_blocks", )
264 - num_tpu_blocks = min(b[0] for b in num_blocks)
265 - num_cpu_blocks = min(b[1] for b in num_blocks)
266 - return num_tpu_blocks, num_cpu_blocks
267 -
268 - def initialize_cache(self, num_gpu_blocks: int,
269 - num_cpu_blocks: int) -> None:
270 - logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
271 - num_cpu_blocks)
272 - self.cache_config.num_gpu_blocks = num_gpu_blocks
273 - self.cache_config.num_cpu_blocks = num_cpu_blocks
274 - self._run_workers("initialize_cache",
275 - num_gpu_blocks=num_gpu_blocks,
276 - num_cpu_blocks=num_cpu_blocks)
277 -
278 - def execute_model(
279 - self,
280 - execute_model_req: ExecuteModelRequest,
281 - ) -> List[SamplerOutput]:
282 - if self.parallel_worker_tasks is None:
283 - self.parallel_worker_tasks = self._run_workers(
284 - "start_worker_execution_loop",
285 - async_run_remote_workers_only=True,
286 - **self.extra_execute_model_run_workers_kwargs)
287 -
288 - # Only the driver worker returns the sampling results.
289 - return self._driver_execute_model(execute_model_req)
290 -
291 - def stop_remote_worker_execution_loop(self) -> None:
292 - if self.parallel_worker_tasks is None:
293 - return
294 -
295 - self._driver_execute_model()
296 - parallel_worker_tasks = self.parallel_worker_tasks
297 - self.parallel_worker_tasks = None
298 - # Ensure that workers exit model loop cleanly
299 - # (this will raise otherwise)
300 - self._wait_for_tasks_completion(parallel_worker_tasks)
301 -
302 -
303 -class RayTPUExecutorAsync(RayTPUExecutor, ExecutorAsyncBase):
304 -
305 - def __init__(self, *args, **kwargs):
306 - super().__init__(*args, **kwargs)
307 - self.driver_exec_method = make_async(self.driver_worker.execute_method)
308 -
309 - async def execute_model_async(
310 - self,
311 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
312 - if self.parallel_worker_tasks is None:
313 - # Start model execution loop running in the parallel workers
314 - self.parallel_worker_tasks = asyncio.create_task(
315 - self._start_worker_execution_loop())
316 -
317 - # Only the driver worker returns the sampling results.
318 - return await self._driver_execute_model_async(execute_model_req)
319 -
320 - async def stop_remote_worker_execution_loop_async(self) -> None:
321 - if self.parallel_worker_tasks is None:
322 - return
323 -
324 - await self._driver_execute_model_async()
325 - parallel_worker_tasks = self.parallel_worker_tasks
326 - self.parallel_worker_tasks = None
327 - # Ensure that workers exit model loop cleanly
328 - # (this will raise otherwise)
329 - await parallel_worker_tasks
330 -
331 - async def _driver_execute_model_async(
332 - self,
333 - execute_model_req: Optional[ExecuteModelRequest] = None
334 - ) -> List[SamplerOutput]:
335 - return await self.driver_exec_method("execute_model",
336 - execute_model_req)
337 -
338 - async def _start_worker_execution_loop(self):
339 - coros = [
340 - worker.execute_method.remote("start_worker_execution_loop")
341 - for worker in self.workers
342 - ]
343 - return await asyncio.gather(*coros)
vllm/executor/ray_utils.py
+25 -1
@@ -1,7 +1,7 @@
1 import os
2 import time
3 from collections import defaultdict
4 -from typing import Dict, List, Optional, Tuple, Union
4 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
5
6 import msgspec
7
@@ -13,6 +13,10 @@ from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13 from vllm.utils import get_ip
14 from vllm.worker.worker_base import WorkerWrapperBase
15
16 +if TYPE_CHECKING:
17 + from vllm.v1.core.scheduler import SchedulerOutput
18 + from vllm.v1.outputs import ModelRunnerOutput
19 +
20 logger = init_logger(__name__)
21 PG_WAIT_TIMEOUT = 1800
22
@@ -95,6 +99,26 @@ try:
99
100 return output
101
102 + def setup_device_if_necessary(self):
103 + # TODO(swang): This is needed right now because Ray CG executes
104 + # on a background thread, so we need to reset torch's current
105 + # device.
106 + # We can remove this API after it is fixed in compiled graph.
107 + import torch
108 + assert self.worker is not None, "Worker is not initialized"
109 + if not self.compiled_dag_cuda_device_set:
110 + torch.cuda.set_device(self.worker.device)
111 + self.compiled_dag_cuda_device_set = True
112 +
113 + def execute_model(
114 + self,
115 + scheduler_output: "SchedulerOutput",
116 + ) -> "ModelRunnerOutput":
117 + self.setup_device_if_necessary()
118 + assert self.worker is not None, "Worker is not initialized"
119 + output = self.worker.model_runner.execute_model(scheduler_output)
120 + return output
121 +
122 def override_env_vars(self, vars: Dict[str, str]):
123 os.environ.update(vars)
124
vllm/executor/ray_xpu_executor.py deleted
-40
@@ -1,40 +0,0 @@
1 -import asyncio
2 -from typing import List, Optional
3 -
4 -import ray
5 -
6 -import vllm.envs as envs
7 -from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
8 -from vllm.executor.xpu_executor import XPUExecutor
9 -from vllm.logger import init_logger
10 -from vllm.utils import make_async
11 -
12 -logger = init_logger(__name__)
13 -
14 -
15 -class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
16 -
17 - def _get_env_vars_to_be_updated(self):
18 - # Get the set of GPU IDs used on each node.
19 - worker_node_and_gpu_ids = []
20 - for worker in [self.driver_dummy_worker] + self.workers:
21 - if worker is None:
22 - # driver_dummy_worker can be None when using ray spmd worker.
23 - continue
24 - worker_node_and_gpu_ids.append(
25 - ray.get(worker.get_node_and_gpu_ids.remote())) # type: ignore
26 -
27 - # Set environment variables for the driver and workers.
28 - all_args_to_update_environment_variables = [({
29 - "VLLM_TRACE_FUNCTION":
30 - str(envs.VLLM_TRACE_FUNCTION),
31 - }, ) for (_, _) in worker_node_and_gpu_ids]
32 - return all_args_to_update_environment_variables
33 -
34 -
35 -class RayXPUExecutorAsync(RayXPUExecutor, RayGPUExecutorAsync):
36 -
37 - def __init__(self, *args, **kwargs):
38 - super().__init__(*args, **kwargs)
39 - self.driver_exec_method = make_async(self.driver_worker.execute_method)
40 - self.pp_locks: Optional[List[asyncio.Lock]] = None
vllm/executor/tpu_executor.py deleted
-142
@@ -1,142 +0,0 @@
1 -from typing import Any, Dict, List, Optional, Set, Tuple
2 -
3 -import torch
4 -
5 -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 -from vllm.logger import init_logger
7 -from vllm.lora.request import LoRARequest
8 -from vllm.model_executor.layers.sampler import SamplerOutput
9 -from vllm.sequence import ExecuteModelRequest
10 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11 - make_async)
12 -
13 -logger = init_logger(__name__)
14 -
15 -
16 -class TPUExecutor(ExecutorBase):
17 -
18 - uses_ray: bool = False
19 -
20 - def _init_executor(self) -> None:
21 - assert not self.scheduler_config.chunked_prefill_enabled, (
22 - "Chunked prefill is not yet supported for TPU backend")
23 - assert not self.speculative_config, (
24 - "Speculative decoding is not yet supported for TPU backend")
25 - if self.model_config.dtype in (torch.float16, torch.float32):
26 - logger.warning(
27 - "The TPU backend currently does not support %s. "
28 - "Using bfloat16 instead.", self.model_config.dtype)
29 - self.model_config.dtype = torch.bfloat16
30 -
31 - # Instantiate the worker and load the model to the device.
32 - self.driver_worker = self._create_worker()
33 - self.driver_worker.init_device()
34 - self.driver_worker.load_model()
35 -
36 - def _get_worker_kwargs(
37 - self,
38 - local_rank: int = 0,
39 - rank: int = 0,
40 - distributed_init_method: Optional[str] = None,
41 - ) -> Dict[str, Any]:
42 - """Return worker init args for a given rank."""
43 - if distributed_init_method is None:
44 - distributed_init_method = get_distributed_init_method(
45 - get_ip(), get_open_port())
46 - return dict(
47 - vllm_config=self.vllm_config,
48 - local_rank=local_rank,
49 - rank=rank,
50 - distributed_init_method=distributed_init_method,
51 - is_driver_worker=rank == 0,
52 - )
53 -
54 - def _create_worker(
55 - self,
56 - local_rank: int = 0,
57 - rank: int = 0,
58 - distributed_init_method: Optional[str] = None,
59 - ):
60 - if self.scheduler_config.is_multi_step:
61 - from vllm.worker.multi_step_tpu_worker import MultiStepTPUWorker
62 - worker = MultiStepTPUWorker(**self._get_worker_kwargs(
63 - local_rank, rank, distributed_init_method))
64 - return worker
65 - else:
66 - from vllm.worker.tpu_worker import TPUWorker
67 -
68 - worker = TPUWorker(**self._get_worker_kwargs(
69 - local_rank, rank, distributed_init_method))
70 - return worker
71 -
72 - def initialize_cache(
73 - self,
74 - num_gpu_blocks: int,
75 - num_cpu_blocks: int,
76 - ) -> None:
77 - """Initialize the KV cache by invoking the underlying worker."""
78 - # NOTE: This is logged in the executor because there can be >1 worker
79 - # with other executors. We could log in the engine level, but work
80 - # remains to abstract away the device for non-GPU configurations.
81 - logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
82 - num_cpu_blocks)
83 - self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
84 -
85 - def determine_num_available_blocks(self) -> Tuple[int, int]:
86 - """Determine the number of available KV blocks by invoking the
87 - underlying worker."""
88 - return self.driver_worker.determine_num_available_blocks()
89 -
90 - def execute_model(
91 - self,
92 - execute_model_req: ExecuteModelRequest,
93 - ) -> List[SamplerOutput]:
94 - output = self.driver_worker.execute_model(execute_model_req)
95 - return output
96 -
97 - def add_lora(self, lora_request: LoRARequest) -> bool:
98 - raise NotImplementedError(
99 - "LoRA is currently not supported by the TPU backend.")
100 -
101 - def remove_lora(self, lora_id: int) -> bool:
102 - raise NotImplementedError(
103 - "LoRA is currently not supported by the TPU backend.")
104 -
105 - def pin_lora(self, lora_id: int) -> bool:
106 - raise NotImplementedError(
107 - "LoRA is currently not supported by the TPU backend.")
108 -
109 - def list_loras(self) -> Set[int]:
110 - raise NotImplementedError(
111 - "LoRA is currently not supported by the TPU backend.")
112 -
113 - def add_prompt_adapter(self, prompt_adapter_request) -> bool:
114 - raise NotImplementedError(
115 - "Soft prompt is currently not supported by the TPU backend.")
116 -
117 - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
118 - raise NotImplementedError(
119 - "Soft prompt is currently not supported by the TPU backend.")
120 -
121 - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
122 - raise NotImplementedError(
123 - "Soft prompt is currently not supported by the TPU backend.")
124 -
125 - def list_prompt_adapters(self) -> Set[int]:
126 - raise NotImplementedError(
127 - "Soft prompt is currently not supported by the TPU backend.")
128 -
129 - def check_health(self) -> None:
130 - # TPUExecutor will always be healthy as long as it's running.
131 - return
132 -
133 -
134 -class TPUExecutorAsync(TPUExecutor, ExecutorAsyncBase):
135 -
136 - async def execute_model_async(
137 - self,
138 - sexecute_model_req: ExecuteModelRequest,
139 - ) -> SamplerOutput:
140 - output = await make_async(self.driver_worker.execute_model
141 - )(sexecute_model_req)
142 - return output
vllm/executor/uniproc_executor.py new
+57
@@ -0,0 +1,57 @@
1 +from typing import Any, Dict, List, Optional, Tuple
2 +
3 +from vllm.executor.executor_base import ExecutorBase
4 +from vllm.logger import init_logger
5 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6 +from vllm.worker.worker_base import WorkerWrapperBase
7 +
8 +logger = init_logger(__name__)
9 +
10 +
11 +class UniProcExecutor(ExecutorBase):
12 +
13 + uses_ray: bool = False
14 +
15 + def _init_executor(self) -> None:
16 + """Initialize the worker and load the model.
17 + """
18 + self.driver_worker = WorkerWrapperBase(vllm_config=self.vllm_config,
19 + rank=0)
20 + distributed_init_method = get_distributed_init_method(
21 + get_ip(), get_open_port())
22 + local_rank = 0
23 + rank = 0
24 + kwargs = dict(
25 + vllm_config=self.vllm_config,
26 + local_rank=local_rank,
27 + rank=rank,
28 + distributed_init_method=distributed_init_method,
29 + is_driver_worker=(not self.parallel_config)
30 + or (rank % self.parallel_config.tensor_parallel_size == 0),
31 + )
32 + self.collective_rpc("init_worker", args=([kwargs], ))
33 + self.collective_rpc("init_device")
34 + self.collective_rpc("load_model")
35 +
36 + def collective_rpc(self,
37 + method: str,
38 + timeout: Optional[float] = None,
39 + args: Tuple = (),
40 + kwargs: Optional[Dict] = None) -> List[Any]:
41 + if kwargs is None:
42 + kwargs = {}
43 + try:
44 + func = getattr(self.driver_worker, method)
45 + except AttributeError:
46 + raise NotImplementedError(f"Method {method} is not implemented.") \
47 + from None
48 + answer = func(*args, **kwargs)
49 + return [answer]
50 +
51 + def check_health(self) -> None:
52 + # UniProcExecutor will always be healthy as long as
53 + # it's running.
54 + return
55 +
56 +
57 +UniProcExecutorAsync = UniProcExecutor
vllm/executor/xpu_executor.py deleted
-39
@@ -1,39 +0,0 @@
1 -from typing import List, Optional, Union
2 -
3 -from vllm.executor.executor_base import ExecutorAsyncBase
4 -from vllm.executor.gpu_executor import GPUExecutor
5 -from vllm.logger import init_logger
6 -from vllm.model_executor.layers.sampler import SamplerOutput
7 -from vllm.sequence import ExecuteModelRequest, PoolerOutput
8 -from vllm.utils import make_async
9 -
10 -logger = init_logger(__name__)
11 -
12 -
13 -class XPUExecutor(GPUExecutor):
14 -
15 - uses_ray: bool = False
16 -
17 - def _init_executor(self) -> None:
18 - assert self.device_config.device_type == "xpu"
19 - assert self.speculative_config is None, (
20 - "Speculative decoding not yet supported for XPU backend")
21 -
22 - GPUExecutor._init_executor(self)
23 -
24 - def execute_model(
25 - self, execute_model_req: ExecuteModelRequest
26 - ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
27 - output = self.driver_worker.execute_model(execute_model_req)
28 - return output
29 -
30 -
31 -class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase):
32 -
33 - async def execute_model_async(
34 - self,
35 - execute_model_req: ExecuteModelRequest,
36 - ) -> List[SamplerOutput]:
37 - output = await make_async(self.driver_worker.execute_model
38 - )(execute_model_req=execute_model_req)
39 - return output
vllm/platforms/cpu.py
+27
@@ -1,3 +1,4 @@
1 +import os
2 from typing import TYPE_CHECKING, Optional
3
4 import psutil
@@ -105,6 +106,32 @@ class CpuPlatform(Platform):
106 else:
107 parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
108
109 + assert vllm_config.device_config.device_type == "cpu"
110 +
111 + #
112 + # Environment variables for CPU executor
113 + #
114 +
115 + # Disable torch async compiling which won't work with daemonic processes
116 + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
117 +
118 + # Intel OpenMP setting
119 + ld_prealod_str = os.getenv("LD_PRELOAD", "")
120 + if "libiomp5.so" in ld_prealod_str:
121 + # The time(milliseconds) that a thread should wait after
122 + # completing the execution of a parallel region, before sleeping.
123 + os.environ['KMP_BLOCKTIME'] = "1"
124 + # Prevents the CPU to run into low performance state
125 + os.environ['KMP_TPAUSE'] = "0"
126 + # Provides fine granularity parallelism
127 + os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
128 + os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
129 + os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
130 +
131 + # To hint IPEX uses shared memory based AllReduce
132 + os.environ["LOCAL_WORLD_SIZE"] = str(
133 + vllm_config.parallel_config.tensor_parallel_size)
134 +
135 @classmethod
136 def is_pin_memory_available(cls) -> bool:
137 logger.warning("Pin memory is not supported on CPU.")
vllm/platforms/cuda.py
+22
@@ -139,6 +139,28 @@ class CudaPlatformBase(Platform):
139 else:
140 parallel_config.worker_cls = "vllm.worker.worker.Worker"
141
142 + world_size = parallel_config.world_size
143 + tensor_parallel_size = parallel_config.tensor_parallel_size
144 +
145 + from vllm.utils import (cuda_device_count_stateless,
146 + update_environment_variables)
147 +
148 + # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
149 + if "CUDA_VISIBLE_DEVICES" not in os.environ:
150 + update_environment_variables({
151 + "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
152 + })
153 +
154 + cuda_device_count = cuda_device_count_stateless()
155 + # Use confusing message for more common TP-only case.
156 + assert tensor_parallel_size <= cuda_device_count, (
157 + f"please set tensor_parallel_size ({tensor_parallel_size}) "
158 + f"to less than max local gpu count ({cuda_device_count})")
159 +
160 + assert world_size <= cuda_device_count, (
161 + f"please ensure that world_size ({world_size}) "
162 + f"is less than than max local gpu count ({cuda_device_count})")
163 +
164 cache_config = vllm_config.cache_config
165 if cache_config and cache_config.block_size is None:
166 cache_config.block_size = 16
vllm/platforms/neuron.py
+8
@@ -35,6 +35,14 @@ class NeuronPlatform(Platform):
35 parallel_config.worker_cls = \
36 "vllm.worker.neuron_worker.NeuronWorker"
37
38 + if parallel_config.world_size > 1:
39 + parallel_config.distributed_executor_backend = "uni"
40 +
41 + assert (vllm_config.lora_config is
42 + None), "LoRA is not supported for Neuron backend."
43 + assert (not vllm_config.speculative_config
44 + ), "Speculative decoding not yet supported for Neuron backend."
45 +
46 cache_config = vllm_config.cache_config
47 if cache_config:
48 # neuron needs block_size = max_model_len
vllm/platforms/openvino.py
+9 -3
@@ -66,9 +66,8 @@ class OpenVinoPlatform(Platform):
66 from vllm.utils import GiB_bytes
67
68 parallel_config = vllm_config.parallel_config
69 - assert (
70 - parallel_config.world_size == 1
71 - ), "OpenVINOExecutor only supports single CPU socket currently."
69 + assert (parallel_config.world_size == 1
70 + ), "OpenVINO only supports single CPU socket currently."
71
72 if parallel_config.worker_cls == "auto":
73 parallel_config.worker_cls = \
@@ -141,3 +140,10 @@ class OpenVinoPlatform(Platform):
140 raise RuntimeError(
141 "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
142 f" {kv_cache_space}, expect a positive integer value.")
143 +
144 + assert vllm_config.device_config.device_type == "openvino"
145 + assert vllm_config.lora_config is None, \
146 + "OpenVINO backend doesn't support LoRA"
147 + assert cls.is_openvino_cpu() or \
148 + cls.is_openvino_gpu(), \
149 + "OpenVINO backend supports only CPU and GPU devices"
vllm/platforms/tpu.py
+10
@@ -72,6 +72,16 @@ class TpuPlatform(Platform):
72 assert vllm_config.speculative_config is None, \
73 "TPU does not support speculative decoding"
74
75 + assert not vllm_config.scheduler_config.chunked_prefill_enabled, (
76 + "Chunked prefill is not yet supported for TPU backend")
77 + assert not vllm_config.speculative_config, (
78 + "Speculative decoding is not yet supported for TPU backend")
79 + if vllm_config.model_config.dtype in (torch.float16, torch.float32):
80 + logger.warning(
81 + "The TPU backend currently does not support %s. "
82 + "Using bfloat16 instead.", vllm_config.model_config.dtype)
83 + vllm_config.model_config.dtype = torch.bfloat16
84 +
85 parallel_config = vllm_config.parallel_config
86 scheduler_config = vllm_config.scheduler_config
87 if parallel_config.worker_cls == "auto":
vllm/platforms/xpu.py
+18 -4
@@ -78,17 +78,31 @@ class XPUPlatform(Platform):
78 raise NotImplementedError(
79 "XPU does not support speculative decoding")
80
81 + if vllm_config.device_config is not None:
82 + assert vllm_config.device_config.device_type == "xpu"
83 +
84 # check and update parallel config
85 parallel_config = vllm_config.parallel_config
83 - if (parallel_config.distributed_executor_backend is not None
84 - and parallel_config.distributed_executor_backend != "ray"):
86 + if parallel_config.worker_cls == "auto":
87 + parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
88 +
89 + if parallel_config.distributed_executor_backend is None:
90 + parallel_config.distributed_executor_backend = "ray"
91 + elif parallel_config.distributed_executor_backend == "mp":
92 + # FIXME(kunshang):
93 + # spawn needs calling `if __name__ == '__main__':``
94 + # fork is not supported for xpu start new process.
95 + logger.error(
96 + "Both start methods (spawn and fork) have issue "
97 + "on XPU if you use mp backend, setting it to ray instead.")
98 + parallel_config.distributed_executor_backend = "ray"
99 +
100 + elif parallel_config.distributed_executor_backend != "ray":
101 logger.warning(
102 "%s is not supported on XPU, fallback to ray distributed"
103 " executor backend.",
104 parallel_config.distributed_executor_backend)
105 parallel_config.distributed_executor_backend = "ray"
90 - if parallel_config.worker_cls == "auto":
91 - parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
106
107 @classmethod
108 def is_pin_memory_available(cls):
vllm/spec_decode/medusa_worker.py
+3 -5
@@ -9,17 +9,15 @@ from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
9 from vllm.spec_decode.interfaces import SpeculativeProposals
10 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11 from vllm.spec_decode.top1_proposer import Top1Proposer
12 -from vllm.worker.worker_base import WorkerWrapperBase
12 +from vllm.worker.worker_base import DelegateWorkerBase
13
14
15 -class MedusaWorker(NonLLMProposerWorkerBase, WorkerWrapperBase):
15 +class MedusaWorker(NonLLMProposerWorkerBase, DelegateWorkerBase):
16 """Worker for Medusa.
17 """
18
19 def __init__(self, *args, **kwargs):
20 - super().__init__(kwargs.get("vllm_config"))
21 - self.init_worker(*args, **kwargs)
22 -
20 + DelegateWorkerBase.__init__(self, *args, **kwargs)
21 # Lazy initialization list.
22 self._proposer: Top1Proposer
23
vllm/spec_decode/multi_step_worker.py
+3 -18
@@ -16,10 +16,10 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
16 SpeculativeProposer)
17 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
18 from vllm.spec_decode.top1_proposer import Top1Proposer
19 -from vllm.worker.worker_base import WorkerWrapperBase
19 +from vllm.worker.worker_base import DelegateWorkerBase
20
21
22 -class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
22 +class MultiStepWorker(ProposerWorkerBase, DelegateWorkerBase):
23 """The MultiStepWorker is equivalent to a Worker except that it allows
24 multiple forward passes in a single call, assuming the scheduler has
25 allocated enough space to store the additional KV. This reduces overhead
@@ -32,15 +32,12 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
32 """
33
34 def __init__(self, *args, **kwargs):
35 - super().__init__(kwargs.get("vllm_config"))
36 - self.init_worker(*args, **kwargs)
37 -
35 + DelegateWorkerBase.__init__(self, *args, **kwargs)
36 # Lazy initialization list.
37 self._proposer: SpeculativeProposer
38
39 def init_device(self) -> None:
40 self.worker.init_device()
43 -
41 self._proposer = Top1Proposer(
42 weakref.proxy(self), # type: ignore[arg-type]
43 self.device,
@@ -56,18 +53,6 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
53 self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
54 True)
55
59 - def determine_num_available_blocks(self) -> Tuple[int, int]:
60 - return self.worker.determine_num_available_blocks()
61 -
62 - def get_cache_block_size_bytes(self) -> int:
63 - return self.worker.get_cache_block_size_bytes()
64 -
65 - def initialize_cache(self, *args, **kwargs) -> None:
66 - self.worker.initialize_cache(*args, **kwargs)
67 -
68 - def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
69 - return self.worker.execute_model(*args, **kwargs)
70 -
56 @torch.inference_mode()
57 def sampler_output(
58 self,
vllm/spec_decode/spec_decode_worker.py
+5 -4
@@ -40,8 +40,8 @@ from vllm.spec_decode.util import (Timer, create_logprobs_output,
40 get_all_num_logprobs,
41 get_sampled_token_logprobs, nvtx_range,
42 split_batch_by_proposal_len)
43 -from vllm.worker.worker_base import (LoraNotSupportedWorkerBase, WorkerBase,
44 - WorkerWrapperBase)
43 +from vllm.utils import resolve_obj_by_qualname
44 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
45
46 logger = init_logger(__name__)
47
@@ -64,8 +64,9 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
64 target_worker_config = copy.deepcopy(vllm_config)
65 target_worker_config.parallel_config.worker_cls =\
66 target_worker_config.parallel_config.sd_worker_cls
67 - target_worker = WorkerWrapperBase(vllm_config=target_worker_config)
68 - target_worker.init_worker(*args, **kwargs)
67 + cls = resolve_obj_by_qualname(
68 + target_worker_config.parallel_config.worker_cls)
69 + target_worker = cls(*args, **kwargs)
70 # Set the disable_logprobs variable in the TargetModelRunner instance
71 # as per its value specified in the SpeculativeConfig.
72 target_worker.model_runner.disable_logprobs =\
vllm/v1/executor/abstract.py
+3 -2
@@ -14,8 +14,9 @@ class Executor(ABC):
14 distributed_executor_backend = (
15 vllm_config.parallel_config.distributed_executor_backend)
16 if distributed_executor_backend == "ray":
17 - from vllm.v1.executor.ray_executor import RayExecutor
18 - executor_class = RayExecutor
17 + from vllm.executor.ray_distributed_executor import ( # noqa
18 + RayDistributedExecutor)
19 + executor_class = RayDistributedExecutor
20 elif distributed_executor_backend == "mp":
21 from vllm.v1.executor.multiproc_executor import MultiprocExecutor
22 executor_class = MultiprocExecutor
vllm/v1/executor/multiproc_executor.py
+13 -4
@@ -246,9 +246,18 @@ class WorkerProc:
246 ready_path: str,
247 ):
248 self.rank = rank
249 - wrapper = WorkerWrapperBase(vllm_config=vllm_config)
250 - wrapper.init_worker(vllm_config, local_rank, rank,
251 - distributed_init_method)
249 + wrapper = WorkerWrapperBase(vllm_config=vllm_config, rank=rank)
250 + # TODO: move `init_worker` to executor level as a collective rpc call
251 + all_kwargs: List[Dict] = [
252 + {} for _ in range(vllm_config.parallel_config.world_size)
253 + ]
254 + all_kwargs[rank] = {
255 + "vllm_config": vllm_config,
256 + "local_rank": local_rank,
257 + "rank": rank,
258 + "distributed_init_method": distributed_init_method,
259 + }
260 + wrapper.init_worker(all_kwargs)
261 self.worker = wrapper.worker
262
263 pid = os.getpid()
@@ -270,7 +279,7 @@ class WorkerProc:
279 ready_socket.send_string(WorkerProc.READY_STR)
280 ready_socket.send(payload)
281
273 - self.worker.initialize()
282 + self.worker.init_device()
283 self.worker.load_model()
284
285 @staticmethod
vllm/v1/executor/uniproc_executor.py
+1 -1
@@ -27,7 +27,7 @@ class UniprocExecutor(Executor):
27 self.observability_config = vllm_config.observability_config
28
29 self.worker: Worker = self._create_worker()
30 - self.worker.initialize()
30 + self.worker.init_device()
31 self.worker.load_model()
32
33 def _create_worker(
vllm/v1/worker/gpu_worker.py
+2 -1
@@ -33,6 +33,7 @@ class Worker:
33 local_rank: int,
34 rank: int,
35 distributed_init_method: str,
36 + is_driver_worker: bool = False,
37 ):
38
39 # TODO: use WorkerBase.__init__(self, vllm_config=vllm_config)
@@ -75,7 +76,7 @@ class Worker:
76 else:
77 self.profiler = None
78
78 - def initialize(self):
79 + def init_device(self):
80 if self.device_config.device.type == "cuda":
81 # torch.distributed.all_reduce does not free the input tensor until
82 # the synchronization point. This causes the memory usage to grow
vllm/worker/hpu_worker.py
+66
@@ -2,6 +2,7 @@
2 # Copyright (C) 2024 Habana Labs, Ltd. an Intel Company
3 ###############################################################################
4
5 +import contextlib
6 import gc
7 import os
8 from typing import List, Optional, Set, Tuple, Type
@@ -18,6 +19,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
19 from vllm.logger import init_logger
20 from vllm.lora.request import LoRARequest
21 from vllm.model_executor import set_random_seed
22 +from vllm.model_executor.layers.sampler import SamplerOutput
23 from vllm.prompt_adapter.request import PromptAdapterRequest
24 from vllm.sequence import ExecuteModelRequest
25 from vllm.utils import bind_kv_cache
@@ -124,6 +126,70 @@ class HPUWorker(LocalOrDistributedWorkerBase):
126 def load_model(self):
127 self.model_runner.load_model()
128
129 + def execute_model(
130 + self,
131 + execute_model_req: Optional[ExecuteModelRequest] = None,
132 + ) -> Optional[List[SamplerOutput]]:
133 + assert execute_model_req is not None
134 + # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION - will log graph compilations per engine step, only when there was any - highly recommended to use alongside PT_HPU_METRICS_GC_DETAILS! # noqa:E501
135 + # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL - will log graph compilations per engine step, always, even if there were none # noqa:E501
136 + # VLLM_HPU_LOG_STEP_CPU_FALLBACKS - will log cpu fallbacks per engine step, only when there was any # noqa:E501
137 + # VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL - will log cpu fallbacks per engine step, always, even if there were none # noqa:E501
138 + log_graph_compilation_all = os.environ.get(
139 + 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL', '0') != '0'
140 + log_graph_compilation = os.environ.get(
141 + 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION',
142 + '0') != '0' or log_graph_compilation_all
143 + log_cpu_fallbacks_all = os.environ.get(
144 + 'VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL', '0') != '0'
145 + log_cpu_fallbacks = os.environ.get('VLLM_HPU_LOG_STEP_CPU_FALLBACKS',
146 + '0') != '0' or log_cpu_fallbacks_all
147 + if log_graph_compilation or log_cpu_fallbacks:
148 + from habana_frameworks.torch.hpu.metrics import metric_localcontext
149 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
150 + is_prompt = any([
151 + seq_group_metadata.is_prompt
152 + for seq_group_metadata in seq_group_metadata_list
153 + ])
154 + max_context_len = max([
155 + max([
156 + len(v.prompt_token_ids) + len(v.output_token_ids)
157 + for v in seq_group_metadata.seq_data.values()
158 + ]) for seq_group_metadata in seq_group_metadata_list
159 + ]) # whoa, that's some spicy stuff right here
160 + max_num_blocks = (
161 + (max_context_len - 1) // self.cache_config.block_size) + 1
162 + input_stats = (f'is_prompt: {is_prompt}, '
163 + f'num_seqs: {len(seq_group_metadata_list)}, '
164 + f'max_context_len: {max_context_len}, '
165 + f'max_num_blocks {max_num_blocks}')
166 + gc_ctx = metric_localcontext(
167 + "graph_compilation"
168 + ) if log_graph_compilation else contextlib.nullcontext()
169 + cpu_fallback_ctx = metric_localcontext(
170 + "cpu_fallback"
171 + ) if log_cpu_fallbacks else contextlib.nullcontext()
172 + with gc_ctx as gc_local_metric, \
173 + cpu_fallback_ctx as cpu_fallback_local_metric:
174 + output = LocalOrDistributedWorkerBase.execute_model(
175 + self, execute_model_req)
176 + if (log_graph_compilation and gc_local_metric.stats()[0][1] > 0
177 + ) or log_graph_compilation_all:
178 + msg = ("VLLM_HPU_STEP_GRAPH_COMPILATION: "
179 + f"{gc_local_metric.stats()}, {input_stats}")
180 + logger.warning(msg)
181 + if (log_cpu_fallbacks and cpu_fallback_local_metric.stats()[0][1] >
182 + 0) or log_cpu_fallbacks_all:
183 + msg = ("VLLM_HPU_STEP_CPU_FALLBACK: "
184 + f"{cpu_fallback_local_metric.stats()}, {input_stats}")
185 + logger.warning(msg)
186 +
187 + return output
188 +
189 + output = LocalOrDistributedWorkerBase.execute_model(
190 + self, execute_model_req)
191 + return output
192 +
193 @torch.inference_mode()
194 def determine_num_available_blocks(self) -> Tuple[int, int]:
195 """Profiles the peak memory usage of the model to determine how many
vllm/worker/neuron_worker.py
+23 -5
@@ -8,6 +8,7 @@ from vllm.config import VllmConfig
8 from vllm.distributed import (ensure_model_parallel_initialized,
9 init_distributed_environment)
10 from vllm.model_executor import set_random_seed
11 +from vllm.model_executor.layers.sampler import SamplerOutput
12 from vllm.sequence import ExecuteModelRequest
13 from vllm.worker.neuron_model_runner import NeuronModelRunner
14 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
@@ -25,6 +26,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
26 local_rank: int,
27 rank: int,
28 distributed_init_method: str,
29 + is_driver_worker: bool = True,
30 ) -> None:
31 WorkerBase.__init__(self, vllm_config=vllm_config)
32 self.local_rank = local_rank
@@ -37,7 +39,22 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
39
40 self.model_runner: NeuronModelRunner = NeuronModelRunner(
41 vllm_config=vllm_config)
40 - self.is_driver_worker = True
42 + self.is_driver_worker = is_driver_worker
43 +
44 + def execute_model(
45 + self,
46 + execute_model_req: Optional[ExecuteModelRequest] = None,
47 + ) -> Optional[List[SamplerOutput]]:
48 + assert execute_model_req is not None
49 + assert (not execute_model_req.blocks_to_swap_in
50 + and not execute_model_req.blocks_to_swap_out
51 + and not execute_model_req.blocks_to_copy), (
52 + "Cache operations are not supported for Neuron backend.")
53 + assert execute_model_req.num_lookahead_slots == 0, (
54 + "lookahead not supported for Neuron backend.")
55 + output = LocalOrDistributedWorkerBase.execute_model(
56 + self, execute_model_req)
57 + return output
58
59 def init_device(self) -> None:
60 self.init_distributed_environment()
@@ -103,13 +120,14 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
120
121 def init_distributed_environment(self):
122 """Neuron uses transformers-neuronx for tensor parallelism.
106 -
107 - vLLM still needs the environment inited when TP/PP > 1
123 + It has only one process to control multiple devices.
124 + vLLM still needs the environment initialized when TP/PP > 1,
125 + so we initialize a distributed environment with one process.
126 """
127 init_distributed_environment(
128 world_size=1,
111 - rank=self.rank,
112 - local_rank=self.local_rank,
129 + rank=0,
130 + local_rank=0,
131 distributed_init_method=self.distributed_init_method,
132 backend="gloo",
133 )
vllm/worker/openvino_worker.py
+2 -4
@@ -211,16 +211,14 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
211
212 def __init__(
213 self,
214 - ov_core: ov.Core,
214 vllm_config: VllmConfig,
215 local_rank: int,
216 rank: int,
217 distributed_init_method: str,
219 - kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined,
218 is_driver_worker: bool = False,
219 ) -> None:
222 - self.ov_core = ov_core
220 WorkerBase.__init__(self, vllm_config)
221 + self.ov_core = ov.Core()
222 self.parallel_config.rank = rank
223 self.local_rank = local_rank
224 self.rank = rank
@@ -237,7 +235,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
235 self.model_runner = OpenVINOModelRunner(
236 self.ov_core,
237 vllm_config=self.vllm_config,
240 - kv_cache_dtype=kv_cache_dtype,
238 + kv_cache_dtype=self.vllm_config.cache_config.cache_dtype,
239 is_driver_worker=is_driver_worker,
240 )
241 # Uninitialized cache engine. Will be initialized by
vllm/worker/worker_base.py
+76 -10
@@ -88,7 +88,6 @@ class WorkerBase(ABC):
88 if output is None:
89 return None
90
91 - @abstractmethod
91 def execute_model(
92 self,
93 execute_model_req: Optional[ExecuteModelRequest] = None
@@ -119,6 +118,58 @@ class WorkerBase(ABC):
118 raise NotImplementedError
119
120
121 +class DelegateWorkerBase(WorkerBase):
122 + """
123 + A class that delegates all methods to another WorkerBase instance. This is
124 + useful for creating a WorkerBase that wraps another WorkerBase instance,
125 + e.g. speculative decoding.
126 + """
127 + worker: WorkerBase
128 +
129 + def __init__(
130 + self,
131 + *args,
132 + **kwargs,
133 + ) -> None:
134 + vllm_config: VllmConfig = kwargs.get("vllm_config")
135 + cls = resolve_obj_by_qualname(vllm_config.parallel_config.worker_cls)
136 + self.worker = cls(*args, **kwargs)
137 +
138 + def init_device(self) -> None:
139 + self.worker.init_device()
140 +
141 + def determine_num_available_blocks(self) -> Tuple[int, int]:
142 + return self.worker.determine_num_available_blocks()
143 +
144 + def initialize_cache(self, num_gpu_blocks: int,
145 + num_cpu_blocks: int) -> None:
146 + self.worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
147 +
148 + def execute_model(
149 + self,
150 + execute_model_req: Optional[ExecuteModelRequest] = None
151 + ) -> Optional[List[SamplerOutput]]:
152 + return self.worker.execute_model(execute_model_req)
153 +
154 + def get_cache_block_size_bytes(self) -> int:
155 + return self.worker.get_cache_block_size_bytes()
156 +
157 + def add_lora(self, lora_request: LoRARequest) -> bool:
158 + return self.worker.add_lora(lora_request)
159 +
160 + def remove_lora(self, lora_id: int) -> bool:
161 + return self.worker.remove_lora(lora_id)
162 +
163 + def pin_lora(self, lora_id: int) -> bool:
164 + return self.worker.pin_lora(lora_id)
165 +
166 + def list_loras(self) -> Set[int]:
167 + return self.worker.list_loras()
168 +
169 + def __getattr__(self, attr):
170 + return getattr(self.worker, attr)
171 +
172 +
173 class LoraNotSupportedWorkerBase(WorkerBase):
174 """Partial implementation of WorkerBase that raises exceptions when LoRA
175 methods are invoked.
@@ -419,17 +470,31 @@ class WorkerWrapperBase:
470 def __init__(
471 self,
472 vllm_config: VllmConfig,
473 + rank: int = 0,
474 ) -> None:
475 + self.rank = rank
476 self.vllm_config = vllm_config
424 - trust_remote_code = vllm_config.model_config.trust_remote_code
477 self.worker: Optional[WorkerBase] = None
426 - if trust_remote_code:
427 - # note: lazy import to avoid importing torch before initializing
428 - from vllm.utils import init_cached_hf_modules
429 - init_cached_hf_modules()
478 + if vllm_config.model_config is not None:
479 + # it can be None in tests
480 + trust_remote_code = vllm_config.model_config.trust_remote_code
481 + if trust_remote_code:
482 + # note: lazy import to avoid importing torch before initializing
483 + from vllm.utils import init_cached_hf_modules
484 + init_cached_hf_modules()
485 +
486 + def adjust_rank(self, rank_mapping: Dict[int, int]) -> None:
487 + """
488 + Adjust the rank based on the given mapping.
489 + It is only used during the initialization of the executor,
490 + to adjust the rank of workers after we create all workers.
491 + """
492 + if self.rank in rank_mapping:
493 + self.rank = rank_mapping[self.rank]
494
431 - @staticmethod
432 - def update_environment_variables(envs: Dict[str, str]) -> None:
495 + def update_environment_variables(self, envs_list: List[Dict[str,
496 + str]]) -> None:
497 + envs = envs_list[self.rank]
498 key = 'CUDA_VISIBLE_DEVICES'
499 if key in envs and key in os.environ:
500 # overwriting CUDA_VISIBLE_DEVICES is desired behavior
@@ -437,11 +502,12 @@ class WorkerWrapperBase:
502 del os.environ[key]
503 update_environment_variables(envs)
504
440 - def init_worker(self, *args, **kwargs):
505 + def init_worker(self, all_kwargs: List[Dict[str, Any]]) -> None:
506 """
507 Here we inject some common logic before initializing the worker.
508 Arguments are passed to the worker class constructor.
509 """
510 + kwargs = all_kwargs[self.rank]
511 enable_trace_function_call_for_thread(self.vllm_config)
512
513 # see https://github.com/NVIDIA/nccl/issues/1234
@@ -452,7 +518,7 @@ class WorkerWrapperBase:
518
519 worker_class = resolve_obj_by_qualname(
520 self.vllm_config.parallel_config.worker_cls)
455 - self.worker = worker_class(*args, **kwargs)
521 + self.worker = worker_class(**kwargs)
522 assert self.worker is not None
523
524 def execute_method(self, method: str, *args, **kwargs):